예제 #1
0
    def get_deactivate_list(self):
        logger = logging.getLogger('essarch.storagemaintenance')
        # Create unique obj_list
        obj_list = []
        for obj in self.object_list_with_writetapes:
            if not any(d['ObjectUUID'] == obj['ObjectUUID'] for d in obj_list):
                obj_list.append(obj)

        # Add sm_list(sm+storage) to obj_list
        for num, obj in enumerate(obj_list):

            #Prepare storage method list
            sm_objs = []
            current_mediumid_search = self.dt_data.get('sSearch_%s' % '4',
                                                       'xxx')
            logger.debug('col4 serach: %s' % current_mediumid_search)

            # Check whether the criteria for replacement of media target prefix is met
            if len(current_mediumid_search
                   ) == 8 and current_mediumid_search.startswith(
                       '-') and current_mediumid_search.__contains__('+'):
                media_target_replace_flag = 1
                # add corresponding new target medium prefix
                sm_obj = sm()
                sm_obj.id = 1
                sm_obj.status = 1
                sm_obj.type = 300
                sm_obj.target = current_mediumid_search[5:8]
                sm_objs.append(sm_obj)
                # add old target medium prefix
                sm_obj = sm()
                sm_obj.id = 2
                sm_obj.status = 0  # Set status to 0 for SM when items on this target medium is replaced
                sm_obj.type = 300
                sm_obj.target = current_mediumid_search[1:4]
                sm_objs.append(sm_obj)
            else:
                media_target_replace_flag = 0
                ArchivePolicy_obj = ArchivePolicy.objects.get(
                    PolicyID=obj['PolicyId__PolicyID'])
                StorageMethod_objs = ArchivePolicy_obj.storagemethod_set.filter(
                    status=1)
                for StorageMethod_obj in StorageMethod_objs:
                    st_objs = StorageMethod_obj.storagetarget_set.filter(
                        status=1)
                    if st_objs.count() == 1:
                        st_obj = st_objs[0]
                    elif st_objs.count() == 0:
                        logger.error(
                            'The storage method %s has no enabled target configured'
                            % sm_obj.name)
                        continue
                    elif st_objs.count() > 1:
                        logger.error(
                            'The storage method %s has too many targets configured with the status enabled'
                            % sm_obj.name)
                        continue
                    if st_obj.target.status == 1:
                        target_obj = st_obj.target
                    else:
                        logger.error('The target %s is disabled' %
                                     st_obj.target.name)
                        continue
                    sm_obj = sm()
                    sm_obj.id = StorageMethod_obj.id
                    sm_obj.status = st_obj.status
                    sm_obj.type = target_obj.type
                    sm_obj.target = target_obj.target
                    sm_objs.append(sm_obj)
                '''
                for i in [1,2,3,4]:
                    sm_obj = sm()
                    sm_obj.id = i
                    sm_obj.status = obj['PolicyId__sm_%s' % i]
                    sm_obj.type = obj['PolicyId__sm_type_%s' % i]
                    #sm_obj.format = getattr(ep_obj,'sm_format_%s' % i)
                    #sm_obj.blocksize = getattr(ep_obj,'sm_blocksize_%s' % i)
                    #sm_obj.maxCapacity = getattr(ep_obj,'sm_maxCapacity_%s' % i)
                    #sm_obj.minChunkSize = getattr(ep_obj,'sm_minChunkSize_%s' % i)
                    #sm_obj.minContainerSize = getattr(ep_obj,'sm_minContainerSize_%s' % i)
                    #sm_obj.minCapacityWarning = getattr(ep_obj,'sm_minCapacityWarning_%s' % i)
                    sm_obj.target = obj['PolicyId__sm_target_%s' % i]
                    sm_objs.append(sm_obj)
            '''
            sm_list = []
            if media_target_replace_flag:
                storage_list = []

            for sm_obj in sm_objs:
                storage_list = []
                if sm_obj.status == 1:
                    for d in self.object_list_with_writetapes:
                        if d['Storage_set__storagemedium__storageMediumID'] is not None:
                            if (sm_obj.type in range(300,330) and
                                d['Storage_set__storagemedium__storageMediumID'].startswith(sm_obj.target) and
                                d['ObjectUUID'] == obj['ObjectUUID']
                                ) or\
                                (media_target_replace_flag and sm_obj.type == 300 and
                                d['Storage_set__storagemedium__storageMediumID'].startswith(current_mediumid_search[1:4]) and
                                d['ObjectUUID'] == obj['ObjectUUID']
                                ) or\
                               (sm_obj.type == 200 and
                                d['Storage_set__storagemedium__storageMediumID'] == 'disk' and
                                d['ObjectUUID'] == obj['ObjectUUID']
                                ):
                                storage_list.append({
                                    'storagemedium__storageMediumID':
                                    d['Storage_set__storagemedium__storageMediumID'],
                                    'storagemedium__CreateDate':
                                    d['Storage_set__storagemedium__CreateDate'],
                                    'contentLocationValue':
                                    d['Storage_set__contentLocationValue'],
                                    'archiveobject__ObjectIdentifierValue':
                                    obj['ObjectIdentifierValue'],
                                    'archiveobject__ObjectUUID':
                                    obj['ObjectUUID'],
                                })
                                #print 'd - storage__storageMediumUUID__storageMediumID: %s' % d['storage__storageMediumUUID__storageMediumID']
                                #print 'o - storage__storageMediumUUID__storageMediumID: %s' % obj['storage__storageMediumUUID__storageMediumID']
                    #print 'ObjectUUID: %s, target:%s , s_count:%s' % (obj['ObjectUUID'],sm_obj.target,len(storage_list))
                    sm_list.append({
                        'id': sm_obj.id,
                        'status': sm_obj.status,
                        #'type': sm_obj.type,
                        'target': sm_obj.target,
                        'storage_list': storage_list
                    })

            for x in sm_list:
                logger.debug('sm_list_x: %s' % repr(x))

            obj_list[num]['sm_list'] = sm_list

        # Create redundant storage list
        redundant_storage_list = {}
        for obj in obj_list:
            for sm_obj in obj['sm_list']:
                active_storage_obj = None
                if len(sm_obj['storage_list']) > 1:
                    #print '##################################################################################### %s, count:%s' % (sm_obj['storage_list'],len(sm_obj['storage_list']))
                    for storage_obj in sm_obj['storage_list']:
                        if active_storage_obj is None:
                            active_storage_obj = storage_obj
                        elif storage_obj[
                                'storagemedium__CreateDate'] > active_storage_obj[
                                    'storagemedium__CreateDate']:
                            active_storage_obj = storage_obj
                    for storage_obj in sm_obj['storage_list']:
                        if storage_obj[
                                'storagemedium__CreateDate'] < active_storage_obj[
                                    'storagemedium__CreateDate']:
                            if not storage_obj[
                                    'storagemedium__storageMediumID'] in redundant_storage_list.keys(
                                    ):
                                redundant_storage_list[storage_obj[
                                    'storagemedium__storageMediumID']] = []
                            redundant_storage_list[storage_obj[
                                'storagemedium__storageMediumID']].append(
                                    storage_obj)

        logger.debug('redundant_storage_list: %s' %
                     repr(redundant_storage_list))

        # Create deactivate_media_list and need_to_migrate_dict
        deactivate_media_list = []
        need_to_migrate_list = []
        #need_to_migrate_dict = {}
        for storageMediumID in redundant_storage_list.keys():
            storage_list = storage.objects.exclude(
                storagemedium__storageMediumStatus=0).filter(
                    storagemedium__storageMediumID=storageMediumID).values(
                        'storagemedium__storageMediumID',
                        'storagemedium__CreateDate',
                        'contentLocationValue',
                        'archiveobject__ObjectIdentifierValue',
                        'archiveobject__ObjectUUID',
                    )
            storage_list2 = list(storage_list)
            for storage_values in storage_list:
                #print 'storage_list_len loop: %s' % len(storage_list)
                for redundant_storage_values in redundant_storage_list[
                        storageMediumID]:
                    if storage_values == redundant_storage_values:
                        #print 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyfound ObjectUUID: %s, storageMediumID: %s, contentLocationValue: %s' % (redundant_storage_values['ObjectUUID__ObjectUUID'],storageMediumID,redundant_storage_values['contentLocationValue'])
                        #print 'storage_list2_len before: %s' % len(storage_list2)
                        #print 'storage_values: %s' % storage_values
                        if storage_values in storage_list2:
                            storage_list2.remove(storage_values)
                            #print 'remove %s' % storage_values
                        #else:
                        #print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!not found!, skip remove %s' % storage_values
                        #print 'storage_list2_len after: %s' % len(storage_list2)
                        #pass
            if len(storage_list2) == 0:
                deactivate_media_list.append([
                    '', '', '', '', storageMediumID, '', '', '', '', '', '',
                    '', '', '', '', '', ''
                ])
            else:
                #need_to_migrate_dict[storageMediumID] = storage_list2
                for m in storage_list2:
                    tmp_list = []
                    keys = [
                        'storagemedium__storageMediumID',
                        'storagemedium__CreateDate', 'contentLocationValue',
                        'archiveobject__ObjectIdentifierValue',
                        'archiveobject__ObjectUUID'
                    ]
                    for key in keys:
                        tmp_list.append(m[key])
                    need_to_migrate_list.append(tmp_list)

        #print '#####################################obj_list: %s count: %s' % (obj_list,len(obj_list))
        #print '*************************************redundant_list: %s count: %s' % (redundant_storage_list,len(redundant_storage_list))
        #print '*************************************redundant_list: %s count: %s, storage_count: %s' % (redundant_storage_list,len(redundant_storage_list),len(redundant_storage_list['ESA001']))
        #deactivate_media_list.append(['ESA001'])
        #print 'deactivate_media_list: %s' % deactivate_media_list
        #print 'need_to_migrate_list: %s' % need_to_migrate_list
        return deactivate_media_list, need_to_migrate_list
예제 #2
0
파일: views.py 프로젝트: shsdev/ESSArch_EPP
    def get_deactivate_list(self):
        logger = logging.getLogger('essarch.storagemaintenance')
        #
        # Create unique obj_list
        obj_list = []
        for obj in self.object_list_with_writetapes:
        #for obj in self.object_list:
            if not any(d['ObjectUUID'] == obj['ObjectUUID'] for d in obj_list):
                obj_list.append(obj)
        
        # Add sm_list(sm+storage) to obj_list 
        for num, obj in enumerate(obj_list):
            
            #Prepare storage method list
            sm_objs = []

            #current_mediumid_search = self.dt_data['sSearch_%s' % '4']
            current_mediumid_search = self.dt_data.get('sSearch_%s' % '4','xxx')
            logger.debug('col4 serach: %s' % current_mediumid_search)

            # Check whether the criteria for replacement of media target prefix is met
            if len(current_mediumid_search) == 8 and current_mediumid_search.startswith('-') and current_mediumid_search.__contains__('+'):
                media_target_replace_flag = 1
                # add corresponding new target medium prefix
                sm_obj = sm()
                sm_obj.id = 1
                sm_obj.status = 1
                sm_obj.type = 300
                sm_obj.target = current_mediumid_search[5:8]
                sm_objs.append(sm_obj)
                # add old target medium prefix
                sm_obj = sm()
                sm_obj.id = 2
                sm_obj.status = 0 # Set status to 0 for SM when items on this target medium is replaced
                sm_obj.type = 300
                sm_obj.target = current_mediumid_search[1:4]
                sm_objs.append(sm_obj)
            else:
                media_target_replace_flag = 0    
                for i in [1,2,3,4]:
                    sm_obj = sm()
                    sm_obj.id = i
                    sm_obj.status = obj['PolicyId__sm_%s' % i]
                    sm_obj.type = obj['PolicyId__sm_type_%s' % i]
                    #sm_obj.format = getattr(ep_obj,'sm_format_%s' % i)
                    #sm_obj.blocksize = getattr(ep_obj,'sm_blocksize_%s' % i)
                    #sm_obj.maxCapacity = getattr(ep_obj,'sm_maxCapacity_%s' % i)
                    #sm_obj.minChunkSize = getattr(ep_obj,'sm_minChunkSize_%s' % i)
                    #sm_obj.minContainerSize = getattr(ep_obj,'sm_minContainerSize_%s' % i)
                    #sm_obj.minCapacityWarning = getattr(ep_obj,'sm_minCapacityWarning_%s' % i)
                    sm_obj.target = obj['PolicyId__sm_target_%s' % i]
                    sm_objs.append(sm_obj)
                            
            sm_list = []
            if media_target_replace_flag:
                storage_list = []
                
            for sm_obj in sm_objs:
                storage_list = []
                if sm_obj.status == 1:
                    for d in self.object_list_with_writetapes:
                        if d['storage__storageMediumUUID__storageMediumID'] is not None:
                            if (sm_obj.type in range(300,330) and
                                d['storage__storageMediumUUID__storageMediumID'].startswith(sm_obj.target) and
                                d['ObjectUUID'] == obj['ObjectUUID']
                                ) or\
                                (media_target_replace_flag and sm_obj.type == 300 and
                                d['storage__storageMediumUUID__storageMediumID'].startswith(current_mediumid_search[1:4]) and
                                d['ObjectUUID'] == obj['ObjectUUID']
                                ) or\
                               (sm_obj.type == 200 and
                                d['storage__storageMediumUUID__storageMediumID'] == 'disk' and
                                d['ObjectUUID'] == obj['ObjectUUID']
                                ):
                                    storage_list.append({'storageMediumUUID__storageMediumID': d['storage__storageMediumUUID__storageMediumID'],
                                                         #'storageMediumUUID__storageMedium': sm_obj.type,
                                                         #'storageMediumUUID__storageMediumDate': d['storage__storageMediumUUID__storageMediumDate'],
                                                         'storageMediumUUID__CreateDate': d['storage__storageMediumUUID__CreateDate'],
                                                         'contentLocationValue': d['storage__contentLocationValue'],
                                                         'ObjectUUID__ObjectIdentifierValue': obj['ObjectIdentifierValue'],
                                                         'ObjectUUID__ObjectUUID': obj['ObjectUUID'],
                                                         })
                                    #print 'd - storage__storageMediumUUID__storageMediumID: %s' % d['storage__storageMediumUUID__storageMediumID']
                                    #print 'o - storage__storageMediumUUID__storageMediumID: %s' % obj['storage__storageMediumUUID__storageMediumID']
                    #print 'ObjectUUID: %s, target:%s , s_count:%s' % (obj['ObjectUUID'],sm_obj.target,len(storage_list))
                    sm_list.append({'id': sm_obj.id,
                                    'status': sm_obj.status,
                                    #'type': sm_obj.type,
                                    'target': sm_obj.target,
                                    'storage_list': storage_list})
                
            for x in sm_list:
                logger.debug('sm_list_x: %s' % repr(x))

            obj_list[num]['sm_list'] = sm_list
        
        # Create redundant storage list
        redundant_storage_list = {}
        for obj in obj_list:
            for sm_obj in obj['sm_list']:
                active_storage_obj = None
                if len(sm_obj['storage_list']) > 1:
                    #print '##################################################################################### %s, count:%s' % (sm_obj['storage_list'],len(sm_obj['storage_list']))
                    for storage_obj in sm_obj['storage_list']:
                        if active_storage_obj is None:
                            active_storage_obj = storage_obj
                        #elif storage_obj['storageMediumUUID__storageMediumDate'] > active_storage_obj['storageMediumUUID__storageMediumDate']:
                        elif storage_obj['storageMediumUUID__CreateDate'] > active_storage_obj['storageMediumUUID__CreateDate']:
                            active_storage_obj = storage_obj
                    for storage_obj in sm_obj['storage_list']:
                        #if storage_obj['storageMediumUUID__storageMediumDate'] < active_storage_obj['storageMediumUUID__storageMediumDate']:
                        if storage_obj['storageMediumUUID__CreateDate'] < active_storage_obj['storageMediumUUID__CreateDate']:
                            if not storage_obj['storageMediumUUID__storageMediumID'] in redundant_storage_list.keys():
                                redundant_storage_list[storage_obj['storageMediumUUID__storageMediumID']] = []
                            redundant_storage_list[storage_obj['storageMediumUUID__storageMediumID']].append(storage_obj)
        
        logger.debug('redundant_storage_list: %s' % repr(redundant_storage_list))

        # Create deactivate_media_list and need_to_migrate_dict
        deactivate_media_list = []
        need_to_migrate_list = []
        #need_to_migrate_dict = {}
        for storageMediumID in redundant_storage_list.keys():
            storage_list = storage.objects.exclude(storageMediumUUID__storageMediumStatus=0).filter(storageMediumUUID__storageMediumID=storageMediumID).values('storageMediumUUID__storageMediumID',
                                                                                                          #'storageMediumUUID__storageMedium',
                                                                                                          #'storageMediumUUID__storageMediumDate',
                                                                                                          'storageMediumUUID__CreateDate',
                                                                                                          'contentLocationValue',
                                                                                                          'ObjectUUID__ObjectIdentifierValue',
                                                                                                          'ObjectUUID__ObjectUUID',
                                                                                                          )
            storage_list2 = list(storage_list)
            for storage_values in storage_list:
                #print 'storage_list_len loop: %s' % len(storage_list)
                for redundant_storage_values in redundant_storage_list[storageMediumID]:
                    if storage_values == redundant_storage_values:
                        #print 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyfound ObjectUUID: %s, storageMediumID: %s, contentLocationValue: %s' % (redundant_storage_values['ObjectUUID__ObjectUUID'],storageMediumID,redundant_storage_values['contentLocationValue'])
                        #print 'storage_list2_len before: %s' % len(storage_list2)
                        #print 'storage_values: %s' % storage_values
                        if storage_values in storage_list2:
                            storage_list2.remove(storage_values)
                            #print 'remove %s' % storage_values
                        #else:
                            #print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!not found!, skip remove %s' % storage_values
                        #print 'storage_list2_len after: %s' % len(storage_list2)
                        #pass
            if len(storage_list2) == 0:
                #deactivate_media_list.append([storageMediumID])
                deactivate_media_list.append(['','','','',storageMediumID,'','','','','','','','','','','',''])
            else:
                #need_to_migrate_dict[storageMediumID] = storage_list2
                for m in storage_list2:
                    tmp_list = []
                    keys = ['storageMediumUUID__storageMediumID',
                          #'storageMediumUUID__storageMedium',
                          #'storageMediumUUID__storageMediumDate',
                          'storageMediumUUID__CreateDate',
                          'contentLocationValue',
                          'ObjectUUID__ObjectIdentifierValue',
                          'ObjectUUID__ObjectUUID']
                    for key in keys:
                        tmp_list.append(m[key])
                    need_to_migrate_list.append(tmp_list)
                                               
        #print '#####################################obj_list: %s count: %s' % (obj_list,len(obj_list))
        #print '*************************************redundant_list: %s count: %s' % (redundant_storage_list,len(redundant_storage_list))
        #print '*************************************redundant_list: %s count: %s, storage_count: %s' % (redundant_storage_list,len(redundant_storage_list),len(redundant_storage_list['ESA001']))
        #deactivate_media_list.append(['ESA001'])
        #print 'deactivate_media_list: %s' % deactivate_media_list
        #print 'need_to_migrate_list: %s' % need_to_migrate_list
        return deactivate_media_list, need_to_migrate_list
예제 #3
0
파일: tasks.py 프로젝트: shsdev/ESSArch_EPP
    def calculate_result(self, email="admin"):

        tasks = robot.objects.all()
        tasks_todo = list(tasks)
        num_tasks = len(tasks)

        ArchTape_list = []
        WriteTape_list = []
        EmptyTape_list = []
        ErrorTape_list = []
        AddTape_list = []

        # Let folks know we started
        self.update_progress(0, num_tasks)

        # Populating MinTape_dict and target_dict
        target_dict = {}
        MinTape_dict = {}
        ESSArchPolicy_objs = ESSArchPolicy.objects.filter(PolicyStat=1)
        for ep_obj in ESSArchPolicy_objs:
            sm_objs = []
            for i in [1, 2, 3, 4]:
                sm_obj = sm()
                sm_obj.id = i
                sm_obj.status = getattr(ep_obj, "sm_%s" % i)
                sm_obj.type = getattr(ep_obj, "sm_type_%s" % i)
                sm_obj.format = getattr(ep_obj, "sm_format_%s" % i)
                sm_obj.blocksize = getattr(ep_obj, "sm_blocksize_%s" % i)
                sm_obj.maxCapacity = getattr(ep_obj, "sm_maxCapacity_%s" % i)
                sm_obj.minChunkSize = getattr(ep_obj, "sm_minChunkSize_%s" % i)
                sm_obj.minContainerSize = getattr(ep_obj, "sm_minContainerSize_%s" % i)
                sm_obj.minCapacityWarning = getattr(ep_obj, "sm_minCapacityWarning_%s" % i)
                sm_obj.target = getattr(ep_obj, "sm_target_%s" % i)
                sm_objs.append(sm_obj)

            for sm_obj in sm_objs:
                if sm_obj.status == 1 and sm_obj.minCapacityWarning > 0:
                    try:
                        if not MinTape_dict[sm_obj.target] > sm_obj.minCapacityWarning:
                            MinTape_dict[sm_obj.target] = sm_obj.minCapacityWarning
                    except KeyError:
                        MinTape_dict[sm_obj.target] = sm_obj.minCapacityWarning
                        target_dict[sm_obj.target] = 0

        # Create all tasks
        for counter, task in enumerate(tasks):
            alarm_flag = None
            alarm_sub = ""
            alarm_msg = ""
            if task == tasks_todo.pop(0):
                if task.status == "ArchTape":
                    event_info = "%s - Slot:%s" % (task.t_id, task.slot_id)
                    ArchTape_list.append(event_info)
                elif task.status == "WriteTape":
                    event_info = "%s - Slot:%s" % (task.t_id, task.slot_id)
                    WriteTape_list.append(event_info)
                elif task.status == "Ready":
                    event_info = "%s - Slot:%s" % (task.t_id, task.slot_id)
                    EmptyTape_list.append(event_info)
                    for target in target_dict.keys():
                        if task.t_id.startswith(target):
                            target_dict[target] += 1
            else:
                event_info = "check proc logfile: %s but task not match pop..." % task
                logger.error(event_info)

            # event_info = 'sleep 10'
            # logger.debug(event_info)
            # time.sleep(10)
            self.update_progress(counter, num_tasks, update_frequency=1)

        # Check if minimum number of new tapes is available
        for target in target_dict.keys():
            if target_dict[target] < MinTape_dict[target]:
                AddTape_list.append(target)

        # Populating ErrorTape_list
        error_tape_objs = storageMedium.objects.exclude(storageMediumStatus=0).filter(storageMediumStatus__gt=30)
        for error_tape_obj in error_tape_objs:
            slot_id = None
            for task in tasks:
                if error_tape_obj.storageMediumID == task.t_id:
                    slot_id = task.slot_id
                    break
            event_info = "%s - Slot:%s" % (error_tape_obj.storageMediumID, slot_id)
            ErrorTape_list.append(event_info)

        if email == "admin":
            # Send mail report to admins
            timestamp = datetime.now()
            alarm_sub = "Daily media report %s" % timestamp
            alarm_msg = "%s\n" % alarm_sub
            alarm_msg += "\nArchive tapes online:\n"
            for item in ArchTape_list:
                alarm_msg += "%s\n" % item
            alarm_msg += "\nAdd new tapes starting with:\n"
            for item in AddTape_list:
                alarm_msg += "%s\n" % item
            alarm_msg += "\nWrite tapes:\n"
            for item in WriteTape_list:
                alarm_msg += "%s\n" % item
            alarm_msg += "\nEmpty tapes:\n"
            for item in EmptyTape_list:
                alarm_msg += "%s\n" % item
            alarm_msg += "\nError tapes:\n"
            for item in ErrorTape_list:
                alarm_msg += "%s\n" % item
            mail_admins(alarm_sub, alarm_msg, fail_silently=False)

        return {
            "ArchTape": ArchTape_list,
            "WriteTape": WriteTape_list,
            "EmptyTape": EmptyTape_list,
            "ErrorTape": ErrorTape_list,
            "AddTape": AddTape_list,
        }