def get_deactivate_list(self):
logger = logging.getLogger('essarch.storagemaintenance')
# Create unique obj_list
obj_list = []
for obj in self.object_list_with_writetapes:
if not any(d['ObjectUUID'] == obj['ObjectUUID'] for d in obj_list):
obj_list.append(obj)
# Add sm_list(sm+storage) to obj_list
for num, obj in enumerate(obj_list):
#Prepare storage method list
sm_objs = []
current_mediumid_search = self.dt_data.get('sSearch_%s' % '4',
'xxx')
logger.debug('col4 serach: %s' % current_mediumid_search)
# Check whether the criteria for replacement of media target prefix is met
if len(current_mediumid_search
) == 8 and current_mediumid_search.startswith(
'-') and current_mediumid_search.__contains__('+'):
media_target_replace_flag = 1
# add corresponding new target medium prefix
sm_obj = sm()
sm_obj.id = 1
sm_obj.status = 1
sm_obj.type = 300
sm_obj.target = current_mediumid_search[5:8]
sm_objs.append(sm_obj)
# add old target medium prefix
sm_obj = sm()
sm_obj.id = 2
sm_obj.status = 0 # Set status to 0 for SM when items on this target medium is replaced
sm_obj.type = 300
sm_obj.target = current_mediumid_search[1:4]
sm_objs.append(sm_obj)
else:
media_target_replace_flag = 0
ArchivePolicy_obj = ArchivePolicy.objects.get(
PolicyID=obj['PolicyId__PolicyID'])
StorageMethod_objs = ArchivePolicy_obj.storagemethod_set.filter(
status=1)
for StorageMethod_obj in StorageMethod_objs:
st_objs = StorageMethod_obj.storagetarget_set.filter(
status=1)
if st_objs.count() == 1:
st_obj = st_objs[0]
elif st_objs.count() == 0:
logger.error(
'The storage method %s has no enabled target configured'
% sm_obj.name)
continue
elif st_objs.count() > 1:
logger.error(
'The storage method %s has too many targets configured with the status enabled'
% sm_obj.name)
continue
if st_obj.target.status == 1:
target_obj = st_obj.target
else:
logger.error('The target %s is disabled' %
st_obj.target.name)
continue
sm_obj = sm()
sm_obj.id = StorageMethod_obj.id
sm_obj.status = st_obj.status
sm_obj.type = target_obj.type
sm_obj.target = target_obj.target
sm_objs.append(sm_obj)
'''
for i in [1,2,3,4]:
sm_obj = sm()
sm_obj.id = i
sm_obj.status = obj['PolicyId__sm_%s' % i]
sm_obj.type = obj['PolicyId__sm_type_%s' % i]
#sm_obj.format = getattr(ep_obj,'sm_format_%s' % i)
#sm_obj.blocksize = getattr(ep_obj,'sm_blocksize_%s' % i)
#sm_obj.maxCapacity = getattr(ep_obj,'sm_maxCapacity_%s' % i)
#sm_obj.minChunkSize = getattr(ep_obj,'sm_minChunkSize_%s' % i)
#sm_obj.minContainerSize = getattr(ep_obj,'sm_minContainerSize_%s' % i)
#sm_obj.minCapacityWarning = getattr(ep_obj,'sm_minCapacityWarning_%s' % i)
sm_obj.target = obj['PolicyId__sm_target_%s' % i]
sm_objs.append(sm_obj)
'''
sm_list = []
if media_target_replace_flag:
storage_list = []
for sm_obj in sm_objs:
storage_list = []
if sm_obj.status == 1:
for d in self.object_list_with_writetapes:
if d['Storage_set__storagemedium__storageMediumID'] is not None:
if (sm_obj.type in range(300,330) and
d['Storage_set__storagemedium__storageMediumID'].startswith(sm_obj.target) and
d['ObjectUUID'] == obj['ObjectUUID']
) or\
(media_target_replace_flag and sm_obj.type == 300 and
d['Storage_set__storagemedium__storageMediumID'].startswith(current_mediumid_search[1:4]) and
d['ObjectUUID'] == obj['ObjectUUID']
) or\
(sm_obj.type == 200 and
d['Storage_set__storagemedium__storageMediumID'] == 'disk' and
d['ObjectUUID'] == obj['ObjectUUID']
):
storage_list.append({
'storagemedium__storageMediumID':
d['Storage_set__storagemedium__storageMediumID'],
'storagemedium__CreateDate':
d['Storage_set__storagemedium__CreateDate'],
'contentLocationValue':
d['Storage_set__contentLocationValue'],
'archiveobject__ObjectIdentifierValue':
obj['ObjectIdentifierValue'],
'archiveobject__ObjectUUID':
obj['ObjectUUID'],
})
#print 'd - storage__storageMediumUUID__storageMediumID: %s' % d['storage__storageMediumUUID__storageMediumID']
#print 'o - storage__storageMediumUUID__storageMediumID: %s' % obj['storage__storageMediumUUID__storageMediumID']
#print 'ObjectUUID: %s, target:%s , s_count:%s' % (obj['ObjectUUID'],sm_obj.target,len(storage_list))
sm_list.append({
'id': sm_obj.id,
'status': sm_obj.status,
#'type': sm_obj.type,
'target': sm_obj.target,
'storage_list': storage_list
})
for x in sm_list:
logger.debug('sm_list_x: %s' % repr(x))
obj_list[num]['sm_list'] = sm_list
# Create redundant storage list
redundant_storage_list = {}
for obj in obj_list:
for sm_obj in obj['sm_list']:
active_storage_obj = None
if len(sm_obj['storage_list']) > 1:
#print '##################################################################################### %s, count:%s' % (sm_obj['storage_list'],len(sm_obj['storage_list']))
for storage_obj in sm_obj['storage_list']:
if active_storage_obj is None:
active_storage_obj = storage_obj
elif storage_obj[
'storagemedium__CreateDate'] > active_storage_obj[
'storagemedium__CreateDate']:
active_storage_obj = storage_obj
for storage_obj in sm_obj['storage_list']:
if storage_obj[
'storagemedium__CreateDate'] < active_storage_obj[
'storagemedium__CreateDate']:
if not storage_obj[
'storagemedium__storageMediumID'] in redundant_storage_list.keys(
):
redundant_storage_list[storage_obj[
'storagemedium__storageMediumID']] = []
redundant_storage_list[storage_obj[
'storagemedium__storageMediumID']].append(
storage_obj)
logger.debug('redundant_storage_list: %s' %
repr(redundant_storage_list))
# Create deactivate_media_list and need_to_migrate_dict
deactivate_media_list = []
need_to_migrate_list = []
#need_to_migrate_dict = {}
for storageMediumID in redundant_storage_list.keys():
storage_list = storage.objects.exclude(
storagemedium__storageMediumStatus=0).filter(
storagemedium__storageMediumID=storageMediumID).values(
'storagemedium__storageMediumID',
'storagemedium__CreateDate',
'contentLocationValue',
'archiveobject__ObjectIdentifierValue',
'archiveobject__ObjectUUID',
)
storage_list2 = list(storage_list)
for storage_values in storage_list:
#print 'storage_list_len loop: %s' % len(storage_list)
for redundant_storage_values in redundant_storage_list[
storageMediumID]:
if storage_values == redundant_storage_values:
#print 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyfound ObjectUUID: %s, storageMediumID: %s, contentLocationValue: %s' % (redundant_storage_values['ObjectUUID__ObjectUUID'],storageMediumID,redundant_storage_values['contentLocationValue'])
#print 'storage_list2_len before: %s' % len(storage_list2)
#print 'storage_values: %s' % storage_values
if storage_values in storage_list2:
storage_list2.remove(storage_values)
#print 'remove %s' % storage_values
#else:
#print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!not found!, skip remove %s' % storage_values
#print 'storage_list2_len after: %s' % len(storage_list2)
#pass
if len(storage_list2) == 0:
deactivate_media_list.append([
'', '', '', '', storageMediumID, '', '', '', '', '', '',
'', '', '', '', '', ''
])
else:
#need_to_migrate_dict[storageMediumID] = storage_list2
for m in storage_list2:
tmp_list = []
keys = [
'storagemedium__storageMediumID',
'storagemedium__CreateDate', 'contentLocationValue',
'archiveobject__ObjectIdentifierValue',
'archiveobject__ObjectUUID'
]
for key in keys:
tmp_list.append(m[key])
need_to_migrate_list.append(tmp_list)
#print '#####################################obj_list: %s count: %s' % (obj_list,len(obj_list))
#print '*************************************redundant_list: %s count: %s' % (redundant_storage_list,len(redundant_storage_list))
#print '*************************************redundant_list: %s count: %s, storage_count: %s' % (redundant_storage_list,len(redundant_storage_list),len(redundant_storage_list['ESA001']))
#deactivate_media_list.append(['ESA001'])
#print 'deactivate_media_list: %s' % deactivate_media_list
#print 'need_to_migrate_list: %s' % need_to_migrate_list
return deactivate_media_list, need_to_migrate_list
def get_deactivate_list(self):
logger = logging.getLogger('essarch.storagemaintenance')
#
# Create unique obj_list
obj_list = []
for obj in self.object_list_with_writetapes:
#for obj in self.object_list:
if not any(d['ObjectUUID'] == obj['ObjectUUID'] for d in obj_list):
obj_list.append(obj)
# Add sm_list(sm+storage) to obj_list
for num, obj in enumerate(obj_list):
#Prepare storage method list
sm_objs = []
#current_mediumid_search = self.dt_data['sSearch_%s' % '4']
current_mediumid_search = self.dt_data.get('sSearch_%s' % '4','xxx')
logger.debug('col4 serach: %s' % current_mediumid_search)
# Check whether the criteria for replacement of media target prefix is met
if len(current_mediumid_search) == 8 and current_mediumid_search.startswith('-') and current_mediumid_search.__contains__('+'):
media_target_replace_flag = 1
# add corresponding new target medium prefix
sm_obj = sm()
sm_obj.id = 1
sm_obj.status = 1
sm_obj.type = 300
sm_obj.target = current_mediumid_search[5:8]
sm_objs.append(sm_obj)
# add old target medium prefix
sm_obj = sm()
sm_obj.id = 2
sm_obj.status = 0 # Set status to 0 for SM when items on this target medium is replaced
sm_obj.type = 300
sm_obj.target = current_mediumid_search[1:4]
sm_objs.append(sm_obj)
else:
media_target_replace_flag = 0
for i in [1,2,3,4]:
sm_obj = sm()
sm_obj.id = i
sm_obj.status = obj['PolicyId__sm_%s' % i]
sm_obj.type = obj['PolicyId__sm_type_%s' % i]
#sm_obj.format = getattr(ep_obj,'sm_format_%s' % i)
#sm_obj.blocksize = getattr(ep_obj,'sm_blocksize_%s' % i)
#sm_obj.maxCapacity = getattr(ep_obj,'sm_maxCapacity_%s' % i)
#sm_obj.minChunkSize = getattr(ep_obj,'sm_minChunkSize_%s' % i)
#sm_obj.minContainerSize = getattr(ep_obj,'sm_minContainerSize_%s' % i)
#sm_obj.minCapacityWarning = getattr(ep_obj,'sm_minCapacityWarning_%s' % i)
sm_obj.target = obj['PolicyId__sm_target_%s' % i]
sm_objs.append(sm_obj)
sm_list = []
if media_target_replace_flag:
storage_list = []
for sm_obj in sm_objs:
storage_list = []
if sm_obj.status == 1:
for d in self.object_list_with_writetapes:
if d['storage__storageMediumUUID__storageMediumID'] is not None:
if (sm_obj.type in range(300,330) and
d['storage__storageMediumUUID__storageMediumID'].startswith(sm_obj.target) and
d['ObjectUUID'] == obj['ObjectUUID']
) or\
(media_target_replace_flag and sm_obj.type == 300 and
d['storage__storageMediumUUID__storageMediumID'].startswith(current_mediumid_search[1:4]) and
d['ObjectUUID'] == obj['ObjectUUID']
) or\
(sm_obj.type == 200 and
d['storage__storageMediumUUID__storageMediumID'] == 'disk' and
d['ObjectUUID'] == obj['ObjectUUID']
):
storage_list.append({'storageMediumUUID__storageMediumID': d['storage__storageMediumUUID__storageMediumID'],
#'storageMediumUUID__storageMedium': sm_obj.type,
#'storageMediumUUID__storageMediumDate': d['storage__storageMediumUUID__storageMediumDate'],
'storageMediumUUID__CreateDate': d['storage__storageMediumUUID__CreateDate'],
'contentLocationValue': d['storage__contentLocationValue'],
'ObjectUUID__ObjectIdentifierValue': obj['ObjectIdentifierValue'],
'ObjectUUID__ObjectUUID': obj['ObjectUUID'],
})
#print 'd - storage__storageMediumUUID__storageMediumID: %s' % d['storage__storageMediumUUID__storageMediumID']
#print 'o - storage__storageMediumUUID__storageMediumID: %s' % obj['storage__storageMediumUUID__storageMediumID']
#print 'ObjectUUID: %s, target:%s , s_count:%s' % (obj['ObjectUUID'],sm_obj.target,len(storage_list))
sm_list.append({'id': sm_obj.id,
'status': sm_obj.status,
#'type': sm_obj.type,
'target': sm_obj.target,
'storage_list': storage_list})
for x in sm_list:
logger.debug('sm_list_x: %s' % repr(x))
obj_list[num]['sm_list'] = sm_list
# Create redundant storage list
redundant_storage_list = {}
for obj in obj_list:
for sm_obj in obj['sm_list']:
active_storage_obj = None
if len(sm_obj['storage_list']) > 1:
#print '##################################################################################### %s, count:%s' % (sm_obj['storage_list'],len(sm_obj['storage_list']))
for storage_obj in sm_obj['storage_list']:
if active_storage_obj is None:
active_storage_obj = storage_obj
#elif storage_obj['storageMediumUUID__storageMediumDate'] > active_storage_obj['storageMediumUUID__storageMediumDate']:
elif storage_obj['storageMediumUUID__CreateDate'] > active_storage_obj['storageMediumUUID__CreateDate']:
active_storage_obj = storage_obj
for storage_obj in sm_obj['storage_list']:
#if storage_obj['storageMediumUUID__storageMediumDate'] < active_storage_obj['storageMediumUUID__storageMediumDate']:
if storage_obj['storageMediumUUID__CreateDate'] < active_storage_obj['storageMediumUUID__CreateDate']:
if not storage_obj['storageMediumUUID__storageMediumID'] in redundant_storage_list.keys():
redundant_storage_list[storage_obj['storageMediumUUID__storageMediumID']] = []
redundant_storage_list[storage_obj['storageMediumUUID__storageMediumID']].append(storage_obj)
logger.debug('redundant_storage_list: %s' % repr(redundant_storage_list))
# Create deactivate_media_list and need_to_migrate_dict
deactivate_media_list = []
need_to_migrate_list = []
#need_to_migrate_dict = {}
for storageMediumID in redundant_storage_list.keys():
storage_list = storage.objects.exclude(storageMediumUUID__storageMediumStatus=0).filter(storageMediumUUID__storageMediumID=storageMediumID).values('storageMediumUUID__storageMediumID',
#'storageMediumUUID__storageMedium',
#'storageMediumUUID__storageMediumDate',
'storageMediumUUID__CreateDate',
'contentLocationValue',
'ObjectUUID__ObjectIdentifierValue',
'ObjectUUID__ObjectUUID',
)
storage_list2 = list(storage_list)
for storage_values in storage_list:
#print 'storage_list_len loop: %s' % len(storage_list)
for redundant_storage_values in redundant_storage_list[storageMediumID]:
if storage_values == redundant_storage_values:
#print 'yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyfound ObjectUUID: %s, storageMediumID: %s, contentLocationValue: %s' % (redundant_storage_values['ObjectUUID__ObjectUUID'],storageMediumID,redundant_storage_values['contentLocationValue'])
#print 'storage_list2_len before: %s' % len(storage_list2)
#print 'storage_values: %s' % storage_values
if storage_values in storage_list2:
storage_list2.remove(storage_values)
#print 'remove %s' % storage_values
#else:
#print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!not found!, skip remove %s' % storage_values
#print 'storage_list2_len after: %s' % len(storage_list2)
#pass
if len(storage_list2) == 0:
#deactivate_media_list.append([storageMediumID])
deactivate_media_list.append(['','','','',storageMediumID,'','','','','','','','','','','',''])
else:
#need_to_migrate_dict[storageMediumID] = storage_list2
for m in storage_list2:
tmp_list = []
keys = ['storageMediumUUID__storageMediumID',
#'storageMediumUUID__storageMedium',
#'storageMediumUUID__storageMediumDate',
'storageMediumUUID__CreateDate',
'contentLocationValue',
'ObjectUUID__ObjectIdentifierValue',
'ObjectUUID__ObjectUUID']
for key in keys:
tmp_list.append(m[key])
need_to_migrate_list.append(tmp_list)
#print '#####################################obj_list: %s count: %s' % (obj_list,len(obj_list))
#print '*************************************redundant_list: %s count: %s' % (redundant_storage_list,len(redundant_storage_list))
#print '*************************************redundant_list: %s count: %s, storage_count: %s' % (redundant_storage_list,len(redundant_storage_list),len(redundant_storage_list['ESA001']))
#deactivate_media_list.append(['ESA001'])
#print 'deactivate_media_list: %s' % deactivate_media_list
#print 'need_to_migrate_list: %s' % need_to_migrate_list
return deactivate_media_list, need_to_migrate_list
def calculate_result(self, email="admin"):
tasks = robot.objects.all()
tasks_todo = list(tasks)
num_tasks = len(tasks)
ArchTape_list = []
WriteTape_list = []
EmptyTape_list = []
ErrorTape_list = []
AddTape_list = []
# Let folks know we started
self.update_progress(0, num_tasks)
# Populating MinTape_dict and target_dict
target_dict = {}
MinTape_dict = {}
ESSArchPolicy_objs = ESSArchPolicy.objects.filter(PolicyStat=1)
for ep_obj in ESSArchPolicy_objs:
sm_objs = []
for i in [1, 2, 3, 4]:
sm_obj = sm()
sm_obj.id = i
sm_obj.status = getattr(ep_obj, "sm_%s" % i)
sm_obj.type = getattr(ep_obj, "sm_type_%s" % i)
sm_obj.format = getattr(ep_obj, "sm_format_%s" % i)
sm_obj.blocksize = getattr(ep_obj, "sm_blocksize_%s" % i)
sm_obj.maxCapacity = getattr(ep_obj, "sm_maxCapacity_%s" % i)
sm_obj.minChunkSize = getattr(ep_obj, "sm_minChunkSize_%s" % i)
sm_obj.minContainerSize = getattr(ep_obj, "sm_minContainerSize_%s" % i)
sm_obj.minCapacityWarning = getattr(ep_obj, "sm_minCapacityWarning_%s" % i)
sm_obj.target = getattr(ep_obj, "sm_target_%s" % i)
sm_objs.append(sm_obj)
for sm_obj in sm_objs:
if sm_obj.status == 1 and sm_obj.minCapacityWarning > 0:
try:
if not MinTape_dict[sm_obj.target] > sm_obj.minCapacityWarning:
MinTape_dict[sm_obj.target] = sm_obj.minCapacityWarning
except KeyError:
MinTape_dict[sm_obj.target] = sm_obj.minCapacityWarning
target_dict[sm_obj.target] = 0
# Create all tasks
for counter, task in enumerate(tasks):
alarm_flag = None
alarm_sub = ""
alarm_msg = ""
if task == tasks_todo.pop(0):
if task.status == "ArchTape":
event_info = "%s - Slot:%s" % (task.t_id, task.slot_id)
ArchTape_list.append(event_info)
elif task.status == "WriteTape":
event_info = "%s - Slot:%s" % (task.t_id, task.slot_id)
WriteTape_list.append(event_info)
elif task.status == "Ready":
event_info = "%s - Slot:%s" % (task.t_id, task.slot_id)
EmptyTape_list.append(event_info)
for target in target_dict.keys():
if task.t_id.startswith(target):
target_dict[target] += 1
else:
event_info = "check proc logfile: %s but task not match pop..." % task
logger.error(event_info)
# event_info = 'sleep 10'
# logger.debug(event_info)
# time.sleep(10)
self.update_progress(counter, num_tasks, update_frequency=1)
# Check if minimum number of new tapes is available
for target in target_dict.keys():
if target_dict[target] < MinTape_dict[target]:
AddTape_list.append(target)
# Populating ErrorTape_list
error_tape_objs = storageMedium.objects.exclude(storageMediumStatus=0).filter(storageMediumStatus__gt=30)
for error_tape_obj in error_tape_objs:
slot_id = None
for task in tasks:
if error_tape_obj.storageMediumID == task.t_id:
slot_id = task.slot_id
break
event_info = "%s - Slot:%s" % (error_tape_obj.storageMediumID, slot_id)
ErrorTape_list.append(event_info)
if email == "admin":
# Send mail report to admins
timestamp = datetime.now()
alarm_sub = "Daily media report %s" % timestamp
alarm_msg = "%s\n" % alarm_sub
alarm_msg += "\nArchive tapes online:\n"
for item in ArchTape_list:
alarm_msg += "%s\n" % item
alarm_msg += "\nAdd new tapes starting with:\n"
for item in AddTape_list:
alarm_msg += "%s\n" % item
alarm_msg += "\nWrite tapes:\n"
for item in WriteTape_list:
alarm_msg += "%s\n" % item
alarm_msg += "\nEmpty tapes:\n"
for item in EmptyTape_list:
alarm_msg += "%s\n" % item
alarm_msg += "\nError tapes:\n"
for item in ErrorTape_list:
alarm_msg += "%s\n" % item
mail_admins(alarm_sub, alarm_msg, fail_silently=False)
return {
"ArchTape": ArchTape_list,
"WriteTape": WriteTape_list,
"EmptyTape": EmptyTape_list,
"ErrorTape": ErrorTape_list,
"AddTape": AddTape_list,
}