def build_exp_list(cur_loc, num, grace_period, serverPath, removeOnly, log, autoArchiveAck): #removeOnly needs to be implemented
'''
Build a list of experiments to archive or delete
Filter out grace period runs based on start time, runs marked Keep
Return n oldest experiments to archive or delete
Remove only mode is important feature to protect against the condition wherein the archive volume is not available
and there are runs marked for deletion. Without remove only mode, the list of runs to process could get filled with
Archive runs, and the Delete Runs would never get processed.
'''
exp = models.Experiment.objects.all().order_by('date')
log.debug ("Total Experiments: %d" % len(exp))
if removeOnly:
exp = exp.filter(storage_options='D').exclude(expName__in = models.Backup.objects.all().values('backupName'))
log.debug ("Deletable %d" % len(exp))
else:
exp = exp.exclude(storage_options='KI').exclude(expName__in = models.Backup.objects.all().values('backupName'))
log.debug ("Deletable or Archivable %d" % len(exp))
# backupConfig
# local time from which to measure time difference
timenow = time.localtime(time.time())
experiments = []
for e in exp:
log.debug('Experiment date %s' % str(e.date))
if len(experiments) < num: # only want to loop until we have the correct number
location = server_and_location(e)
if location == None:
continue
# TODO: instead of using the experiment date field, which is set to when the experiment was performed
# use the file timestamp of the first (or last?) .dat file when calculating grace period
EXPERIMENTAL = True
if not EXPERIMENTAL:
diff = time.mktime(timenow) - time.mktime(datetime.datetime.timetuple(e.date))
log.debug ('Time since experiment was run %d' % diff)
else:
stat = os.stat(e.expDir)
diff = time.mktime(timenow) - stat.st_mtime
log.debug ('Time since folder created is %d' % diff)
# grace_period units are hours
if diff < (grace_period * 3600): #convert hours to seconds
log.info ('Within grace period: %s' % e.expName)
continue
experiment = Experiment(e,
str(e.expName),
str(e.date),
str(e.star),
str(e.storage_options),
str(e.user_ack),
str(e.expDir),
location,
e.pk)
try:
# don't add anything to the list if its already archived
bk = models.Backup.objects.get(backupName=experiment.get_exp_name())
log.debug('This has been archived')
continue
except:
# check that the path exists, and double check that its not marked to 'Keep'
if not path.islink(experiment.get_exp_path()) and \
path.exists(experiment.get_exp_path()) and \
experiment.location==cur_loc and \
experiment.get_storage_option() != 'KI' and \
serverPath in experiment.dir:
# change user ack status from unset to Selected: triggers an email notification
if e.user_ack == 'U':
e.user_ack = 'S'
e.save()
experiment.user_ack = 'S'
# Runs marked Archive are automatically handled so set them as Acknowledged:
if experiment.get_storage_option() == 'A':
e.user_ack = 'A'
e.save()
experiment.user_ack = 'A'
# If global settings are set to auto-acknowledge, then set user_ack as acknowledged
if autoArchiveAck:
e.user_ack = 'A'
e.save()
experiment.user_ack = 'A'
experiments.append(experiment)
#DEBUG MODE: show why the experiment is or is not valid for archiving
log.debug("Path: %s" % experiment.get_exp_path())
log.debug("Does path exist? %s" % ('yes' if path.exists(experiment.get_exp_path()) else 'no'))
log.debug("Is path a link? %s" % ('yes' if path.islink(experiment.get_exp_path()) else 'no'))
log.debug("Is exp location same as loc? %s" % ('yes' if experiment.location==cur_loc else 'no'))
log.debug("Is storage option not Keep? %s" % ('yes' if experiment.get_storage_option() != 'KI' else 'no'))
log.debug("Is %s in %s? %s" % (serverPath,experiment.dir,'yes' if serverPath in experiment.dir else 'no'))
log.debug("User Ack is set to %s" % experiment.user_ack)
else:
log.debug("Number of experiments: %d" % len(experiments))
return experiments
log.debug("Number of experiments: %d" % len(experiments))
return experiments
def build_exp_list(num, grace_period, serverPath, removeOnly, log, autoArchiveAck): #removeOnly needs to be implemented
'''
Build a list of experiments to archive or delete
Filter out grace period runs based on start time, runs marked Keep
TODO: Filter out runs whose FTP status indicates they are still transferring
Return n oldest experiments to archive or delete
Remove only mode is important feature to protect against the condition wherein the archive volume is not available
and there are runs marked for deletion. Without remove only mode, the list of runs to process could get filled with
Archive runs, and the Delete Runs would never get processed.
'''
exp = models.Experiment.objects.all().order_by('date')
log.debug ("Total Experiments: %d" % len(exp))
if removeOnly:
exp = exp.filter(storage_options='D').exclude(expName__in = models.Backup.objects.all().values('backupName'))
log.debug ("Deletable %d" % len(exp))
else:
exp = exp.exclude(storage_options='KI').exclude(expName__in = models.Backup.objects.all().values('backupName'))
log.debug ("Deletable or Archivable %d" % len(exp))
# backupConfig
# local time from which to measure time difference
timenow = time.localtime(time.time())
experiments = []
for e in exp:
log.debug('Experiment date %s' % str(e.date))
if len(experiments) < num: # only want to loop until we have the correct number
if not os.path.isdir(e.expDir):
#Create an entry in the Backup db.
kwargs = {"experiment": e,
"backupName": e.expName,
# This is True when the data has been archived. Since its missing, it hasn't been archived
"isBackedUp": False,
"backupDate": datetime.datetime.now(),
"backupPath": "DELETED"
}
ret = models.Backup(**kwargs)
ret.save()
log.info ("Raw Data missing: %s. Creating Backup object" % e.expDir)
continue
# Instead of using the experiment date field, which is set to when the experiment was performed
# use the file timestamp of the first (or last?) .dat file when calculating grace period
diff = time.mktime(timenow) - time.mktime(datetime.datetime.timetuple(e.date))
log.debug ('Time since experiment was run %d' % diff)
# grace_period units are hours
if diff < (grace_period * 3600): #convert hours to seconds
log.info ('Within grace period: %s' % e.expName)
continue
# TS-2736 Do not archive/delete Runs that are still FTP transferring.
# What about runs that are stuck forever? Without this test, those types of runs eventually get archived or deleted
# with this test in place, they will stay on server forever.
# # If run is still transferring, skip
# if e.ftpStatus.strip() == RUN_STATUS_COMPLETE or \
# e.ftpStatus.strip() == RUN_STATUS_ABORT or \
# e.ftpStatus.strip() == RUN_STATUS_MISSING or \
# e.ftpStatus.strip() == RUN_STATUS_SYS_CRIT:
# pass
# else:
# logger.errors.debug("Skip this one, still transferring: %s" % e.expName)
# continue
experiment = Experiment(e,
str(e.expName),
str(e.date),
str(e.star),
str(e.storage_options),
str(e.user_ack),
str(e.expDir),
e.pk,
str(e.rawdatastyle))
try:
# don't add anything to the list if its already archived
bk = models.Backup.objects.get(backupName=experiment.get_exp_name())
log.debug('This has been archived')
continue
except:
# check that the path exists, and double check that its not marked to 'Keep'
if not path.islink(experiment.get_exp_path()) and \
path.exists(experiment.get_exp_path()) and \
experiment.get_storage_option() != 'KI' and \
experiment.dir.startswith(serverPath):
# change user ack status from unset to Selected: triggers an email notification
if e.user_ack == 'U':
e.user_ack = 'S'
e.save()
experiment.user_ack = 'S'
# Runs marked Archive are automatically handled so set them as Acknowledged:
if experiment.get_storage_option() == 'A':
e.user_ack = 'A'
e.save()
experiment.user_ack = 'A'
# If global settings are set to auto-acknowledge, then set user_ack as acknowledged
if autoArchiveAck:
e.user_ack = 'A'
e.save()
experiment.user_ack = 'A'
experiments.append(experiment)
#DEBUG MODE: show why the experiment is or is not valid for archiving
log.debug("Path: %s" % experiment.get_exp_path())
log.debug("Does path exist? %s" % ('yes' if path.exists(experiment.get_exp_path()) else 'no'))
log.debug("Is path a link? %s" % ('yes' if path.islink(experiment.get_exp_path()) else 'no'))
log.debug("Is storage option not Keep? %s" % ('yes' if experiment.get_storage_option() != 'KI' else 'no'))
log.debug("Is %s in %s? %s" % (serverPath,experiment.dir,'yes' if serverPath in experiment.dir else 'no'))
log.debug("User Ack is set to %s" % experiment.user_ack)
else:
log.debug("Number of experiments: %d" % len(experiments))
return experiments
log.debug("Number of experiments: %d" % len(experiments))
return experiments
def build_exp_list(num, grace_period, serverPath, removeOnly, log,
autoArchiveAck): # removeOnly needs to be implemented
'''
Build a list of experiments to archive or delete
Filter out grace period runs based on start time, runs marked Keep
TODO: Filter out runs whose FTP status indicates they are still transferring
Return n oldest experiments to archive or delete
Remove only mode is important feature to protect against the condition wherein the archive volume is not available
and there are runs marked for deletion. Without remove only mode, the list of runs to process could get filled with
Archive runs, and the Delete Runs would never get processed.
When auto-acknowledge is disabled (the default) and an archive volume is configured, there can be a situation where
the list of experiments is filled with Delete and the Archive experiments are not included and thus never processed -
if the user fails to manually acknowledge deletion.
But also, the remove_experiments function handles the Delete first because those are faster and free space quicky while
archive can take a long time copying.
'''
exp = models.Experiment.objects.all().order_by('date')
log.debug("Total Experiments: %d" % len(exp))
if removeOnly:
exp = exp.filter(storage_options='D').exclude(
expName__in=models.Backup.objects.all().values('backupName'))
log.debug("Deletable %d" % len(exp))
else:
exp = exp.exclude(storage_options='KI').exclude(
expName__in=models.Backup.objects.all().values('backupName'))
log.debug("Deletable or Archivable %d" % len(exp))
# backupConfig
# local time from which to measure time difference
timenow = time.localtime(time.time())
experiments = []
for e in exp:
log.debug('Experiment date %s' % str(e.date))
if len(experiments
) < num: # only want to loop until we have the correct number
if not os.path.isdir(e.expDir):
#Create an entry in the Backup db.
kwargs = {
"experiment": e,
"backupName": e.expName,
# This is True when the data has been archived. Since its missing, it hasn't been archived
"isBackedUp": False,
"backupDate": datetime.datetime.now(),
"backupPath": "DELETED"
}
ret = models.Backup(**kwargs)
ret.save()
log.info("Raw Data missing: %s. Creating Backup object" %
e.expDir)
continue
# Instead of using the experiment date field, which is set to when the experiment was performed
# use the file timestamp of the first (or last?) .dat file when calculating grace period
diff = time.mktime(timenow) - time.mktime(
datetime.datetime.timetuple(e.date))
log.debug('Time since experiment was run %d' % diff)
# grace_period units are hours
if diff < (grace_period * 3600): # convert hours to seconds
log.info('Within grace period: %s' % e.expName)
continue
# TS-2736 Do not archive/delete Runs that are still FTP transferring.
# What about runs that are stuck forever? Without this test, those types of runs eventually get archived or deleted
# with this test in place, they will stay on server forever.
# # If run is still transferring, skip
# if e.ftpStatus.strip() == RUN_STATUS_COMPLETE or \
# e.ftpStatus.strip() == RUN_STATUS_ABORT or \
# e.ftpStatus.strip() == RUN_STATUS_MISSING or \
# e.ftpStatus.strip() == RUN_STATUS_SYS_CRIT:
# pass
# else:
# logger.errors.debug("Skip this one, still transferring: %s" % e.expName)
# continue
experiment = Experiment(
e, str(e.expName), str(e.date), str(e.star),
str(e.storage_options), str(e.user_ack), str(e.expDir), e.pk,
str(e.rawdatastyle),
str(e.diskusage) if e.diskusage is not None else "Unknown")
try:
# don't add anything to the list if its already archived
bk = models.Backup.objects.get(
backupName=experiment.get_exp_name())
log.debug('This has been archived')
continue
except:
# check that the path exists, and double check that its not marked to 'Keep'
if not path.islink(experiment.get_exp_path()) and \
path.exists(experiment.get_exp_path()) and \
experiment.get_storage_option() != 'KI' and \
experiment.dir.startswith(serverPath):
# change user ack status from unset to Selected: triggers an email notification
if e.user_ack == 'U':
e.user_ack = 'S'
e.save()
experiment.user_ack = 'S'
# Runs marked Archive are automatically handled so set them as Acknowledged:
if experiment.get_storage_option() == 'A':
e.user_ack = 'A'
e.save()
experiment.user_ack = 'A'
# If global settings are set to auto-acknowledge, then set user_ack as acknowledged
if autoArchiveAck:
e.user_ack = 'A'
e.save()
experiment.user_ack = 'A'
experiments.append(experiment)
#DEBUG MODE: show why the experiment is or is not valid for archiving
log.debug("Path: %s" % experiment.get_exp_path())
log.debug("Does path exist? %s" % ('yes' if path.exists(
experiment.get_exp_path()) else 'no'))
log.debug("Is path a link? %s" % ('yes' if path.islink(
experiment.get_exp_path()) else 'no'))
log.debug("Is storage option not Keep? %s" %
('yes' if experiment.get_storage_option() != 'KI'
else 'no'))
log.debug("Is %s in %s? %s" %
(serverPath, experiment.dir,
'yes' if serverPath in experiment.dir else 'no'))
log.debug("User Ack is set to %s" % experiment.user_ack)
else:
log.debug("Number of experiments: %d" % len(experiments))
return experiments
log.debug("Number of experiments: %d" % len(experiments))
return experiments
