def build_exp_list(cur_loc, num, grace_period, serverPath, removeOnly, log, autoArchiveAck): #removeOnly needs to be implemented ''' Build a list of experiments to archive or delete Filter out grace period runs based on start time, runs marked Keep Return n oldest experiments to archive or delete Remove only mode is important feature to protect against the condition wherein the archive volume is not available and there are runs marked for deletion. Without remove only mode, the list of runs to process could get filled with Archive runs, and the Delete Runs would never get processed. ''' exp = models.Experiment.objects.all().order_by('date') log.debug ("Total Experiments: %d" % len(exp)) if removeOnly: exp = exp.filter(storage_options='D').exclude(expName__in = models.Backup.objects.all().values('backupName')) log.debug ("Deletable %d" % len(exp)) else: exp = exp.exclude(storage_options='KI').exclude(expName__in = models.Backup.objects.all().values('backupName')) log.debug ("Deletable or Archivable %d" % len(exp)) # backupConfig # local time from which to measure time difference timenow = time.localtime(time.time()) experiments = [] for e in exp: log.debug('Experiment date %s' % str(e.date)) if len(experiments) < num: # only want to loop until we have the correct number location = server_and_location(e) if location == None: continue # TODO: instead of using the experiment date field, which is set to when the experiment was performed # use the file timestamp of the first (or last?) .dat file when calculating grace period EXPERIMENTAL = True if not EXPERIMENTAL: diff = time.mktime(timenow) - time.mktime(datetime.datetime.timetuple(e.date)) log.debug ('Time since experiment was run %d' % diff) else: stat = os.stat(e.expDir) diff = time.mktime(timenow) - stat.st_mtime log.debug ('Time since folder created is %d' % diff) # grace_period units are hours if diff < (grace_period * 3600): #convert hours to seconds log.info ('Within grace period: %s' % e.expName) continue experiment = Experiment(e, str(e.expName), str(e.date), str(e.star), str(e.storage_options), str(e.user_ack), str(e.expDir), location, e.pk) try: # don't add anything to the list if its already archived bk = models.Backup.objects.get(backupName=experiment.get_exp_name()) log.debug('This has been archived') continue except: # check that the path exists, and double check that its not marked to 'Keep' if not path.islink(experiment.get_exp_path()) and \ path.exists(experiment.get_exp_path()) and \ experiment.location==cur_loc and \ experiment.get_storage_option() != 'KI' and \ serverPath in experiment.dir: # change user ack status from unset to Selected: triggers an email notification if e.user_ack == 'U': e.user_ack = 'S' e.save() experiment.user_ack = 'S' # Runs marked Archive are automatically handled so set them as Acknowledged: if experiment.get_storage_option() == 'A': e.user_ack = 'A' e.save() experiment.user_ack = 'A' # If global settings are set to auto-acknowledge, then set user_ack as acknowledged if autoArchiveAck: e.user_ack = 'A' e.save() experiment.user_ack = 'A' experiments.append(experiment) #DEBUG MODE: show why the experiment is or is not valid for archiving log.debug("Path: %s" % experiment.get_exp_path()) log.debug("Does path exist? %s" % ('yes' if path.exists(experiment.get_exp_path()) else 'no')) log.debug("Is path a link? %s" % ('yes' if path.islink(experiment.get_exp_path()) else 'no')) log.debug("Is exp location same as loc? %s" % ('yes' if experiment.location==cur_loc else 'no')) log.debug("Is storage option not Keep? %s" % ('yes' if experiment.get_storage_option() != 'KI' else 'no')) log.debug("Is %s in %s? %s" % (serverPath,experiment.dir,'yes' if serverPath in experiment.dir else 'no')) log.debug("User Ack is set to %s" % experiment.user_ack) else: log.debug("Number of experiments: %d" % len(experiments)) return experiments log.debug("Number of experiments: %d" % len(experiments)) return experiments
def build_exp_list(num, grace_period, serverPath, removeOnly, log, autoArchiveAck): #removeOnly needs to be implemented ''' Build a list of experiments to archive or delete Filter out grace period runs based on start time, runs marked Keep TODO: Filter out runs whose FTP status indicates they are still transferring Return n oldest experiments to archive or delete Remove only mode is important feature to protect against the condition wherein the archive volume is not available and there are runs marked for deletion. Without remove only mode, the list of runs to process could get filled with Archive runs, and the Delete Runs would never get processed. ''' exp = models.Experiment.objects.all().order_by('date') log.debug ("Total Experiments: %d" % len(exp)) if removeOnly: exp = exp.filter(storage_options='D').exclude(expName__in = models.Backup.objects.all().values('backupName')) log.debug ("Deletable %d" % len(exp)) else: exp = exp.exclude(storage_options='KI').exclude(expName__in = models.Backup.objects.all().values('backupName')) log.debug ("Deletable or Archivable %d" % len(exp)) # backupConfig # local time from which to measure time difference timenow = time.localtime(time.time()) experiments = [] for e in exp: log.debug('Experiment date %s' % str(e.date)) if len(experiments) < num: # only want to loop until we have the correct number if not os.path.isdir(e.expDir): #Create an entry in the Backup db. kwargs = {"experiment": e, "backupName": e.expName, # This is True when the data has been archived. Since its missing, it hasn't been archived "isBackedUp": False, "backupDate": datetime.datetime.now(), "backupPath": "DELETED" } ret = models.Backup(**kwargs) ret.save() log.info ("Raw Data missing: %s. Creating Backup object" % e.expDir) continue # Instead of using the experiment date field, which is set to when the experiment was performed # use the file timestamp of the first (or last?) .dat file when calculating grace period diff = time.mktime(timenow) - time.mktime(datetime.datetime.timetuple(e.date)) log.debug ('Time since experiment was run %d' % diff) # grace_period units are hours if diff < (grace_period * 3600): #convert hours to seconds log.info ('Within grace period: %s' % e.expName) continue # TS-2736 Do not archive/delete Runs that are still FTP transferring. # What about runs that are stuck forever? Without this test, those types of runs eventually get archived or deleted # with this test in place, they will stay on server forever. # # If run is still transferring, skip # if e.ftpStatus.strip() == RUN_STATUS_COMPLETE or \ # e.ftpStatus.strip() == RUN_STATUS_ABORT or \ # e.ftpStatus.strip() == RUN_STATUS_MISSING or \ # e.ftpStatus.strip() == RUN_STATUS_SYS_CRIT: # pass # else: # logger.errors.debug("Skip this one, still transferring: %s" % e.expName) # continue experiment = Experiment(e, str(e.expName), str(e.date), str(e.star), str(e.storage_options), str(e.user_ack), str(e.expDir), e.pk, str(e.rawdatastyle)) try: # don't add anything to the list if its already archived bk = models.Backup.objects.get(backupName=experiment.get_exp_name()) log.debug('This has been archived') continue except: # check that the path exists, and double check that its not marked to 'Keep' if not path.islink(experiment.get_exp_path()) and \ path.exists(experiment.get_exp_path()) and \ experiment.get_storage_option() != 'KI' and \ experiment.dir.startswith(serverPath): # change user ack status from unset to Selected: triggers an email notification if e.user_ack == 'U': e.user_ack = 'S' e.save() experiment.user_ack = 'S' # Runs marked Archive are automatically handled so set them as Acknowledged: if experiment.get_storage_option() == 'A': e.user_ack = 'A' e.save() experiment.user_ack = 'A' # If global settings are set to auto-acknowledge, then set user_ack as acknowledged if autoArchiveAck: e.user_ack = 'A' e.save() experiment.user_ack = 'A' experiments.append(experiment) #DEBUG MODE: show why the experiment is or is not valid for archiving log.debug("Path: %s" % experiment.get_exp_path()) log.debug("Does path exist? %s" % ('yes' if path.exists(experiment.get_exp_path()) else 'no')) log.debug("Is path a link? %s" % ('yes' if path.islink(experiment.get_exp_path()) else 'no')) log.debug("Is storage option not Keep? %s" % ('yes' if experiment.get_storage_option() != 'KI' else 'no')) log.debug("Is %s in %s? %s" % (serverPath,experiment.dir,'yes' if serverPath in experiment.dir else 'no')) log.debug("User Ack is set to %s" % experiment.user_ack) else: log.debug("Number of experiments: %d" % len(experiments)) return experiments log.debug("Number of experiments: %d" % len(experiments)) return experiments
def build_exp_list(num, grace_period, serverPath, removeOnly, log, autoArchiveAck): # removeOnly needs to be implemented ''' Build a list of experiments to archive or delete Filter out grace period runs based on start time, runs marked Keep TODO: Filter out runs whose FTP status indicates they are still transferring Return n oldest experiments to archive or delete Remove only mode is important feature to protect against the condition wherein the archive volume is not available and there are runs marked for deletion. Without remove only mode, the list of runs to process could get filled with Archive runs, and the Delete Runs would never get processed. When auto-acknowledge is disabled (the default) and an archive volume is configured, there can be a situation where the list of experiments is filled with Delete and the Archive experiments are not included and thus never processed - if the user fails to manually acknowledge deletion. But also, the remove_experiments function handles the Delete first because those are faster and free space quicky while archive can take a long time copying. ''' exp = models.Experiment.objects.all().order_by('date') log.debug("Total Experiments: %d" % len(exp)) if removeOnly: exp = exp.filter(storage_options='D').exclude( expName__in=models.Backup.objects.all().values('backupName')) log.debug("Deletable %d" % len(exp)) else: exp = exp.exclude(storage_options='KI').exclude( expName__in=models.Backup.objects.all().values('backupName')) log.debug("Deletable or Archivable %d" % len(exp)) # backupConfig # local time from which to measure time difference timenow = time.localtime(time.time()) experiments = [] for e in exp: log.debug('Experiment date %s' % str(e.date)) if len(experiments ) < num: # only want to loop until we have the correct number if not os.path.isdir(e.expDir): #Create an entry in the Backup db. kwargs = { "experiment": e, "backupName": e.expName, # This is True when the data has been archived. Since its missing, it hasn't been archived "isBackedUp": False, "backupDate": datetime.datetime.now(), "backupPath": "DELETED" } ret = models.Backup(**kwargs) ret.save() log.info("Raw Data missing: %s. Creating Backup object" % e.expDir) continue # Instead of using the experiment date field, which is set to when the experiment was performed # use the file timestamp of the first (or last?) .dat file when calculating grace period diff = time.mktime(timenow) - time.mktime( datetime.datetime.timetuple(e.date)) log.debug('Time since experiment was run %d' % diff) # grace_period units are hours if diff < (grace_period * 3600): # convert hours to seconds log.info('Within grace period: %s' % e.expName) continue # TS-2736 Do not archive/delete Runs that are still FTP transferring. # What about runs that are stuck forever? Without this test, those types of runs eventually get archived or deleted # with this test in place, they will stay on server forever. # # If run is still transferring, skip # if e.ftpStatus.strip() == RUN_STATUS_COMPLETE or \ # e.ftpStatus.strip() == RUN_STATUS_ABORT or \ # e.ftpStatus.strip() == RUN_STATUS_MISSING or \ # e.ftpStatus.strip() == RUN_STATUS_SYS_CRIT: # pass # else: # logger.errors.debug("Skip this one, still transferring: %s" % e.expName) # continue experiment = Experiment( e, str(e.expName), str(e.date), str(e.star), str(e.storage_options), str(e.user_ack), str(e.expDir), e.pk, str(e.rawdatastyle), str(e.diskusage) if e.diskusage is not None else "Unknown") try: # don't add anything to the list if its already archived bk = models.Backup.objects.get( backupName=experiment.get_exp_name()) log.debug('This has been archived') continue except: # check that the path exists, and double check that its not marked to 'Keep' if not path.islink(experiment.get_exp_path()) and \ path.exists(experiment.get_exp_path()) and \ experiment.get_storage_option() != 'KI' and \ experiment.dir.startswith(serverPath): # change user ack status from unset to Selected: triggers an email notification if e.user_ack == 'U': e.user_ack = 'S' e.save() experiment.user_ack = 'S' # Runs marked Archive are automatically handled so set them as Acknowledged: if experiment.get_storage_option() == 'A': e.user_ack = 'A' e.save() experiment.user_ack = 'A' # If global settings are set to auto-acknowledge, then set user_ack as acknowledged if autoArchiveAck: e.user_ack = 'A' e.save() experiment.user_ack = 'A' experiments.append(experiment) #DEBUG MODE: show why the experiment is or is not valid for archiving log.debug("Path: %s" % experiment.get_exp_path()) log.debug("Does path exist? %s" % ('yes' if path.exists( experiment.get_exp_path()) else 'no')) log.debug("Is path a link? %s" % ('yes' if path.islink( experiment.get_exp_path()) else 'no')) log.debug("Is storage option not Keep? %s" % ('yes' if experiment.get_storage_option() != 'KI' else 'no')) log.debug("Is %s in %s? %s" % (serverPath, experiment.dir, 'yes' if serverPath in experiment.dir else 'no')) log.debug("User Ack is set to %s" % experiment.user_ack) else: log.debug("Number of experiments: %d" % len(experiments)) return experiments log.debug("Number of experiments: %d" % len(experiments)) return experiments