def _get_deletable_paths(self, path_list, older_than,
deletable_regexps):
'''
Return a list with the file paths that can be deleted.
'''
today = dt.datetime.today()
to_delete = []
for file_path in path_list:
timeslot = utilities.extract_timeslot(file_path)
if timeslot is not None:
date_diff = today - timeslot
if date_diff.days > older_than:
if self._is_deletable(file_path, deletable_regexps, older_than):
to_delete.append(file_path)
return to_delete
def _get_deletable_paths(self, path_list, older_than, deletable_regexps):
'''
Return a list with the file paths that can be deleted.
'''
today = dt.datetime.today()
to_delete = []
for file_path in path_list:
timeslot = utilities.extract_timeslot(file_path)
if timeslot is not None:
date_diff = today - timeslot
if date_diff.days > older_than:
if self._is_deletable(file_path, deletable_regexps,
older_than):
to_delete.append(file_path)
return to_delete
def delete_old_files(self, older_than, emergency=False):
'''
Delete system input and output files older than the input
number of days.
Inputs:
older_than - An integer specifying the difference in days that
files must have fom the current day in order to be deleted.
This way, files whose timeslot is more recent will not be
deleted.
emergency - A boolean specifying if each file's own
'delete_older_than' setting is to be ignored.
This method only operates on files whose file name matches defined
systemsettings.models.File search patterns and only if these files
have their 'toDelete' attribute set to True. This method will also
respect individual file tolerances for deletion.
If a file defines a variable called 'delete_older_than' in its
extra infos section in the settings, then this method will take that
into account and the file will only be deleted if its own
'delete_older_than' variable is smaller than the specified input
'older_than' for this method.
However, if called with the 'emergency' argument, the files will get
deleted no matter if they define their own 'delete_older_than'
tolerance.
'''
to_delete = []
data_dir = os.path.join(self.dataPath, 'OUTPUT_DATA')
today = dt.datetime.today()
deletable_regexps = self._get_deletable_regexps(older_than,
emergency=emergency)
for root, dirs, files in os.walk(data_dir):
for file_path in files:
timeslot = utilities.extract_timeslot(file_path)
if timeslot is not None:
date_diff = today - timeslot
if date_diff.days > older_than:
if self._is_deletable(file_path, deletable_regexps,
older_than):
to_delete.append(os.path.join(root, file_path))
#self.delete_files(to_delete)
return to_delete
def delete_old_files(self, older_than, emergency=False):
'''
Delete system input and output files older than the input
number of days.
Inputs:
older_than - An integer specifying the difference in days that
files must have fom the current day in order to be deleted.
This way, files whose timeslot is more recent will not be
deleted.
emergency - A boolean specifying if each file's own
'delete_older_than' setting is to be ignored.
This method only operates on files whose file name matches defined
systemsettings.models.File search patterns and only if these files
have their 'toDelete' attribute set to True. This method will also
respect individual file tolerances for deletion.
If a file defines a variable called 'delete_older_than' in its
extra infos section in the settings, then this method will take that
into account and the file will only be deleted if its own
'delete_older_than' variable is smaller than the specified input
'older_than' for this method.
However, if called with the 'emergency' argument, the files will get
deleted no matter if they define their own 'delete_older_than'
tolerance.
'''
to_delete = []
data_dir = os.path.join(self.dataPath, 'OUTPUT_DATA')
today = dt.datetime.today()
deletable_regexps = self._get_deletable_regexps(older_than,
emergency=emergency)
for root, dirs, files in os.walk(data_dir):
for file_path in files:
timeslot = utilities.extract_timeslot(file_path)
if timeslot is not None:
date_diff = today - timeslot
if date_diff.days > older_than:
if self._is_deletable(file_path, deletable_regexps, older_than):
to_delete.append(os.path.join(root, file_path))
#self.delete_files(to_delete)
return to_delete
def delete_logs(self, older_than):
'''
Delete log files older than the input number of days.
'''
logs_dir = os.path.join(self.dataPath, 'LOGS')
today = dt.datetime.today()
to_delete = []
for root, dirs, files in os.walk(logs_dir):
for file_path in files:
timeslot = utilities.extract_timeslot(file_path)
if timeslot is not None:
date_diff = today - timeslot
if date_diff.days > older_than:
full_path = os.path.join(root, file_path)
to_delete.append(full_path)
self.delete_files(to_delete)
def delete_logs(self, older_than):
'''
Delete log files older than the input number of days.
'''
logs_dir = os.path.join(self.dataPath, 'LOGS')
today = dt.datetime.today()
to_delete = []
for root, dirs, files in os.walk(logs_dir):
for file_path in files:
timeslot = utilities.extract_timeslot(file_path)
if timeslot is not None:
date_diff = today - timeslot
if date_diff.days > older_than:
full_path = os.path.join(root, file_path)
to_delete.append(full_path)
self.delete_files(to_delete)
def _sort_latest_timeslot(self, paths, timeUnit):
'''
Return the latest file.
Inputs:
paths - A list of file paths to analyze.
timeUnit - A string specifying the type of latest file
to choose. Accepted values:
'absolute' - The latest file, according to this instance's
own timeslot.
'month' - The latest file belonging to the same month as
the instance's own timeslot.
Returns:
A string with the path of the latest file.
'''
latestPath = None
# 20 years (in seconds), a large initialization value
latestDiff = 60 * 60 * 24 * 365 * 20
for path in paths:
pTimeslot = utilities.extract_timeslot(path)
timeDiff = self.timeslot - pTimeslot
absTimeDiff = timeDiff.seconds + (timeDiff.days * 24 * 60 * 60)
if timeUnit == 'absolute':
if (absTimeDiff <= latestDiff) and (absTimeDiff >= 0):
latestPath = path
latestDiff = absTimeDiff
elif timeUnit == 'month':
if (pTimeslot.month == self.timeslot.month) and \
(absTimeDiff <= latestDiff):
latestPath = path
latestDiff = absTimeDiff
return latestPath
def _sort_latest_timeslot(self, paths, timeUnit):
'''
Return the latest file.
Inputs:
paths - A list of file paths to analyze.
timeUnit - A string specifying the type of latest file
to choose. Accepted values:
'absolute' - The latest file, according to this instance's
own timeslot.
'month' - The latest file belonging to the same month as
the instance's own timeslot.
Returns:
A string with the path of the latest file.
'''
latestPath = None
# 20 years (in seconds), a large initialization value
latestDiff = 60 * 60 * 24 * 365 * 20
for path in paths:
pTimeslot = utilities.extract_timeslot(path)
timeDiff = self.timeslot - pTimeslot
absTimeDiff = timeDiff.seconds + (timeDiff.days * 24 * 60 * 60)
if timeUnit == 'absolute':
if (absTimeDiff <= latestDiff) and (absTimeDiff >= 0):
latestPath = path
latestDiff = absTimeDiff
elif timeUnit == 'month':
if (pTimeslot.month == self.timeslot.month) and \
(absTimeDiff <= latestDiff):
latestPath = path
latestDiff = absTimeDiff
return latestPath