def backfill_dmfilestats_diskspace():
''' Backfill records with DMFileStat.diskspace = None, one at a time
These could be older data sets or new ones where update_diskusage task failed
'''
dmfilestats = DMFileStat.objects.filter(diskspace=None, action_state='L', files_in_use='').order_by('-created')
if dmfilestats.count() > 0:
dmactions.update_diskspace(dmfilestats[0])
def backfill_dmfilestats_diskspace():
''' Backfill records with DMFileStat.diskspace = None, one at a time
These could be older data sets or new ones where update_diskusage task failed
'''
dmfilestats = DMFileStat.objects.filter(
diskspace=None, action_state='L', files_in_use='').order_by('-created')
if dmfilestats.count() > 0:
dmactions.update_diskspace(dmfilestats[0])
def update_diskusage(resultpk):
'''
Task to update DMFileStat.diskspace for all associated with this resultpk
NOTE: This can be a long-lived task
'''
try:
result = Results.objects.get(pk=resultpk)
for type in FILESET_TYPES:
dmfilestat = result.get_filestat(type)
dmactions.update_diskspace(dmfilestat)
except:
raise
def update_diskusage(resultpk):
'''
Task to update DMFileStat.diskspace for all associated with this resultpk
NOTE: This can be a long-lived task
'''
try:
result = Results.objects.get(pk=resultpk)
for type in FILESET_TYPES:
dmfilestat = result.get_filestat(type)
dmactions.update_diskspace(dmfilestat)
except:
raise
def update_diskusage(resultpk):
'''
Task to update DMFileStat.diskspace for all associated with this resultpk
NOTE: This can be a long-lived task
'''
try:
result = Results.objects.get(pk=resultpk)
search_dirs = [result.get_report_dir(), result.experiment.expDir]
cached_file_list = dmactions.get_walk_filelist(search_dirs)
for type in FILESET_TYPES:
dmfilestat = result.get_filestat(type)
dmactions.update_diskspace(dmfilestat, cached=cached_file_list)
except SoftTimeLimitExceeded:
logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk,result.resultsName), extra = d)
except:
raise
def update_diskusage(resultpk):
'''
Task to update DMFileStat.diskspace for all associated with this resultpk
NOTE: This can be a long-lived task
'''
try:
result = Results.objects.get(pk=resultpk)
search_dirs = [result.get_report_dir(), result.experiment.expDir]
cached_file_list = dmactions.get_walk_filelist(search_dirs)
for type in FILESET_TYPES:
dmfilestat = result.get_filestat(type)
dmactions.update_diskspace(dmfilestat, cached=cached_file_list)
except SoftTimeLimitExceeded:
logger.warn("Time exceeded update_diskusage for (%d) %s" %
(resultpk, result.resultsName),
extra=d)
except:
raise
def update_dmfilestats_diskspace(dmfilestat):
''' Task to update DMFileStat.diskspace '''
dmactions.update_diskspace(dmfilestat)
def update_dmfilestats_diskspace(dmfilestat):
''' Task to update DMFileStat.diskspace '''
dmactions.update_diskspace(dmfilestat)
def update_dmfilestats_diskspace(dmfilestat):
""" Task to update DMFileStat.diskspace """
dmactions.update_diskspace(dmfilestat)
threshdate = datetime.now(pytz.UTC) - timedelta(days=AGE_THRESH_DAYS)
exceed_thresh_stats = stats.filter(created__lt=threshdate)
within_thresh_stats = stats.filter(created__gte=threshdate)
howmuchexceeds = 0
howmuchsafe = 0
unknownexceeds = 0
unknownsafe = 0
for item in exceed_thresh_stats:
if item.diskspace != None:
howmuchexceeds += item.diskspace
else:
if BACKFILL:
print "getting %s diskspace for %s" % (type,
item.result.resultsName)
try:
howmuchexceeds += update_diskspace(item)
except:
print traceback.format_exc()
else:
unknownexceeds += 1
for item in within_thresh_stats:
if item.diskspace != None:
howmuchsafe += item.diskspace
else:
'''
if BACKFILL:
print "getting diskspace for %s" % (item.result.resultsName)
try:
update_diskspace(item)
howmuchsafe += item.diskspace
else: AGE_THRESH_DAYS = THRESHOLD_DAYS
threshdate = datetime.now(pytz.UTC) - timedelta(days=AGE_THRESH_DAYS)
exceed_thresh_stats = stats.filter(created__lt=threshdate)
within_thresh_stats = stats.filter(created__gte=threshdate)
howmuchexceeds = 0
howmuchsafe = 0
unknownexceeds = 0
unknownsafe = 0
for item in exceed_thresh_stats:
if item.diskspace != None:
howmuchexceeds += item.diskspace
else:
if BACKFILL:
print "getting %s diskspace for %s" % (type, item.result.resultsName)
try:
howmuchexceeds += update_diskspace(item)
except:
print traceback.format_exc()
else:
unknownexceeds += 1
for item in within_thresh_stats:
if item.diskspace != None:
howmuchsafe += item.diskspace
else:
'''
if BACKFILL:
print "getting diskspace for %s" % (item.result.resultsName)
try:
update_diskspace(item)
howmuchsafe += item.diskspace
