def backfill_dmfilestats_diskspace(): ''' Backfill records with DMFileStat.diskspace = None, one at a time These could be older data sets or new ones where update_diskusage task failed ''' dmfilestats = DMFileStat.objects.filter(diskspace=None, action_state='L', files_in_use='').order_by('-created') if dmfilestats.count() > 0: dmactions.update_diskspace(dmfilestats[0])
def backfill_dmfilestats_diskspace(): ''' Backfill records with DMFileStat.diskspace = None, one at a time These could be older data sets or new ones where update_diskusage task failed ''' dmfilestats = DMFileStat.objects.filter( diskspace=None, action_state='L', files_in_use='').order_by('-created') if dmfilestats.count() > 0: dmactions.update_diskspace(dmfilestats[0])
def update_diskusage(resultpk): ''' Task to update DMFileStat.diskspace for all associated with this resultpk NOTE: This can be a long-lived task ''' try: result = Results.objects.get(pk=resultpk) for type in FILESET_TYPES: dmfilestat = result.get_filestat(type) dmactions.update_diskspace(dmfilestat) except: raise
def update_diskusage(resultpk): ''' Task to update DMFileStat.diskspace for all associated with this resultpk NOTE: This can be a long-lived task ''' try: result = Results.objects.get(pk=resultpk) search_dirs = [result.get_report_dir(), result.experiment.expDir] cached_file_list = dmactions.get_walk_filelist(search_dirs) for type in FILESET_TYPES: dmfilestat = result.get_filestat(type) dmactions.update_diskspace(dmfilestat, cached=cached_file_list) except SoftTimeLimitExceeded: logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk,result.resultsName), extra = d) except: raise
def update_diskusage(resultpk): ''' Task to update DMFileStat.diskspace for all associated with this resultpk NOTE: This can be a long-lived task ''' try: result = Results.objects.get(pk=resultpk) search_dirs = [result.get_report_dir(), result.experiment.expDir] cached_file_list = dmactions.get_walk_filelist(search_dirs) for type in FILESET_TYPES: dmfilestat = result.get_filestat(type) dmactions.update_diskspace(dmfilestat, cached=cached_file_list) except SoftTimeLimitExceeded: logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk, result.resultsName), extra=d) except: raise
def update_dmfilestats_diskspace(dmfilestat): ''' Task to update DMFileStat.diskspace ''' dmactions.update_diskspace(dmfilestat)
def update_dmfilestats_diskspace(dmfilestat): """ Task to update DMFileStat.diskspace """ dmactions.update_diskspace(dmfilestat)
threshdate = datetime.now(pytz.UTC) - timedelta(days=AGE_THRESH_DAYS) exceed_thresh_stats = stats.filter(created__lt=threshdate) within_thresh_stats = stats.filter(created__gte=threshdate) howmuchexceeds = 0 howmuchsafe = 0 unknownexceeds = 0 unknownsafe = 0 for item in exceed_thresh_stats: if item.diskspace != None: howmuchexceeds += item.diskspace else: if BACKFILL: print "getting %s diskspace for %s" % (type, item.result.resultsName) try: howmuchexceeds += update_diskspace(item) except: print traceback.format_exc() else: unknownexceeds += 1 for item in within_thresh_stats: if item.diskspace != None: howmuchsafe += item.diskspace else: ''' if BACKFILL: print "getting diskspace for %s" % (item.result.resultsName) try: update_diskspace(item) howmuchsafe += item.diskspace
else: AGE_THRESH_DAYS = THRESHOLD_DAYS threshdate = datetime.now(pytz.UTC) - timedelta(days=AGE_THRESH_DAYS) exceed_thresh_stats = stats.filter(created__lt=threshdate) within_thresh_stats = stats.filter(created__gte=threshdate) howmuchexceeds = 0 howmuchsafe = 0 unknownexceeds = 0 unknownsafe = 0 for item in exceed_thresh_stats: if item.diskspace != None: howmuchexceeds += item.diskspace else: if BACKFILL: print "getting %s diskspace for %s" % (type, item.result.resultsName) try: howmuchexceeds += update_diskspace(item) except: print traceback.format_exc() else: unknownexceeds += 1 for item in within_thresh_stats: if item.diskspace != None: howmuchsafe += item.diskspace else: ''' if BACKFILL: print "getting diskspace for %s" % (item.result.resultsName) try: update_diskspace(item) howmuchsafe += item.diskspace