def copy_files_to_destination(source_dir, destination, dmfileset, log): cached_file_list = get_walk_filelist([source_dir]) to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list) logger.debug("[Data Import] Importing %d files from %s to %s" % (len(to_process), source_dir, destination)) log.write("Copy files to %s ...\n" % destination) for i, filepath in enumerate(to_process): log.write("%s.\n" % filepath) try: _copy_to_dir(filepath, source_dir, destination) except Exception as exception: logger.error(traceback.format_exc()) log.write("%s" % exception) log.flush() if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, "plugin_out") if not os.path.isdir(plugin_out): oldmask = os.umask(0000) # grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) if dmfileset.type == dmactions_types.BASE: # for onboard results need to make sigproc_results link if os.path.exists(os.path.join(destination, "onboard_results")): os.symlink( os.path.join(destination, "onboard_results", "sigproc_results"), os.path.join(destination, "sigproc_results"), ) log.write("Copy files done.\n")
def update_diskusage(resultpk): ''' Task to update DMFileStat.diskspace for all associated with this resultpk NOTE: This can be a long-lived task ''' try: result = Results.objects.get(pk=resultpk) search_dirs = [result.get_report_dir(), result.experiment.expDir] cached_file_list = dmactions.get_walk_filelist(search_dirs) for type in FILESET_TYPES: dmfilestat = result.get_filestat(type) dmactions.update_diskspace(dmfilestat, cached=cached_file_list) except SoftTimeLimitExceeded: logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk,result.resultsName), extra = d) except: raise
def update_diskusage(resultpk): ''' Task to update DMFileStat.diskspace for all associated with this resultpk NOTE: This can be a long-lived task ''' try: result = Results.objects.get(pk=resultpk) search_dirs = [result.get_report_dir(), result.experiment.expDir] cached_file_list = dmactions.get_walk_filelist(search_dirs) for type in FILESET_TYPES: dmfilestat = result.get_filestat(type) dmactions.update_diskspace(dmfilestat, cached=cached_file_list) except SoftTimeLimitExceeded: logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk, result.resultsName), extra=d) except: raise
def copy_files_to_destination(source_dir, destination, dmfileset, log): cached_file_list = get_walk_filelist([source_dir]) to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list) logger.debug('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination)) log.write('Copy files to %s ...\n' % destination) for i, filepath in enumerate(to_process): log.write('%s.\n' % filepath) try: _copy_to_dir(filepath, source_dir, destination) except Exception as exception: logger.error(traceback.format_exc()) log.write("%s" % exception) log.flush() if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, 'plugin_out') if not os.path.isdir(plugin_out): oldmask = os.umask(0000) #grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) if dmfileset.type == dmactions_types.BASE: # for onboard results need to make sigproc_results link if os.path.exists(os.path.join(destination, 'onboard_results')): os.symlink( os.path.join(destination, 'onboard_results', 'sigproc_results'), os.path.join(destination, 'sigproc_results')) log.write('Copy files done.\n')