def copy_files_to_destination(source_dir, destination, dmfileset, log):
cached_file_list = get_walk_filelist([source_dir])
to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list)
logger.debug("[Data Import] Importing %d files from %s to %s" % (len(to_process), source_dir, destination))
log.write("Copy files to %s ...\n" % destination)
for i, filepath in enumerate(to_process):
log.write("%s.\n" % filepath)
try:
_copy_to_dir(filepath, source_dir, destination)
except Exception as exception:
logger.error(traceback.format_exc())
log.write("%s" % exception)
log.flush()
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, "plugin_out")
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) # grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
if dmfileset.type == dmactions_types.BASE:
# for onboard results need to make sigproc_results link
if os.path.exists(os.path.join(destination, "onboard_results")):
os.symlink(
os.path.join(destination, "onboard_results", "sigproc_results"),
os.path.join(destination, "sigproc_results"),
)
log.write("Copy files done.\n")
def update_diskusage(resultpk):
'''
Task to update DMFileStat.diskspace for all associated with this resultpk
NOTE: This can be a long-lived task
'''
try:
result = Results.objects.get(pk=resultpk)
search_dirs = [result.get_report_dir(), result.experiment.expDir]
cached_file_list = dmactions.get_walk_filelist(search_dirs)
for type in FILESET_TYPES:
dmfilestat = result.get_filestat(type)
dmactions.update_diskspace(dmfilestat, cached=cached_file_list)
except SoftTimeLimitExceeded:
logger.warn("Time exceeded update_diskusage for (%d) %s" % (resultpk,result.resultsName), extra = d)
except:
raise
def update_diskusage(resultpk):
'''
Task to update DMFileStat.diskspace for all associated with this resultpk
NOTE: This can be a long-lived task
'''
try:
result = Results.objects.get(pk=resultpk)
search_dirs = [result.get_report_dir(), result.experiment.expDir]
cached_file_list = dmactions.get_walk_filelist(search_dirs)
for type in FILESET_TYPES:
dmfilestat = result.get_filestat(type)
dmactions.update_diskspace(dmfilestat, cached=cached_file_list)
except SoftTimeLimitExceeded:
logger.warn("Time exceeded update_diskusage for (%d) %s" %
(resultpk, result.resultsName),
extra=d)
except:
raise
def copy_files_to_destination(source_dir, destination, dmfileset, log):
cached_file_list = get_walk_filelist([source_dir])
to_process, to_keep = _file_selector(source_dir,
dmfileset.include,
dmfileset.exclude, [],
cached=cached_file_list)
logger.debug('[Data Import] Importing %d files from %s to %s' %
(len(to_process), source_dir, destination))
log.write('Copy files to %s ...\n' % destination)
for i, filepath in enumerate(to_process):
log.write('%s.\n' % filepath)
try:
_copy_to_dir(filepath, source_dir, destination)
except Exception as exception:
logger.error(traceback.format_exc())
log.write("%s" % exception)
log.flush()
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, 'plugin_out')
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) #grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
if dmfileset.type == dmactions_types.BASE:
# for onboard results need to make sigproc_results link
if os.path.exists(os.path.join(destination, 'onboard_results')):
os.symlink(
os.path.join(destination, 'onboard_results',
'sigproc_results'),
os.path.join(destination, 'sigproc_results'))
log.write('Copy files done.\n')
