def copy_files_to_destination(source_dir, destination, dmfileset, log):
cached_file_list = get_walk_filelist([source_dir])
to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list)
logger.debug("[Data Import] Importing %d files from %s to %s" % (len(to_process), source_dir, destination))
log.write("Copy files to %s ...\n" % destination)
for i, filepath in enumerate(to_process):
log.write("%s.\n" % filepath)
try:
_copy_to_dir(filepath, source_dir, destination)
except Exception as exception:
logger.error(traceback.format_exc())
log.write("%s" % exception)
log.flush()
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, "plugin_out")
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) # grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
if dmfileset.type == dmactions_types.BASE:
# for onboard results need to make sigproc_results link
if os.path.exists(os.path.join(destination, "onboard_results")):
os.symlink(
os.path.join(destination, "onboard_results", "sigproc_results"),
os.path.join(destination, "sigproc_results"),
)
log.write("Copy files done.\n")
def generate_pdf_from_archived_report(source_dir):
from iondb.rundb.data.dmactions import _copy_to_dir
def get_reportPK(directory):
# Isolate the Report Directory from the fullpath
reportnamedir = os.path.split(os.path.abspath(directory))[1]
# Strip off the underscore and experiment PK to get the Report Name
reportname = reportnamedir.replace("_" + reportnamedir.rsplit("_")[-1],
"")
# Lookup the Report Name in the database
reportPK = models.Results.objects.get(resultsName=reportname)
return reportPK
result = get_reportPK(source_dir)
dmfilestat = result.get_filestat('Output Files')
report_dir = result.get_report_dir()
# set archivepath for get_report_dir to find files when generating pdf
print "Current archivepath: %s" % (dmfilestat.archivepath)
print "Changing to: %s" % (source_dir)
dmfilestat.archivepath = source_dir
dmfilestat.save()
if False:
pdfpath = write_summary_pdf(result.pk)
print "Wrote file: %s" % (pdfpath)
shutil.copyfile(
pdfpath,
os.path.join(os.path.abspath(os.path.split(pdfpath)[0]),
'backupPDF.pdf'))
else:
# create report pdf via latex
latex_filepath = os.path.join(
'/tmp',
os.path.basename(report_dir) + '-full.tex')
url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1"
urllib.urlretrieve(url, latex_filepath)
pdf = [
"pdflatex", "-output-directory", "/tmp", "-interaction",
"batchmode", latex_filepath
]
proc = subprocess.Popen(pdf,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=source_dir)
_, stderr = proc.communicate()
if stderr:
logger.error('Error: ' + stderr)
else:
_copy_to_dir(
os.path.join('/tmp',
os.path.basename(report_dir) + '-full.pdf'),
'/tmp', report_dir)
shutil.copyfile(
os.path.join('/tmp',
os.path.basename(report_dir) + '-full.pdf'),
os.path.join(report_dir, 'backupPDF.pdf'))
return
def copy_files_to_destination(source_dir, destination, dmfileset,
cached_file_list, log, add_warning):
to_process, to_keep = dm_utils._file_selector(source_dir,
dmfileset.include,
dmfileset.exclude, [],
cached=cached_file_list)
logger.info('[Data Import] Importing %d files from %s to %s' %
(len(to_process), source_dir, destination))
log('Copy files to destination: %d files, source=%s destination=%s' %
(len(to_process), source_dir, destination))
plugin_warnings = {}
for i, filepath in enumerate(to_process):
log('%s' % filepath, flush=True)
try:
_copy_to_dir(filepath, source_dir, destination)
except Exception as e:
# log and ignore errors from plugin files
if 'plugin_out' in filepath:
plugin_name = filepath.split('plugin_out/')[1].split('_out')[0]
plugin_warnings[plugin_name] = plugin_warnings.get(
plugin_name, 0) + 1
log(traceback.format_exc())
else:
raise
for plugin, count in plugin_warnings.items():
add_warning('Unable to copy %d files for plugin %s' % (count, plugin))
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, 'plugin_out')
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) #grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
# remove pdf folder, it may have incorrect permissions
pdf_dir = os.path.join(destination, 'pdf')
if os.path.exists(pdf_dir):
shutil.rmtree(pdf_dir, ignore_errors=True)
# for onboard results need to create sigproc_results link
if dmfileset.type == dmactions_types.BASE:
if os.path.exists(os.path.join(destination, 'onboard_results')):
os.symlink(
os.path.join(destination, 'onboard_results',
'sigproc_results'),
os.path.join(destination, 'sigproc_results'))
log('Copy files to destination %s done.' % dmfileset.type)
def copy_files_to_destination(source_dir, destination, dmfileset, cached_file_list, log, add_warning):
to_process, to_keep = dm_utils._file_selector(
source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list)
logger.info('[Data Import] Importing %d files from %s to %s' %
(len(to_process), source_dir, destination))
log('Copy files to destination: %d files, source=%s destination=%s' %
(len(to_process), source_dir, destination))
plugin_warnings = {}
for i, filepath in enumerate(to_process):
log('%s' % filepath, flush=True)
try:
_copy_to_dir(filepath, source_dir, destination)
except Exception as e:
# log and ignore errors from plugin files
if 'plugin_out' in filepath:
plugin_name = filepath.split('plugin_out/')[1].split('_out')[0]
plugin_warnings[plugin_name] = plugin_warnings.get(plugin_name, 0) + 1
log(traceback.format_exc())
else:
raise
for plugin, count in plugin_warnings.items():
add_warning('Unable to copy %d files for plugin %s' % (count, plugin))
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, 'plugin_out')
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) # grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
# remove pdf folder, it may have incorrect permissions
pdf_dir = os.path.join(destination, 'pdf')
if os.path.exists(pdf_dir):
shutil.rmtree(pdf_dir, ignore_errors=True)
# for onboard results need to create sigproc_results link
if dmfileset.type == dmactions_types.BASE:
if os.path.exists(os.path.join(destination, 'onboard_results')):
os.symlink(os.path.join(destination, 'onboard_results', 'sigproc_results'),
os.path.join(destination, 'sigproc_results'))
log('Copy files to destination %s done.' % dmfileset.type)
def copy_files_to_destination(source_dir, destination, dmfileset, log):
to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [])
logger.debug('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination) )
log.write('Copy files to %s ...\n' % destination)
for i, filepath in enumerate(to_process):
log.write('%s.\n' % filepath)
_copy_to_dir(filepath, source_dir, destination)
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, 'plugin_out')
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) #grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
log.write('Copy files done.\n')
def copy_files_to_destination(source_dir, destination, dmfileset, log):
to_process, to_keep = _file_selector(source_dir, dmfileset.include,
dmfileset.exclude, [])
logger.debug('[Data Import] Importing %d files from %s to %s' %
(len(to_process), source_dir, destination))
log.write('Copy files to %s ...\n' % destination)
for i, filepath in enumerate(to_process):
log.write('%s.\n' % filepath)
_copy_to_dir(filepath, source_dir, destination)
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, 'plugin_out')
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) #grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
log.write('Copy files done.\n')
def generate_report_pdf(source_dir, result, dmfilestat, log, add_warning):
# copy report pdf from archived if exists, otherwise create
report_dir = result.get_report_dir()
source_dir = source_dir.rstrip('/')
pdf_filepath = os.path.join(report_dir, os.path.basename(report_dir)+'-full.pdf')
logger.debug('[Data Import] Generating report pdf %s.' % pdf_filepath)
log('Generating report pdf %s' % pdf_filepath)
if os.path.exists(os.path.join(source_dir, os.path.basename(source_dir)+'-full.pdf')):
_copy_to_dir(os.path.join(source_dir, os.path.basename(source_dir)+'-full.pdf'), source_dir, report_dir)
os.rename(os.path.join(report_dir, os.path.basename(source_dir)+'-full.pdf'), pdf_filepath)
elif os.path.exists(os.path.join(source_dir, 'report.pdf')):
_copy_to_dir(os.path.join(source_dir, 'report.pdf'), source_dir, report_dir)
os.rename(os.path.join(report_dir, 'report.pdf'), pdf_filepath)
else:
# set archivepath for get_report_dir to find files when generating pdf
dmfilestat.archivepath = source_dir
dmfilestat.save()
# create report pdf via latex
latex_filepath = os.path.join('/tmp', os.path.basename(report_dir)+'-full.tex' )
url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1"
urllib.urlretrieve(url , latex_filepath)
pdf = ["pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath]
proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir)
stdout, stderr = proc.communicate()
if stderr:
add_warning('Error creating report pdf: %s' % stderr)
else:
_copy_to_dir(os.path.join('/tmp', os.path.basename(report_dir)+'-full.pdf' ), '/tmp', report_dir)
log('Generate report pdf done.')
def generate_pdf_from_archived_report(source_dir):
from iondb.rundb.data.dmactions import _copy_to_dir
import shutil
def get_reportPK(directory):
# Isolate the Report Directory from the fullpath
reportnamedir = os.path.split(os.path.abspath(directory))[1]
# Strip off the underscore and experiment PK to get the Report Name
reportname = reportnamedir.replace("_"+reportnamedir.rsplit("_")[-1],"")
# Lookup the Report Name in the database
reportPK = models.Results.objects.get(resultsName=reportname)
return reportPK
result = get_reportPK(source_dir)
dmfilestat = result.get_filestat('Output Files')
report_dir = result.get_report_dir()
# set archivepath for get_report_dir to find files when generating pdf
print "Current archivepath: %s" % (dmfilestat.archivepath)
print "Changing to: %s" % (source_dir)
dmfilestat.archivepath = source_dir
dmfilestat.save()
if False:
pdfpath = write_summary_pdf(result.pk)
print "Wrote file: %s" % (pdfpath)
shutil.copyfile(pdfpath, os.path.join(os.path.abspath(os.path.split(pdfpath)[0]), 'backupPDF.pdf'))
else:
# create report pdf via latex
latex_filepath = os.path.join('/tmp', os.path.basename(report_dir)+'-full.tex' )
url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1"
urllib.urlretrieve(url , latex_filepath)
pdf = ["pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath]
proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir)
stdout, stderr = proc.communicate()
if stderr:
log.write('Error: '+ stderr)
else:
_copy_to_dir(os.path.join('/tmp', os.path.basename(report_dir)+'-full.pdf' ), '/tmp', report_dir)
shutil.copyfile(os.path.join('/tmp', os.path.basename(report_dir)+'-full.pdf' ),
os.path.join(report_dir, 'backupPDF.pdf'))
return
def generate_report_pdf(source_dir, result, dmfilestat, log, add_warning):
# copy report pdf from archived if exists, otherwise create
report_dir = result.get_report_dir()
source_dir = source_dir.rstrip('/')
pdf_filepath = os.path.join(report_dir, os.path.basename(report_dir) + '-full.pdf')
logger.info('[Data Import] Generating report pdf %s.' % pdf_filepath)
log('Generating report pdf %s' % pdf_filepath)
if os.path.exists(os.path.join(source_dir, os.path.basename(source_dir) + '-full.pdf')):
_copy_to_dir(
os.path.join(source_dir, os.path.basename(source_dir) + '-full.pdf'), source_dir, report_dir)
os.rename(os.path.join(report_dir, os.path.basename(source_dir) + '-full.pdf'), pdf_filepath)
elif os.path.exists(os.path.join(source_dir, 'report.pdf')):
_copy_to_dir(os.path.join(source_dir, 'report.pdf'), source_dir, report_dir)
os.rename(os.path.join(report_dir, 'report.pdf'), pdf_filepath)
else:
# set archivepath for get_report_dir to find files when generating pdf
dmfilestat.archivepath = source_dir
dmfilestat.save()
# create report pdf via latex
latex_filepath = os.path.join('/tmp', os.path.basename(report_dir) + '-full.tex')
url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1"
urllib.urlretrieve(url, latex_filepath)
pdf = ["pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath]
proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir)
stdout, stderr = proc.communicate()
if stderr:
add_warning('Error creating report pdf: %s' % stderr)
else:
_copy_to_dir(os.path.join('/tmp', os.path.basename(report_dir) + '-full.pdf'), '/tmp', report_dir)
log('Generate report pdf done.')
def copy_files_to_destination(source_dir, destination, dmfileset, log):
cached_file_list = get_walk_filelist([source_dir])
to_process, to_keep = _file_selector(source_dir,
dmfileset.include,
dmfileset.exclude, [],
cached=cached_file_list)
logger.debug('[Data Import] Importing %d files from %s to %s' %
(len(to_process), source_dir, destination))
log.write('Copy files to %s ...\n' % destination)
for i, filepath in enumerate(to_process):
log.write('%s.\n' % filepath)
try:
_copy_to_dir(filepath, source_dir, destination)
except Exception as exception:
logger.error(traceback.format_exc())
log.write("%s" % exception)
log.flush()
if dmfileset.type == dmactions_types.OUT:
# make sure we have plugin_out folder
plugin_out = os.path.join(destination, 'plugin_out')
if not os.path.isdir(plugin_out):
oldmask = os.umask(0000) #grant write permission to plugin user
os.mkdir(plugin_out)
os.umask(oldmask)
if dmfileset.type == dmactions_types.BASE:
# for onboard results need to make sigproc_results link
if os.path.exists(os.path.join(destination, 'onboard_results')):
os.symlink(
os.path.join(destination, 'onboard_results',
'sigproc_results'),
os.path.join(destination, 'sigproc_results'))
log.write('Copy files done.\n')
