def copy_files_to_destination(source_dir, destination, dmfileset, log): cached_file_list = get_walk_filelist([source_dir]) to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list) logger.debug("[Data Import] Importing %d files from %s to %s" % (len(to_process), source_dir, destination)) log.write("Copy files to %s ...\n" % destination) for i, filepath in enumerate(to_process): log.write("%s.\n" % filepath) try: _copy_to_dir(filepath, source_dir, destination) except Exception as exception: logger.error(traceback.format_exc()) log.write("%s" % exception) log.flush() if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, "plugin_out") if not os.path.isdir(plugin_out): oldmask = os.umask(0000) # grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) if dmfileset.type == dmactions_types.BASE: # for onboard results need to make sigproc_results link if os.path.exists(os.path.join(destination, "onboard_results")): os.symlink( os.path.join(destination, "onboard_results", "sigproc_results"), os.path.join(destination, "sigproc_results"), ) log.write("Copy files done.\n")
def generate_pdf_from_archived_report(source_dir): from iondb.rundb.data.dmactions import _copy_to_dir def get_reportPK(directory): # Isolate the Report Directory from the fullpath reportnamedir = os.path.split(os.path.abspath(directory))[1] # Strip off the underscore and experiment PK to get the Report Name reportname = reportnamedir.replace("_" + reportnamedir.rsplit("_")[-1], "") # Lookup the Report Name in the database reportPK = models.Results.objects.get(resultsName=reportname) return reportPK result = get_reportPK(source_dir) dmfilestat = result.get_filestat('Output Files') report_dir = result.get_report_dir() # set archivepath for get_report_dir to find files when generating pdf print "Current archivepath: %s" % (dmfilestat.archivepath) print "Changing to: %s" % (source_dir) dmfilestat.archivepath = source_dir dmfilestat.save() if False: pdfpath = write_summary_pdf(result.pk) print "Wrote file: %s" % (pdfpath) shutil.copyfile( pdfpath, os.path.join(os.path.abspath(os.path.split(pdfpath)[0]), 'backupPDF.pdf')) else: # create report pdf via latex latex_filepath = os.path.join( '/tmp', os.path.basename(report_dir) + '-full.tex') url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1" urllib.urlretrieve(url, latex_filepath) pdf = [ "pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath ] proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir) _, stderr = proc.communicate() if stderr: logger.error('Error: ' + stderr) else: _copy_to_dir( os.path.join('/tmp', os.path.basename(report_dir) + '-full.pdf'), '/tmp', report_dir) shutil.copyfile( os.path.join('/tmp', os.path.basename(report_dir) + '-full.pdf'), os.path.join(report_dir, 'backupPDF.pdf')) return
def copy_files_to_destination(source_dir, destination, dmfileset, cached_file_list, log, add_warning): to_process, to_keep = dm_utils._file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list) logger.info('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination)) log('Copy files to destination: %d files, source=%s destination=%s' % (len(to_process), source_dir, destination)) plugin_warnings = {} for i, filepath in enumerate(to_process): log('%s' % filepath, flush=True) try: _copy_to_dir(filepath, source_dir, destination) except Exception as e: # log and ignore errors from plugin files if 'plugin_out' in filepath: plugin_name = filepath.split('plugin_out/')[1].split('_out')[0] plugin_warnings[plugin_name] = plugin_warnings.get( plugin_name, 0) + 1 log(traceback.format_exc()) else: raise for plugin, count in plugin_warnings.items(): add_warning('Unable to copy %d files for plugin %s' % (count, plugin)) if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, 'plugin_out') if not os.path.isdir(plugin_out): oldmask = os.umask(0000) #grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) # remove pdf folder, it may have incorrect permissions pdf_dir = os.path.join(destination, 'pdf') if os.path.exists(pdf_dir): shutil.rmtree(pdf_dir, ignore_errors=True) # for onboard results need to create sigproc_results link if dmfileset.type == dmactions_types.BASE: if os.path.exists(os.path.join(destination, 'onboard_results')): os.symlink( os.path.join(destination, 'onboard_results', 'sigproc_results'), os.path.join(destination, 'sigproc_results')) log('Copy files to destination %s done.' % dmfileset.type)
def copy_files_to_destination(source_dir, destination, dmfileset, cached_file_list, log, add_warning): to_process, to_keep = dm_utils._file_selector( source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list) logger.info('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination)) log('Copy files to destination: %d files, source=%s destination=%s' % (len(to_process), source_dir, destination)) plugin_warnings = {} for i, filepath in enumerate(to_process): log('%s' % filepath, flush=True) try: _copy_to_dir(filepath, source_dir, destination) except Exception as e: # log and ignore errors from plugin files if 'plugin_out' in filepath: plugin_name = filepath.split('plugin_out/')[1].split('_out')[0] plugin_warnings[plugin_name] = plugin_warnings.get(plugin_name, 0) + 1 log(traceback.format_exc()) else: raise for plugin, count in plugin_warnings.items(): add_warning('Unable to copy %d files for plugin %s' % (count, plugin)) if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, 'plugin_out') if not os.path.isdir(plugin_out): oldmask = os.umask(0000) # grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) # remove pdf folder, it may have incorrect permissions pdf_dir = os.path.join(destination, 'pdf') if os.path.exists(pdf_dir): shutil.rmtree(pdf_dir, ignore_errors=True) # for onboard results need to create sigproc_results link if dmfileset.type == dmactions_types.BASE: if os.path.exists(os.path.join(destination, 'onboard_results')): os.symlink(os.path.join(destination, 'onboard_results', 'sigproc_results'), os.path.join(destination, 'sigproc_results')) log('Copy files to destination %s done.' % dmfileset.type)
def copy_files_to_destination(source_dir, destination, dmfileset, log): to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, []) logger.debug('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination) ) log.write('Copy files to %s ...\n' % destination) for i, filepath in enumerate(to_process): log.write('%s.\n' % filepath) _copy_to_dir(filepath, source_dir, destination) if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, 'plugin_out') if not os.path.isdir(plugin_out): oldmask = os.umask(0000) #grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) log.write('Copy files done.\n')
def copy_files_to_destination(source_dir, destination, dmfileset, log): to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, []) logger.debug('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination)) log.write('Copy files to %s ...\n' % destination) for i, filepath in enumerate(to_process): log.write('%s.\n' % filepath) _copy_to_dir(filepath, source_dir, destination) if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, 'plugin_out') if not os.path.isdir(plugin_out): oldmask = os.umask(0000) #grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) log.write('Copy files done.\n')
def generate_report_pdf(source_dir, result, dmfilestat, log, add_warning): # copy report pdf from archived if exists, otherwise create report_dir = result.get_report_dir() source_dir = source_dir.rstrip('/') pdf_filepath = os.path.join(report_dir, os.path.basename(report_dir)+'-full.pdf') logger.debug('[Data Import] Generating report pdf %s.' % pdf_filepath) log('Generating report pdf %s' % pdf_filepath) if os.path.exists(os.path.join(source_dir, os.path.basename(source_dir)+'-full.pdf')): _copy_to_dir(os.path.join(source_dir, os.path.basename(source_dir)+'-full.pdf'), source_dir, report_dir) os.rename(os.path.join(report_dir, os.path.basename(source_dir)+'-full.pdf'), pdf_filepath) elif os.path.exists(os.path.join(source_dir, 'report.pdf')): _copy_to_dir(os.path.join(source_dir, 'report.pdf'), source_dir, report_dir) os.rename(os.path.join(report_dir, 'report.pdf'), pdf_filepath) else: # set archivepath for get_report_dir to find files when generating pdf dmfilestat.archivepath = source_dir dmfilestat.save() # create report pdf via latex latex_filepath = os.path.join('/tmp', os.path.basename(report_dir)+'-full.tex' ) url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1" urllib.urlretrieve(url , latex_filepath) pdf = ["pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath] proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir) stdout, stderr = proc.communicate() if stderr: add_warning('Error creating report pdf: %s' % stderr) else: _copy_to_dir(os.path.join('/tmp', os.path.basename(report_dir)+'-full.pdf' ), '/tmp', report_dir) log('Generate report pdf done.')
def generate_pdf_from_archived_report(source_dir): from iondb.rundb.data.dmactions import _copy_to_dir import shutil def get_reportPK(directory): # Isolate the Report Directory from the fullpath reportnamedir = os.path.split(os.path.abspath(directory))[1] # Strip off the underscore and experiment PK to get the Report Name reportname = reportnamedir.replace("_"+reportnamedir.rsplit("_")[-1],"") # Lookup the Report Name in the database reportPK = models.Results.objects.get(resultsName=reportname) return reportPK result = get_reportPK(source_dir) dmfilestat = result.get_filestat('Output Files') report_dir = result.get_report_dir() # set archivepath for get_report_dir to find files when generating pdf print "Current archivepath: %s" % (dmfilestat.archivepath) print "Changing to: %s" % (source_dir) dmfilestat.archivepath = source_dir dmfilestat.save() if False: pdfpath = write_summary_pdf(result.pk) print "Wrote file: %s" % (pdfpath) shutil.copyfile(pdfpath, os.path.join(os.path.abspath(os.path.split(pdfpath)[0]), 'backupPDF.pdf')) else: # create report pdf via latex latex_filepath = os.path.join('/tmp', os.path.basename(report_dir)+'-full.tex' ) url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1" urllib.urlretrieve(url , latex_filepath) pdf = ["pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath] proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir) stdout, stderr = proc.communicate() if stderr: log.write('Error: '+ stderr) else: _copy_to_dir(os.path.join('/tmp', os.path.basename(report_dir)+'-full.pdf' ), '/tmp', report_dir) shutil.copyfile(os.path.join('/tmp', os.path.basename(report_dir)+'-full.pdf' ), os.path.join(report_dir, 'backupPDF.pdf')) return
def generate_report_pdf(source_dir, result, dmfilestat, log, add_warning): # copy report pdf from archived if exists, otherwise create report_dir = result.get_report_dir() source_dir = source_dir.rstrip('/') pdf_filepath = os.path.join(report_dir, os.path.basename(report_dir) + '-full.pdf') logger.info('[Data Import] Generating report pdf %s.' % pdf_filepath) log('Generating report pdf %s' % pdf_filepath) if os.path.exists(os.path.join(source_dir, os.path.basename(source_dir) + '-full.pdf')): _copy_to_dir( os.path.join(source_dir, os.path.basename(source_dir) + '-full.pdf'), source_dir, report_dir) os.rename(os.path.join(report_dir, os.path.basename(source_dir) + '-full.pdf'), pdf_filepath) elif os.path.exists(os.path.join(source_dir, 'report.pdf')): _copy_to_dir(os.path.join(source_dir, 'report.pdf'), source_dir, report_dir) os.rename(os.path.join(report_dir, 'report.pdf'), pdf_filepath) else: # set archivepath for get_report_dir to find files when generating pdf dmfilestat.archivepath = source_dir dmfilestat.save() # create report pdf via latex latex_filepath = os.path.join('/tmp', os.path.basename(report_dir) + '-full.tex') url = "http://127.0.0.1/report/" + str(result.pk) + "/?latex=1" urllib.urlretrieve(url, latex_filepath) pdf = ["pdflatex", "-output-directory", "/tmp", "-interaction", "batchmode", latex_filepath] proc = subprocess.Popen(pdf, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir) stdout, stderr = proc.communicate() if stderr: add_warning('Error creating report pdf: %s' % stderr) else: _copy_to_dir(os.path.join('/tmp', os.path.basename(report_dir) + '-full.pdf'), '/tmp', report_dir) log('Generate report pdf done.')
def copy_files_to_destination(source_dir, destination, dmfileset, log): cached_file_list = get_walk_filelist([source_dir]) to_process, to_keep = _file_selector(source_dir, dmfileset.include, dmfileset.exclude, [], cached=cached_file_list) logger.debug('[Data Import] Importing %d files from %s to %s' % (len(to_process), source_dir, destination)) log.write('Copy files to %s ...\n' % destination) for i, filepath in enumerate(to_process): log.write('%s.\n' % filepath) try: _copy_to_dir(filepath, source_dir, destination) except Exception as exception: logger.error(traceback.format_exc()) log.write("%s" % exception) log.flush() if dmfileset.type == dmactions_types.OUT: # make sure we have plugin_out folder plugin_out = os.path.join(destination, 'plugin_out') if not os.path.isdir(plugin_out): oldmask = os.umask(0000) #grant write permission to plugin user os.mkdir(plugin_out) os.umask(oldmask) if dmfileset.type == dmactions_types.BASE: # for onboard results need to make sigproc_results link if os.path.exists(os.path.join(destination, 'onboard_results')): os.symlink( os.path.join(destination, 'onboard_results', 'sigproc_results'), os.path.join(destination, 'sigproc_results')) log.write('Copy files done.\n')