def crop_pdf(pdf, margins=0): mcdp_tmp_dir = get_mcdp_tmp_dir() prefix = 'crop_pdf()' d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix) try: f_pdf = os.path.join(d, 'file.pdf') with open(f_pdf, 'w') as f: f.write(pdf) f_pdf_crop = os.path.join(d, 'file_crop.pdf') cmd = [ 'pdfcrop', '--margins', str(margins), f_pdf, f_pdf_crop, ] system_cmd_result(d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_pdf_crop) as f: data = f.read() return data finally: shutil.rmtree(d)
def run_bibtex2html(contents): res = AugmentedResult() erase = True with tmpdir(prefix='bibtex', erase=erase, keep_on_exception=True) as d: fn = os.path.join(d, 'input.bib') fno = os.path.join(d, 'out') fno1 = fno + '.html' # fno2 = fno + '_bib.html' with open(fn, 'w') as f: f.write(contents) cmd = ['bibtex2html', '-unicode', '--dl', '-o', fno, fn] system_cmd_result( '.', cmd, display_stdout=False, display_stderr=False, raise_on_error=True, display_prefix=None, # leave it there env=None) bibtex2html_output = open(fno1).read() fixed = bibtex2html_output.replace('<p>\n</dd>', '</dd><!--fix-->') with open(os.path.join(d, 'fixed.html'), 'w') as f: f.write(fixed) out = process_bibtex2html_output(fixed, d) write_data_to_file(out, os.path.join(d, 'processed.html')) res.set_result(out) return res
def graphviz_run(filename_dot, output, prog='dot'): suff = os.path.splitext(output)[1][1:] if not suff in ['png', 'pdf', 'ps', 'svg']: raise ValueError((output, suff)) encoder = suff cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot] with timeit_wall('running graphviz on %s' % filename_dot, 1.0): try: # print('running graphviz') system_cmd_result( cwd='.', cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True, ) # print('done') except (CmdException, KeyboardInterrupt): emergency = 'emergency.dot' logger.error('saving to %r' % emergency) # XXX contents = open(filename_dot).read() with open(emergency, 'w') as f: f.write(contents) raise
def write_coverage_report(outdir, covdata, module): print('Writing coverage data to %s' % outdir) outdir = os.path.abspath(outdir) if not os.path.exists(outdir): os.makedirs(outdir) with create_tmp_dir() as cwd: coverage_file = os.path.join(cwd, '.coverage') with open(coverage_file, 'wb') as f: f.write(covdata) cmd = ['coverage', 'html', '-d', outdir, '--include=*/%s/*' % module] res = system_cmd_result( cwd=cwd, cmd=cmd, display_stdout=True, display_stderr=True, raise_on_error=True) print(res.stdout) print(res.stderr) system_cmd_result( cwd=cwd, cmd=['find', '.'], display_stdout=True, display_stderr=True, raise_on_error=True)
def get_ast_as_pdf(s, parse_expr): s = s.replace('\t', ' ') contents = ast_to_html(s, ignore_line=None, parse_expr=parse_expr, add_line_gutter=False) html = get_minimal_document(contents) mcdp_tmp_dir = get_mcdp_tmp_dir() prefix = 'get_ast_as_pdf()' d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix) try: f_html = os.path.join(d, 'file.html') with open(f_html, 'w') as f: f.write(html) try: f_pdf = os.path.join(d, 'file.pdf') cmd= ['wkhtmltopdf','-s','A1',f_html,f_pdf] system_cmd_result( d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_pdf) as f: data = f.read() data = crop_pdf(data, margins=0) return data except CmdException as e: raise e finally: shutil.rmtree(d)
def get_ast_as_pdf(s, parse_expr): s = s.replace('\t', ' ') contents = ast_to_html(s, complete_document=False, extra_css=None, ignore_line=None, parse_expr=parse_expr, add_line_gutter=False, add_css=False) html = get_minimal_document(contents) d = mkdtemp() f_html = os.path.join(d, 'file.html') with open(f_html, 'w') as f: f.write(html) try: f_pdf = os.path.join(d, 'file.pdf') cmd= ['wkhtmltopdf','-s','A1',f_html,f_pdf] system_cmd_result( d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_pdf) as f: data = f.read() data = crop_pdf(data, margins=0) return data except CmdException as e: raise e
def resize_icon(filename, size): check_isinstance(filename, str) tmppath = get_mcdp_tmp_dir() res = os.path.join(tmppath, 'resized', str(size)) safe_makedirs(res) resized = os.path.join(res, os.path.basename(filename)) if not os.path.exists(resized): cmd = ['convert', filename, '-resize', '%s' % size, # remove alpha - otherwise lulu complains '-background', 'white', '-alpha','remove', '-alpha','off', resized] try: system_cmd_result(cwd='.', cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True) except CmdException: raise return resized
def get_nodejs_bin(): """ Raises NodeNotFound (XXX) """ tries = ['nodejs', 'node'] try: cmd = [tries[0], '--version'] _res = system_cmd_result(os.getcwd(), cmd, display_stdout=False, display_stderr=False, raise_on_error=True) return tries[0] # pragma: no cover except CmdException as e: try: cmd = [tries[1], '--version'] _res = system_cmd_result(os.getcwd(), cmd, display_stdout=False, display_stderr=False, raise_on_error=True) return tries[1] except CmdException as e: # pragma: no cover msg = 'Node.js executable "node" or "nodejs" not found.' msg += '\nOn Ubuntu, it can be installed using:' msg += '\n\n\tsudo apt-get install -y nodejs' raise_wrapped(PrerenderError, e, msg, compact=True)
def convert_to_mov_prores(filename, out=None, quiet=True, profile=3, qv=None, timestamp=None, metadata={}): my_ext = '.mov' out = validate_args(filename, out, my_ext) cmds = ['ffmpeg'] cmds += ['-y'] cmds += ['-i', filename] cmds += ['-f', 'mov'] cmds += ['-an'] # no audio cmds += get_prores_encoder_params(profile, qv) cmds += get_ffmpeg_metadata_args(metadata, timestamp) cmds += [out] try: system_cmd_result('.', cmds, display_stdout=not quiet, display_stderr=not quiet, raise_on_error=True, capture_keyboard_interrupt=False) except CmdException: if os.path.exists(out): os.unlink(out) raise assert os.path.exists(out)
def convert_to_mkv_mp4(filename, out=None, quiet=True, crf=18, preset='medium', timestamp=None, metadata={}): """ Converts to a matrioska file with all metadata. """ my_ext = '.mkv' out = validate_args(filename, out, my_ext) cmds = ['ffmpeg'] cmds += ['-y'] cmds += ['-i', filename] cmds += get_x264_encoder_params(crf, preset) cmds += get_ffmpeg_metadata_args(metadata, timestamp) cmds += [out] try: system_cmd_result('.', cmds, display_stdout=not quiet, display_stderr=not quiet, raise_on_error=True, capture_keyboard_interrupt=False) except CmdException: if os.path.exists(out): os.unlink(out) raise assert os.path.exists(out)
def call_nosetests(module): import nose with create_tmp_dir() as cwd: cmd = ['nosetests', module] system_cmd_result( cwd=cwd, cmd=cmd, display_stdout=True, display_stderr=True, raise_on_error=True)
def test_example_package(): from system_cmd import system_cmd_result # make sure it's installed # noinspection PyUnresolvedReferences import example_package # @UnusedImport with create_tmp_dir() as cwd: print('Working in %r ' % cwd) cmd = ['comptests', '--contracts', '--coverage', # '--nonose', 'example_package'] res = system_cmd_result(cwd, cmd, display_stdout=True, display_stderr=True, raise_on_error=False) assert res.ret != 0, res fs = [ 'out-comptests/report.html', 'out-comptests/report/reportclass1single/' 'reportclass1single-c1a-checkclass1dynamic-examplepackage-exampleclass1.html', ] # # if False: # # these are for reports # fs += ['out-comptests/report/single/single-checkclass1dynamic' # '-examplepackage-exampleclass1.html', # 'out-comptests/report/reportclass1single/reportclass1single' # '-checkclass1dynamic-c1a-examplepackage-exampleclass1.html',] errors = [] for f in fs: fn = os.path.join(cwd, f) print('Testing %r' % f) if not os.path.exists(fn): errors.append(fn) if errors: msg = 'Files not found:\n' + '\n'.join(errors) raise Exception(msg) print('now calling comptests-to-junit') cmd = ['comptests-to-junit', 'out-comptests/compmake'] system_cmd_result(cwd, cmd, display_stdout=True, display_stderr=True, raise_on_error=True)
def run_example(name, command, expect_fail=False): examples = get_examples_path() pyfile = os.path.join(examples, '%s.py' % name) if not os.path.exists(pyfile): msg = 'Example file does not exist: %s' % pyfile raise Exception(msg) with create_tmp_dir() as cwd: cmd = [pyfile, command] try: res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) if expect_fail: msg = 'Expected failure of %s but everything OK.' % name msg += '\n cwd = %s' % cwd msg += '\n' + indent(res.stderr, 'stderr| ') msg += '\n' + indent(res.stdout, 'stdout| ') raise Exception(msg) return res except CmdException as e: stderr = e.res.stderr stdout = e.res.stdout if not expect_fail: msg = ('Example %r: Command %r failed unexpectedly.' % (name, command)) msg += '\n retcode: %r' % e.res.ret msg += '\n' + indent(stderr, 'stderr| ') msg += '\n' + indent(stdout, 'stdout| ') raise Exception(msg)
def check_git_supports_superproject(): res = system_cmd_result('.', ['git', '--version'], display_stdout=False, display_stderr=False, raise_on_error=True, capture_keyboard_interrupt=False, env=None)
def crop_pdf(pdf, margins=0): d = mkdtemp() f_pdf = os.path.join(d, 'file.pdf') with open(f_pdf, 'w') as f: f.write(pdf) f_pdf_crop = os.path.join(d, 'file_crop.pdf') cmd = ['pdfcrop', '--margins', str(margins), f_pdf, f_pdf_crop] system_cmd_result( d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_pdf_crop) as f: data = f.read() return data
def get_qacct(sge_id): """ Only provides results if the job already rann. Raises JobNotRunYet if the job didn't run. """ cmd = ['qacct', '-j', sge_id] cwd = os.getcwd() res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=False) if res.ret == 1: if 'startup' in res.stderr or 'startup' in res.stdout: # /opt/sge6/default/common/accounting: No such file or directory # no jobs running since startup raise JobNotRunYet(sge_id) if 'not found' in res.stderr: # todo raise JobNotRunYet(sge_id) raise Exception('qcct failed: %s' % res) if res.ret != 0: raise Exception('qcct failed: %s' % res) values = {} for line in res.stdout.split('\n'): tokens = line.split() if len(tokens) >= 2: # XXX k = tokens[0] v = " ".join(tokens[1:]) if k == 'failed': v = tokens[1] values[k] = v return values
def run0(self, cmd, cwd=None, errmsg=None, raise_on_error=True): try: if cwd is None: cwd = self.destination display_prefix = '%-30s: ' % self.short_path if self.show_operations: print(display_prefix + ' %s ' % str(cmd)) display_prefix = '%-30s ' % '' res = system_cmd_result(cwd, cmd, raise_on_error=raise_on_error, display_stdout=self.show_operations, display_stderr=self.show_operations, display_prefix=display_prefix ) return res except CmdException as e: if errmsg: s = '%s\n' % errmsg else: s = 'Command %r failed with error code %d.' % (cmd, e.res.ret) if e.res.stdout: s += ' stdout: %r ' % e.res.stdout if e.res.stderr: s += ' stderr: %r ' % e.res.stderr s = 'On resource:\n\t%s\n' % self +s raise ActionException(s)
def get_compmake_bin(): """ Returns the path to the compmake executable. """ if SGEJob.compmake_bin is None: cwd = os.path.abspath(os.getcwd()) compmake_bin = system_cmd_result(cwd, 'which compmake').stdout.strip() compmake_bin = os.path.abspath(compmake_bin) SGEJob.compmake_bin = compmake_bin return SGEJob.compmake_bin
def find_command_path(prog): res = system_cmd_result(cwd=os.getcwd(), cmd=['which', prog], display_stdout=False, display_stderr=False, raise_on_error=True) prog = res.stdout return prog
def get_command_output(cmd): cwd = os.getcwd() res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) return res.stdout
def parmake_job2_new_process(args): """ Starts the job in a new compmake process. """ (job_id, context) = args compmake_bin = which("compmake") db = context.get_compmake_db() storage = db.basepath # XXX: where = os.path.join(storage, "parmake_job2_new_process") if not os.path.exists(storage): try: os.makedirs(storage) except: pass out_result = os.path.join(where, "%s.results.pickle" % job_id) out_result = os.path.abspath(out_result) cmd = [compmake_bin, storage] if not all_disabled(): cmd += ["--contracts"] cmd += [ "--status_line_enabled", "0", "--colorize", "0", "-c", "make_single out_result=%s %s" % (out_result, job_id), ] cwd = os.getcwd() cmd_res = system_cmd_result( cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=False ) ret = cmd_res.ret if ret == CompmakeConstants.RET_CODE_JOB_FAILED: # XXX: msg = "Job %r failed in external process" % job_id msg += indent(cmd_res.stdout, "stdout| ") msg += indent(cmd_res.stderr, "stderr| ") res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) raise JobFailed.from_dict(res) elif ret != 0: msg = "Host failed while doing %r" % job_id msg += "\n cmd: %s" % " ".join(cmd) msg += "\n" + indent(cmd_res.stdout, "stdout| ") msg += "\n" + indent(cmd_res.stderr, "stderr| ") raise CompmakeBug(msg) # XXX: res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) return res
def test_example_package(): from system_cmd import system_cmd_result # make sure it's installed import example_package # @UnusedImport with create_tmp_dir() as cwd: print('Working in %r ' % cwd) cmd = [ 'comptests', '--contracts', #'--nonose', 'example_package' ] system_cmd_result(cwd, cmd, display_stdout=True, display_stderr=True, raise_on_error=True) fs = [ 'out-comptests/report.html', 'out-comptests/report/reportclass1single/' 'reportclass1single-c1a-checkclass1dynamic-examplepackage-exampleclass1.html', ] # # if False: # # these are for reports # fs += ['out-comptests/report/single/single-checkclass1dynamic' # '-examplepackage-exampleclass1.html', # 'out-comptests/report/reportclass1single/reportclass1single' # '-checkclass1dynamic-c1a-examplepackage-exampleclass1.html',] errors = [] for f in fs: fn = os.path.join(cwd, f) print('Testing %r' % f) if not os.path.exists(fn): errors.append(fn) if errors: msg = 'Files not found:\n' + '\n'.join(errors) raise Exception(msg)
def jobs_nosetests_single(context, module): with create_tmp_dir() as cwd: out = os.path.join(cwd, '%s.pickle' % module) cmd = ['nosetests', '--collect-only', '--with-xunitext', '--xunitext-file' , out, '-v', '-s', module] system_cmd_result( cwd=cwd, cmd=cmd, display_stdout=True, display_stderr=True, raise_on_error=True) tests = safe_pickle_load(out) print('found %d mcdp_lang_tests ' % len(tests)) for t in tests: context.comp(execute, t)
def call_nosetests_plus_coverage(module): """ This also calls the coverage module. It returns the .coverage file as a string. """ with create_tmp_dir() as cwd: prog = find_command_path('nosetests') cmd = [prog, module] # note: coverage -> python-coverage in Ubuntu14 cmd = ['coverage', 'run'] + cmd system_cmd_result( cwd=cwd, cmd=cmd, display_stdout=True, display_stderr=True, raise_on_error=True) coverage_file = os.path.join(cwd, '.coverage') with open(coverage_file) as f: res = f.read() #print('read %d bytes in %s' % (len(res), coverage_file)) return res
def render_pdf(data_aug): data = data_aug.get_result() prefix = 'prince_render' with tmpdir(prefix=prefix) as d: f_html = os.path.join(d, 'file.html') with open(f_html, 'w') as f: f.write(data) f_out = os.path.join(d, 'out.pdf') cmd = ['prince', '--javascript', '-o', f_out, f_html] pwd = os.getcwd() system_cmd_result(pwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_out) as f: data = f.read() return data
def delete_job(self): # cmd = ['qdel', self.sge_id] cmd = ['qdel', self.sge_job_name] cwd = os.path.abspath(os.getcwd()) # TODO: check errors try: _ = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True, capture_keyboard_interrupt=False) except CmdException as e: error('Error while deleting job:\n%s' % e)
def graphviz_run(filename_dot, output, prog='dot'): suff = os.path.splitext(output)[1][1:] if not suff in ['png', 'pdf', 'ps', 'svg']: raise ValueError((output, suff)) encoder = suff cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot] try: # print('running graphviz') system_cmd_result(cwd='.', cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True) # print('done') except (CmdException, KeyboardInterrupt): emergency = 'emergency.dot' logger.error('saving to %r' % emergency) # XXX contents = open(filename_dot).read() with open(emergency, 'w') as f: f.write(contents) # print(contents) raise
def syntax_pdf(data): """ Returns a PDF string """ from mcdp_report.html import ast_to_html s = data['s'] s = s.replace('\t', ' ') lines_to_hide = get_lines_to_hide(data['params']) def ignore_line(lineno): return lineno in lines_to_hide contents = ast_to_html(s, ignore_line=ignore_line, parse_expr=Syntax.ndpt_dp_rvalue, ) html = get_minimal_document(contents) d = mkdtemp() f_html = os.path.join(d, 'file.html') with open(f_html, 'w') as f: f.write(html) try: f_pdf = os.path.join(d, 'file.pdf') cmd= ['wkhtmltopdf','-s','A1',f_html,f_pdf] system_cmd_result( d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) f_pdf_crop = os.path.join(d, 'file_crop.pdf') cmd = ['pdfcrop', '--margins', '4', f_pdf, f_pdf_crop] system_cmd_result( d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_pdf_crop) as f: data = f.read() f_png = os.path.join(d, 'file.png') cmd = ['convert', '-density', '600', f_pdf_crop, '-background', 'white', '-alpha', 'remove', '-resize', '50%', f_png] system_cmd_result( d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_png) as f: pngdata = f.read() return [('pdf', 'syntax_pdf', data), ('png', 'syntax_pdf', pngdata)] except CmdException as e: raise e
def get_active_groups(username=None): cmd = ['groups'] if username: cmd.append(username) try: res = system_cmd_result('.', cmd, display_stdout=False, display_stderr=False, raise_on_error=True, capture_keyboard_interrupt=False, env=None) except CmdException as e: raise Exception(str(e)) active_groups = res.stdout.split() # XXX return active_groups
def get_qacct(self): cmd = ['qacct', '-j', self.sge_id] cwd = os.getcwd() res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=True, capture_keyboard_interrupt=False) values = {} for line in res.stdout.split('\n'): tokens = line.split() if len(tokens) >= 2: # XXX k = tokens[0] v = " ".join(tokens[1:]) if k == 'failed': v = tokens[1] values[k] = v return values
def lessc_string(s1): """ Runs "node lessc" on the string Raises LesscError. """ # print(indent(s1, 'lessc input: ')) mcdp_tmp_dir = get_mcdp_tmp_dir() prefix = 'prerender_lessc' d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix) try: f1 = os.path.join(d, 'input.less') f2 = os.path.join(d, 'out.css') with open(f1, 'w') as f: f.write(s1) try: cmd = ['lessc', f1, f2] res = system_cmd_result(d, cmd, display_stdout=True, display_stderr=True, raise_on_error=False) if res.ret: msg = 'lessc error (return value = %d).' % res.ret msg += '\n\n' + indent(res.stderr, '', 'stderr:') msg += '\n\n' + indent(res.stdout, '', 'stdout:') raise LesscError(msg) with open(f2) as f: data = f.read() return data except CmdException as e: raise e finally: shutil.rmtree(d)
def get_last_version_fresh(): url = 'https://pypi.org/pypi/duckietown-shell/json' try: req = urllib2.Request(url) try: res = urllib2.urlopen(req, timeout=1) data = res.read() except urllib2.URLError as e: # msg = 'Cannot connect to server %s: %s' % (url, (e)) # raise Exception(msg) if which('curl') is not None: res = system_cmd_result('.', ['curl', url]) data = res.stdout else: raise CouldNotGetVersion() info = json.loads(data) last_version = info['info']['version'] return last_version except Exception as e: raise CouldNotGetVersion(str(e))
def src_cites_pkg(src, pkg): cmd = ['grep', '-r', ' ' + pkg, src] print " ".join(cmd) res = system_cmd_result('.', cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=True) if res.ret == 1: return False # not found elif res.ret == 0: # found out = res.stdout if 'import' in out: return True else: return False # print out # # print res.stdout.split('\n')[-1] return True elif res.ret == -1: # error raise Exception(res)
def plot_pkgs(pkgs, out, ignore_repo=[], cluster=False, link_same_repo=True): dirname = os.path.dirname(out) if dirname and not os.path.exists(dirname): os.makedirs(dirname) import gvgen graph = gvgen.GvGen() repo2node = {} pkg2node = {} def get_repo_node(repo): if not repo in repo2node: repo2node[repo] = graph.newItem(repo) return repo2node[repo] def get_pkg_node(id_pkg): if not id_pkg in pkg2node: if cluster: pkg2node[id_pkg] = graph.newItem(id_pkg, get_repo_node(repo)) else: pkg2node[id_pkg] = graph.newItem(id_pkg) return pkg2node[id_pkg] # if cluster: # repo2node = {} # for id_pkg, pkg in pkgs.items(): # repo = pkg['repo'] # if repo in ignore_repo: # continue # if not repo in repo2node: # repo2node[repo] = graph.newItem(repo) # if cluster: # pkg2node['procgraph'] = graph.newItem('procgraph', repo2node['procgraph']) # else: # pkg2node['procgraph'] = graph.newItem('procgraph') for id_pkg, pkg in pkgs.items(): repo = pkg['repo'] if repo in ignore_repo: continue # if repo == 'procgraph': # pkg2node[id_pkg] = pkg2node['procgraph'] # else: get_pkg_node(id_pkg) for id_pkg, pkg in pkgs.items(): for child in pkg['requires']: if not child in pkgs: print('warning, %s dependence %s not found in list' % (id_pkg, child)) else: child_repo = pkgs[child]['repo'] if child_repo in ignore_repo: continue same_repo = pkgs[id_pkg]['repo'] == child_repo # if not same_repo: if link_same_repo or not same_repo: l = graph.newLink(get_pkg_node(id_pkg), get_pkg_node(child)) d12 = child in pkgs[id_pkg]['requires'] d21 = id_pkg in pkgs[child]['requires'] loop = d12 and d21 if loop: graph.propertyAppend(l,"color","red") # TODO: add check? with open(out, 'w') as f: graph.dot(f) cmd = ['dot', out, '-Tpng', '-o', out + '.png'] system_cmd_result('.', cmd, raise_on_error=True)
def prerender_mathjax_(html): """ Runs the prerender.js script to pre-render the MathJax into images. Raises PrerenderError. """ assert not '<html>' in html, html use = get_nodejs_bin() html = html.replace('<p>$$', '\n$$') html = html.replace('$$</p>', '$$\n') script = get_prerender_js() mcdp_tmp_dir = get_mcdp_tmp_dir() prefix = 'prerender_mathjax_' d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix) try: f_html = os.path.join(d, 'file.html') with open(f_html, 'w') as f: f.write(html) try: f_out = os.path.join(d, 'out.html') cmd = [use, script, f_html, f_out] pwd = os.getcwd() res = system_cmd_result(pwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False) if res.ret: # pragma: no cover if 'Error: Cannot find module' in res.stderr: msg = 'You have to install the MathJax and/or jsdom libraries.' msg += '\nOn Ubuntu, you can install them using:' msg += '\n\n\tsudo apt-get install npm' msg += '\n\n\tnpm install MathJax-node jsdom' msg += '\n\n' + indent(res.stderr, ' |') raise PrerenderError(msg) if 'parse error' in res.stderr: lines = [ _ for _ in res.stderr.split('\n') if 'parse error' in _ ] assert lines msg = 'LaTeX conversion errors:\n\n' + '\n'.join(lines) raise PrerenderError(msg) msg = 'Unknown error (ret = %d).' % res.ret msg += '\n\n' + indent(res.stderr, ' |') raise PrerenderError(msg) with open(f_out) as f: data = f.read() # Read the data soup = bs(data) # find this and move it at the end # <style id="MathJax_SVG_styles" tag_style = soup.find(id='MathJax_SVG_styles') if not tag_style: msg = 'Expected to find style MathJax_SVG_styles' raise_desc(Exception, msg, soup=str(soup)) # <svg style="display: none;"><defs id="MathJax_SVG_glyphs"> tag_svg_defs = soup.find('svg', style="display: none;") if not tag_svg_defs: msg = 'Expected to find tag <svg display=none>' raise_desc(Exception, msg, soup=str(soup)) other_tag = soup.find('div', style="display:none") if not other_tag: msg = 'Expected to find tag <div style="display:none">' raise_desc(Exception, msg, soup=str(soup)) #<div style="display:none">Because of mathjax bug</div> soup.append(other_tag.extract()) soup.append(tag_svg_defs.extract()) soup.append(tag_style.extract()) data = to_html_stripping_fragment(soup) return data except CmdException as e: # pragma: no cover raise e finally: shutil.rmtree(d)
def command(shell, args): parser = argparse.ArgumentParser() parser.add_argument( '--image', default= 'andreacensi/mcdp_books:duckuments@sha256:ae2fcdbb8ce409e4817ed74c67b04bb91cd14ca96bed887e75e5275fa2efc933', help="Which image to use") parsed = parser.parse_args(args=args) image = parsed.image check_docker_environment() # check_git_supports_superproject() from system_cmd import system_cmd_result pwd = os.getcwd() bookdir = os.path.join(pwd, 'book') if not os.path.exists(bookdir): msg = 'Could not find "book" directory %r.' % bookdir DTCommandAbs.fail(msg) # check that the resources directory is present resources = os.path.join(pwd, 'resources') if not os.path.exists(os.path.join(resources, 'templates')): msg = 'It looks like that the "resources" repo is not checked out.' msg += '\nMaybe try:\n' msg += '\n git submodule init' msg += '\n git submodule update' raise Exception(msg) # XXX entries = list(os.listdir(bookdir)) entries = [_ for _ in entries if not _[0] == '.'] if len(entries) > 1: msg = 'Found more than one directory in "book": %s' % entries DTCommandAbs.fail(msg) bookname = entries[0] src = os.path.join(bookdir, bookname) res = system_cmd_result(pwd, ['git', '--version'], raise_on_error=True) git_version = res.stdout print('git version: %s' % git_version) cmd = ['git', 'rev-parse', '--show-superproject-working-tree'] res = system_cmd_result(pwd, cmd, raise_on_error=True) gitdir_super = res.stdout.strip() if '--show' in gitdir_super or not gitdir_super: msg = "Your git version is too low, as it does not support --show-superproject-working-tree" msg += '\n\nDetected: %s' % git_version raise InvalidEnvironment(msg) print('gitdir_super: %r' % gitdir_super) res = system_cmd_result(pwd, ['git', 'rev-parse', '--show-toplevel'], raise_on_error=True) gitdir = res.stdout.strip() if '--show' in gitdir or not gitdir: msg = "Your git version is too low, as it does not support --show-toplevel" msg += '\n\nDetected: %s' % git_version raise InvalidEnvironment(msg) print('gitdir: %r' % gitdir) pwd1 = os.path.realpath(pwd) user = getpass.getuser() tmpdir = '/tmp' fake_home = os.path.join(tmpdir, 'fake-%s-home' % user) if not os.path.exists(fake_home): os.makedirs(fake_home) resources = 'resources' uid1 = os.getuid() if sys.platform == 'darwin': flag = ':delegated' else: flag = '' cmd = [ 'docker', 'run', '-v', '%s:%s%s' % (gitdir, gitdir, flag), '-v', '%s:%s%s' % (gitdir_super, gitdir_super, flag), '-v', '%s:%s%s' % (pwd1, pwd1, flag), '-v', '%s:%s%s' % (fake_home, '/home/%s' % user, flag), '-e', 'USER=%s' % user, '-e', 'USERID=%s' % uid1, '-m', '4GB', '--user', '%s' % uid1 ] interactive = True if interactive: cmd.append('-it') cmd += [ image, '/project/run-book-native.sh', bookname, src, resources, pwd1 ] print('executing:\nls ' + " ".join(cmd)) # res = system_cmd_result(pwd, cmd, raise_on_error=True) try: p = subprocess.Popen(cmd, bufsize=0, executable=None, stdin=None, stdout=None, stderr=None, preexec_fn=None, shell=False, cwd=pwd, env=None) except OSError as e: if e.errno == 2: msg = 'Could not find "docker" executable.' DTCommandAbs.fail(msg) raise p.communicate() print('\n\nCompleted.')
def test_false2(): res = system_cmd_result('.', 'cat UTF-8-test.txt', display_stderr=True, display_stdout=True) print(res)
def main(): try: documents_data = sys.stdin.read() documents = yaml.load(documents_data) out = 'pdfs' if not os.path.exists(out): os.makedirs(out) errors = [] pdfs = [] for d in documents: id_document = get_id(d) pdf_url = make_pdf_url(id_document) pdf_file = os.path.join(out, id_document + '.pdf') if os.path.exists(pdf_file): pdfs.append(pdf_file) else: logger.info('Downloading %s' % pdf_file) response = urllib2.urlopen(pdf_url) data = response.read() is_valid = (data[1:4] == 'PDF') if is_valid: with open(pdf_file, 'w') as f: f.write(data) pdfs.append(pdf_file) else: logger.error('Invalid response for document %r' % d) errors.append(id_document) with open(pdf_file + '.invalid-response.html', 'w') as f: f.write(data) pdfout = 'joined.pdf' cmd = ['pdftk'] cmd.extend(pdfs) cmd.extend(['cat', 'output', pdfout]) system_cmd_result(cwd='.', cmd=cmd, display_stdout=True, display_stderr=True, raise_on_error=True) with open(pdfout) as f: data = f.read() logger.info('Writing on stdout %s' % len(data)) sys.stdout.write(data) if errors: logger.error('Could not download all files.') sys.exit(2) except Exception as e: import traceback logger.error(traceback.format_exc(e)) sys.exit(-1)
def execute(self, spool): db = self.db # Todo: check its this one storage = os.path.abspath(db.basepath) # create a new spool directory for each execution # otherwise we get confused! self.stderr = os.path.join(spool, '%s.stderr' % self.job_id) self.stdout = os.path.join(spool, '%s.stdout' % self.job_id) self.retcode = os.path.join(spool, '%s.retcode' % self.job_id) self.out_results = os.path.join(spool, '%s.results.pickle' % self.job_id) if os.path.exists(self.stderr): os.remove(self.stderr) if os.path.exists(self.stdout): os.remove(self.stdout) if os.path.exists(self.retcode): os.remove(self.retcode) if os.path.exists(self.out_results): os.remove(self.out_results) options = [] cwd = os.path.abspath(os.getcwd()) variables = dict(SGE_O_WORKDIR=cwd, PYTHONPATH=os.getenv('PYTHONPATH', '') + ':' + cwd) # nice-looking name self.sge_job_name = 'cm%s-%s' % (os.getpid(), self.job_id) # Note that we get the official "id" later and we store it in # self.sge_id options.extend( ['-v', ",".join('%s=%s' % x for x in variables.items())]) # XXX: spaces options.extend(['-e', self.stderr]) options.extend(['-o', self.stdout]) options.extend(['-N', self.sge_job_name]) options.extend(['-wd', cwd]) options.extend(['-V']) # pass all environment options.extend(['-terse']) compmake_bin = SGEJob.get_compmake_bin() compmake_options = [ compmake_bin, storage, '--retcodefile', self.retcode, '--status_line_enabled', '0', '--colorize', '0', '-c', '"make_single out_result=%s %s"' % (self.out_results, self.job_id), ] if not all_disabled(): compmake_options += ['--contracts'] # XXX: spaces in variable out_result write_stdin = ' '.join(compmake_options) cmd = ['qsub'] + options res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=True, raise_on_error=True, write_stdin=write_stdin, capture_keyboard_interrupt=False) self.sge_id = res.stdout.strip() self.already_read = False self.npolls = 0 self.told_you_ready = False
def parmake_job2_new_process(args): """ Starts the job in a new compmake process. """ (job_id, context) = args compmake_bin = which('compmake') db = context.get_compmake_db() storage = db.basepath # XXX: where = os.path.join(storage, 'parmake_job2_new_process') if not os.path.exists(storage): try: os.makedirs(storage) except: pass out_result = os.path.join(where, '%s.results.pickle' % job_id) out_result = os.path.abspath(out_result) cmd = [compmake_bin, storage] if not all_disabled(): cmd += ['--contracts'] cmd += [ '--status_line_enabled', '0', '--colorize', '0', '-c', 'make_single out_result=%s %s' % (out_result, job_id), ] cwd = os.getcwd() cmd_res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=False) ret = cmd_res.ret if ret == CompmakeConstants.RET_CODE_JOB_FAILED: # XXX: msg = 'Job %r failed in external process' % job_id msg += indent(cmd_res.stdout, 'stdout| ') msg += indent(cmd_res.stderr, 'stderr| ') res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) raise JobFailed.from_dict(res) elif ret != 0: msg = 'Host failed while doing %r' % job_id msg += '\n cmd: %s' % " ".join(cmd) msg += '\n' + indent(cmd_res.stdout, 'stdout| ') msg += '\n' + indent(cmd_res.stderr, 'stderr| ') raise CompmakeBug(msg) # XXX: res = safe_pickle_load(out_result) os.unlink(out_result) result_dict_check(res) return res
def test_false(): res = system_cmd_result('.', 'cp not-existing done') print(res)
def convert_to_mp4(filename, mp4=None, quiet=True, crf=18, preset='medium', timestamp=None, metadata={}): """ Creates a web-ready mp4 using ffmpeg. Needs either qtquickstart (from the python package) or qt-quickstart from ffmpeg. On ubuntu 12.04: need 'sudo apt-get install x264 libavcodec-extra-53' to install necessary codecs. You can see a list of supported presets by using: 'x264 --help'. (other packages that might be helpful: # libavdevice-extra-52 libavfilter-extra-0 libavformat-extra-52 libavutil-extra-49 libpostproc-extra-51 libswscale-extra-0) """ my_ext = '.mp4' if os.path.splitext(mp4)[1] != my_ext: msg = 'I expect a %r as out (%r)' % (my_ext, mp4) raise ValueError(msg) if os.path.splitext(filename)[1] == my_ext: msg = 'Need a file that does not end in %r (%r)' % (my_ext, filename) raise ValueError(msg) basename, _ = os.path.splitext(filename) if mp4 is None: mp4 = basename + my_ext # need .mp4 at the end otherwise ffmpeg gets confused tmp = basename + '.mp4.firstpass.mp4' if not os.path.exists(filename): raise ValueError("File does not exist: %s" % filename) # if (os.path.exists(mp4) and # (os.path.getmtime(mp4) > os.path.getmtime(filename))): # return if os.path.exists(tmp): os.unlink(tmp) if os.path.exists(mp4): os.unlink(mp4) cmds = ['ffmpeg'] cmds += ['-y'] cmds += ['-i', filename] cmds += get_x264_encoder_params(crf, preset) cmds += get_ffmpeg_metadata_args(metadata, timestamp) cmds += [tmp] # print cmds try: system_cmd_result('.', cmds, display_stdout=not quiet, display_stderr=not quiet, raise_on_error=True, capture_keyboard_interrupt=False) except CmdException: if os.path.exists(tmp): os.unlink(tmp) raise do_quickstart(source=tmp, target=mp4) if os.path.exists(tmp): os.unlink(tmp)
def process_svg_file(filename, target, preamble, proc_dir): if not os.path.exists(proc_dir): try: os.makedirs(proc_dir) except: pass logger.debug('Copying SVG file to temp directory %s '% proc_dir) shutil.copy(filename, join(proc_dir, 'in.svg.tmp')) logger.debug('Running Inkscape.') cmd = [inkscape_cmd, '-D', '-z', '--file', join(proc_dir, 'in.svg.tmp'), '--export-pdf', join(proc_dir, 'out.pdf'), '--export-latex'] system_cmd_result(cwd=proc_dir, cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True) assert os.path.exists(join(proc_dir, 'out.pdf')) assert os.path.exists(join(proc_dir, 'out.pdf_tex')) logger.debug('Creating template file..') # ok now we have the pdf and pdf_tex files ready. # Let's build the temp tex file to generate the figure s = r""" \documentclass[]{article} \usepackage{graphicx} \usepackage{color} \begin{document} \pagestyle{empty} \input{PREAMBLE} \input{out.pdf_tex} \end{document} """ s = s.strip() s = s.replace('PREAMBLE', os.path.realpath(preamble)) fn = join(proc_dir, 'file.tex') with open(fn, 'w') as f: f.write(s) # compile logger.debug('Compile using latex..') cmd = [latex_cmd, 'file.tex'] system_cmd_result(cwd=proc_dir, cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True) # crop logger.debug('Crop the pdf..') cmd = [crop_cmd, 'file.pdf', 'cropped.pdf'] system_cmd_result(cwd=proc_dir, cmd=cmd, display_stdout=False, display_stderr=False, raise_on_error=True) logger.debug('Copying to target %r..' % target) shutil.copy(join(proc_dir, 'cropped.pdf'), target)
def test_one(): system_cmd_result('.', 'ls -a')
def execute(self): db = get_compmake_db() # Todo: check its this one storage = os.path.abspath(db.basepath) spool = os.path.join(storage, 'sge_spool') if not os.path.exists(spool): os.makedirs(spool) self.stderr = os.path.join(spool, '%s.stderr' % self.job_id) self.stdout = os.path.join(spool, '%s.stdout' % self.job_id) self.retcode = os.path.join(spool, '%s.retcode' % self.job_id) if os.path.exists(self.stderr): os.remove(self.stderr) if os.path.exists(self.stdout): os.remove(self.stdout) if os.path.exists(self.retcode): os.remove(self.retcode) options = [] cwd = os.path.abspath(os.getcwd()) variables = dict(SGE_O_WORKDIR=cwd, PYTHONPATH=os.getenv('PYTHONPATH', '') + ':' + cwd) options.extend(['-v', ",". join('%s=%s' % x for x in variables.items())]) # warning: spaces options.extend(['-e', self.stderr]) options.extend(['-o', self.stdout]) options.extend(['-N', self.job_id]) options.extend(['-wd', cwd]) options.extend(['-V']) # pass all environment options.extend(['-terse']) compmake_bin = SGEJob.get_compmake_bin() compmake_options = [compmake_bin, storage, '--retcodefile', self.retcode, '--status_line_enabled', '0', '--colorize', '0', '-c', '"make %s"' % self.job_id] write_stdin = ' '.join(compmake_options) cmd = ['qsub'] + options res = system_cmd_result(cwd, cmd, display_stdout=False, display_stderr=True, raise_on_error=True, write_stdin=write_stdin, capture_keyboard_interrupt=False) self.sge_id = res.stdout.strip() self.already_read = False self.npolls = 0
def get_candidate_user(user_db, result, provider_name): ''' Returns a candidate UserInfo structure as observed from Oauth response ''' # get the data name = result.user.name email = result.user.email if result.user.id is None: msg = 'Cannot obtain ID for authentication.' raise Exception(msg) # if email is None: # msg = 'Cannot obtain email address' # raise Exception(msg) if provider_name == 'github': candidate_username = result.user.username website = result.user.data['blog'] affiliation = result.user.data['company'] unique_id = result.user.id picture = result.user.picture elif provider_name == 'facebook': if name is None: msg = 'Could not get name from Facebook so cannot create username.' raise Exception(msg) candidate_username = name.encode('utf8').replace(' ', '_').lower() website = None affiliation = None unique_id = result.user.id picture = result.user.picture elif provider_name == 'google': if name is None: msg = 'Could not get name from Google so cannot create username.' raise Exception(msg) candidate_username = name.encode('utf8').replace(' ', '_').lower() website = result.user.link affiliation = None unique_id = result.user.id picture = result.user.picture elif provider_name == 'linkedin': candidate_username = name.encode('utf8').replace(' ', '_').lower() website = result.user.link affiliation = None unique_id = result.user.id picture = result.user.picture elif provider_name == 'amazon': candidate_username = name.encode('utf8').replace(' ', '_').lower() website = result.user.link affiliation = None unique_id = result.user.id picture = None else: assert False, provider_name candidate_usernames = [candidate_username] username = user_db.find_available_user_name(candidate_usernames) res = {} res['username'] = username res['name'] = name res['email'] = email res['website'] = website res['affiliation'] = affiliation res['account_last_active'] = datetime.datetime.now() res['account_created'] = datetime.datetime.now() res['authentication_ids'] = [{ 'provider': provider_name, 'id': unique_id, 'password': None }] logger.info('Reading picture %s' % picture) if picture is None: jpg = None else: try: local_filename = 'pic.jpg' cwd = '.' cmd = ['wget', '-O', local_filename, picture] system_cmd_result(cwd, cmd, display_stdout=True, display_stderr=True, raise_on_error=True) # local_filename, headers = urlretrieve(picture) with open(local_filename, 'rb') as f: jpg = f.read() logger.info('read %s bytes' % len(jpg)) except BaseException as exc: logger.error(exc) jpg = None for k in res: if isinstance(res[k], unicode): res[k] = res[k].encode('utf8') res['subscriptions'] = [] res['groups'] = [ 'account_created_automatically', 'account_created_using_%s' % provider_name ] data = { 'info': res, 'images': { 'user': { 'jpg': jpg, 'png': None, 'svg': None, 'pdf': None } } } # logger.debug('new user:\n%s' % yaml_dump(data)) user = DB.view_manager.create_view_instance(DB.user, data) user.set_root() return user
def syntax_pdf(data): """ Returns a PDF string """ from mcdp_report.html import ast_to_html s = data['s'] s = s.replace('\t', ' ') lines_to_hide = get_lines_to_hide(data['params']) def ignore_line(lineno): return lineno in lines_to_hide contents = ast_to_html( s, ignore_line=ignore_line, parse_expr=Syntax.ndpt_dp_rvalue, ) html = get_minimal_document(contents) d = mkdtemp() f_html = os.path.join(d, 'file.html') with open(f_html, 'w') as f: f.write(html) try: f_pdf = os.path.join(d, 'file.pdf') cmd = ['wkhtmltopdf', '-s', 'A1', f_html, f_pdf] system_cmd_result(d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) f_pdf_crop = os.path.join(d, 'file_crop.pdf') cmd = ['pdfcrop', '--margins', '4', f_pdf, f_pdf_crop] system_cmd_result(d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_pdf_crop) as f: data = f.read() f_png = os.path.join(d, 'file.png') cmd = [ 'convert', '-density', '600', f_pdf_crop, '-background', 'white', '-alpha', 'remove', '-resize', '50%', f_png ] system_cmd_result(d, cmd, display_stdout=False, display_stderr=False, raise_on_error=True) with open(f_png) as f: pngdata = f.read() return [('pdf', 'syntax_pdf', data), ('png', 'syntax_pdf', pngdata)] except CmdException as e: raise e