示例#1
0
文件: pdf_ops.py 项目: rusi/mcdp
def crop_pdf(pdf, margins=0):

    mcdp_tmp_dir = get_mcdp_tmp_dir()
    prefix = 'crop_pdf()'
    d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix)

    try:
        f_pdf = os.path.join(d, 'file.pdf')
        with open(f_pdf, 'w') as f:
            f.write(pdf)
        f_pdf_crop = os.path.join(d, 'file_crop.pdf')
        cmd = [
            'pdfcrop',
            '--margins',
            str(margins),
            f_pdf,
            f_pdf_crop,
        ]
        system_cmd_result(d,
                          cmd,
                          display_stdout=False,
                          display_stderr=False,
                          raise_on_error=True)

        with open(f_pdf_crop) as f:
            data = f.read()
        return data
    finally:
        shutil.rmtree(d)
示例#2
0
def run_bibtex2html(contents):
    res = AugmentedResult()
    erase = True
    with tmpdir(prefix='bibtex', erase=erase, keep_on_exception=True) as d:
        fn = os.path.join(d, 'input.bib')
        fno = os.path.join(d, 'out')
        fno1 = fno + '.html'
        # fno2 = fno + '_bib.html'
        with open(fn, 'w') as f:
            f.write(contents)

        cmd = ['bibtex2html', '-unicode', '--dl', '-o', fno, fn]

        system_cmd_result(
            '.',
            cmd,
            display_stdout=False,
            display_stderr=False,
            raise_on_error=True,
            display_prefix=None,  # leave it there
            env=None)

        bibtex2html_output = open(fno1).read()

        fixed = bibtex2html_output.replace('<p>\n</dd>', '</dd><!--fix-->')

        with open(os.path.join(d, 'fixed.html'), 'w') as f:
            f.write(fixed)

        out = process_bibtex2html_output(fixed, d)

        write_data_to_file(out, os.path.join(d, 'processed.html'))

        res.set_result(out)
        return res
示例#3
0
def graphviz_run(filename_dot, output, prog='dot'):
    suff = os.path.splitext(output)[1][1:]
    if not suff in ['png', 'pdf', 'ps', 'svg']:
        raise ValueError((output, suff))

    encoder = suff

    cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot]

    with timeit_wall('running graphviz on %s' % filename_dot, 1.0):
        try:
            # print('running graphviz')
            system_cmd_result(
                cwd='.',
                cmd=cmd,
                display_stdout=False,
                display_stderr=False,
                raise_on_error=True,
            )
            # print('done')
        except (CmdException, KeyboardInterrupt):
            emergency = 'emergency.dot'
            logger.error('saving to %r' % emergency)  # XXX
            contents = open(filename_dot).read()
            with open(emergency, 'w') as f:
                f.write(contents)
            raise
示例#4
0
def write_coverage_report(outdir, covdata, module):
    print('Writing coverage data to %s' % outdir)
    outdir = os.path.abspath(outdir)
    if not os.path.exists(outdir):
        os.makedirs(outdir)
    with create_tmp_dir() as cwd:
        coverage_file = os.path.join(cwd, '.coverage')
        with open(coverage_file, 'wb') as f:
            f.write(covdata)

        cmd = ['coverage', 'html', '-d', outdir,
               '--include=*/%s/*' % module]
        res = system_cmd_result(
            cwd=cwd, cmd=cmd,
            display_stdout=True,
            display_stderr=True,
            raise_on_error=True)
        print(res.stdout)
        print(res.stderr)

        system_cmd_result(
            cwd=cwd, cmd=['find', '.'],
            display_stdout=True,
            display_stderr=True,
            raise_on_error=True)
示例#5
0
def get_ast_as_pdf(s, parse_expr):
    s = s.replace('\t', '    ')
    contents = ast_to_html(s,
                       ignore_line=None, parse_expr=parse_expr,
                       add_line_gutter=False)
    html = get_minimal_document(contents)
    
    mcdp_tmp_dir = get_mcdp_tmp_dir()
    prefix = 'get_ast_as_pdf()'
    d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix)
    
    try:
        f_html = os.path.join(d, 'file.html')
        with open(f_html, 'w') as f:
            f.write(html)
            
        try:
            f_pdf = os.path.join(d, 'file.pdf')
            cmd= ['wkhtmltopdf','-s','A1',f_html,f_pdf]
            system_cmd_result(
                    d, cmd, 
                    display_stdout=False,
                    display_stderr=False,
                    raise_on_error=True)
    
            with open(f_pdf) as f:
                data = f.read()
            
            data = crop_pdf(data, margins=0)
    
            return data
        except CmdException as e:
            raise e
    finally:
        shutil.rmtree(d)
示例#6
0
def get_ast_as_pdf(s, parse_expr):
    s = s.replace('\t', '    ')
    contents = ast_to_html(s, complete_document=False, extra_css=None,
                       ignore_line=None, parse_expr=parse_expr,
                       add_line_gutter=False, add_css=False)
    html = get_minimal_document(contents)
    d = mkdtemp()
    
    f_html = os.path.join(d, 'file.html')
    with open(f_html, 'w') as f:
        f.write(html)
        
    try:
        f_pdf = os.path.join(d, 'file.pdf')
        cmd= ['wkhtmltopdf','-s','A1',f_html,f_pdf]
        system_cmd_result(
                d, cmd, 
                display_stdout=False,
                display_stderr=False,
                raise_on_error=True)

        with open(f_pdf) as f:
            data = f.read()
        
        data = crop_pdf(data, margins=0)

        return data
    except CmdException as e:
        raise e
示例#7
0
def resize_icon(filename, size):
    check_isinstance(filename, str)
    
    tmppath = get_mcdp_tmp_dir()
    res = os.path.join(tmppath, 'resized', str(size))

    safe_makedirs(res)
    resized = os.path.join(res, os.path.basename(filename))
    if not os.path.exists(resized):
        cmd = ['convert', 
               filename, 
               '-resize', '%s' % size, 
               # remove alpha - otherwise lulu complains
               '-background', 'white',
                '-alpha','remove',
                '-alpha','off', 
               resized]
        try:

            system_cmd_result(cwd='.', cmd=cmd,
                     display_stdout=False,
                     display_stderr=False,
                     raise_on_error=True)

        except CmdException:
            raise
    return resized
示例#8
0
def get_nodejs_bin():
    """ Raises NodeNotFound (XXX) """
    tries = ['nodejs', 'node']
    try:
        cmd = [tries[0], '--version']
        _res = system_cmd_result(os.getcwd(),
                                 cmd,
                                 display_stdout=False,
                                 display_stderr=False,
                                 raise_on_error=True)
        return tries[0]  # pragma: no cover
    except CmdException as e:
        try:
            cmd = [tries[1], '--version']
            _res = system_cmd_result(os.getcwd(),
                                     cmd,
                                     display_stdout=False,
                                     display_stderr=False,
                                     raise_on_error=True)
            return tries[1]
        except CmdException as e:  # pragma: no cover
            msg = 'Node.js executable "node" or "nodejs" not found.'
            msg += '\nOn Ubuntu, it can be installed using:'
            msg += '\n\n\tsudo apt-get install -y nodejs'
            raise_wrapped(PrerenderError, e, msg, compact=True)
示例#9
0
def convert_to_mov_prores(filename, out=None, quiet=True, profile=3, qv=None, timestamp=None, metadata={}):
    my_ext = '.mov'    
    out = validate_args(filename, out, my_ext)

    cmds = ['ffmpeg']
    cmds += ['-y']
    cmds += ['-i', filename]
    cmds += ['-f', 'mov']
    cmds += ['-an']  # no audio
    cmds += get_prores_encoder_params(profile, qv)
    cmds += get_ffmpeg_metadata_args(metadata, timestamp)
    cmds += [out]
    
    try:
        system_cmd_result('.', cmds,
                  display_stdout=not quiet,
                  display_stderr=not quiet,
                  raise_on_error=True,
                  capture_keyboard_interrupt=False)
    except CmdException:
        if os.path.exists(out):
            os.unlink(out)
        raise
    
    assert os.path.exists(out)
示例#10
0
def convert_to_mkv_mp4(filename, out=None, quiet=True, crf=18, preset='medium', timestamp=None, metadata={}):
    """
        Converts to a matrioska file with all metadata.
        
    """
    my_ext = '.mkv'
    out = validate_args(filename, out, my_ext)
    

    cmds = ['ffmpeg']
    cmds += ['-y']
    cmds += ['-i', filename]
    cmds += get_x264_encoder_params(crf, preset)
    cmds += get_ffmpeg_metadata_args(metadata, timestamp)
    cmds += [out]
    
    try:
        system_cmd_result('.', cmds,
                  display_stdout=not quiet,
                  display_stderr=not quiet,
                  raise_on_error=True,
                  capture_keyboard_interrupt=False)
    except CmdException:
        if os.path.exists(out):
            os.unlink(out)
        raise
    
    assert os.path.exists(out)
示例#11
0
def call_nosetests(module):
    import nose
    with create_tmp_dir() as cwd:
        cmd = ['nosetests', module]
        system_cmd_result(
            cwd=cwd, cmd=cmd,
            display_stdout=True,
            display_stderr=True,
            raise_on_error=True)
def test_example_package():
    from system_cmd import system_cmd_result

    # make sure it's installed
    # noinspection PyUnresolvedReferences
    import example_package  # @UnusedImport

    with create_tmp_dir() as cwd:
        print('Working in %r ' % cwd)
        cmd = ['comptests',
               '--contracts',
               '--coverage',
               # '--nonose',
               'example_package']


        res = system_cmd_result(cwd, cmd,
                          display_stdout=True,
                          display_stderr=True,
                          raise_on_error=False)
        assert res.ret != 0, res

        fs = [
            'out-comptests/report.html',
            'out-comptests/report/reportclass1single/'
            'reportclass1single-c1a-checkclass1dynamic-examplepackage-exampleclass1.html',
        ]
        #
        #         if False:
        #             # these are for reports
        #             fs += ['out-comptests/report/single/single-checkclass1dynamic'
        #                   '-examplepackage-exampleclass1.html',
        #                   'out-comptests/report/reportclass1single/reportclass1single'
        #                   '-checkclass1dynamic-c1a-examplepackage-exampleclass1.html',]

        errors = []
        for f in fs:
            fn = os.path.join(cwd, f)
            print('Testing %r' % f)
            if not os.path.exists(fn):
                errors.append(fn)

        if errors:
            msg = 'Files not found:\n' + '\n'.join(errors)
            raise Exception(msg)

        print('now calling comptests-to-junit')
        cmd = ['comptests-to-junit', 'out-comptests/compmake']
        system_cmd_result(cwd, cmd,
                          display_stdout=True,
                          display_stderr=True,
                          raise_on_error=True)
示例#13
0
def run_example(name, command, expect_fail=False):
    examples = get_examples_path()
    pyfile = os.path.join(examples, '%s.py' % name)
    if not os.path.exists(pyfile):
        msg = 'Example file does not exist: %s' % pyfile
        raise Exception(msg)

    with create_tmp_dir() as cwd:
        cmd = [pyfile, command]
        try:
            res = system_cmd_result(cwd, cmd, 
                              display_stdout=False,
                              display_stderr=False,
                              raise_on_error=True)
            if expect_fail:
                msg = 'Expected failure of %s but everything OK.' % name
                msg += '\n cwd = %s'  % cwd 
                msg += '\n' + indent(res.stderr, 'stderr| ')
                msg += '\n' + indent(res.stdout, 'stdout| ')
                raise Exception(msg)
            return res
        except CmdException as e:
            stderr = e.res.stderr
            stdout = e.res.stdout
            if not expect_fail:
                msg = ('Example %r: Command %r failed unexpectedly.' % 
                       (name, command))
                msg += '\n retcode: %r' % e.res.ret
                msg += '\n' + indent(stderr, 'stderr| ')
                msg += '\n' + indent(stdout, 'stdout| ')
                raise Exception(msg)
示例#14
0
def check_git_supports_superproject():
    res = system_cmd_result('.', ['git', '--version'],
                            display_stdout=False,
                            display_stderr=False,
                            raise_on_error=True,
                            capture_keyboard_interrupt=False,
                            env=None)
示例#15
0
def crop_pdf(pdf, margins=0):
    d = mkdtemp()
    f_pdf = os.path.join(d, 'file.pdf')
    with open(f_pdf, 'w') as f:
        f.write(pdf)
    f_pdf_crop = os.path.join(d, 'file_crop.pdf')
    cmd = ['pdfcrop', '--margins', str(margins), f_pdf, f_pdf_crop]
    system_cmd_result(
            d, cmd,
            display_stdout=False,
            display_stderr=False,
            raise_on_error=True)

    with open(f_pdf_crop) as f:
        data = f.read()
    return data
示例#16
0
def get_qacct(sge_id):
    """ 
        Only provides results if the job already rann.
    
        Raises JobNotRunYet if the job didn't run.
    """
    cmd = ['qacct', '-j', sge_id]
    cwd = os.getcwd()
    res = system_cmd_result(cwd,
                            cmd,
                            display_stdout=False,
                            display_stderr=False,
                            raise_on_error=False,
                            capture_keyboard_interrupt=False)
    if res.ret == 1:
        if 'startup' in res.stderr or 'startup' in res.stdout:
            # /opt/sge6/default/common/accounting: No such file or directory
            # no jobs running since startup
            raise JobNotRunYet(sge_id)
        if 'not found' in res.stderr:
            # todo
            raise JobNotRunYet(sge_id)
        raise Exception('qcct failed: %s' % res)
    if res.ret != 0:
        raise Exception('qcct failed: %s' % res)
    values = {}
    for line in res.stdout.split('\n'):
        tokens = line.split()
        if len(tokens) >= 2:  # XXX
            k = tokens[0]
            v = " ".join(tokens[1:])
            if k == 'failed':
                v = tokens[1]
            values[k] = v
    return values
示例#17
0
def get_qacct(sge_id):
    """ 
        Only provides results if the job already rann.
    
        Raises JobNotRunYet if the job didn't run.
    """
    cmd = ['qacct', '-j', sge_id]
    cwd = os.getcwd()
    res = system_cmd_result(cwd, cmd,
                            display_stdout=False,
                            display_stderr=False,
                            raise_on_error=False,
                            capture_keyboard_interrupt=False)
    if res.ret == 1:
        if 'startup' in res.stderr or 'startup' in res.stdout:
            # /opt/sge6/default/common/accounting: No such file or directory
            # no jobs running since startup
            raise JobNotRunYet(sge_id)
        if 'not found' in res.stderr:
            # todo
            raise JobNotRunYet(sge_id)
        raise Exception('qcct failed: %s' % res)
    if res.ret != 0:
        raise Exception('qcct failed: %s' % res)
    values = {}
    for line in res.stdout.split('\n'):
        tokens = line.split()
        if len(tokens) >= 2:  # XXX
            k = tokens[0]
            v = " ".join(tokens[1:])
            if k == 'failed':
                v = tokens[1]
            values[k] = v
    return values
示例#18
0
 def run0(self, cmd, cwd=None, errmsg=None, raise_on_error=True): 
     try:
         if cwd is None: 
             cwd = self.destination
             
         display_prefix = '%-30s: ' % self.short_path
         
         if self.show_operations:
             print(display_prefix + ' %s ' % str(cmd))
            
         display_prefix = '%-30s  ' % ''
         res = system_cmd_result(cwd, cmd,
                                 raise_on_error=raise_on_error,
                                 display_stdout=self.show_operations,
                                 display_stderr=self.show_operations,
                                 display_prefix=display_prefix
                                 )
         return res
     except CmdException as e:
         if errmsg:
             s = '%s\n' % errmsg
         else:
             s = 'Command %r failed with error code %d.' % (cmd, e.res.ret)
             if e.res.stdout:
                 s += ' stdout: %r ' % e.res.stdout
             if e.res.stderr:
                 s += ' stderr: %r ' % e.res.stderr          
              
         s = 'On resource:\n\t%s\n' % self +s
         raise ActionException(s)
示例#19
0
def run_example(name, command, expect_fail=False):
    examples = get_examples_path()
    pyfile = os.path.join(examples, '%s.py' % name)
    if not os.path.exists(pyfile):
        msg = 'Example file does not exist: %s' % pyfile
        raise Exception(msg)

    with create_tmp_dir() as cwd:
        cmd = [pyfile, command]
        try:
            res = system_cmd_result(cwd,
                                    cmd,
                                    display_stdout=False,
                                    display_stderr=False,
                                    raise_on_error=True)
            if expect_fail:
                msg = 'Expected failure of %s but everything OK.' % name
                msg += '\n cwd = %s' % cwd
                msg += '\n' + indent(res.stderr, 'stderr| ')
                msg += '\n' + indent(res.stdout, 'stdout| ')
                raise Exception(msg)
            return res
        except CmdException as e:
            stderr = e.res.stderr
            stdout = e.res.stdout
            if not expect_fail:
                msg = ('Example %r: Command %r failed unexpectedly.' %
                       (name, command))
                msg += '\n retcode: %r' % e.res.ret
                msg += '\n' + indent(stderr, 'stderr| ')
                msg += '\n' + indent(stdout, 'stdout| ')
                raise Exception(msg)
示例#20
0
 def get_compmake_bin():
     """ Returns the path to the compmake executable. """
     if SGEJob.compmake_bin is None:
         cwd = os.path.abspath(os.getcwd())
         compmake_bin = system_cmd_result(cwd, 'which compmake').stdout.strip()
         compmake_bin = os.path.abspath(compmake_bin)
         SGEJob.compmake_bin = compmake_bin
     return SGEJob.compmake_bin 
示例#21
0
def find_command_path(prog):
    res = system_cmd_result(cwd=os.getcwd(),
                             cmd=['which', prog],
                             display_stdout=False,
                            display_stderr=False,
                            raise_on_error=True)
    prog = res.stdout
    return prog
示例#22
0
def get_command_output(cmd):
    cwd = os.getcwd()
    res = system_cmd_result(cwd,
                            cmd,
                            display_stdout=False,
                            display_stderr=False,
                            raise_on_error=True)
    return res.stdout
示例#23
0
def parmake_job2_new_process(args):
    """ Starts the job in a new compmake process. """
    (job_id, context) = args
    compmake_bin = which("compmake")

    db = context.get_compmake_db()
    storage = db.basepath  # XXX:
    where = os.path.join(storage, "parmake_job2_new_process")
    if not os.path.exists(storage):
        try:
            os.makedirs(storage)
        except:
            pass

    out_result = os.path.join(where, "%s.results.pickle" % job_id)
    out_result = os.path.abspath(out_result)
    cmd = [compmake_bin, storage]

    if not all_disabled():
        cmd += ["--contracts"]

    cmd += [
        "--status_line_enabled",
        "0",
        "--colorize",
        "0",
        "-c",
        "make_single out_result=%s %s" % (out_result, job_id),
    ]

    cwd = os.getcwd()
    cmd_res = system_cmd_result(
        cwd, cmd, display_stdout=False, display_stderr=False, raise_on_error=False, capture_keyboard_interrupt=False
    )
    ret = cmd_res.ret

    if ret == CompmakeConstants.RET_CODE_JOB_FAILED:  # XXX:
        msg = "Job %r failed in external process" % job_id
        msg += indent(cmd_res.stdout, "stdout| ")
        msg += indent(cmd_res.stderr, "stderr| ")

        res = safe_pickle_load(out_result)
        os.unlink(out_result)
        result_dict_check(res)

        raise JobFailed.from_dict(res)

    elif ret != 0:
        msg = "Host failed while doing %r" % job_id
        msg += "\n cmd: %s" % " ".join(cmd)
        msg += "\n" + indent(cmd_res.stdout, "stdout| ")
        msg += "\n" + indent(cmd_res.stderr, "stderr| ")
        raise CompmakeBug(msg)  # XXX:

    res = safe_pickle_load(out_result)
    os.unlink(out_result)
    result_dict_check(res)
    return res
示例#24
0
def test_example_package():
    from system_cmd import system_cmd_result

    # make sure it's installed
    import example_package  # @UnusedImport

    with create_tmp_dir() as cwd:
        print('Working in %r ' % cwd)
        cmd = [
            'comptests',
            '--contracts',
            #'--nonose',
            'example_package'
        ]
        system_cmd_result(cwd,
                          cmd,
                          display_stdout=True,
                          display_stderr=True,
                          raise_on_error=True)

        fs = [
            'out-comptests/report.html',
            'out-comptests/report/reportclass1single/'
            'reportclass1single-c1a-checkclass1dynamic-examplepackage-exampleclass1.html',
        ]
        #
        #         if False:
        #             # these are for reports
        #             fs += ['out-comptests/report/single/single-checkclass1dynamic'
        #                   '-examplepackage-exampleclass1.html',
        #                   'out-comptests/report/reportclass1single/reportclass1single'
        #                   '-checkclass1dynamic-c1a-examplepackage-exampleclass1.html',]

        errors = []
        for f in fs:
            fn = os.path.join(cwd, f)
            print('Testing %r' % f)
            if not os.path.exists(fn):
                errors.append(fn)

        if errors:
            msg = 'Files not found:\n' + '\n'.join(errors)
            raise Exception(msg)
示例#25
0
def jobs_nosetests_single(context, module):

    with create_tmp_dir() as cwd:
        out = os.path.join(cwd, '%s.pickle' % module)
        cmd = ['nosetests',
               '--collect-only',
               '--with-xunitext',
               '--xunitext-file' ,
               out,
               '-v', '-s', module]
        system_cmd_result(
            cwd=cwd, cmd=cmd,
            display_stdout=True,
            display_stderr=True,
            raise_on_error=True)

        tests = safe_pickle_load(out)
        print('found %d mcdp_lang_tests ' % len(tests))

        for t in tests:
            context.comp(execute, t)
示例#26
0
def call_nosetests_plus_coverage(module):
    """
        This also calls the coverage module.
        It returns the .coverage file as a string.
    """
    with create_tmp_dir() as cwd:
        prog = find_command_path('nosetests')
        cmd = [prog, module]

        # note: coverage -> python-coverage in Ubuntu14
        cmd = ['coverage', 'run'] + cmd
        system_cmd_result(
            cwd=cwd, cmd=cmd,
            display_stdout=True,
            display_stderr=True,
            raise_on_error=True)
        coverage_file = os.path.join(cwd, '.coverage')
        with open(coverage_file) as f:
            res = f.read()
        #print('read %d bytes in %s' % (len(res), coverage_file))
        return res
示例#27
0
def render_pdf(data_aug):
    data = data_aug.get_result()
    prefix = 'prince_render'
    with tmpdir(prefix=prefix) as d:
        f_html = os.path.join(d, 'file.html')
        with open(f_html, 'w') as f:
            f.write(data)

        f_out = os.path.join(d, 'out.pdf')
        cmd = ['prince', '--javascript', '-o', f_out, f_html]
        pwd = os.getcwd()
        system_cmd_result(pwd,
                          cmd,
                          display_stdout=False,
                          display_stderr=False,
                          raise_on_error=True)

        with open(f_out) as f:
            data = f.read()

        return data
示例#28
0
 def delete_job(self):
     # cmd = ['qdel', self.sge_id]
     cmd = ['qdel', self.sge_job_name]
     cwd = os.path.abspath(os.getcwd())
     # TODO: check errors
     try:
         _ = system_cmd_result(cwd, cmd,
                               display_stdout=False,
                               display_stderr=False,
                               raise_on_error=True,
                               capture_keyboard_interrupt=False)
     except CmdException as e:
         error('Error while deleting job:\n%s' % e)
示例#29
0
def graphviz_run(filename_dot, output, prog='dot'):
    suff = os.path.splitext(output)[1][1:]
    if not suff in ['png', 'pdf', 'ps', 'svg']:
        raise ValueError((output, suff))

    encoder = suff

    cmd = [prog, '-T%s' % encoder, '-o', output, filename_dot]
    try:
        # print('running graphviz')
        system_cmd_result(cwd='.', cmd=cmd,
                 display_stdout=False,
                 display_stderr=False,
                 raise_on_error=True)
        # print('done')
    except (CmdException, KeyboardInterrupt):
        emergency = 'emergency.dot'
        logger.error('saving to %r' % emergency)  # XXX
        contents = open(filename_dot).read()
        with open(emergency, 'w') as f:
            f.write(contents)
#         print(contents)
        raise
示例#30
0
 def delete_job(self):
     # cmd = ['qdel', self.sge_id]
     cmd = ['qdel', self.sge_job_name]
     cwd = os.path.abspath(os.getcwd())
     # TODO: check errors
     try:
         _ = system_cmd_result(cwd,
                               cmd,
                               display_stdout=False,
                               display_stderr=False,
                               raise_on_error=True,
                               capture_keyboard_interrupt=False)
     except CmdException as e:
         error('Error while deleting job:\n%s' % e)
示例#31
0
文件: plot.py 项目: AndreaCensi/mcdp
def syntax_pdf(data):
    """ Returns a PDF string """
    from mcdp_report.html import ast_to_html
    s = data['s']
    s = s.replace('\t', '    ')
    
    lines_to_hide = get_lines_to_hide(data['params'])
    def ignore_line(lineno):
        return lineno  in lines_to_hide
    contents = ast_to_html(s,  
                       ignore_line=ignore_line, parse_expr=Syntax.ndpt_dp_rvalue,
                       )
    html = get_minimal_document(contents)

    d = mkdtemp()
    
    f_html = os.path.join(d, 'file.html')
    with open(f_html, 'w') as f:
        f.write(html)
        
    try:
        f_pdf = os.path.join(d, 'file.pdf')
        cmd= ['wkhtmltopdf','-s','A1',f_html,f_pdf]
        system_cmd_result(
                d, cmd, 
                display_stdout=False,
                display_stderr=False,
                raise_on_error=True)
          
        f_pdf_crop = os.path.join(d, 'file_crop.pdf')
        cmd = ['pdfcrop', '--margins', '4', f_pdf, f_pdf_crop]
        system_cmd_result(
                d, cmd, 
                display_stdout=False,
                display_stderr=False,
                raise_on_error=True)
    
        with open(f_pdf_crop) as f:
            data = f.read()
    
        f_png = os.path.join(d, 'file.png')
        cmd = ['convert', '-density', '600', f_pdf_crop,
                '-background', 'white', '-alpha', 'remove',
                '-resize', '50%', f_png]
        system_cmd_result(
                d, cmd,
                display_stdout=False,
                display_stderr=False,
                raise_on_error=True)
    
        with open(f_png) as f:
            pngdata = f.read()
    
        return [('pdf', 'syntax_pdf', data), ('png', 'syntax_pdf', pngdata)]
    except CmdException as e:
        raise e
示例#32
0
def get_active_groups(username=None):
    cmd = ['groups']

    if username:
        cmd.append(username)

    try:
        res = system_cmd_result('.', cmd,
                                display_stdout=False,
                                display_stderr=False,
                                raise_on_error=True,
                                capture_keyboard_interrupt=False,
                                env=None)
    except CmdException as e:
        raise Exception(str(e))
    active_groups = res.stdout.split()  # XXX
    return active_groups
示例#33
0
 def get_qacct(self):
     cmd = ['qacct', '-j', self.sge_id]
     cwd = os.getcwd()
     res = system_cmd_result(cwd, cmd,
                             display_stdout=False,
                             display_stderr=False,
                             raise_on_error=True,
                             capture_keyboard_interrupt=False)
     values = {}
     for line in res.stdout.split('\n'):
         tokens = line.split()
         if len(tokens) >= 2:  # XXX
             k = tokens[0]
             v = " ".join(tokens[1:])
             if k == 'failed':
                 v = tokens[1]
             values[k] = v
     return values
示例#34
0
def lessc_string(s1):
    """
        Runs "node lessc" on the string

        Raises LesscError.
    """
    #     print(indent(s1, 'lessc input: '))
    mcdp_tmp_dir = get_mcdp_tmp_dir()
    prefix = 'prerender_lessc'
    d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix)
    try:
        f1 = os.path.join(d, 'input.less')
        f2 = os.path.join(d, 'out.css')
        with open(f1, 'w') as f:
            f.write(s1)

        try:
            cmd = ['lessc', f1, f2]
            res = system_cmd_result(d,
                                    cmd,
                                    display_stdout=True,
                                    display_stderr=True,
                                    raise_on_error=False)

            if res.ret:
                msg = 'lessc error (return value = %d).' % res.ret
                msg += '\n\n' + indent(res.stderr, '', 'stderr:')
                msg += '\n\n' + indent(res.stdout, '', 'stdout:')
                raise LesscError(msg)

            with open(f2) as f:
                data = f.read()

            return data
        except CmdException as e:
            raise e
    finally:
        shutil.rmtree(d)
def get_last_version_fresh():
    url = 'https://pypi.org/pypi/duckietown-shell/json'

    try:
        req = urllib2.Request(url)
        try:
            res = urllib2.urlopen(req, timeout=1)
            data = res.read()

        except urllib2.URLError as e:
            # msg = 'Cannot connect to server %s: %s' % (url, (e))
            # raise Exception(msg)
            if which('curl') is not None:
                res = system_cmd_result('.', ['curl', url])
                data = res.stdout
            else:
                raise CouldNotGetVersion()

        info = json.loads(data)
        last_version = info['info']['version']
        return last_version
    except Exception as e:
        raise CouldNotGetVersion(str(e))
示例#36
0
def src_cites_pkg(src, pkg):
    cmd = ['grep', '-r', ' ' + pkg, src]
    print " ".join(cmd)
    res = system_cmd_result('.', cmd,
                      display_stdout=False,
                      display_stderr=False,
                      raise_on_error=False,
                      capture_keyboard_interrupt=True)  
    if res.ret == 1:
        return False
        # not found
    elif res.ret == 0:
        # found

        out = res.stdout
        if 'import' in out:
            return True
        else:
            return False
        # print out
        # # print res.stdout.split('\n')[-1]
        return True
    elif res.ret == -1: # error 
        raise Exception(res)
示例#37
0
def plot_pkgs(pkgs, out, ignore_repo=[], cluster=False,
              link_same_repo=True):
    
    dirname = os.path.dirname(out)
    if dirname and not os.path.exists(dirname):
        os.makedirs(dirname)
        
    import gvgen
    graph = gvgen.GvGen()
    repo2node = {}
    pkg2node = {}

    def get_repo_node(repo):
        if not repo in repo2node:
            repo2node[repo] = graph.newItem(repo)
        return repo2node[repo] 

    def get_pkg_node(id_pkg):
        if not id_pkg in pkg2node:
            if cluster:
                pkg2node[id_pkg] = graph.newItem(id_pkg, get_repo_node(repo))
            else:
                pkg2node[id_pkg] = graph.newItem(id_pkg)        
        return pkg2node[id_pkg]


    # if cluster:
    #     repo2node = {}
    #     for id_pkg, pkg in pkgs.items():
    #         repo = pkg['repo']
    #         if repo in ignore_repo:
    #             continue
    #         if not repo in repo2node:
    #             repo2node[repo] = graph.newItem(repo)


    # if cluster:
    #     pkg2node['procgraph'] = graph.newItem('procgraph', repo2node['procgraph'])
    # else:
    #     pkg2node['procgraph'] = graph.newItem('procgraph')
    

    for id_pkg, pkg in pkgs.items():
        repo = pkg['repo']
        if repo in ignore_repo:
            continue

        # if repo == 'procgraph':
        #     pkg2node[id_pkg]  = pkg2node['procgraph']
        # else:
        get_pkg_node(id_pkg)

    for id_pkg, pkg in pkgs.items():
        for child in pkg['requires']:
            if not child in pkgs:
                print('warning, %s dependence %s not found in list' % (id_pkg, child))
            else:    
                child_repo = pkgs[child]['repo']
                if child_repo in ignore_repo:
                    continue
            same_repo = pkgs[id_pkg]['repo'] == child_repo
#             if not same_repo:
            if link_same_repo or not same_repo:
                l = graph.newLink(get_pkg_node(id_pkg), get_pkg_node(child))
                
                d12 = child in pkgs[id_pkg]['requires']
                d21 = id_pkg in pkgs[child]['requires']
                loop = d12 and d21
                if loop: 
                    graph.propertyAppend(l,"color","red")

    # TODO: add check?
    
    with open(out, 'w') as f:
        graph.dot(f)

    cmd = ['dot', out, '-Tpng', '-o', out + '.png']
    system_cmd_result('.', cmd, raise_on_error=True)
示例#38
0
def prerender_mathjax_(html):
    """
        Runs the prerender.js script to pre-render the MathJax into images.

        Raises PrerenderError.
    """
    assert not '<html>' in html, html

    use = get_nodejs_bin()

    html = html.replace('<p>$$', '\n$$')
    html = html.replace('$$</p>', '$$\n')
    script = get_prerender_js()
    mcdp_tmp_dir = get_mcdp_tmp_dir()
    prefix = 'prerender_mathjax_'
    d = mkdtemp(dir=mcdp_tmp_dir, prefix=prefix)

    try:
        f_html = os.path.join(d, 'file.html')
        with open(f_html, 'w') as f:
            f.write(html)

        try:
            f_out = os.path.join(d, 'out.html')
            cmd = [use, script, f_html, f_out]
            pwd = os.getcwd()
            res = system_cmd_result(pwd,
                                    cmd,
                                    display_stdout=False,
                                    display_stderr=False,
                                    raise_on_error=False)

            if res.ret:  # pragma: no cover
                if 'Error: Cannot find module' in res.stderr:
                    msg = 'You have to install the MathJax and/or jsdom libraries.'
                    msg += '\nOn Ubuntu, you can install them using:'
                    msg += '\n\n\tsudo apt-get install npm'
                    msg += '\n\n\tnpm install MathJax-node jsdom'
                    msg += '\n\n' + indent(res.stderr, '  |')
                    raise PrerenderError(msg)

                if 'parse error' in res.stderr:
                    lines = [
                        _ for _ in res.stderr.split('\n') if 'parse error' in _
                    ]
                    assert lines
                    msg = 'LaTeX conversion errors:\n\n' + '\n'.join(lines)
                    raise PrerenderError(msg)

                msg = 'Unknown error (ret = %d).' % res.ret
                msg += '\n\n' + indent(res.stderr, '  |')
                raise PrerenderError(msg)

            with open(f_out) as f:
                data = f.read()

            # Read the data
            soup = bs(data)
            # find this and move it at the end
            # <style id="MathJax_SVG_styles"
            tag_style = soup.find(id='MathJax_SVG_styles')
            if not tag_style:
                msg = 'Expected to find style MathJax_SVG_styles'
                raise_desc(Exception, msg, soup=str(soup))
            # <svg style="display: none;"><defs id="MathJax_SVG_glyphs">
            tag_svg_defs = soup.find('svg', style="display: none;")
            if not tag_svg_defs:
                msg = 'Expected to find tag <svg display=none>'
                raise_desc(Exception, msg, soup=str(soup))

            other_tag = soup.find('div', style="display:none")
            if not other_tag:
                msg = 'Expected to find tag <div style="display:none">'
                raise_desc(Exception, msg, soup=str(soup))

            #<div style="display:none">Because of mathjax bug</div>
            soup.append(other_tag.extract())
            soup.append(tag_svg_defs.extract())
            soup.append(tag_style.extract())
            data = to_html_stripping_fragment(soup)

            return data
        except CmdException as e:  # pragma: no cover
            raise e
    finally:
        shutil.rmtree(d)
    def command(shell, args):

        parser = argparse.ArgumentParser()

        parser.add_argument(
            '--image',
            default=
            'andreacensi/mcdp_books:duckuments@sha256:ae2fcdbb8ce409e4817ed74c67b04bb91cd14ca96bed887e75e5275fa2efc933',
            help="Which image to use")

        parsed = parser.parse_args(args=args)
        image = parsed.image

        check_docker_environment()
        # check_git_supports_superproject()

        from system_cmd import system_cmd_result

        pwd = os.getcwd()
        bookdir = os.path.join(pwd, 'book')

        if not os.path.exists(bookdir):
            msg = 'Could not find "book" directory %r.' % bookdir
            DTCommandAbs.fail(msg)

        # check that the resources directory is present

        resources = os.path.join(pwd, 'resources')
        if not os.path.exists(os.path.join(resources, 'templates')):
            msg = 'It looks like that the "resources" repo is not checked out.'
            msg += '\nMaybe try:\n'
            msg += '\n   git submodule init'
            msg += '\n   git submodule update'
            raise Exception(msg)  # XXX

        entries = list(os.listdir(bookdir))
        entries = [_ for _ in entries if not _[0] == '.']
        if len(entries) > 1:
            msg = 'Found more than one directory in "book": %s' % entries
            DTCommandAbs.fail(msg)
        bookname = entries[0]
        src = os.path.join(bookdir, bookname)

        res = system_cmd_result(pwd, ['git', '--version'], raise_on_error=True)
        git_version = res.stdout
        print('git version: %s' % git_version)

        cmd = ['git', 'rev-parse', '--show-superproject-working-tree']
        res = system_cmd_result(pwd, cmd, raise_on_error=True)
        gitdir_super = res.stdout.strip()

        if '--show' in gitdir_super or not gitdir_super:
            msg = "Your git version is too low, as it does not support --show-superproject-working-tree"
            msg += '\n\nDetected: %s' % git_version
            raise InvalidEnvironment(msg)

        print('gitdir_super: %r' % gitdir_super)
        res = system_cmd_result(pwd, ['git', 'rev-parse', '--show-toplevel'],
                                raise_on_error=True)
        gitdir = res.stdout.strip()

        if '--show' in gitdir or not gitdir:
            msg = "Your git version is too low, as it does not support --show-toplevel"
            msg += '\n\nDetected: %s' % git_version
            raise InvalidEnvironment(msg)

        print('gitdir: %r' % gitdir)

        pwd1 = os.path.realpath(pwd)
        user = getpass.getuser()

        tmpdir = '/tmp'
        fake_home = os.path.join(tmpdir, 'fake-%s-home' % user)
        if not os.path.exists(fake_home):
            os.makedirs(fake_home)
        resources = 'resources'
        uid1 = os.getuid()

        if sys.platform == 'darwin':
            flag = ':delegated'
        else:
            flag = ''

        cmd = [
            'docker', 'run', '-v',
            '%s:%s%s' % (gitdir, gitdir, flag), '-v',
            '%s:%s%s' % (gitdir_super, gitdir_super, flag), '-v',
            '%s:%s%s' % (pwd1, pwd1, flag), '-v',
            '%s:%s%s' % (fake_home, '/home/%s' % user, flag), '-e',
            'USER=%s' % user, '-e',
            'USERID=%s' % uid1, '-m', '4GB', '--user',
            '%s' % uid1
        ]

        interactive = True

        if interactive:
            cmd.append('-it')

        cmd += [
            image, '/project/run-book-native.sh', bookname, src, resources,
            pwd1
        ]

        print('executing:\nls ' + " ".join(cmd))
        # res = system_cmd_result(pwd, cmd, raise_on_error=True)

        try:
            p = subprocess.Popen(cmd,
                                 bufsize=0,
                                 executable=None,
                                 stdin=None,
                                 stdout=None,
                                 stderr=None,
                                 preexec_fn=None,
                                 shell=False,
                                 cwd=pwd,
                                 env=None)
        except OSError as e:
            if e.errno == 2:
                msg = 'Could not find "docker" executable.'
                DTCommandAbs.fail(msg)
            raise

        p.communicate()
        print('\n\nCompleted.')
示例#40
0
def test_false2():
    res = system_cmd_result('.',
                            'cat UTF-8-test.txt',
                            display_stderr=True,
                            display_stdout=True)
    print(res)
示例#41
0
def main():
    try:

        documents_data = sys.stdin.read()
        documents = yaml.load(documents_data)

        out = 'pdfs'

        if not os.path.exists(out):
            os.makedirs(out)

        errors = []
        pdfs = []

        for d in documents:
            id_document = get_id(d)

            pdf_url = make_pdf_url(id_document)

            pdf_file = os.path.join(out, id_document + '.pdf')

            if os.path.exists(pdf_file):
                pdfs.append(pdf_file)
            else:
                logger.info('Downloading %s' % pdf_file)
                response = urllib2.urlopen(pdf_url)
                data = response.read()

                is_valid = (data[1:4] == 'PDF')

                if is_valid:
                    with open(pdf_file, 'w') as f:
                        f.write(data)
                    pdfs.append(pdf_file)
                else:
                    logger.error('Invalid response for document %r' % d)
                    errors.append(id_document)
                    with open(pdf_file + '.invalid-response.html', 'w') as f:
                        f.write(data)

        pdfout = 'joined.pdf'

        cmd = ['pdftk']
        cmd.extend(pdfs)
        cmd.extend(['cat', 'output', pdfout])

        system_cmd_result(cwd='.',
                          cmd=cmd,
                          display_stdout=True,
                          display_stderr=True,
                          raise_on_error=True)

        with open(pdfout) as f:
            data = f.read()
        logger.info('Writing on stdout %s' % len(data))
        sys.stdout.write(data)

        if errors:
            logger.error('Could not download all files.')
            sys.exit(2)

    except Exception as e:
        import traceback
        logger.error(traceback.format_exc(e))
        sys.exit(-1)
示例#42
0
    def execute(self, spool):
        db = self.db
        # Todo: check its this one

        storage = os.path.abspath(db.basepath)

        # create a new spool directory for each execution
        # otherwise we get confused!

        self.stderr = os.path.join(spool, '%s.stderr' % self.job_id)
        self.stdout = os.path.join(spool, '%s.stdout' % self.job_id)
        self.retcode = os.path.join(spool, '%s.retcode' % self.job_id)
        self.out_results = os.path.join(spool,
                                        '%s.results.pickle' % self.job_id)

        if os.path.exists(self.stderr):
            os.remove(self.stderr)

        if os.path.exists(self.stdout):
            os.remove(self.stdout)

        if os.path.exists(self.retcode):
            os.remove(self.retcode)

        if os.path.exists(self.out_results):
            os.remove(self.out_results)

        options = []
        cwd = os.path.abspath(os.getcwd())
        variables = dict(SGE_O_WORKDIR=cwd,
                         PYTHONPATH=os.getenv('PYTHONPATH', '') + ':' + cwd)

        # nice-looking name
        self.sge_job_name = 'cm%s-%s' % (os.getpid(), self.job_id)
        # Note that we get the official "id" later and we store it in
        # self.sge_id
        options.extend(
            ['-v', ",".join('%s=%s' % x for x in variables.items())])
        # XXX: spaces
        options.extend(['-e', self.stderr])
        options.extend(['-o', self.stdout])
        options.extend(['-N', self.sge_job_name])
        options.extend(['-wd', cwd])

        options.extend(['-V'])  # pass all environment

        options.extend(['-terse'])

        compmake_bin = SGEJob.get_compmake_bin()

        compmake_options = [
            compmake_bin, storage,
            '--retcodefile', self.retcode,
            '--status_line_enabled', '0',
            '--colorize', '0',
            '-c',
            '"make_single out_result=%s %s"' % (self.out_results, self.job_id),
        ]

        if not all_disabled():
            compmake_options += ['--contracts']

        # XXX: spaces in variable out_result

        write_stdin = ' '.join(compmake_options)

        cmd = ['qsub'] + options

        res = system_cmd_result(cwd, cmd,
                                display_stdout=False,
                                display_stderr=True,
                                raise_on_error=True,
                                write_stdin=write_stdin,
                                capture_keyboard_interrupt=False)

        self.sge_id = res.stdout.strip()

        self.already_read = False
        self.npolls = 0

        self.told_you_ready = False
示例#43
0
    def execute(self, spool):
        db = self.db
        # Todo: check its this one

        storage = os.path.abspath(db.basepath)

        # create a new spool directory for each execution
        # otherwise we get confused!

        self.stderr = os.path.join(spool, '%s.stderr' % self.job_id)
        self.stdout = os.path.join(spool, '%s.stdout' % self.job_id)
        self.retcode = os.path.join(spool, '%s.retcode' % self.job_id)
        self.out_results = os.path.join(spool,
                                        '%s.results.pickle' % self.job_id)

        if os.path.exists(self.stderr):
            os.remove(self.stderr)

        if os.path.exists(self.stdout):
            os.remove(self.stdout)

        if os.path.exists(self.retcode):
            os.remove(self.retcode)

        if os.path.exists(self.out_results):
            os.remove(self.out_results)

        options = []
        cwd = os.path.abspath(os.getcwd())
        variables = dict(SGE_O_WORKDIR=cwd,
                         PYTHONPATH=os.getenv('PYTHONPATH', '') + ':' + cwd)

        # nice-looking name
        self.sge_job_name = 'cm%s-%s' % (os.getpid(), self.job_id)
        # Note that we get the official "id" later and we store it in
        # self.sge_id
        options.extend(
            ['-v', ",".join('%s=%s' % x for x in variables.items())])
        # XXX: spaces
        options.extend(['-e', self.stderr])
        options.extend(['-o', self.stdout])
        options.extend(['-N', self.sge_job_name])
        options.extend(['-wd', cwd])

        options.extend(['-V'])  # pass all environment

        options.extend(['-terse'])

        compmake_bin = SGEJob.get_compmake_bin()

        compmake_options = [
            compmake_bin,
            storage,
            '--retcodefile',
            self.retcode,
            '--status_line_enabled',
            '0',
            '--colorize',
            '0',
            '-c',
            '"make_single out_result=%s %s"' % (self.out_results, self.job_id),
        ]

        if not all_disabled():
            compmake_options += ['--contracts']

        # XXX: spaces in variable out_result

        write_stdin = ' '.join(compmake_options)

        cmd = ['qsub'] + options

        res = system_cmd_result(cwd,
                                cmd,
                                display_stdout=False,
                                display_stderr=True,
                                raise_on_error=True,
                                write_stdin=write_stdin,
                                capture_keyboard_interrupt=False)

        self.sge_id = res.stdout.strip()

        self.already_read = False
        self.npolls = 0

        self.told_you_ready = False
示例#44
0
def parmake_job2_new_process(args):
    """ Starts the job in a new compmake process. """
    (job_id, context) = args
    compmake_bin = which('compmake')

    db = context.get_compmake_db()
    storage = db.basepath  # XXX:
    where = os.path.join(storage, 'parmake_job2_new_process')
    if not os.path.exists(storage):
        try:
            os.makedirs(storage)
        except:
            pass

    out_result = os.path.join(where, '%s.results.pickle' % job_id)
    out_result = os.path.abspath(out_result)
    cmd = [compmake_bin, storage]

    if not all_disabled():
        cmd += ['--contracts']

    cmd += [
        '--status_line_enabled',
        '0',
        '--colorize',
        '0',
        '-c',
        'make_single out_result=%s %s' % (out_result, job_id),
    ]

    cwd = os.getcwd()
    cmd_res = system_cmd_result(cwd,
                                cmd,
                                display_stdout=False,
                                display_stderr=False,
                                raise_on_error=False,
                                capture_keyboard_interrupt=False)
    ret = cmd_res.ret

    if ret == CompmakeConstants.RET_CODE_JOB_FAILED:  # XXX:
        msg = 'Job %r failed in external process' % job_id
        msg += indent(cmd_res.stdout, 'stdout| ')
        msg += indent(cmd_res.stderr, 'stderr| ')

        res = safe_pickle_load(out_result)
        os.unlink(out_result)
        result_dict_check(res)

        raise JobFailed.from_dict(res)

    elif ret != 0:
        msg = 'Host failed while doing %r' % job_id
        msg += '\n cmd: %s' % " ".join(cmd)
        msg += '\n' + indent(cmd_res.stdout, 'stdout| ')
        msg += '\n' + indent(cmd_res.stderr, 'stderr| ')
        raise CompmakeBug(msg)  # XXX:

    res = safe_pickle_load(out_result)
    os.unlink(out_result)
    result_dict_check(res)
    return res
示例#45
0
def test_false():
    res = system_cmd_result('.', 'cp not-existing done')
    print(res)
示例#46
0
def convert_to_mp4(filename, mp4=None, quiet=True, crf=18, preset='medium', timestamp=None, metadata={}):
    """ 
        Creates a web-ready mp4 using ffmpeg.
    
        Needs either qtquickstart (from the python package) or qt-quickstart from ffmpeg.
        
        On ubuntu 12.04: need 'sudo apt-get install x264 libavcodec-extra-53'
        to install necessary codecs.
        You can see a list of supported presets by using: 'x264 --help'.
    
        
        (other packages that might be helpful: # libavdevice-extra-52  libavfilter-extra-0 
         libavformat-extra-52 libavutil-extra-49 libpostproc-extra-51 libswscale-extra-0)
       
    """
    my_ext = '.mp4'
    
    if os.path.splitext(mp4)[1] != my_ext:
        msg = 'I expect a %r as out (%r)' % (my_ext, mp4)
        raise ValueError(msg)

    if os.path.splitext(filename)[1] == my_ext:
        msg = 'Need a file that does not end in %r (%r)' % (my_ext, filename)
        raise ValueError(msg)
    
    basename, _ = os.path.splitext(filename)

    if mp4 is None:
        mp4 = basename + my_ext

    # need .mp4 at the end otherwise ffmpeg gets confused
    tmp = basename + '.mp4.firstpass.mp4'

    if not os.path.exists(filename):
        raise ValueError("File does not exist: %s" % filename)

#    if (os.path.exists(mp4) and
#        (os.path.getmtime(mp4) > os.path.getmtime(filename))):
#        return

    if os.path.exists(tmp):
        os.unlink(tmp)
    if os.path.exists(mp4):
        os.unlink(mp4)

    cmds = ['ffmpeg']
    cmds += ['-y']
    cmds += ['-i', filename]
    cmds += get_x264_encoder_params(crf, preset)
    cmds += get_ffmpeg_metadata_args(metadata, timestamp)
    cmds += [tmp]
    
    # print cmds
    
    try:
        system_cmd_result('.', cmds,
                  display_stdout=not quiet,
                  display_stderr=not quiet,
                  raise_on_error=True,
                  capture_keyboard_interrupt=False)
    except CmdException:
        if os.path.exists(tmp):
            os.unlink(tmp)
        raise

    do_quickstart(source=tmp, target=mp4)
    
    if os.path.exists(tmp):
        os.unlink(tmp)
示例#47
0
def process_svg_file(filename, target, preamble, proc_dir):
    if not os.path.exists(proc_dir):
        try:
            os.makedirs(proc_dir)
        except:
            pass

    logger.debug('Copying SVG file to temp directory %s '% proc_dir)
    shutil.copy(filename, join(proc_dir, 'in.svg.tmp'))

    logger.debug('Running Inkscape.')
    cmd = [inkscape_cmd,
           '-D',
           '-z',
           '--file', join(proc_dir, 'in.svg.tmp'),
           '--export-pdf', join(proc_dir, 'out.pdf'),
           '--export-latex']

    system_cmd_result(cwd=proc_dir, cmd=cmd,
                      display_stdout=False,
                      display_stderr=False,
                      raise_on_error=True)
    
    assert os.path.exists(join(proc_dir, 'out.pdf'))
    assert os.path.exists(join(proc_dir, 'out.pdf_tex'))

    logger.debug('Creating template file..')
    # ok now we have the pdf and pdf_tex files ready.
    # Let's build the temp tex file to generate the figure
    s = r"""
\documentclass[]{article}
\usepackage{graphicx}
\usepackage{color}
\begin{document}
\pagestyle{empty}
\input{PREAMBLE}
\input{out.pdf_tex}
\end{document}
"""
    s = s.strip() 
    s = s.replace('PREAMBLE', os.path.realpath(preamble))

    fn = join(proc_dir, 'file.tex')
    with open(fn, 'w') as f:
        f.write(s)

    # compile
    logger.debug('Compile using latex..')
    cmd = [latex_cmd, 'file.tex']
    system_cmd_result(cwd=proc_dir, cmd=cmd,
                      display_stdout=False,
                      display_stderr=False,
                      raise_on_error=True)
    # crop
    logger.debug('Crop the pdf..')
    cmd = [crop_cmd, 'file.pdf', 'cropped.pdf']
    system_cmd_result(cwd=proc_dir, cmd=cmd,
                      display_stdout=False,
                      display_stderr=False,
                      raise_on_error=True)
    
    
    logger.debug('Copying to target %r..' % target)
    shutil.copy(join(proc_dir, 'cropped.pdf'), target)
示例#48
0
def test_one():
    system_cmd_result('.', 'ls -a')
示例#49
0
    def execute(self):
        db = get_compmake_db()
        # Todo: check its this one
        storage = os.path.abspath(db.basepath)
        
        spool = os.path.join(storage, 'sge_spool')
        if not os.path.exists(spool):
            os.makedirs(spool)
            
        self.stderr = os.path.join(spool, '%s.stderr' % self.job_id)
        self.stdout = os.path.join(spool, '%s.stdout' % self.job_id)
        self.retcode = os.path.join(spool, '%s.retcode' % self.job_id)
        
        if os.path.exists(self.stderr):
            os.remove(self.stderr)
        
        if os.path.exists(self.stdout):
            os.remove(self.stdout)
        
        if os.path.exists(self.retcode):
            os.remove(self.retcode)
             
        options = []
        cwd = os.path.abspath(os.getcwd())
        variables = dict(SGE_O_WORKDIR=cwd,
                         PYTHONPATH=os.getenv('PYTHONPATH', '') + ':' + cwd)
        
        options.extend(['-v', ",". join('%s=%s' % x for x in variables.items())])
        # warning: spaces
        options.extend(['-e', self.stderr])
        options.extend(['-o', self.stdout])
        options.extend(['-N', self.job_id])
        options.extend(['-wd', cwd])
        
        options.extend(['-V'])  # pass all environment
        
        options.extend(['-terse'])
        
        compmake_bin = SGEJob.get_compmake_bin()
        
        compmake_options = [compmake_bin, storage,
                            '--retcodefile', self.retcode,
                            '--status_line_enabled', '0',
                            '--colorize', '0',
                            '-c', '"make %s"' % self.job_id]
        
        write_stdin = ' '.join(compmake_options)
        
        cmd = ['qsub'] + options 

        res = system_cmd_result(cwd, cmd,
                      display_stdout=False,
                      display_stderr=True,
                      raise_on_error=True,
                      write_stdin=write_stdin,
                      capture_keyboard_interrupt=False)
     
        self.sge_id = res.stdout.strip()
        
        self.already_read = False
        self.npolls = 0
示例#50
0
def get_candidate_user(user_db, result, provider_name):
    ''' Returns a candidate UserInfo structure as observed from Oauth response '''
    # get the data
    name = result.user.name
    email = result.user.email

    if result.user.id is None:
        msg = 'Cannot obtain ID for authentication.'
        raise Exception(msg)
    #     if email is None:
    #         msg = 'Cannot obtain email address'
    #         raise Exception(msg)

    if provider_name == 'github':
        candidate_username = result.user.username
        website = result.user.data['blog']
        affiliation = result.user.data['company']
        unique_id = result.user.id
        picture = result.user.picture
    elif provider_name == 'facebook':
        if name is None:
            msg = 'Could not get name from Facebook so cannot create username.'
            raise Exception(msg)

        candidate_username = name.encode('utf8').replace(' ', '_').lower()
        website = None
        affiliation = None
        unique_id = result.user.id
        picture = result.user.picture
    elif provider_name == 'google':
        if name is None:
            msg = 'Could not get name from Google so cannot create username.'
            raise Exception(msg)
        candidate_username = name.encode('utf8').replace(' ', '_').lower()
        website = result.user.link
        affiliation = None
        unique_id = result.user.id
        picture = result.user.picture
    elif provider_name == 'linkedin':
        candidate_username = name.encode('utf8').replace(' ', '_').lower()
        website = result.user.link
        affiliation = None
        unique_id = result.user.id
        picture = result.user.picture
    elif provider_name == 'amazon':
        candidate_username = name.encode('utf8').replace(' ', '_').lower()
        website = result.user.link
        affiliation = None
        unique_id = result.user.id
        picture = None
    else:
        assert False, provider_name
    candidate_usernames = [candidate_username]

    username = user_db.find_available_user_name(candidate_usernames)
    res = {}
    res['username'] = username
    res['name'] = name
    res['email'] = email
    res['website'] = website
    res['affiliation'] = affiliation
    res['account_last_active'] = datetime.datetime.now()
    res['account_created'] = datetime.datetime.now()
    res['authentication_ids'] = [{
        'provider': provider_name,
        'id': unique_id,
        'password': None
    }]
    logger.info('Reading picture %s' % picture)
    if picture is None:
        jpg = None
    else:
        try:
            local_filename = 'pic.jpg'
            cwd = '.'
            cmd = ['wget', '-O', local_filename, picture]
            system_cmd_result(cwd,
                              cmd,
                              display_stdout=True,
                              display_stderr=True,
                              raise_on_error=True)
            # local_filename, headers = urlretrieve(picture)
            with open(local_filename, 'rb') as f:
                jpg = f.read()
            logger.info('read %s bytes' % len(jpg))
        except BaseException as exc:
            logger.error(exc)
            jpg = None
    for k in res:
        if isinstance(res[k], unicode):
            res[k] = res[k].encode('utf8')
    res['subscriptions'] = []
    res['groups'] = [
        'account_created_automatically',
        'account_created_using_%s' % provider_name
    ]
    data = {
        'info': res,
        'images': {
            'user': {
                'jpg': jpg,
                'png': None,
                'svg': None,
                'pdf': None
            }
        }
    }

    #     logger.debug('new user:\n%s' % yaml_dump(data))

    user = DB.view_manager.create_view_instance(DB.user, data)
    user.set_root()
    return user
示例#51
0
文件: plot.py 项目: kannode/mcdp
def syntax_pdf(data):
    """ Returns a PDF string """
    from mcdp_report.html import ast_to_html
    s = data['s']
    s = s.replace('\t', '    ')

    lines_to_hide = get_lines_to_hide(data['params'])

    def ignore_line(lineno):
        return lineno in lines_to_hide

    contents = ast_to_html(
        s,
        ignore_line=ignore_line,
        parse_expr=Syntax.ndpt_dp_rvalue,
    )
    html = get_minimal_document(contents)

    d = mkdtemp()

    f_html = os.path.join(d, 'file.html')
    with open(f_html, 'w') as f:
        f.write(html)

    try:
        f_pdf = os.path.join(d, 'file.pdf')
        cmd = ['wkhtmltopdf', '-s', 'A1', f_html, f_pdf]
        system_cmd_result(d,
                          cmd,
                          display_stdout=False,
                          display_stderr=False,
                          raise_on_error=True)

        f_pdf_crop = os.path.join(d, 'file_crop.pdf')
        cmd = ['pdfcrop', '--margins', '4', f_pdf, f_pdf_crop]
        system_cmd_result(d,
                          cmd,
                          display_stdout=False,
                          display_stderr=False,
                          raise_on_error=True)

        with open(f_pdf_crop) as f:
            data = f.read()

        f_png = os.path.join(d, 'file.png')
        cmd = [
            'convert', '-density', '600', f_pdf_crop, '-background', 'white',
            '-alpha', 'remove', '-resize', '50%', f_png
        ]
        system_cmd_result(d,
                          cmd,
                          display_stdout=False,
                          display_stderr=False,
                          raise_on_error=True)

        with open(f_png) as f:
            pngdata = f.read()

        return [('pdf', 'syntax_pdf', data), ('png', 'syntax_pdf', pngdata)]
    except CmdException as e:
        raise e