Beispiel #1
0
def get_subtitle(url, path):
    in_data = urllib2.urlopen(url)
    temp_file = NamedTemporaryFile()

    temp_file.write(in_data.read())
    in_data.close()
    temp_file.seek(0)

    if is_zipfile(temp_file.name):
        zip_file = ZipFile(temp_file)
        for name in zip_file.namelist():
            # don't unzip stub __MACOSX folders
            if '.srt' in name and '__MACOSX' not in name:
                logger.info(' '.join(['Unpacking zipped subtitle', name, 'to', os.path.dirname(path)]))
                zip_file.extract(name, os.path.dirname(path))

        zip_file.close()
    elif is_rarfile(temp_file.name):
        rar_path = path + '.rar'
        logger.info('Saving rared subtitle as %s' % rar_path)
        with open(rar_path, 'w') as out_file:
            out_file.write(temp_file.read())

        try:
            import subprocess
            #extract all .srt in the rared file
            ret_code = subprocess.call(['unrar', 'e', '-n*srt', rar_path])
            if ret_code == 0:
                logger.info('Unpacking rared subtitle to %s' % os.path.dirname(path))
                os.remove(rar_path)
        except OSError:
            logger.info('Unpacking rared subtitle failed.'
                        'Please, install unrar to automate this step.')
    temp_file.close()
 def _get_tmp_file(self):
     ''' Creates a tmp file with the file data '''
     data = self.request.files['xml_file'][0]['body']
     tmp_file = NamedTemporaryFile(delete=False)
     tmp_file.write(data)
     tmp_file.close()
     return tmp_file.name
def main():
    logging.basicConfig(format='%(message)s', level=logging.INFO)

    args = parse_args()

    from_date = args.from_date
    to_date = args.to_date

    s3_client = S3Client.connect_with_profile(args.aws_profile)
    s3_bucket = args.s3_bucket
    s3_key = "{}_{}.logs.json".format(from_date.isoformat(), to_date.isoformat())
    sumo_access_id = args.sumo_access_id or os.environ["SUMOLOGIC_ACCESS_ID"]
    sumo_access_key = args.sumo_access_key or os.environ["SUMOLOGIC_ACCESS_KEY"]
    sumo_logic_client = SumoLogicClient(sumo_access_id, sumo_access_key)

    logging.info(
        "Searching for request logs in {} in range {} to {}".format(
            args.environment, from_date.isoformat(), to_date.isoformat()))

    request_logs = sumo_logic_client.search(
        sumo_logic_query(args.environment), start_dt=from_date, end_dt=to_date, fields=REQUEST_FIELDS)

    tmp_file = NamedTemporaryFile(mode='w', encoding='utf-8', delete=False)
    logging.debug("Writing out logs to temporary file {}".format(tmp_file.name))

    try:
        for request in request_logs:
            tmp_file.write(json.dumps(request, default=json_serialize) + '\n')

        s3_client.upload_from_file(s3_bucket, s3_key, tmp_file.name)
    finally:
        logging.debug("Removing temporary file {}".format(tmp_file.name))
        os.remove(tmp_file.name)
    def test_main(self):
        xml = """<record>
            <datafield tag="999" ind1="C" ind2="5">
                <subfield code="s">Test Journal Name,100,10</subfield>
            </datafield>
        </record>"""
        xml_temp_file = NamedTemporaryFile(dir=CFG_TMPDIR)
        xml_temp_file.write(xml)
        xml_temp_file.flush()

        kb = "TEST JOURNAL NAME---Converted"
        kb_temp_file = NamedTemporaryFile(dir=CFG_TMPDIR)
        kb_temp_file.write(kb)
        kb_temp_file.flush()

        dest_temp_fd, dest_temp_path = mkstemp(dir=CFG_TMPDIR)
        try:
            os.close(dest_temp_fd)

            process = subprocess.Popen([self.bin_path, xml_temp_file.name,
                                       '--kb', kb_temp_file.name,
                                       '-o', dest_temp_path],
                                       stderr=subprocess.PIPE,
                                       stdout=subprocess.PIPE)
            process.wait()

            transformed_xml = open(dest_temp_path).read()
            self.assertXmlEqual(transformed_xml, """<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record><datafield ind1="C" ind2="5" tag="999"><subfield code="s">Converted,100,10</subfield></datafield></record>
</collection>""")
        finally:
            os.unlink(dest_temp_path)
Beispiel #5
0
	def run(self):
		active_view = self.window.active_view()
		text = "\n\n".join(getSelectedText(active_view)).strip()

		tf = NamedTemporaryFile(mode="w", delete=False)
		try:
			tf.write(text)
			tf.close()

			res = subprocess.check_output(["m4", tf.name],
			                              stderr=subprocess.STDOUT,
			                              cwd=os.path.dirname(os.path.abspath(active_view.file_name())))
			res = res.decode('utf-8').replace('\r', '').strip()

			panel_name = "m4expand.results"
			panel = self.window.create_output_panel(panel_name)
			self.window.run_command("show_panel", {"panel": "output." + panel_name})

			panel.set_read_only(False)
			panel.set_syntax_file(active_view.settings().get("syntax"))
			panel.run_command("append", {"characters": res})
			panel.set_read_only(True)
		except Exception as e:
			print("M4Expand - An error occurred: ", e)
		finally:
			os.unlink(tf.name)
Beispiel #6
0
 def testLogfile(self):
     """Test logging into a logfile"""
     f = NamedTemporaryFile(delete=False)
     filename = f.name
     try:
         set_log_level("error")  # avoid using the console logger
         f.write(":-P\n")
         f.close()
         start_logfile(f.name, "devinfo")
         log = getLogger("prosoda.test.integration.test_logger")
         log.debug("Should not be in logfile! :-( ")
         log.info("Should be in logfile :-) ")
         log.devinfo("Should be in logfile :-) ")
         log.warning("Should really be in logfile :-D ")
         stop_logfile(f.name)
         contents = file(f.name).read()
         self.assertNotIn(":-(", contents)
         self.assertNotIn(":-P", contents)
         self.assertIn(":-)", contents)
         self.assertIn(":-D", contents)
         # Make sure no colour codes are leaked into the logfile
         self.assertNotIn("\033", contents)
     finally:
         set_log_level("debug")
         unlink(filename)
Beispiel #7
0
    def compile_inline(self,data,ext):
        """
        Compile inline css. Have to compile to a file, because some css compilers
        may not output to stdout, but we know they all output to a file. It's a
        little hackish, but you shouldn't be compiling in production anyway,
        right?
        """
        compiler = settings.COMPILER_FORMATS[ext]
        try:
            bin = compiler['binary_path']
        except:
            raise Exception("Path to CSS compiler must be included in COMPILER_FORMATS")
        
        tmp_file = NamedTemporaryFile(mode='w',suffix=ext)
        tmp_file.write(dedent(data))
        tmp_file.flush()
        path, ext = os.path.splitext(tmp_file.name)
        tmp_css = ''.join((path,'.css'))
        
        self.compile(path,compiler)
        data = open(tmp_css,'r').read()
        
        # cleanup
        tmp_file.close()
        os.remove(tmp_css)

        return data  
Beispiel #8
0
def tmpfile(stream, mode=None):
    """Context manager that writes a :class:`Stream` object to a named
    temporary file and yield it's filename. Cleanup deletes from the temporary
    file from disk.

    Args:
        stream (Stream): Stream object to write to disk as temporary file.
        mode (int, optional): File mode to set on temporary file.

    Returns:
        str: Temporoary file name
    """
    tmp = NamedTemporaryFile(delete=False)

    if mode is not None:
        oldmask = os.umask(0)

        try:
            os.chmod(tmp.name, mode)
        finally:
            os.umask(oldmask)

    for data in stream:
        tmp.write(to_bytes(data))

    tmp.close()

    yield tmp.name

    os.remove(tmp.name)
Beispiel #9
0
    def test_safe_md5(self):
        """Make sure we have the expected md5 with varied input types

        This method is ported from PyCogent (http://www.pycogent.org). PyCogent
        is a GPL project, but we obtained permission from the authors of this
        method to port it to the BIOM Format project (and keep it under BIOM's
        BSD license).
        """
        exp = 'd3b07384d113edec49eaa6238ad5ff00'

        tmp_f = NamedTemporaryFile(
            mode='w',
            prefix='test_safe_md5',
            suffix='txt')
        tmp_f.write('foo\n')
        tmp_f.flush()

        obs = safe_md5(open(tmp_f.name, 'U'))
        self.assertEqual(obs, exp)

        obs = safe_md5(['foo\n'])
        self.assertEqual(obs, exp)

        # unsupported type raises TypeError
        self.assertRaises(TypeError, safe_md5, 42)
    def run_solver(self, conflicts, election, deletion_handler, outfile=None):
        if not conflicts:
            return [], 0

        self.deletion_handler = deletion_handler

        instance = self.generate_instance(conflicts, election)

        f = NamedTemporaryFile(delete=False)
        f.write(instance.encode(code))
        f.close()

        process = Popen([self.cmd, f.name], stdout=PIPE)
        out, err = process.communicate()

        conflict_variables, optimum = self.parse_instance(out)

        if outfile:
            candidates = election[0]
            votes = election[1]
            votecounts = election[2]

            votemap = self.delete_votes(votes, votecounts, conflict_variables)
            votesum = sum(votemap.values())

            write_map(candidates, votesum, votemap, open(outfile, "w"))

        remove(f.name)
        return conflict_variables, optimum
Beispiel #11
0
        def odt_subreport(name=None, obj=None):
            if not aeroo_ooo:
                return _("Error! Subreports not available!")
            report_xml_ids = ir_obj.search(cr, uid, [('report_name', '=', name)], context=context)
            if report_xml_ids:
                service = netsvc.Service._services['report.%s' % name]
                report_xml = ir_obj.browse(cr, uid, report_xml_ids[0], context=context)
                data = {'model': obj._table_name, 'id': obj.id, 'report_type': 'aeroo', 'in_format': 'oo-odt'}
                ### Get new printing object ###
                sub_aeroo_print = AerooPrint()
                service.active_prints[sub_aeroo_print.id] = sub_aeroo_print
                context['print_id'] = sub_aeroo_print.id
                ###############################
                sub_aeroo_print.start_time = time.time()
                report, output = service.create_aeroo_report(cr, uid, \
                                                             [obj.id], data, report_xml, context=context,
                                                             output='odt')  # change for OpenERP 6.0 - Service class usage

                ### Delete printing object ###
                AerooPrint.print_ids.remove(sub_aeroo_print.id)
                del service.active_prints[sub_aeroo_print.id]
                ##############################
                temp_file = NamedTemporaryFile(suffix='.odt', prefix='aeroo-report-', delete=False)
                try:
                    temp_file.write(report)
                finally:
                    temp_file.close()
                # self.oo_subreports[print_id].append(temp_file.name)
                # aeroo_print.subreports.append(temp_file.name)
                self.active_prints[aeroo_print.id].subreports.append(temp_file.name)
                return "<insert_doc('%s')>" % temp_file.name
            return None
Beispiel #12
0
 def get_tempfile(self, **kwargs):
     kwargs.setdefault('suffix', '.vrt')
     tempfile = NamedTemporaryFile(**kwargs)
     tempfile.write(self.content)
     tempfile.flush()
     tempfile.seek(0)
     return tempfile
Beispiel #13
0
    def render_to_temporary_file(self, template_name, mode='w+b', bufsize=-1,
                                 suffix='.html', prefix='tmp', dir=None,
                                 delete=True):
        template = self.resolve_template(template_name)

        context = self.resolve_context(self.context_data)

        content = smart_str(template.render(context))
        content = make_absolute_paths(content)

        try:
            tempfile = NamedTemporaryFile(mode=mode, bufsize=bufsize,
                                      suffix=suffix, prefix=prefix,
                                      dir=dir, delete=delete)
        except TypeError:
            tempfile = NamedTemporaryFile(mode=mode, buffering=bufsize,
                                      suffix=suffix, prefix=prefix,
                                      dir=dir, delete=delete)

        try:
            tempfile.write(content)
            tempfile.flush()
            return tempfile
        except TypeError:
            tempfile.write(bytes(content, 'UTF-8'))
            tempfile.flush()
            return tempfile
        except:
            # Clean-up tempfile if an Exception is raised.
            tempfile.close()
            raise
Beispiel #14
0
    def create_temp_file(self, edid_binary):
        edid_file = NamedTemporaryFile(delete=False)
        edid_file.write(edid_binary)
        edid_file.flush()
        edid_file.seek(0)

        return edid_file
    def _write_local_schema_file(self, cursor):
        """
        Takes a cursor, and writes the BigQuery schema for the results to a
        local file system.

        :return: A dictionary where key is a filename to be used as an object
            name in GCS, and values are file handles to local files that
            contains the BigQuery schema fields in .json format.
        """
        schema = []
        for field in cursor.description:
            # See PEP 249 for details about the description tuple.
            field_name = field[0]
            field_type = self.type_map(field[1])
            field_mode = 'REPEATED' if field[1] in (1009, 1005, 1007,
                                                    1016) else 'NULLABLE'
            schema.append({
                'name': field_name,
                'type': field_type,
                'mode': field_mode,
            })

        self.log.info('Using schema for %s: %s', self.schema_filename, schema)
        tmp_schema_file_handle = NamedTemporaryFile(delete=True)
        s = json.dumps(schema, sort_keys=True)
        if PY3:
            s = s.encode('utf-8')
        tmp_schema_file_handle.write(s)
        return {self.schema_filename: tmp_schema_file_handle}
    def test_pipeline_run():
        'It tests that the pipeline runs ok'
        pipeline = 'sanger_with_qual'

        fhand_adaptors = NamedTemporaryFile()
        fhand_adaptors.write(ADAPTORS)
        fhand_adaptors.flush()

        arabidopsis_genes = 'arabidopsis_genes+'

        univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes)
        configuration = {'remove_vectors_blastdb': {'vectors': univec},
                         'remove_adaptors': {'adaptors': fhand_adaptors.name}}

        seq_fhand = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'), 'r')
        qual_fhand = open(os.path.join(TEST_DATA_DIR, 'qual.fasta'), 'r')

        seq_iter = seqs_in_file(seq_fhand, qual_fhand)

        filtered_seq_iter = _pipeline_builder(pipeline, seq_iter,
                                              configuration)

        seq_list = list(filtered_seq_iter)
        assert 'CGAtcgggggg' in str(seq_list[0].seq)
        assert len(seq_list) == 6
    def _generate_training_files(self):
        """Returns a tuple of file objects suitable for passing to the
        RdpTrainer application controller.
        """
        tmp_dir = get_qiime_temp_dir()
        training_set = RdpTrainingSet()
        reference_seqs_file = open(self.Params['reference_sequences_fp'], 'U')
        id_to_taxonomy_file = open(self.Params['id_to_taxonomy_fp'], 'U')

        for seq_id, seq in MinimalFastaParser(reference_seqs_file):
            training_set.add_sequence(seq_id, seq)

        for line in id_to_taxonomy_file:
            seq_id, lineage_str = map(strip, line.split('\t'))
            training_set.add_lineage(seq_id, lineage_str)

        training_set.dereplicate_taxa()

        rdp_taxonomy_file = NamedTemporaryFile(
            prefix='RdpTaxonAssigner_taxonomy_', suffix='.txt', dir=tmp_dir)
        rdp_taxonomy_file.write(training_set.get_rdp_taxonomy())
        rdp_taxonomy_file.seek(0)

        rdp_training_seqs_file = NamedTemporaryFile(
            prefix='RdpTaxonAssigner_training_seqs_', suffix='.fasta',
            dir=tmp_dir)
        for rdp_id, seq in training_set.get_training_seqs():
            rdp_training_seqs_file.write('>%s\n%s\n' % (rdp_id, seq))
        rdp_training_seqs_file.seek(0)

        self._training_set = training_set

        return rdp_taxonomy_file, rdp_training_seqs_file
    def test_seq_pipeline_parallel_run(self):
        'It tests that the pipeline runs ok'
        pipeline = 'sanger_without_qual'

        fhand_adaptors = NamedTemporaryFile()
        fhand_adaptors.write(ADAPTORS)
        fhand_adaptors.flush()
        arabidopsis_genes = 'arabidopsis_genes+'
        univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes)
        configuration = {'remove_vectors': {'vectors': univec},
                         'remove_adaptors': {'adaptors': fhand_adaptors.name}}

        in_fhands = {}
        in_fhands['in_seq'] = open(os.path.join(TEST_DATA_DIR, 'seq.fasta'),
                                   'r')
        out_fhand = NamedTemporaryFile()
        writer = SequenceWriter(out_fhand, file_format='fasta')
        writers = {'seq': writer}

        seq_pipeline_runner(pipeline, configuration, in_fhands,
                            processes=4, writers=writers)
        out_fhand = open(out_fhand.name, 'r')

        result_seq = out_fhand.read()
        assert result_seq.count('>') == 6
        #are we keeping the description?
        assert 'mdust' in result_seq
    def test_seq_pipeline_parallel_run_with_fasta_qual(self):
        'The pipeline runs in parallel with fasta and qual'
        pipeline = 'sanger_with_qual'

        fhand_adaptors = NamedTemporaryFile()
        fhand_adaptors.write(ADAPTORS)
        fhand_adaptors.flush()
        arabidopsis_genes = 'arabidopsis_genes+'
        univec = os.path.join(TEST_DATA_DIR, 'blast', arabidopsis_genes)
        configuration = {'remove_vectors': {'vectors': univec},
                         'remove_adaptors': {'adaptors': fhand_adaptors.name}}

        seq1 = create_random_seqwithquality(500, qual_range=50)
        seq2 = create_random_seqwithquality(500, qual_range=51)
        seq3 = create_random_seqwithquality(500, qual_range=52)
        seqs = [seq1, seq2, seq3]
        inseq_fhand, inqual_fhand = create_temp_seq_file(seqs, format='qual')

        in_fhands = {}
        in_fhands['in_seq'] = open(inseq_fhand.name)
        in_fhands['in_qual'] = open(inqual_fhand.name)

        outseq_fhand = NamedTemporaryFile()
        outqual_fhand = NamedTemporaryFile()
        writer = SequenceWriter(outseq_fhand, qual_fhand=outqual_fhand,
                                file_format='fasta')
        writers = {'seq': writer}

        seq_pipeline_runner(pipeline, configuration, in_fhands,
                            processes=4, writers=writers)
        out_fhand = open(outseq_fhand.name, 'r')

        result_seq = out_fhand.read()
        assert result_seq.count('>') == 3
Beispiel #20
0
def write_temp_file(data):
    # create a temp file for use as a config file. This should get cleaned
    # up magically at the end of the run.
    fid = NamedTemporaryFile(mode='w+b', suffix='.tmp')
    fid.write(data)
    fid.seek(0)
    return fid
Beispiel #21
0
def test_toy_corpus():

    keats = ('She dwells with Beauty - Beauty that must die;\n\n'
             'And Joy, whose hand is ever at his lips\n\n' 
             'Bidding adieu; and aching Pleasure nigh,\n\n'
             'Turning to poison while the bee-mouth sips:\n\n'
             'Ay, in the very temple of Delight\n\n'
             'Veil\'d Melancholy has her sovran shrine,\n\n'
             'Though seen of none save him whose strenuous tongue\n\n'
             'Can burst Joy\'s grape against his palate fine;\n\n'
             'His soul shall taste the sadness of her might,\n\n'
             'And be among her cloudy trophies hung.')

    assert toy_corpus(keats)
    assert toy_corpus(keats, nltk_stop=True)
    assert toy_corpus(keats, stop_freq=1)
    assert toy_corpus(keats, add_stop=['and', 'with'])
    assert toy_corpus(keats, nltk_stop=True,
                      stop_freq=1, add_stop=['ay'])

    import os
    from tempfile import NamedTemporaryFile as NFT

    tmp = NFT(delete=False)
    tmp.write(keats)
    tmp.close()

    c = toy_corpus(tmp.name, is_filename=True, 
                   nltk_stop=True, add_stop=['ay'])
    
    assert c
    os.remove(tmp.name)

    return c
    def _write_local_schema_file(self, cursor):
        """
        Takes a cursor, and writes the BigQuery schema for the results to a
        local file system.

        :return: A dictionary where key is a filename to be used as an object
            name in GCS, and values are file handles to local files that
            contains the BigQuery schema fields in .json format.
        """
        schema = []
        for field in cursor.description:
            # See PEP 249 for details about the description tuple.
            field_name = field[0]
            field_type = self.type_map(field[1])
            # Always allow TIMESTAMP to be nullable. MySQLdb returns None types
            # for required fields because some MySQL timestamps can't be
            # represented by Python's datetime (e.g. 0000-00-00 00:00:00).
            field_mode = 'NULLABLE' if field[6] or field_type == 'TIMESTAMP' else 'REQUIRED'
            schema.append({
                'name': field_name,
                'type': field_type,
                'mode': field_mode,
            })

        self.log.info('Using schema for %s: %s', self.schema_filename, schema)
        tmp_schema_file_handle = NamedTemporaryFile(delete=True)
        s = json.dumps(schema, tmp_schema_file_handle)
        if PY3:
            s = s.encode('utf-8')
        tmp_schema_file_handle.write(s)
        return {self.schema_filename: tmp_schema_file_handle}
    def get_logs(self):
        """
        Build the logs entry for the metadata 'output' section

        :return: list, Output instances
        """

        # Collect logs from server
        kwargs = {}
        if self.namespace is not None:
            kwargs['namespace'] = self.namespace
        logs = self.osbs.get_build_logs(self.build_id, **kwargs)

        # Deleted once closed
        logfile = NamedTemporaryFile(prefix=self.build_id,
                                     suffix=".log",
                                     mode='w')
        logfile.write(logs)
        logfile.flush()

        docker_logs = NamedTemporaryFile(prefix="docker-%s" % self.build_id,
                                         suffix=".log",
                                         mode='w')
        docker_logs.write("\n".join(self.workflow.build_logs))
        docker_logs.flush()

        return [Output(file=docker_logs,
                       metadata=self.get_output_metadata(docker_logs.name,
                                                         "build.log")),
                Output(file=logfile,
                       metadata=self.get_output_metadata(logfile.name,
                                                         "openshift-final.log"))]
def testFileAnnotationSpeed(author_testimg_generated, gatewaywrapper):
    """ Tests speed of loading file annotations. See PR: 4176 """
    try:
        f = NamedTemporaryFile()
        f.write("testFileAnnotationSpeed text")
        ns = TESTANN_NS
        image = author_testimg_generated

        # use the same file to create many file annotations
        for i in range(20):
            fileAnn = gatewaywrapper.gateway.createFileAnnfromLocalFile(f.name, mimetype="text/plain", ns=ns)
            image.linkAnnotation(fileAnn)
    finally:
        f.close()

    now = time.time()
    for ann in image.listAnnotations():
        if ann._obj.__class__ == omero.model.FileAnnotationI:
            # mimmic behaviour of templates which call multiple times
            print ann.getId()
            print ann.getFileName()
            print ann.getFileName()
            print ann.getFileSize()
            print ann.getFileSize()
    print time.time() - now
Beispiel #25
0
def SubmitPlay(request):    
    import jinja2
    from tempfile import NamedTemporaryFile
    import os

   
    html = ''

    inventory = """
    [current]
    {{ public_ip_address }}
    """
    # for (name,value) in request.GET:
        
        # if name=='Servers':
            # html=html+str(value+'\n')
   

    html=request.GET['Severs']
    inventory_template = jinja2.Template(inventory)
    rendered_inventory = inventory_template.render({
        'public_ip_address': html    
        # and the rest of our variables
    })

    # Create a temporary file and write the template string to it
    hosts = NamedTemporaryFile(delete=False)
    hosts.write(rendered_inventory)
    hosts.close()
    
    print(hosts.name)

    import ansiblepythonapi as myPlay
    args=['/home/ec2-user/playss/AnsiblePlus/test.yml']
    # args.append('-i')
    # args.append(hosts.name)
    message=myPlay.main(args)

    objects=[]
    for runner_results in myPlay.message:              
        values=[]
        for (host, value) in runner_results.get('dark', {}).iteritems():
            try:
                values.append(host)
                values.append(value['failed'])
                values.append(value['msg'])    
                objects.append(values)
            except:
                pass
        for (host, value) in runner_results.get('contacted', {}).iteritems():
            try:
                values.append(host)
                values.append(value['failed'])
                values.append(value['msg'])    
                objects.append(values)
            except:
                pass
        # for msg in pb.stats.output():   
    context=Context({'Summary':objects})
    return render(request, 'AnsibleResponce.html',context)
Beispiel #26
0
def run_via_pbs(args, pbs):
    assert(pbs in ('condor',))  # for now

    # TODO: RF to support multiple backends, parameters, etc, for now -- just condor, no options
    f = NamedTemporaryFile('w', prefix='datalad-%s-' % pbs, suffix='.submit', delete=False)
    try:
        pwd = getpwd()
        logs = f.name.replace('.submit', '.log')
        exe = args[0]
        # TODO: we might need better way to join them, escaping spaces etc.  There must be a stock helper
        #exe_args = ' '.join(map(repr, args[1:])) if len(args) > 1 else ''
        exe_args = ' '.join(args[1:]) if len(args) > 1 else ''
        f.write("""\
Executable = %(exe)s
Initialdir = %(pwd)s
Output = %(logs)s
Error = %(logs)s
getenv = True

arguments = %(exe_args)s
queue
""" % locals())
        f.close()
        Runner().run(['condor_submit', f.name])
        lgr.info("Scheduled execution via %s.  Logs will be stored under %s" % (pbs, logs))
    finally:
        os.unlink(f.name)
def hmmscan(fasta, database_path, ncpus=10):

    F = NamedTemporaryFile()
    F.write(fasta)
    F.flush()
    OUT = NamedTemporaryFile()
    cmd = '%s --cpu %s -o /dev/null -Z 190000 --tblout %s %s %s' %(HMMSCAN, ncpus, OUT.name, database_path, F.name)
    #print cmd
    sts = subprocess.call(cmd, shell=True)
    byquery = defaultdict(list)

    if sts == 0:
        for line in OUT:
            #['#', '---', 'full', 'sequence', '----', '---', 'best', '1', 'domain', '----', '---', 'domain', 'number', 'estimation', '----']
            #['#', 'target', 'name', 'accession', 'query', 'name', 'accession', 'E-value', 'score', 'bias', 'E-value', 'score', 'bias', 'exp', 'reg', 'clu', 'ov', 'env', 'dom', 'rep', 'inc', 'description', 'of', 'target']
            #['#-------------------', '----------', '--------------------', '----------', '---------', '------', '-----', '---------', '------', '-----', '---', '---', '---', '---', '---', '---', '---', '---', '---------------------']
            #['delNOG20504', '-', '553220', '-', '1.3e-116', '382.9', '6.2', '3.4e-116', '381.6', '6.2', '1.6', '1', '1', '0', '1', '1', '1', '1', '-']
            if line.startswith('#'): continue
            fields = line.split() # output is not tab delimited! Should I trust this split?
            hit, _, query, _ , evalue, score, bias, devalue, dscore, dbias = fields[0:10]
            evalue, score, bias, devalue, dscore, dbias = map(float, [evalue, score, bias, devalue, dscore, dbias])
            byquery[query].append([hit, evalue, score])
            
    OUT.close()
    F.close()
    return byquery
    def _write_local_data_files(self, cursor):
        """
        Takes a cursor, and writes results to a local file.

        :return: A dictionary where keys are filenames to be used as object
            names in GCS, and values are file handles to local files that
            contain the data for the GCS objects.
        """
        schema = list(map(lambda schema_tuple: schema_tuple[0], cursor.description))
        file_no = 0
        tmp_file_handle = NamedTemporaryFile(delete=True)
        tmp_file_handles = {self.filename.format(file_no): tmp_file_handle}

        for row in cursor:
            # Convert datetime objects to utc seconds, and decimals to floats
            row = map(self.convert_types, row)
            row_dict = dict(zip(schema, row))

            # TODO validate that row isn't > 2MB. BQ enforces a hard row size of 2MB.
            s = json.dumps(row_dict)
            if PY3:
                s = s.encode('utf-8')
            tmp_file_handle.write(s)

            # Append newline to make dumps BigQuery compatible.
            tmp_file_handle.write(b'\n')

            # Stop if the file exceeds the file size limit.
            if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
                file_no += 1
                tmp_file_handle = NamedTemporaryFile(delete=True)
                tmp_file_handles[self.filename.format(file_no)] = tmp_file_handle

        return tmp_file_handles
Beispiel #29
0
def get_splice_score(a, s_type=5):
    if s_type not in [3,5]:
        raise Exception("Invalid splice type {}, should be 3 or 5".format(s_type))
    
    maxent = config.maxentpath
    if not maxent:
        raise Exception("Please provide path to the score5.pl and score3.pl maxent scripts in config file")

    tmp = NamedTemporaryFile()
    for name,seq in a:
        tmp.write(">{}\n{}\n".format(name,seq))
    tmp.flush()
    cmd = "perl score{}.pl {}".format(s_type, tmp.name)
    p = sp.Popen(cmd, shell=True, cwd=maxent, stdout=sp.PIPE)
    score = 0
    for line in p.stdout.readlines():
        vals = line.strip().split("\t")
        if len(vals) > 1:
            try:
                score += float(vals[-1])
            except ValueError:
                logger.error("valueError, skipping: {}".format(vals))
            except:
                logger.error("Something unexpected happened")
    return score
Beispiel #30
0
    def _build_and_catch_errors(self, build_func, options_bytes, source=None):
        try:
            return build_func()
        except _cl.RuntimeError as e:
            msg = e.what
            if options_bytes:
                msg = msg + "\n(options: %s)" % options_bytes.decode("utf-8")

            if source is not None:
                from tempfile import NamedTemporaryFile
                srcfile = NamedTemporaryFile(mode="wt", delete=False, suffix=".cl")
                try:
                    srcfile.write(source)
                finally:
                    srcfile.close()

                msg = msg + "\n(source saved as %s)" % srcfile.name

            code = e.code
            routine = e.routine

            err = _cl.RuntimeError(
                    _cl.Error._ErrorRecord(
                        msg=msg,
                        code=code,
                        routine=routine))

        # Python 3.2 outputs the whole list of currently active exceptions
        # This serves to remove one (redundant) level from that nesting.
        raise err
Beispiel #31
0
class GitIndexTests(unittest.TestCase):
    def setUp(self):
        self.dir = TemporaryDirectory()
        self.file = NamedTemporaryFile(dir=self.dir.name, delete=False)
        self.filename = path.basename(self.file.name)
        self.author = Signature('QuitStoreTest', '*****@*****.**')
        self.comitter = Signature('QuitStoreTest', '*****@*****.**')
        self.repo = quit.git.Repository(self.dir.name, create=True)

    def tearDown(self):
        self.file = None
        self.filename = None
        self.dir.cleanup()
        self.dir = None
        self.reop = None

    def addfile(self):
        """Create a repository and add a file to the git index."""
        # Write to file
        self.file.write(b'First Line\n')
        self.file.read()

        # Add file to index
        repo = Repository(self.dir.name)
        index = repo.index
        index.read()
        index.add(self.filename)
        index.write()
        self.repo = quit.git.Repository(self.dir.name)

    def createcommit(self):
        """Prepare a git repository with one existing commit.

        Create a directory, initialize a git Repository, add
        and commit a file.

        Returns:
            A list containing the directory and file
        """
        self.addfile()
        # Create commit
        repo = Repository(self.dir.name)
        index = repo.index
        index.read()
        tree = index.write_tree()
        message = "First commit of temporary test repo"
        repo.create_commit('HEAD', self.author, self.comitter, message, tree,
                           [])
        self.repo = quit.git.Repository(self.dir.name)

    def testIndexSetRevision(self):
        self.createcommit()
        reviosions = self.repo.revisions()
        index = quit.git.Index(self.repo)

        index.set_revision(reviosions[0].id)

        with self.assertRaises(Exception) as context:
            index.set_revision('not.existing.revision')

    def testIndexAddFile(self):
        index = quit.git.Index(self.repo)
        self.assertEqual(len(index.stash), 0)
        index.add(self.filename, b'First Line\n')
        self.assertEqual(len(index.stash), 1)

    def testIndexCommit(self):
        index = quit.git.Index(self.repo)

        self.assertFalse(index.dirty)

        commit = index.commit("First commit from quit test", "QuitTest",
                              "*****@*****.**")

        self.assertTrue(index.dirty)

        with self.assertRaises(Exception) as context:
            index.commit("Second commit from quit test", "QuitTest",
                         "*****@*****.**")
Beispiel #32
0
    def flushPartition(self):
        # Flush a Partition to secondary memory.

        # Choose a random victim from main memory, which size is not empty.
        victim_len = 0
        while (victim_len == 0):
            victim = randint(
                0, (self.left_table.size + self.right_table.size) - 1)
            if (victim < self.left_table.size):
                victim_len = len(self.left_table.partitions[victim].records)
            else:
                victim_len = len(
                    self.right_table.partitions[victim %
                                                self.right_table.size].records)

        if (victim < self.left_table.size):
            file_descriptor = self.fileDescriptor_left
            partition_to_flush = self.left_table.partitions[victim]
            self.memory_left = self.memory_left - len(
                partition_to_flush.records)
            self.left_table.partitions[victim] = Partition()

        else:
            victim = victim % self.right_table.size
            file_descriptor = self.fileDescriptor_right
            partition_to_flush = self.right_table.partitions[victim]
            self.memory_right = self.memory_right - len(
                partition_to_flush.records)
            self.right_table.partitions[victim] = Partition()

        # Update file descriptor
        if (file_descriptor.has_key(victim)):
            lenfile = file_descriptor[victim].size
            file = open(file_descriptor[victim].file.name, 'a')
            file_descriptor.update({
                victim:
                FileDescriptor(file,
                               len(partition_to_flush.records) + lenfile,
                               file_descriptor[victim].timestamps)
            })
        else:
            file = NamedTemporaryFile(suffix=".ht", prefix="", delete=False)
            file_descriptor.update({
                victim:
                FileDescriptor(file, len(partition_to_flush.records), set())
            })

        # Flush partition in file.
        for record in partition_to_flush.records:
            self.timestamp += 1
            tuple = str(record.tuple)
            ats = str(record.ats)
            dts = str(self.timestamp)
            #ats   = "%.40r" % (record.ats)
            #dts   = "%.40r" % (time())

            file.write(tuple + '|')
            file.write(ats + '|')
            file.write(dts + '\n')

        file.close()
Beispiel #33
0
class _TestPymo(unittest.TestCase):
    """Test class for pymo module

    Example usage:

     python pymo.py (to execute all tests)

     python pymo.py -c (for verbose console logging)
     python pymo.py -f mylog.log (for logging to file mylog.log)
     python pymo.py -c -f mylog.log (for both)

     python pymo.py _Test_pymo.test_fetch_smiles # (to execute only the specified test)

     coverage run pymo.py (to run test code coverage analysis)
     coverage report pymo.py (to view the result of the test code coverage analysis)
    """
    def setUp(self):
        """setup test data location, unittest config and logger"""

        # location of test data
        self.test_data_location = root_dir + '/contrib/script/py/mmp/testdata/'

        # temp output file and dir
        self.temp_inp_file = NamedTemporaryFile(encoding='utf-8',
                                                mode='wt',
                                                suffix='.smi',
                                                delete=False)
        self.temp_out_file = NamedTemporaryFile(encoding='utf-8',
                                                mode='wt',
                                                delete=False)
        self.temp_out_dir = mkdtemp()

        test_smiles = {
            # basic test set - all the below id's and structures are CHEMBL
            '3105327': 'Cc1ccc2c(ccn2c3nc(cs3)c4cc(ccc4F)C(F)(F)F)c1',
            '1526778': 'CC(=O)c1c(C)n(c(C)c1C(=O)C)c2nc(c(C)s2)c3ccc(C)c(C)c3',
            '1494678': 'CC(=O)c1c(C)n(c(C)c1C(=O)C)c2nc(c(C)s2)c3ccccc3',
            '472166': 'OC(CCn1ccnc1)(c2ccccc2)c3ccccc3',
            '69798': 'Cc1nccn1CCC(O)(c2ccccc2)c3ccccc3',
            '367346': 'Cc1sc(N)nc1c2cccc(Cl)c2',
            '366881': 'Cc1sc(N)nc1c2ccc(Cl)c(Cl)c2',
            '1477460': 'COc1ccc(cc1)c2nc(sc2C)n3c(C)c(C(=O)C)c(C(=O)C)c3C',
            '1441050': 'COc1ccc(cc1OC)c2nc(sc2C)n3c(C)c(C(=O)C)c(C(=O)C)c3C'
        }

        # write test data to temp file 01
        for smi_id, smi in test_smiles.items():
            string = smi + ' ' + smi_id + '\n'
            self.temp_inp_file.write(string)
        self.temp_inp_file.close()

    def tearDown(self):
        """cleanup test data and settings"""

        # Clean up the directory
        # os.removedirs(self.temp_out_dir)
        shutil.rmtree(self.temp_out_dir)

    def test_fileconv(self):
        log.debug("Testing fileconv")
        exit_status = fileconv(os.path.join(self.test_data_location,
                                            self.temp_inp_file.name),
                               params_dict={
                                   '-v':
                                   '',
                                   '-c':
                                   '10',
                                   '-C':
                                   '20',
                                   '-S':
                                   os.path.join(self.temp_out_dir,
                                                'atomcountfilter'),
                                   '-o':
                                   'smi'
                               })
        log.debug("fileconv return code was: %s" % exit_status)
        self.assertEqual(exit_status, 0)

    def test_make_these_molecules(self):
        log.debug("Testing _make_these_molecules")

        # test data from mmp_enum_mols_from_pairs.py
        context = NamedTemporaryFile(encoding='utf-8',
                                     mode='wt',
                                     suffix='.smi',
                                     delete=False)
        examp_context = "[1CH3]CCCCCC partOne_1\n[1CH3]CCCCCCC partOne_2\n[1CH3]CCCCCCCC partOne_3\n[1CH3]CCCCCCCCC partOne_4\n[1CH3]CCCCCCCCCC partOne_5"
        context.write(examp_context)
        context.close()

        frags = NamedTemporaryFile(encoding='utf-8',
                                   mode='wt',
                                   suffix='.smi',
                                   delete=False)
        examp_frags = "[1OH]CCCCCC partTwo_1\n[1OH]CCCCCCC partTwo_2\n[1OH]CCCCCCCC partTwo_3\n[1OH]CCCCCCCCC partTwo_4\n[1OH]CCCCCCCCCC partTwo_5\n[1OH]CCCCCCCCCCC partTwo_6\n"
        frags.write(examp_frags)
        frags.close()

        make_instr = NamedTemporaryFile(encoding='utf-8',
                                        mode='wt',
                                        delete=False)
        examp_make_instr = "partOne_2 partTwo_3\npartOne_4 partTwo_5\n"
        make_instr.write(examp_make_instr)
        make_instr.close()

        rxn = NamedTemporaryFile(encoding='utf-8',
                                 mode='wt',
                                 suffix='.rxn',
                                 delete=False)
        # this is the reaction specification that trxn needs to combine isotopically labelled mmp fragmentation points
        single_rxn = "(0 Reaction\n (0 Scaffold\n  (A C smarts \"[!0*]\")\n  (A I isotope (0 0))\n )\n"
        single_rxn += " (1 Sidechain\n  (A C smarts \"[!0*]\")\n  (A I isotope (0 0))\n  (A I join (0 0))\n )\n)"
        rxn.write(single_rxn)
        rxn.close()

        exit_status = make_these_molecules([context.name, frags.name],
                                           make_instr.name, [rxn.name])

        log.debug("make_these_molecules return code was: %s" % exit_status)
        self.assertEqual(exit_status, 0)

    def test_dicer(self):
        log.debug("Testing dicer")
        log.debug("Testing %s" % dicer.__name__)
        exit_status = dicer(
            os.path.join(self.test_data_location, self.temp_inp_file.name),
            os.path.join(self.temp_out_dir, 'dicer.out'),
            params_dict={},
        )
        log.debug("dicer return code was: %s" % exit_status)
        self.assertEqual(exit_status, 0)

    def test_alogp(self):
        log.debug("Testing CMI")
        exit_status = alogp(
            os.path.join(self.test_data_location, self.temp_inp_file.name),
            os.path.join(self.temp_out_dir, 'alogp.out'))
        log.debug("CMI return code was: %s" % exit_status)
        self.assertEqual(exit_status, 0)
Beispiel #34
0
def mrjob(options):
    "Generates and executes MR job script"

    user = os.getenv('USER')
    tstamp = int(time.time())
    hdir = hdfs_dir(options.hdir, options.hdfs_prefix)

    if PYDOOP:
        odir = hdfs.path.join(hdir, options.odir)
        idir = hdfs.path.join(hdir, options.idir)
        schema = hdfs.path.join(hdir, options.schema)
        for name in [
                hdir,
                odir,
                idir,
        ]:
            if options.verbose:
                print("Checking %s" % name)
            if not hdfs.path.isdir(name):
                if name in [hdir, idir]:
                    print("ERROR: %s does not exist" % name)
                    sys.exit(1)
                # else:
                #     print(" Creating output directory: %s" % name)
                #     hdfs.mkdir(name)
            elif name == odir:
                # in case odir exists and is not empty, move it somewhere and re-create
                if hdfs.ls(odir):
                    ocache = hdfs.path.normpath(odir) + '_%d' % tstamp
                    if options.verbose:
                        print(
                            " Non-empty output directory exists, saving it in %s"
                            % ocache)
                    hdfs.move(odir, ocache)
                    # hdfs.mkdir(odir)
                # if it's empty, remove it
                else:
                    hdfs.rmr(odir)

        if options.verbose:
            print("Checking %s" % schema)
        if not hdfs.path.isfile(schema):
            print("ERROR: %s does not exist" % schema)
            sys.exit(1)
    else:
        idir = '%s%s' % (hdir, 'data')
        odir = '%s%s' % (hdir, 'mrout')
        schema = '%s%s' % (hdir, options.schema)
        if options.verbose:
            msg = 'pydoop module is not present on this system'
            msg += ', will use input as is without checking'
            print('WARNING:', msg)
    for name in [options.mrpy, options.pydoop, options.avro]:
        if options.verbose:
            print("Checking %s" % name)
        if not os.path.isfile(name):
            print("ERROR: %s does not exist" % name)
            sys.exit(1)


#     module = os.path.basename(os.path.splitext(options.mrpy)[0])
    code = create_mrpy(options.mrpy, options.verbose)

    cmd = """#!/bin/bash
input={input}
output={output}
schema={schema}
ifile=/tmp/mr_{user}_{tstamp}.py
cat << EOF > $ifile
{code}
EOF

module=mr_{user}_{tstamp}
arch_pydoop={pydoop}
arch_avro={avro}
echo "Input URI : $input"
echo "Output URI: $output"
echo "Schema: $schema"
echo "MR script : $ifile"
echo "Module name : $module"
echo "Pydoop archive: $arch_pydoop"
echo "Avro archive  : $arch_avro"
echo "-----------------"
echo "Submitting MR job"
pydoop submit \
    --upload-archive-to-cache $arch_pydoop \
    --upload-archive-to-cache $arch_avro \
    -D avro.schema=$schema \
    --do-not-use-java-record-reader \
    --log-level {loglevel} \
    --job-name WMArchive \
    --num-reducers 1 \
    --upload-file-to-cache $ifile \
    --mrv2 $module $input $output
    """.format(input=idir,
               output=odir,
               user=user,
               tstamp=tstamp,
               code=code,
               schema=schema,
               loglevel=options.loglevel,
               pydoop=os.path.abspath(options.pydoop),
               avro=os.path.abspath(options.avro))

    fobj = NamedTemporaryFile(delete=False)
    fobj.write(cmd)
    fobj.close()

    fstat = os.stat(fobj.name)
    os.chmod(fobj.name, fstat.st_mode | stat.S_IEXEC)

    if options.execute:
        run(fobj.name, options.verbose)
    else:
        if options.verbose:
            print("------- Generated script --------")
        print(open(fobj.name, 'r').read())
        if options.verbose:
            print("---------------------------------")

    # clean up temporary file
    os.unlink(fobj.name)
Beispiel #35
0
def run_p2p_prediction(tx_lat, tx_lng, rx_lat, rx_lng, path_ssn,
                    path_name="",
                    tx_antenna="ISOTROPIC",
                    tx_gos=0.0,
                    rx_antenna="ISOTROPIC",
                    rx_gos=0.0,
                    path_month=None,
                    path_year=None,
                    path_hour=[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24],
                    path_frequency=[10],
                    path_bw=3000,
                    path_SNRr=15,
                    tx_power=100,
                    path_sorl="SHORTPATH",
                    path_manmade_noise="CITY",
                    report_format=["RPT_BCR"],
                    data_path="./data/",
                    report_dict_keys=['BCR'],
                    zeroMidnight=False,
                    returnInputFile=False,
                    returnOutputFile=False
                    ):

    tx_power = 10 * (log10(tx_power/1000.0))

    report_format_str = " | ".join(report_format)
    path_frequency_str = ", ".join([str(n) for n in path_frequency])
    path_hour_str = ", ".join([str(n) for n in path_hour])
    buf = []
    if path_name:
        buf.append('PathName "{:s}"'.format(path_name))
    buf.append('Path.L_tx.lat {:.6f}'.format(tx_lat))
    buf.append('Path.L_tx.lng {:.6f}'.format(tx_lng))
    buf.append('TXAntFilePath "{:s}"'.format(tx_antenna))
    if tx_antenna == "ISOTROPIC":
        buf.append('TXGOS {:.2f}'.format(tx_gos))
    #buf.append('AntennaOrientation TX2RX')

    buf.append('Path.L_rx.lat {:.6f}'.format(rx_lat))
    buf.append('Path.L_rx.lng {:.6f}'.format(rx_lng))
    buf.append('RXAntFilePath "{:s}"'.format(rx_antenna))
    if rx_antenna == "ISOTROPIC":
        buf.append('RXGOS {:.2f}'.format(rx_gos))
    #buf.append('AntennaOrientation RX2TX')

    if not path_year:
        now = datetime.datetime.utcnow()
        path_year = now.year
    buf.append('Path.year {:d}'.format(path_year))
    if not path_month:
        now = datetime.datetime.utcnow()
        path_month = now.month
    buf.append('Path.month {:d}'.format(path_month))

    buf.append('Path.hour {:s}'.format(path_hour_str))
    buf.append('Path.SSN {:d}'.format(path_ssn))
    buf.append('Path.frequency {:s}'.format(path_frequency_str))
    buf.append('Path.txpower {:.2f}'.format(tx_power))
    buf.append('Path.BW {:.2f}'.format(path_bw))
    buf.append('Path.SNRr {:.2f}'.format(path_SNRr))
    buf.append('Path.SNRXXp 90')
    buf.append('Path.ManMadeNoise "{:s}"'.format(path_manmade_noise))
    buf.append('Path.SorL "{:s}"'.format(path_sorl))
    buf.append('RptFileFormat "{:s}"'.format(report_format_str))
    buf.append('LL.lat {:.6f}'.format(rx_lat))
    buf.append('LL.lng {:.6f}'.format(rx_lng))
    buf.append('LR.lat {:.6f}'.format(rx_lat))
    buf.append('LR.lng {:.6f}'.format(rx_lng))
    buf.append('UL.lat {:.6f}'.format(rx_lat))
    buf.append('UL.lng {:.6f}'.format(rx_lng))
    buf.append('UR.lat {:.6f}'.format(rx_lat))
    buf.append('UR.lng {:.6f}'.format(rx_lng))
    buf.append('DataFilePath "{:s}"'.format(data_path))

    input_file = NamedTemporaryFile(mode='w+t', prefix="proppy_", suffix='.in', delete=False)
    text_in = "{:s}\n".format('\n'.join(buf))
    #print(text_in)
    input_file.write(text_in)
    input_file.close()

    FNULL = open(os.devnull, 'w')
    output_file = NamedTemporaryFile(prefix="proppy_", suffix='.out', delete=False)

    return_code = subprocess.call(["ITURHFProp",
        '-c',
        input_file.name,
        output_file.name],
        stderr=subprocess.STDOUT)

    if return_code != 232:
        raise ITURHFPropError("Internal Server Error: Return Code {:d}".format(return_code))

    try:
        prediction_dict = get_predictions_as_dict(output_file.name, report_dict_keys, zeroMidnight=zeroMidnight)
    except:
        print("Unexpected error:", sys.exc_info()[0])
        raise ITURHFPropError("Internal Server Error: Error parsing file")

    print(input_file.name)
    print(output_file.name)
    #os.remove(input_file.name)
    #os.remove(output_file.name)
    return prediction_dict
Beispiel #36
0
def get_readable_fileobj(name_or_obj,
                         encoding=None,
                         cache=False,
                         show_progress=True,
                         remote_timeout=None):
    """
    Given a filename or a readable file-like object, return a context
    manager that yields a readable file-like object.

    This supports passing filenames, URLs, and readable file-like
    objects, any of which can be compressed in gzip or bzip2.

    Notes
    -----

    This function is a context manager, and should be used for example
    as::

        with get_readable_fileobj('file.dat') as f:
            contents = f.read()

    Parameters
    ----------
    name_or_obj : str or file-like object
        The filename of the file to access (if given as a string), or
        the file-like object to access.

        If a file-like object, it must be opened in binary mode.

    encoding : str, optional
        When `None` (default), returns a file-like object with a
        `read` method that on Python 2.x returns `bytes` objects and
        on Python 3.x returns `str` (`unicode`) objects, using
        `locale.getpreferredencoding()` as an encoding.  This matches
        the default behavior of the built-in `open` when no `mode`
        argument is provided.

        When `'binary'`, returns a file-like object where its `read`
        method returns `bytes` objects.

        When another string, it is the name of an encoding, and the
        file-like object's `read` method will return `str` (`unicode`)
        objects, decoded from binary using the given encoding.

    cache : bool, optional
        Whether to cache the contents of remote URLs.

    show_progress : bool, optional
        Whether to display a progress bar if the file is downloaded
        from a remote server.  Default is `True`.

    remote_timeout : float
        Timeout for remote requests in seconds (default is the configurable
        REMOTE_TIMEOUT, which is 3s by default)

    Returns
    -------
    file : readable file-like object
    """

    # close_fds is a list of file handles created by this function
    # that need to be closed.  We don't want to always just close the
    # returned file handle, because it may simply be the file handle
    # passed in.  In that case it is not the responsibility of this
    # function to close it: doing so could result in a "double close"
    # and an "invalid file descriptor" exception.
    close_fds = []
    delete_fds = []

    if remote_timeout is None:
        # use configfile default
        remote_timeout = REMOTE_TIMEOUT()

    # Get a file object to the content
    if isinstance(name_or_obj, six.string_types):
        if _is_url(name_or_obj):
            name_or_obj = download_file(name_or_obj,
                                        cache=cache,
                                        show_progress=show_progress,
                                        timeout=remote_timeout)
        if six.PY3:
            fileobj = io.FileIO(name_or_obj, 'r')
        elif six.PY2:
            fileobj = open(name_or_obj, 'rb')
        close_fds.append(fileobj)
    else:
        fileobj = name_or_obj

    # Check if the file object supports random access, and if not,
    # then wrap it in a BytesIO buffer.  It would be nicer to use a
    # BufferedReader to avoid reading loading the whole file first,
    # but that is not compatible with streams or urllib2.urlopen
    # objects on Python 2.x.
    if not hasattr(fileobj, 'seek'):
        fileobj = io.BytesIO(fileobj.read())

    # Now read enough bytes to look at signature
    signature = fileobj.read(4)
    fileobj.seek(0)

    if signature[:3] == b'\x1f\x8b\x08':  # gzip
        import struct
        try:
            from .compat import gzip
            fileobj_new = gzip.GzipFile(fileobj=fileobj, mode='rb')
            fileobj_new.read(1)  # need to check that the file is really gzip
        except (IOError, EOFError):  # invalid gzip file
            fileobj.seek(0)
            fileobj_new.close()
        except struct.error:  # invalid gzip file on Python 3
            fileobj.seek(0)
            fileobj_new.close()
        else:
            fileobj_new.seek(0)
            fileobj = fileobj_new
    elif signature[:3] == b'BZh':  # bzip2
        try:
            # bz2.BZ2File does not support file objects, only filenames, so we
            # need to write the data to a temporary file
            tmp = NamedTemporaryFile("wb", delete=False)
            tmp.write(fileobj.read())
            tmp.close()
            delete_fds.append(tmp)
            import bz2
            fileobj_new = bz2.BZ2File(tmp.name, mode='rb')
            fileobj_new.read(1)  # need to check that the file is really bzip2
        except IOError:  # invalid bzip2 file
            fileobj.seek(0)
            fileobj_new.close()
        else:
            fileobj_new.seek(0)
            close_fds.append(fileobj_new)
            fileobj = fileobj_new

    # By this point, we have a file, io.FileIO, gzip.GzipFile, or
    # bz2.BZ2File instance opened in binary mode (that is, read
    # returns bytes).  Now we need to, if requested, wrap it in a
    # io.TextIOWrapper so read will return unicode based on the
    # encoding parameter.

    if six.PY3:
        needs_textio_wrapper = encoding != 'binary'
    elif six.PY2:
        needs_textio_wrapper = encoding != 'binary' and encoding is not None

    if needs_textio_wrapper:
        # A bz2.BZ2File can not be wrapped by a TextIOWrapper,
        # so we decompress it to a temporary file and then
        # return a handle to that.
        import bz2
        if isinstance(fileobj, bz2.BZ2File):
            tmp = NamedTemporaryFile("wb", delete=False)
            data = fileobj.read()
            tmp.write(data)
            tmp.close()
            delete_fds.append(tmp)
            if six.PY3:
                fileobj = io.FileIO(tmp.name, 'r')
            elif six.PY2:
                fileobj = open(tmp.name, 'rb')
            close_fds.append(fileobj)

        # On Python 2.x, we need to first wrap the regular `file`
        # instance in a `io.FileIO` object before it can be
        # wrapped in a `TextIOWrapper`.  We don't just create an
        # `io.FileIO` object in the first place, because we can't
        # get a raw file descriptor out of it on Python 2.x, which
        # is required for the XML iterparser.
        if six.PY2 and isinstance(fileobj, file):
            fileobj = io.FileIO(fileobj.fileno())

        fileobj = io.BufferedReader(fileobj)
        fileobj = io.TextIOWrapper(fileobj, encoding=encoding)

        # Ensure that file is at the start - io.FileIO will for
        # example not always be at the start:
        # >>> import io
        # >>> f = open('test.fits', 'rb')
        # >>> f.read(4)
        # 'SIMP'
        # >>> f.seek(0)
        # >>> fileobj = io.FileIO(f.fileno())
        # >>> fileobj.tell()
        # 4096L

        fileobj.seek(0)

    try:
        yield fileobj
    finally:
        for fd in close_fds:
            fd.close()
        for fd in delete_fds:
            os.remove(fd.name)
Beispiel #37
0
def ccopen(source, *args, **kwargs):
    """Guess the identity of a particular log file and return an instance of it.

    Inputs:
        source - a single logfile, a list of logfiles (for a single job),
                 an input stream, or an URL pointing to a log file.
        *args, **kwargs - arguments and keyword arguments passed to filetype

    Returns:
      one of ADF, DALTON, GAMESS, GAMESS UK, Gaussian, Jaguar,
      Molpro, MOPAC, NWChem, ORCA, Psi3, Psi/Psi4, QChem, CJSON or None
      (if it cannot figure it out or the file does not exist).
    """
    inputfile = None
    is_stream = False

    # Check if source is a link or contains links. Retrieve their content.
    # Try to open the logfile(s), using openlogfile, if the source is a string (filename)
    # or list of filenames. If it can be read, assume it is an open file object/stream.
    is_string = isinstance(source, str)
    is_url = True if is_string and URL_PATTERN.match(source) else False
    is_listofstrings = isinstance(source, list) and all(
        [isinstance(s, str) for s in source])
    if is_string or is_listofstrings:
        # Process links from list (download contents into temporary location)
        if is_listofstrings:
            filelist = []
            for filename in source:
                if not URL_PATTERN.match(filename):
                    filelist.append(filename)
                else:
                    try:
                        response = urlopen(filename)
                        tfile = NamedTemporaryFile(delete=False)
                        tfile.write(response.read())
                        # Close the file because Windows won't let open it second time
                        tfile.close()
                        filelist.append(tfile.name)
                        # Delete temporary file when the program finishes
                        atexit.register(os.remove, tfile.name)
                    except (ValueError, URLError) as error:
                        if not kwargs.get('quiet', False):
                            (errno, strerror) = error.args
                        return None
            source = filelist

        if not is_url:
            try:
                inputfile = logfileparser.openlogfile(source)
            except IOError as error:
                if not kwargs.get('quiet', False):
                    (errno, strerror) = error.args
                return None
        else:
            try:
                response = urlopen(source)
                is_stream = True

                # Retrieve filename from URL if possible
                filename = re.findall("\w+\.\w+", source.split('/')[-1])
                filename = filename[0] if filename else ""

                inputfile = logfileparser.openlogfile(filename,
                                                      object=response.read())
            except (ValueError, URLError) as error:
                if not kwargs.get('quiet', False):
                    (errno, strerror) = error.args
                return None

    elif hasattr(source, "read"):
        inputfile = source
        is_stream = True

    # Streams are tricky since they don't have seek methods or seek won't work
    # by design even if it is present. We solve this now by reading in the
    # entire stream and using a StringIO buffer for parsing. This might be
    # problematic for very large streams. Slow streams might also be an issue if
    # the parsing is not instantaneous, but we'll deal with such edge cases
    # as they arise. Ideally, in the future we'll create a class dedicated to
    # dealing with these issues, supporting both files and streams.
    if is_stream:
        try:
            inputfile.seek(0, 0)
        except (AttributeError, IOError):
            contents = inputfile.read()
            try:
                inputfile = io.StringIO(contents)
            except:
                inputfile = io.StringIO(unicode(contents))
            inputfile.seek(0, 0)

    # Proceed to return an instance of the logfile parser only if the filetype
    # could be guessed. Need to make sure the input file is closed before creating
    # an instance, because parsers will handle opening/closing on their own.
    filetype = guess_filetype(inputfile)

    # If the input file isn't a standard compchem log file, try one of
    # the readers, falling back to Open Babel.
    if not filetype:
        if kwargs.get("cjson"):
            filetype = readerclasses['cjson']
        elif source and not is_stream:
            ext = os.path.splitext(source)[1][1:].lower()
            for extension in readerclasses:
                if ext == extension:
                    filetype = readerclasses[extension]

    # Proceed to return an instance of the logfile parser only if the filetype
    # could be guessed. Need to make sure the input file is closed before creating
    # an instance, because parsers will handle opening/closing on their own.
    if filetype:
        # We're going to close and reopen below anyway, so this is just to avoid
        # the missing seek method for fileinput.FileInput. In the long run
        # we need to refactor to support for various input types in a more
        # centralized fashion.
        if is_listofstrings:
            pass
        else:
            inputfile.seek(0, 0)
        if not is_stream:
            if is_listofstrings:
                if filetype == Turbomole:
                    source = sort_turbomole_outputs(source)
            inputfile.close()
            return filetype(source, *args, **kwargs)
        return filetype(inputfile, *args, **kwargs)
Beispiel #38
0
    def submit(self, **kwargs):
        """
            dictionary of keywords to be parsed
        """
        executable = kwargs.get("executable")
        env = dict(kwargs.get("env", {}))
        memory = kwargs.get("memory", 0.)
        cpu = kwargs.get("cpu", 0.)
        dry = bool(kwargs.get("dry", False))
        verbose = bool(kwargs.get("verbose", True))
        wd = str(kwargs.get("workdir", "$(pwd)"))
        part = str(kwargs.get("partition", "debug"))
        #log.error("ENV SETTINGS: %s",str(env))

        if cpu == 0.: raise Exception("must provide cpu time")

        sscript = NamedTemporaryFile(dir=wd,
                                     suffix=".sh",
                                     mode="w",
                                     delete=False)
        sname = abspath(sscript.name)
        sscript.write("#!/bin/sh\n")
        sscript.write("#SBATCH --ntasks=1\n")
        sscript.write("#SBATCH --partition={part}\n".format(part=part))
        sscript.write("#SBATCH -e {exe}.err\n".format(exe=sname))
        sscript.write("#SBATCH -o {exe}.out\n".format(exe=sname))
        sscript.write("#SBATCH --time={cpu}\n#SBATCH --mem={mem}\n\n".format(
            cpu=cpu, mem=memory))
        for key, value in env.iteritems():
            environ[key] = value
            sscript.write("sbatch --export={key} # {value}\n".format(
                key=key, value=value))
        sscript.write(
            "\nsrun bash {executable}\n".format(executable=executable))
        sscript.close()
        chmod(sname, 0o755)
        chdir(dirname(sname))
        cmd = "{sub} ./{fn}".format(sub=self.executor, fn=basename(sname))
        return self.__submit__(cmd, verbose=verbose, dry=dry)
Beispiel #39
0
class JoinPairedEndsTests(TestCase):
    """Tests for join_paired_ends."""
    def setUp(self):
        """set up files to be used in all tests"""

        # set up temp directory that all temp files will
        # be written to:
        self.temp_dir_path = mkdtemp()

        # store files:
        # joined_pairs
        self.joined_pe = NamedTemporaryFile(prefix='joined_',
                                            suffix='.fastq',
                                            dir=self.temp_dir_path,
                                            delete=False)
        self.jpe_fp = self.joined_pe.name
        self.joined_pe.write(joined_reads)
        self.joined_pe.close()

        # all barcodes
        self.all_barcodes = NamedTemporaryFile(prefix='all_bc_',
                                               suffix='.fastq',
                                               dir=self.temp_dir_path,
                                               delete=False)
        self.all_bc_fp = self.all_barcodes.name
        self.all_barcodes.write(all_barcodes)
        self.all_barcodes.close()

        # out of order barcodes
        self.ooo_barcodes = NamedTemporaryFile(prefix='ooo_bc_',
                                               suffix='.fastq',
                                               dir=self.temp_dir_path,
                                               delete=False)
        self.ooo_bc_fp = self.ooo_barcodes.name
        self.ooo_barcodes.write(all_barcodes_out_of_order)
        self.ooo_barcodes.close()

        # missing barcodes
        self.missing_barcodes = NamedTemporaryFile(prefix='missing_bc_',
                                                   suffix='.fastq',
                                                   dir=self.temp_dir_path,
                                                   delete=False)
        self.missing_bc_fp = self.missing_barcodes.name
        self.missing_barcodes.write(missing_barcodes)
        self.missing_barcodes.close()

    def tearDown(self):
        """Remove all temp files"""
        shutil.rmtree(self.temp_dir_path)

    def test_write_synced_barcodes_fastq(self):
        """write_synced_barcodes_fastq: should work properly.
           This function expects the barcodes.fastq and joined_pairs.fastq
           files to be in the same order except for those missing data
           that could not be joined. That is, not all paired-ends will
           assemble.
        """
        filtered_bc_path = write_synced_barcodes_fastq(self.jpe_fp,
                                                       self.all_bc_fp)

        observed_barcodes = open(filtered_bc_path, 'U').read()
        self.assertEqual(observed_barcodes, synced_barcodes)

        os.remove(filtered_bc_path)

    def test_out_of_order_barcodes(self):
        """write_synced_barcodes_fastq: should fail if barcodes out of order
           By out of order I mean that the data in the barcodes file
           are not in the same order as the data within the joined paired-ends
           file.
        """
        self.assertRaises(StopIteration, write_synced_barcodes_fastq,
                          self.jpe_fp, self.ooo_bc_fp)

    def test_missing_barcodes(self):
        """"write_synced_barcodes_fastq: should fail if barcodes are missing."""
        self.assertRaises(StopIteration, write_synced_barcodes_fastq,
                          self.jpe_fp, self.missing_bc_fp)
Beispiel #40
0
class GitRepositoryTests(unittest.TestCase):
    def setUp(self):

        self.dir = TemporaryDirectory()
        self.remotedir = TemporaryDirectory()
        self.file = NamedTemporaryFile(dir=self.dir.name, delete=False)
        self.filename = path.basename(self.file.name)
        self.author = Signature('QuitStoreTest', '*****@*****.**')
        self.comitter = Signature('QuitStoreTest', '*****@*****.**')

        # Initialize repository
        init_repository(self.dir.name, False)

    def tearDown(self):
        self.file = None
        self.filename = None
        self.dir.cleanup()
        self.dir = None
        self.remotedir.cleanup()
        self.remotedir = None

    def addfile(self):
        """Create a repository and add a file to the git index."""
        # Write to file
        self.file.write(b'First Line\n')
        self.file.read()

        # Add file to index
        repo = Repository(self.dir.name)
        index = repo.index
        index.read()
        index.add(self.filename)
        index.write()

    def createcommit(self):
        """Prepare a git repository with one existing commit.

        Create a directory, initialize a git Repository, add
        and commit a file.

        Returns:
            A list containing the directory and file
        """
        self.addfile()
        # Create commit
        repo = Repository(self.dir.name)
        index = repo.index
        index.read()
        tree = index.write_tree()
        message = "First commit of temporary test repo"
        repo.create_commit('HEAD', self.author, self.comitter, message, tree,
                           [])

    def testInitNotExistingsRepo(self):
        dir = TemporaryDirectory()

        repo = quit.git.Repository(dir.name, create=True)
        self.assertFalse(repo.is_bare)
        self.assertEqual(len(repo.revisions()), 0)

        dir.cleanup()

    def testInitEmptyRepo(self):
        self.addfile()
        repo = quit.git.Repository(self.dir.name, create=True)
        self.assertFalse(repo.is_bare)
        self.assertEqual(len(repo.revisions()), 0)

    def testInitRepoWithExistingCommit(self):
        self.createcommit()
        repo = quit.git.Repository(self.dir.name)
        self.assertFalse(repo.is_bare)
        self.assertEqual(len(repo.revisions()), 1)

    def testCloneRepo(self):
        REMOTE_NAME = 'origin'
        REMOTE_URL = 'git://github.com/AKSW/QuitStore.example.git'

        dir = TemporaryDirectory()
        repo = quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
        self.assertTrue(path.exists(path.join(dir.name, 'example.nt')))
        self.assertTrue(path.exists(path.join(dir.name, 'example.nt.graph')))
        self.assertFalse(repo.is_bare)
        dir.cleanup()

    @unittest.skip("Currently fails on travis")
    def testCloneRepoViaSSH(self):
        environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/sshkey/"

        REMOTE_URL = '[email protected]:AKSW/QuitStore.example.git'

        dir = TemporaryDirectory()
        repo = quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
        self.assertTrue(path.exists(path.join(dir.name, 'example.nt')))
        self.assertFalse(repo.is_bare)
        dir.cleanup()

    def testCloneRepoViaSSHNoKeyFiles(self):
        environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/nosshkey/"
        if "SSH_AUTH_SOCK" in environ:
            del environ["SSH_AUTH_SOCK"]

        REMOTE_URL = '[email protected]:AKSW/QuitStore.example.git'

        dir = TemporaryDirectory()
        with self.assertRaises(Exception) as context:
            quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
        dir.cleanup()

    def testCloneNotExistingRepo(self):
        environ["QUIT_SSH_KEY_HOME"] = "./tests/assets/sshkey/"

        REMOTE_URL = '[email protected]:AKSW/ThereIsNoQuitStoreRepo.git'

        dir = TemporaryDirectory()
        with self.assertRaises(Exception) as context:
            quit.git.Repository(dir.name, create=True, origin=REMOTE_URL)
        dir.cleanup()

    def testPushRepo(self):
        """Test if it is possible to push to an empty remote repository."""
        with TemporaryRepository(True) as remote:
            graphContent = """
                <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
            with TemporaryRepositoryFactory().withGraph(
                    "http://example.org/", graphContent) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertTrue(remote.is_empty)
                self.assertFalse(local.is_empty)

                quitRepo.push("origin", "master")

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)

    def testPushRefspecs(self):
        """Test if it is possible to push to an empty remote repository."""
        for refspec in [
                'master', 'refs/heads/master', 'refs/heads/master:master',
                'master:master', 'master:refs/heads/master',
                'refs/heads/master:refs/heads/master'
        ]:
            with TemporaryRepository(True) as remote:
                graphContent = """
                    <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
                with TemporaryRepositoryFactory().withGraph(
                        "http://example.org/", graphContent) as local:
                    local.remotes.create("origin", remote.path)
                    quitRepo = quit.git.Repository(local.workdir)

                    self.assertTrue(remote.is_empty)
                    self.assertFalse(local.is_empty)

                    quitRepo.push("origin", refspec)

                    self.assertFalse(remote.is_empty)
                    self.assertFalse(local.is_empty)

    def testPushRepoNotConfiguredRemote(self):
        """Test if the push failes if the origin remote was not defined."""
        with TemporaryRepository(True) as remote:
            graphContent = """
                <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
            with TemporaryRepositoryFactory().withGraph(
                    "http://example.org/", graphContent) as local:
                local.remotes.create("upstream", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertTrue(remote.is_empty)
                self.assertFalse(local.is_empty)

                with self.assertRaises(RemoteNotFound):
                    quitRepo.push("origin", "master")

                self.assertTrue(remote.is_empty)
                self.assertFalse(local.is_empty)

    def testPushRepoWithRemoteName(self):
        """Test if it is possible to push to a remote repository, which is not called orign."""
        with TemporaryRepository(True) as remote:
            graphContent = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."
            with TemporaryRepositoryFactory().withGraph(
                    "http://example.org/", graphContent) as local:
                local.remotes.create("upstream", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertTrue(remote.is_empty)
                self.assertFalse(local.is_empty)

                quitRepo.push("upstream", "master")

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)

    def testPushRepoNotConfiguredNamedRemote(self):
        """Test if the push failes if the specified remote was not defined."""
        with TemporaryRepository(is_bare=True) as remote:
            graphContent = """
                <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
            with TemporaryRepositoryFactory().withGraph(
                    "http://example.org/", graphContent) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertTrue(remote.is_empty)
                self.assertFalse(local.is_empty)

                with self.assertRaises(RemoteNotFound):
                    quitRepo.push("upstream", "master")

                self.assertTrue(remote.is_empty)
                self.assertFalse(local.is_empty)

    def testPushRepoWithDivergedRemote(self):
        """Test for an exception, if the local and remote repositories are diverged."""
        with TemporaryRepositoryFactory().withEmptyGraph(
                "http://example.org/") as remote:
            graphContent = """
                <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
            with TemporaryRepositoryFactory().withGraph(
                    "http://example.org/", graphContent) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)

                with self.assertRaises(pygit2.GitError):
                    quitRepo.push("origin", "master")

    @unittest.skip("requires a remote with pre-receive hook")
    def testPushRepoWithRemoteReject(self):
        """Test for an exception, if the remote repositories rejects a push.

        CAUTION: This test is disabled, because it requires a remote with pre-receive hook.
        Unfortunately the libgit2 does not execute pre-receive hooks on local repositories.
        """
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as local:
            local.remotes.create("origin", "ssh://[email protected]/testing.git")
            quitRepo = quit.git.Repository(local.workdir)

            self.assertFalse(local.is_empty)

            with self.assertRaises(QuitGitPushError):
                quitRepo.push()

    def testFetchRepo(self):
        """Test if it is possible to fetch from a remote repository."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryRepository(False) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertTrue(local.is_empty)
                self.assertTrue(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                with self.assertRaises(RevisionNotFound):
                    quitRepo.revision('HEAD')

                quitRepo.fetch()

                self.assertEqual(
                    quitRepo.revision('origin/master').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)
                self.assertFalse(quitRepo.is_empty)

    def testFetchUpstreamRepo(self):
        """Test if it is possible to from from a remote, which set as upstream."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/x> <http://ex.org/x> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryRepository(clone_from_repo=remote) as local:
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)
                self.assertFalse(quitRepo.is_empty)

                with open(path.join(remote.workdir, "graph.nt"),
                          "a") as graphFile:
                    graphContent = """
                        <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
                    graphFile.write(graphContent)

                createCommit(repository=remote)

                remoteHead = remote.revparse_single('HEAD').hex
                localHead = local.revparse_single('HEAD').hex

                self.assertNotEqual(localHead, remoteHead)

                quitRepo.fetch()

                self.assertEqual(
                    quitRepo.revision('origin/master').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)
                self.assertFalse(quitRepo.is_empty)

    def testPullRepo(self):
        """Test if it is possible to pull from a remote repository."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryRepository(False) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertTrue(local.is_empty)
                self.assertTrue(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                with self.assertRaises(RevisionNotFound):
                    quitRepo.revision('HEAD')

                quitRepo.pull("origin", "master")

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)
                self.assertFalse(quitRepo.is_empty)

    def testPullRepoWithUnbornHead(self):
        """Test if it is possible to pull from a remote repository."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryRepository(False) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertTrue(local.is_empty)
                self.assertTrue(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                with self.assertRaises(RevisionNotFound):
                    quitRepo.revision('HEAD')

                quitRepo.pull()

                self.assertEqual(
                    quitRepo.revision('origin/master').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)
                self.assertFalse(quitRepo.is_empty)

    def testPullRepoClonedNoChanges(self):
        """Test pull if both repos are at the same state."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryDirectory() as localDirectory:
                quitRepo = quit.git.Repository(localDirectory,
                                               create=True,
                                               origin=remote.path)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                quitRepo.pull("origin", "master")

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

    def testPullRepoClonedAndPullWithChanges(self):
        """Test clone, commit on remote and pull."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryDirectory() as localDirectory:
                quitRepo = quit.git.Repository(localDirectory,
                                               create=True,
                                               origin=remote.path)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                quitRepo.pull()

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteQuitRepo = quit.git.Repository(remote.workdir)
                index = remoteQuitRepo.index(remoteHead)
                graphContent += """
                    <http://ex.org/x> <http://ex.org/z> <http://ex.org/z> ."""
                index.add("graph.nt", graphContent)

                author = Signature('QuitStoreTest', '*****@*****.**')
                commitid = index.commit("from test", author.name, author.email)

                quitRepo.pull()

                self.assertEqual(quitRepo.revision('HEAD').id, str(commitid))

    def testPullRepoClonedAndPullWithThreeWayGitMerge(self):
        """Test clone, commit on remote and pull with merge, which resolves without conflicts."""
        graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n"
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryDirectory() as localDirectory:
                quitRepo = quit.git.Repository(localDirectory,
                                               create=True,
                                               origin=remote.path)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                index = quitRepo.index(remoteHead)
                graph2Content = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n"
                index.add("graph2.nt", graph2Content)
                index.add("graph2.nt.graph", "http://example2.org/")
                author = Signature('QuitStoreTest', '*****@*****.**')
                localCommitid = index.commit("from local", author.name,
                                             author.email)
                quitRepo._repository.checkout_tree(
                    quitRepo._repository.get(localCommitid))

                self.assertEqual(
                    quitRepo.revision('HEAD').id, str(localCommitid))

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteQuitRepo = quit.git.Repository(remote.workdir)
                index = remoteQuitRepo.index(remoteHead)
                graphContent += "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n"
                index.add("graph.nt", graphContent)
                remoteCommitid = index.commit("from remote", author.name,
                                              author.email)
                remoteQuitRepo._repository.checkout_tree(
                    remoteQuitRepo._repository.get(remoteCommitid))

                quitRepo.pull(method="three-way-git")

                self.assertNotEqual(
                    quitRepo.revision('HEAD').id, str(localCommitid))
                self.assertNotEqual(
                    quitRepo.revision('HEAD').id, str(remoteCommitid))

                # check if head has local and remote commit id as ancestor
                self.assertListEqual([
                    parent.id for parent in quitRepo.revision('HEAD').parents
                ], [str(localCommitid),
                    str(remoteCommitid)])

                # check if the merged commit contains all file contents
                with open(path.join(localDirectory, "graph.nt")) as f:
                    self.assertEqual("".join(f.readlines()), graphContent)

                with open(path.join(localDirectory, "graph2.nt")) as f:
                    self.assertEqual("".join(f.readlines()), graph2Content)

    def testPullRepoClonedAndPullWithThreeWayMerge(self):
        """Test clone, commit on remote and pull with merge, which resolves without conflicts."""
        graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n"
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryDirectory() as localDirectory:
                quitRepo = quit.git.Repository(localDirectory,
                                               create=True,
                                               origin=remote.path)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                index = quitRepo.index(remoteHead)
                graph2Content = "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n"
                index.add("graph2.nt", graph2Content)
                index.add("graph2.nt.graph", "http://example2.org/\n")
                author = Signature('QuitStoreTest', '*****@*****.**')
                localCommitid = index.commit("from local", author.name,
                                             author.email)
                quitRepo._repository.checkout_tree(
                    quitRepo._repository.get(localCommitid))

                self.assertEqual(
                    quitRepo.revision('HEAD').id, str(localCommitid))

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteQuitRepo = quit.git.Repository(remote.workdir)
                index = remoteQuitRepo.index(remoteHead)
                graphContent += "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n"
                index.add("graph.nt", graphContent)
                remoteCommitid = index.commit("from remote", author.name,
                                              author.email)
                remoteQuitRepo._repository.checkout_tree(
                    remoteQuitRepo._repository.get(remoteCommitid))

                quitRepo.pull(method="three-way")

                self.assertNotEqual(
                    quitRepo.revision('HEAD').id, str(localCommitid))
                self.assertNotEqual(
                    quitRepo.revision('HEAD').id, str(remoteCommitid))

                # check if head has local and remote commit id as ancestor
                self.assertListEqual([
                    parent.id for parent in quitRepo.revision('HEAD').parents
                ], [str(localCommitid),
                    str(remoteCommitid)])

                # check if the merged commit contains all file contents
                with open(path.join(localDirectory, "graph.nt")) as f:
                    self.assertEqual("".join(f.readlines()), graphContent)

                with open(path.join(localDirectory, "graph2.nt")) as f:
                    self.assertEqual("".join(f.readlines()), graph2Content)

    def testPullRepoClonedAndPullWithConflict(self):
        """Test clone, commit on remote and pull with conflict."""
        graphContent = "<http://ex.org/a> <http://ex.org/b> <http://ex.org/c> .\n"
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryDirectory() as localDirectory:
                quitRepo = quit.git.Repository(localDirectory,
                                               create=True,
                                               origin=remote.path)

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                index = quitRepo._repository.index
                with open(path.join(localDirectory, "graph.nt"),
                          "a") as graphFile:
                    graphFile.write(
                        "<http://ex.org/x> <http://ex.org/y> <http://ex.org/y> .\n"
                    )
                index.add("graph.nt")
                index.write()
                tree = index.write_tree()

                author = Signature('QuitStoreTest', '*****@*****.**')
                localCommitid = quitRepo._repository.create_commit(
                    'HEAD', author, author, "from local", tree, [remoteHead])

                self.assertFalse(remote.is_empty)
                self.assertFalse(quitRepo.is_empty)

                index = remote.index
                with open(path.join(remote.workdir, "graph.nt"),
                          "a") as graphFile:
                    graphFile.write(
                        "<http://ex.org/x> <http://ex.org/z> <http://ex.org/z> .\n"
                    )
                index.add("graph.nt")
                index.write()
                tree = index.write_tree()
                remoteCommitid = remote.create_commit('HEAD', author, author,
                                                      "from remote", tree,
                                                      [remoteHead])

                remoteQuitRepo = quit.git.Repository(remote.workdir)

                with self.assertRaises(QuitMergeConflict):
                    quitRepo.pull(method="context")

                self.assertEqual(
                    quitRepo.revision('HEAD').id, str(localCommitid))
                self.assertEqual(
                    remoteQuitRepo.revision('HEAD').id, str(remoteCommitid))

    def testPullRepoFromNamedRemote(self):
        """Test if it is possible to pull from a remote repository, which is not called origin."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryRepository(False) as local:
                local.remotes.create("upstream", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertTrue(local.is_empty)
                self.assertTrue(quitRepo.is_empty)

                remoteHead = remote.revparse_single('HEAD').hex

                with self.assertRaises(RevisionNotFound):
                    quitRepo.revision('HEAD')

                quitRepo.pull(remote_name='upstream', refspec="master")

                self.assertEqual(quitRepo.revision('HEAD').id, remoteHead)

                self.assertFalse(remote.is_empty)
                self.assertFalse(local.is_empty)
                self.assertFalse(quitRepo.is_empty)

    def testPullRepoFromNotConfiguredRemote(self):
        """Test if it is possible to pull from a remote repository, which is not called origin."""
        graphContent = """
            <http://ex.org/x> <http://ex.org/y> <http://ex.org/z> ."""
        with TemporaryRepositoryFactory().withGraph("http://example.org/",
                                                    graphContent) as remote:
            with TemporaryRepository(False) as local:
                local.remotes.create("origin", remote.path)
                quitRepo = quit.git.Repository(local.workdir)

                self.assertFalse(remote.is_empty)
                self.assertTrue(local.is_empty)
                self.assertTrue(quitRepo.is_empty)

                with self.assertRaises(RemoteNotFound):
                    quitRepo.pull(remote_name='upstream')

                self.assertFalse(remote.is_empty)
                self.assertTrue(local.is_empty)
                self.assertTrue(quitRepo.is_empty)

    def testPullRepoNoFFW(self):
        """TODO"""
        pass

    def testPullRepoFromMasterToDevelop(self):
        """TODO"""
        pass

    def testPullRepoFromDevelopToMaster(self):
        """TODO"""
        pass

    def testPullRepoFromRemoteTrackingBranch(self):
        """TODO"""
        pass

    def testRepositoryIsEmpty(self):
        """Test that adding data causes a new commit."""
        self.addfile()
        repo = quit.git.Repository(self.dir.name)
        self.assertTrue(repo.is_empty)

        self.createcommit()
        repo = quit.git.Repository(self.dir.name)
        self.assertFalse(repo.is_empty)

    def testRepositoryIsBare(self):
        """Test if is_bare is currently done in init/clone tests."""
        pass

    def testNoGCConfiguration(self):
        """Test Garbage Collection configuration."""
        quit.git.Repository(self.dir.name, garbageCollection=False)

        with subprocess.Popen(["git", "config", "gc.auto"],
                              stdout=subprocess.PIPE,
                              cwd=self.dir.name) as getGCAuto:
            stdout, stderr = getGCAuto.communicate()
            response = stdout.decode("UTF-8").strip()

        self.assertEqual(response, '')

    def testGCConfiguration(self):
        """Test Garbage Collection configuration."""
        quit.git.Repository(self.dir.name, garbageCollection=True)

        with subprocess.Popen(["git", "config", "gc.auto"],
                              stdout=subprocess.PIPE,
                              cwd=self.dir.name) as getGCAuto:
            stdout, stderr = getGCAuto.communicate()
            response = stdout.decode("UTF-8").strip()

        self.assertNotEqual(response, '')
        self.assertEqual(response, '256')
Beispiel #41
0
def main():
    from argparse import ArgumentParser
    from os import path
    import numpy as np
    parser = ArgumentParser()
    parser.add_argument('prototxt')
    parser.add_argument('output_caffemodel')
    parser.add_argument(
        '-l',
        '--load',
        help=
        'Load a pretrained model and rescale it [bias and type are not supported]'
    )
    parser.add_argument('-d',
                        '--data',
                        default=None,
                        help='Image list to use [default prototxt data]')
    parser.add_argument('-b', '--bias', type=float, default=0.1, help='Bias')
    parser.add_argument(
        '-t',
        '--type',
        default='elwise',
        help=
        'Type: elwise, pca, zca, kmeans, rand (random input patches). Add fast_ to speed up the initialization, but you might lose in precision.'
    )
    parser.add_argument('-z',
                        action='store_true',
                        help='Zero all weights and reinitialize')
    parser.add_argument('-cs', action='store_true', help='Correct for scaling')
    parser.add_argument('-q', action='store_true', help='Quiet execution')
    parser.add_argument('-s',
                        type=float,
                        default=1.0,
                        help='Scale the input [only custom data "-d"]')
    parser.add_argument('-bs',
                        type=int,
                        default=16,
                        help='Batch size [only custom data "-d"]')
    parser.add_argument('-nit',
                        type=int,
                        default=10,
                        help='Number of iterations')
    parser.add_argument(
        '--mem-limit',
        type=int,
        default=500,
        help='How much memory should we use for the data buffer (MB)?')
    parser.add_argument('--gpu',
                        type=int,
                        default=0,
                        help='What gpu to run it on?')
    args = parser.parse_args()

    if args.q:
        from os import environ
        environ['GLOG_minloglevel'] = '2'
    import caffe, load
    from caffe import NetSpec, layers as L

    caffe.set_mode_gpu()
    if args.gpu is not None:
        caffe.set_device(args.gpu)

    model = load.ProtoDesc(args.prototxt)
    net = NetSpec()
    if args.data is not None:
        fl = getFileList(args.data)
        if len(fl) == 0:
            print("Unknown data type for '%s'" % args.data)
            exit(1)
        from tempfile import NamedTemporaryFile
        f = NamedTemporaryFile('w')
        f.write('\n'.join([path.abspath(i) + ' 0' for i in fl]))
        f.flush()
        net.data, net.label = L.ImageData(source=f.name,
                                          batch_size=args.bs,
                                          new_width=model.input_dim[-1],
                                          new_height=model.input_dim[-1],
                                          transform_param=dict(
                                              mean_value=[104, 117, 123],
                                              scale=args.s),
                                          ntop=2)
        net.out = model(data=net.data, label=net.label)
    else:
        net.out = model()

    n = netFromString('force_backward:true\n' + str(net.to_proto()),
                      caffe.TRAIN)

    if args.load is not None:
        n.copy_from(args.load)
        # Rescale existing layers?
        #if args.fix:
        #magicFix(n, args.nit)

    if args.z:
        # Zero out all layers
        for l in n.layers:
            for b in l.blobs:
                b.data[...] = 0
    if any([
            np.abs(l.blobs[0].data).sum() < 1e-10 for l in n.layers
            if len(l.blobs) > 0
    ]):
        print([
            m for l, m in zip(n.layers, n._layer_names)
            if len(l.blobs) > 0 and np.abs(l.blobs[0].data).sum() < 1e-10
        ])
        magicInitialize(n,
                        args.bias,
                        NIT=args.nit,
                        type=args.type,
                        max_data=args.mem_limit * 1024 * 1024 / 4)
    else:
        print("Network already initialized, skipping magic init")
    if args.cs:
        # A simply helper function that lets you figure out which layers are not
        # homogeneous
        #print( estimateHomogenety(n) )
        calibrateGradientRatio(n)
    n.save(args.output_caffemodel)
def run_playbook(hosts, playbook, tags=[], private_key_file=private_key_file):
    # initialize needed objects
    loader = DataLoader()
    options = Options(connection='ssh',
                      private_key_file=private_key_file,
                      module_path='',
                      forks=100,
                      become=True,
                      become_method='sudo',
                      become_user='******',
                      check=False,
                      tags=tags)

    passwords = dict(vault_pass='')
    results_callback = ResultCallback()

    host_file = NamedTemporaryFile(delete=False)
    host_file.write(b'[servers]\n')
    for i, h in enumerate(hosts):
        print(i, " : ", h)
        host_file.write(bytes('{0} num={1}\n'.format(h, i), encoding='utf8'))
    host_file.close()

    # set inventory
    inventory = InventoryManager(loader=loader, sources=host_file.name)
    variable_manager = VariableManager(loader=loader, inventory=inventory)

    # setup playbook executor, before the run
    pbex = PlaybookExecutor(playbooks=[playbook],
                            inventory=inventory,
                            variable_manager=variable_manager,
                            loader=loader,
                            options=options,
                            passwords=passwords)

    pbex._tqm._stdout_callback = results_callback
    # run playbook and get stats
    result = pbex.run()
    stats = pbex._tqm._stats

    outputs = {
        0: 'deployment successful',
        1: 'error occurred during deployment',
        2: 'one or more hosts failed',
        4: 'one or more hosts unreachable',
        255: 'unknown error occurred during deployment'
    }

    run_success = True
    hosts = sorted(stats.processed.keys())
    for h in hosts:
        t = stats.summarize(h)
        if t['unreachable'] > 0 or t['failures'] > 0:
            run_success = False

    os.remove(host_file.name)

    try:
        out = outputs[result]
    except KeyError:
        out = 'unrecognised error code'
    return result, out
Beispiel #43
0
    def from_file_using_temporary_files(cls,
                                        file,
                                        format=None,
                                        codec=None,
                                        parameters=None,
                                        start_second=None,
                                        duration=None,
                                        **kwargs):
        orig_file = file
        file, close_file = _fd_or_path_or_tempfile(file, 'rb', tempfile=False)

        if format:
            format = format.lower()
            format = AUDIO_FILE_EXT_ALIASES.get(format, format)

        def is_format(f):
            f = f.lower()
            if format == f:
                return True
            if isinstance(orig_file, basestring):
                return orig_file.lower().endswith(".{0}".format(f))
            if isinstance(orig_file, bytes):
                return orig_file.lower().endswith(
                    (".{0}".format(f)).encode('utf8'))
            return False

        if is_format("wav"):
            try:
                obj = cls._from_safe_wav(file)
                if close_file:
                    file.close()
                if start_second is None and duration is None:
                    return obj
                elif start_second is not None and duration is None:
                    return obj[start_second * 1000:]
                elif start_second is None and duration is not None:
                    return obj[:duration * 1000]
                else:
                    return obj[start_second * 1000:(start_second + duration) *
                               1000]
            except:
                file.seek(0)
        elif is_format("raw") or is_format("pcm"):
            sample_width = kwargs['sample_width']
            frame_rate = kwargs['frame_rate']
            channels = kwargs['channels']
            metadata = {
                'sample_width': sample_width,
                'frame_rate': frame_rate,
                'channels': channels,
                'frame_width': channels * sample_width
            }
            obj = cls(data=file.read(), metadata=metadata)
            if close_file:
                file.close()
            if start_second is None and duration is None:
                return obj
            elif start_second is not None and duration is None:
                return obj[start_second * 1000:]
            elif start_second is None and duration is not None:
                return obj[:duration * 1000]
            else:
                return obj[start_second * 1000:(start_second + duration) *
                           1000]

        input_file = NamedTemporaryFile(mode='wb', delete=False)
        try:
            input_file.write(file.read())
        except (OSError):
            input_file.flush()
            input_file.close()
            input_file = NamedTemporaryFile(mode='wb',
                                            delete=False,
                                            buffering=2**31 - 1)
            if close_file:
                file.close()
            close_file = True
            file = open(orig_file, buffering=2**13 - 1, mode='rb')
            reader = file.read(2**31 - 1)
            while reader:
                input_file.write(reader)
                reader = file.read(2**31 - 1)
        input_file.flush()
        if close_file:
            file.close()

        output = NamedTemporaryFile(mode="rb", delete=False)

        conversion_command = [
            cls.converter,
            '-y',  # always overwrite existing files
        ]

        # If format is not defined
        # ffmpeg/avconv will detect it automatically
        if format:
            conversion_command += ["-f", format]

        if codec:
            # force audio decoder
            conversion_command += ["-acodec", codec]

        conversion_command += [
            "-i",
            input_file.name,  # input_file options (filename last)
            "-vn",  # Drop any video streams if there are any
            "-f",
            "wav"  # output options (filename last)
        ]

        if start_second is not None:
            conversion_command += ["-ss", str(start_second)]

        if duration is not None:
            conversion_command += ["-t", str(duration)]

        conversion_command += [output.name]

        if parameters is not None:
            # extend arguments with arbitrary set
            conversion_command.extend(parameters)

        log_conversion(conversion_command)

        with open(os.devnull, 'rb') as devnull:
            p = subprocess.Popen(conversion_command,
                                 stdin=devnull,
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE)
        p_out, p_err = p.communicate()

        log_subprocess_output(p_out)
        log_subprocess_output(p_err)

        try:
            if p.returncode != 0:
                raise CouldntDecodeError(
                    "Decoding failed. ffmpeg returned error code: {0}\n\nOutput from ffmpeg/avlib:\n\n{1}"
                    .format(p.returncode, p_err.decode(errors='ignore')))
            obj = cls._from_safe_wav(output)
        finally:
            input_file.close()
            output.close()
            os.unlink(input_file.name)
            os.unlink(output.name)

        if start_second is None and duration is None:
            return obj
        elif start_second is not None and duration is None:
            return obj[0:]
        elif start_second is None and duration is not None:
            return obj[:duration * 1000]
        else:
            return obj[0:duration * 1000]
Beispiel #44
0
class UcareCommonConfigFileTest(unittest.TestCase):

    def setUp(self):
        self.tmp_config_file = NamedTemporaryFile(mode='w+t', delete=False)

    def tearDown(self):
        conf.pub_key = None
        conf.secret = None

        conf.api_base = 'https://api.uploadcare.com/'
        conf.upload_base = 'https://upload.uploadcare.com/'

        conf.verify_api_ssl = True
        conf.verify_upload_ssl = True

    def test_use_pub_key_from_config_file(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'pub_key = demopublickey'
        )
        self.tmp_config_file.close()

        main(arg_namespace('list_files'), [self.tmp_config_file.name])

        self.assertEqual(conf.pub_key, 'demopublickey')

    def test_redefine_pub_key_by_second_config_file(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'pub_key = demopublickey'
        )
        self.tmp_config_file.close()

        second_tmp_conf_file = NamedTemporaryFile(mode='w+t', delete=False)
        second_tmp_conf_file.write(
            '[ucare]\n'
            'pub_key = demopublickey_modified'
        )
        second_tmp_conf_file.close()

        main(arg_namespace('list_files'),
             [self.tmp_config_file.name, second_tmp_conf_file.name])

        self.assertEqual(conf.pub_key, 'demopublickey_modified')

    def test_use_available_pub_key_from_config_files(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'pub_key = demopublickey'
        )
        self.tmp_config_file.close()

        second_tmp_conf_file = NamedTemporaryFile(mode='w+t', delete=False)
        second_tmp_conf_file.write(
            '[ucare]\n'
            'secret = demosecretkey'
        )
        second_tmp_conf_file.close()

        main(arg_namespace('list_files'),
             [self.tmp_config_file.name, second_tmp_conf_file.name])

        self.assertEqual(conf.pub_key, 'demopublickey')

    def test_redefine_config_pub_key_by_args(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'pub_key = demopublickey'
        )
        self.tmp_config_file.close()

        main(arg_namespace('--pub_key pub list_files'),
             [self.tmp_config_file.name])

        self.assertEqual(conf.pub_key, 'pub')

    def test_load_verify_api_ssl_false_value_from_config(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'verify_api_ssl = false'
        )
        self.tmp_config_file.close()

        main(arg_namespace('list_files'), [self.tmp_config_file.name])

        self.assertFalse(conf.verify_api_ssl)

    def test_load_verify_api_ssl_true_value_from_config(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'verify_api_ssl = true'
        )
        self.tmp_config_file.close()

        main(arg_namespace('list_files'), [self.tmp_config_file.name])

        self.assertTrue(conf.verify_api_ssl)

    def test_redefine_config_verify_api_ssl_by_args(self, request):
        request.return_value = MockListResponse()

        self.tmp_config_file.write(
            '[ucare]\n'
            'verify_api_ssl = true'
        )
        self.tmp_config_file.close()

        main(arg_namespace('--no_check_api_certificate list_files'),
             [self.tmp_config_file.name])

        self.assertFalse(conf.verify_api_ssl)
Beispiel #45
0
def run_fio(remote, config, rbd_test_dir):
    """
    create fio config file with options based on above config
    get the fio from github, generate binary, and use it to run on
    the generated fio config file
    """
    fio_config=NamedTemporaryFile(prefix='fio_rbd_', dir='/tmp/', delete=False)
    fio_config.write('[global]\n')
    if config.get('io-engine'):
        ioengine=config['io-engine']
        fio_config.write('ioengine={ioe}\n'.format(ioe=ioengine))
    else:
        fio_config.write('ioengine=sync\n')
    if config.get('bs'):
        bs=config['bs']
        fio_config.write('bs={bs}\n'.format(bs=bs))
    else:
        fio_config.write('bs=4k\n')
    iodepth = config.get('io-depth', 2)
    fio_config.write('iodepth={iod}\n'.format(iod=iodepth))
    if config.get('fio-io-size'):
        size=config['fio-io-size']
        fio_config.write('size={size}\n'.format(size=size))
    else:
        fio_config.write('size=100m\n')

    fio_config.write('time_based\n')
    if config.get('runtime'):
        runtime=config['runtime']
        fio_config.write('runtime={runtime}\n'.format(runtime=runtime))
    else:
        fio_config.write('runtime=1800\n')
    fio_config.write('allow_file_create=0\n')
    image_size=10240
    if config.get('image_size'):
        image_size=config['image_size']

    formats=[1,2]
    features=[['layering'],['striping'],['exclusive-lock','object-map']]
    fio_version='2.7'
    if config.get('formats'):
        formats=config['formats']
    if config.get('features'):
        features=config['features']
    if config.get('fio-version'):
        fio_version=config['fio-version']

    # handle package required for ioengine, if any
    sn=remote.shortname
    ioengine_pkg = get_ioengine_package_name(ioengine, remote)
    if ioengine_pkg:
        install_package(ioengine_pkg, remote)

    fio_config.write('norandommap\n')
    if ioengine == 'rbd':
        fio_config.write('clientname=admin\n')
        fio_config.write('pool=rbd\n')
        fio_config.write('invalidate=0\n')
    elif ioengine == 'libaio':
        fio_config.write('direct=1\n')
    for frmt in formats:
        for feature in features:
           log.info("Creating rbd images on {sn}".format(sn=sn))
           feature_name = '-'.join(feature)
           rbd_name = 'i{i}f{f}{sn}'.format(i=frmt,f=feature_name,sn=sn)
           rbd_snap_name = 'i{i}f{f}{sn}@i{i}f{f}{sn}Snap'.format(i=frmt,f=feature_name,sn=sn)
           rbd_clone_name = 'i{i}f{f}{sn}Clone'.format(i=frmt,f=feature_name,sn=sn)
           create_args=['rbd', 'create',
                        '--size', '{size}'.format(size=image_size),
                        '--image', rbd_name,
                        '--image-format', '{f}'.format(f=frmt)]
           map(lambda x: create_args.extend(['--image-feature', x]), feature)
           remote.run(args=create_args)
           remote.run(args=['rbd', 'info', rbd_name])
           if ioengine != 'rbd':
               rbd_dev = run_rbd_map(remote, rbd_name, iodepth)
               if config.get('test-clone-io'):
                    log.info("Testing clones using fio")
                    remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
                    remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
                    remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
                    rbd_clone_dev = run_rbd_map(remote, rbd_clone_name, iodepth)
               fio_config.write('[{rbd_dev}]\n'.format(rbd_dev=rbd_dev))
               if config.get('rw'):
                   rw=config['rw']
                   fio_config.write('rw={rw}\n'.format(rw=rw))
               else:
                   fio_config .write('rw=randrw\n')
               fio_config.write('filename={rbd_dev}\n'.format(rbd_dev=rbd_dev))
               if config.get('test-clone-io'):
                   fio_config.write('[{rbd_clone_dev}]\n'.format(rbd_clone_dev=rbd_clone_dev))
                   fio_config.write('rw={rw}\n'.format(rw=rw))
                   fio_config.write('filename={rbd_clone_dev}\n'.format(rbd_clone_dev=rbd_clone_dev))
           else:
               if config.get('test-clone-io'):
                    log.info("Testing clones using fio")
                    remote.run(args=['rbd', 'snap', 'create', rbd_snap_name])
                    remote.run(args=['rbd', 'snap', 'protect', rbd_snap_name])
                    remote.run(args=['rbd', 'clone', rbd_snap_name, rbd_clone_name])
               fio_config.write('[{img_name}]\n'.format(img_name=rbd_name))
               if config.get('rw'):
                   rw=config['rw']
                   fio_config.write('rw={rw}\n'.format(rw=rw))
               else:
                   fio_config.write('rw=randrw\n')
               fio_config.write('rbdname={img_name}\n'.format(img_name=rbd_name))
               if config.get('test-clone-io'):
                   fio_config.write('[{clone_img_name}]\n'.format(clone_img_name=rbd_clone_name))
                   fio_config.write('rw={rw}\n'.format(rw=rw))
                   fio_config.write('rbdname={clone_img_name}\n'.format(clone_img_name=rbd_clone_name))


    fio_config.close()
    remote.put_file(fio_config.name,fio_config.name)
    try:
        log.info("Running rbd feature - fio test on {sn}".format(sn=sn))
        fio = "https://github.com/axboe/fio/archive/fio-" + fio_version + ".tar.gz"
        remote.run(args=['mkdir', run.Raw(rbd_test_dir),])
        remote.run(args=['cd' , run.Raw(rbd_test_dir),
                         run.Raw(';'), 'wget' , fio , run.Raw(';'), run.Raw('tar -xvf fio*tar.gz'), run.Raw(';'),
                         run.Raw('cd fio-fio*'), 'configure', run.Raw(';') ,'make'])
        remote.run(args=['ceph', '-s'])
        remote.run(args=[run.Raw('{tdir}/fio-fio-{v}/fio --showcmd {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
        remote.run(args=['sudo', run.Raw('{tdir}/fio-fio-{v}/fio {f}'.format(tdir=rbd_test_dir,v=fio_version,f=fio_config.name))])
        remote.run(args=['ceph', '-s'])
    finally:
        out=StringIO.StringIO()
        remote.run(args=['rbd','showmapped', '--format=json'], stdout=out)
        mapped_images = json.loads(out.getvalue())
        if mapped_images:
            log.info("Unmapping rbd images on {sn}".format(sn=sn))
            for image in mapped_images.itervalues():
                remote.run(args=['sudo', 'rbd', 'unmap', str(image['device'])])
        log.info("Cleaning up fio install")
        remote.run(args=['rm','-rf', run.Raw(rbd_test_dir)])
        if ioengine_pkg:
            remove_package(ioengine_pkg, remote)
Beispiel #46
0
def gen_keys():
    # good account key
    account_key = NamedTemporaryFile()
    Popen(["openssl", "genrsa", "-out", account_key.name, "2048"]).wait()

    # weak 1024 bit key
    weak_key = NamedTemporaryFile()
    Popen(["openssl", "genrsa", "-out", weak_key.name, "1024"]).wait()

    # good domain key
    domain_key = NamedTemporaryFile()
    domain_csr = NamedTemporaryFile()
    Popen([
        "openssl", "req", "-newkey", "rsa:2048", "-nodes", "-keyout",
        domain_key.name, "-subj", "/CN={0}".format(DOMAIN), "-out",
        domain_csr.name
    ]).wait()

    # subject alt-name domain
    san_csr = NamedTemporaryFile()
    san_conf = NamedTemporaryFile()
    san_conf.write(open("/etc/ssl/openssl.cnf").read().encode("utf8"))
    san_conf.write(
        "\n[SAN]\nsubjectAltName=DNS:{0}\n".format(DOMAIN).encode("utf8"))
    san_conf.seek(0)
    Popen([
        "openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj",
        "/", "-reqexts", "SAN", "-config", san_conf.name, "-out", san_csr.name
    ]).wait()

    # invalid domain csr
    invalid_csr = NamedTemporaryFile()
    Popen([
        "openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj",
        "/CN=\xC3\xA0\xC2\xB2\xC2\xA0_\xC3\xA0\xC2\xB2\xC2\xA0.com", "-out",
        invalid_csr.name
    ]).wait()

    # nonexistent domain csr
    nonexistent_csr = NamedTemporaryFile()
    Popen([
        "openssl", "req", "-new", "-sha256", "-key", domain_key.name, "-subj",
        "/CN=404.{0}".format(DOMAIN), "-out", nonexistent_csr.name
    ]).wait()

    #account-signed domain csr
    account_csr = NamedTemporaryFile()
    Popen([
        "openssl", "req", "-new", "-sha256", "-key", account_key.name, "-subj",
        "/CN={0}".format(DOMAIN), "-out", account_csr.name
    ]).wait()

    return {
        "account_key": account_key,
        "weak_key": weak_key,
        "domain_key": domain_key,
        "domain_csr": domain_csr,
        "san_csr": san_csr,
        "invalid_csr": invalid_csr,
        "nonexistent_csr": nonexistent_csr,
        "account_csr": account_csr,
    }
Beispiel #47
0
def write(data, path, html=None):

    with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip:

        if html is not None:
            zip.writestr('index.html', html)

        content = io.StringIO()
        content.write('Manifest-Version: 1.0\n')
        content.write('Created-By: jamovi\n')
        content.write('Data-Archive-Version: 1.0.2\n')
        content.write('jamovi-Archive-Version: 1.0\n')
        zip.writestr('META-INF/MANIFEST.MF',
                     bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED)

        content = None

        fields = []
        for column in data.dataset:
            field = {}
            field['name'] = column.name
            field['columnType'] = ColumnType.stringify(column.column_type)
            field['measureType'] = MeasureType.stringify(column.measure_type)
            field['formula'] = column.formula
            field['formulaMessage'] = column.formula_message
            if column.measure_type == MeasureType.CONTINUOUS:
                field['type'] = 'number'
            else:
                field['type'] = 'integer'
            field['importName'] = column.import_name
            fields.append(field)

        metadata = {}

        metadataset = {}
        metadataset['rowCount'] = data.dataset.row_count
        metadataset['columnCount'] = data.dataset.column_count
        metadataset['fields'] = fields

        metadata['dataSet'] = metadataset

        zip.writestr('metadata.json', json.dumps(metadata),
                     zipfile.ZIP_DEFLATED)

        metadata = None

        xdata = {}
        for column in data.dataset:
            if column.has_levels:
                xdata[column.name] = {'labels': column.levels}
        zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED)
        xdata = None

        row_count = data.dataset.row_count
        required_bytes = 0
        for column in data.dataset:
            if column.measure_type == MeasureType.CONTINUOUS:
                required_bytes += (8 * row_count)
            else:
                required_bytes += (4 * row_count)

        temp_file = NamedTemporaryFile(delete=False)
        temp_file.truncate(required_bytes)

        for column in data.dataset:
            if column.measure_type == MeasureType.CONTINUOUS:
                for i in range(0, row_count):
                    value = column.raw(i)
                    byts = struct.pack('<d', value)
                    temp_file.write(byts)
            else:
                for i in range(0, row_count):
                    value = column.raw(i)
                    byts = struct.pack('<i', value)
                    temp_file.write(byts)

        temp_file.close()

        zip.write(temp_file.name, 'data.bin')
        os.remove(temp_file.name)

        resources = []

        for analysis in data.analyses:
            if analysis.has_results is False:
                continue
            analysis_dir = '{:02} {}/analysis'.format(analysis.id,
                                                      analysis.name)
            zip.writestr(analysis_dir, analysis.serialize(),
                         zipfile.ZIP_DEFLATED)
            resources += analysis.resources

        for rel_path in resources:
            abs_path = os.path.join(data.instance_path, rel_path)
            zip.write(abs_path, rel_path)
Beispiel #48
0
    CN_FILE = abspath(expanduser(CONFIG_DICT['contact_network_file']))
    CONFIG_DICT['contact_network_file'] = '/FAVITES_MOUNT/%s' % CN_FILE.split('/')[-1]
TN_FILE = None
if 'transmission_network_file' in CONFIG_DICT:
    TN_FILE = abspath(expanduser(CONFIG_DICT['transmission_network_file']))
    CONFIG_DICT['transmission_network_file'] = '/FAVITES_MOUNT/%s' % TN_FILE.split('/')[-1]
TREE_FILE = None
if 'tree_file' in CONFIG_DICT:
    TREE_FILE = abspath(expanduser(CONFIG_DICT['tree_file']))
    CONFIG_DICT['tree_file'] = '/FAVITES_MOUNT/%s' % TREE_FILE.split('/')[-1]
ERRORFREE_SEQ_FILE = None
if 'errorfree_sequence_file' in CONFIG_DICT:
    ERRORFREE_SEQ_FILE = abspath(expanduser(CONFIG_DICT['errorfree_sequence_file']))
    CONFIG_DICT['errorfree_sequence_file'] = '/FAVITES_MOUNT/%s' % ERRORFREE_SEQ_FILE.split('/')[-1]
TMP_CONFIG = NamedTemporaryFile('w')
TMP_CONFIG.write(str(CONFIG_DICT))
TMP_CONFIG.flush()

# pull the newest versioned Docker image (if applicable)
if args.update is None:
    version = None
    try:
        o = check_output(['docker','images']).decode().splitlines()
        for l in o:
            if l.startswith(DOCKER_IMAGE):
                version = '%s:%s' % (DOCKER_IMAGE,l.split()[1]); break
    except CalledProcessError as e:
        raise RuntimeError("docker images command failed\n%s"%e.output)
    if version is None:
        args.update = []
if args.update is not None:
Beispiel #49
0
class Git:
    def __init__(self, git_repo, repo_dir, pkey=None):
        self.git_repo = git_repo
        self.repo_dir = repo_dir
        self.repo = None
        self.pkey = pkey
        self.fd = None
        self.env = {}

    def archive(self, filepath, commit):
        with open(filepath, 'wb') as f:
            self.repo.archive(f, commit)

    def fetch_branches_tags(self):
        self.fetch()
        branches, tags = {}, {}
        for ref in self.repo.references:
            if isinstance(ref, RemoteReference):
                if ref.remote_head != 'HEAD':
                    branches[ref.remote_head] = self._get_commits(
                        f'origin/{ref.remote_head}')
            elif isinstance(ref, TagReference):
                tags[ref.name] = {
                    'id': ref.tag.hexsha,
                    'author': ref.tag.tagger.name,
                    'date': self._format_date(ref.tag.tagged_date),
                    'message': ref.tag.message.strip()
                } if ref.tag else {
                    'id': ref.commit.binsha.hex(),
                    'author': ref.commit.author.name,
                    'date': self._format_date(ref.commit.authored_date),
                    'message': ref.commit.message.strip()
                }
        tags = sorted(tags.items(), key=lambda x: x[1]['date'], reverse=True)
        return branches, dict(tags)

    def fetch(self):
        try:
            self.repo.remotes.origin.fetch(f=True, p=True, P=True)
        except GitCommandError as e:
            if self.env:
                self.repo.remotes.origin.fetch(env=self.env,
                                               f=True,
                                               p=True,
                                               P=True)
            else:
                raise e

    def _get_repo(self):
        if os.path.exists(self.repo_dir):
            try:
                return Repo(self.repo_dir)
            except InvalidGitRepositoryError:
                if os.path.isdir(self.repo_dir):
                    shutil.rmtree(self.repo_dir)
                else:
                    os.remove(self.repo_dir)
        try:
            repo = Repo.clone_from(self.git_repo, self.repo_dir)
        except GitCommandError as e:
            if self.env:
                repo = Repo.clone_from(self.git_repo,
                                       self.repo_dir,
                                       env=self.env)
            else:
                raise e
        return repo

    def _get_commits(self, branch, count=10):
        commits = []
        for commit in self.repo.iter_commits(branch):
            if len(commits) == count:
                break
            commits.append({
                'id': commit.hexsha,
                'author': commit.author.name,
                'date': self._format_date(commit.committed_date),
                'message': commit.message.strip()
            })
        return commits

    def _format_date(self, timestamp):
        if isinstance(timestamp, int):
            date = datetime.fromtimestamp(timestamp)
            return date.strftime('%Y-%m-%d %H:%M')
        return timestamp

    def __enter__(self):
        if self.pkey:
            self.fd = NamedTemporaryFile()
            self.fd.write(self.pkey.encode())
            self.fd.flush()
            self.env = {'GIT_SSH_COMMAND': f'ssh -i {self.fd.name}'}
        self.repo = self._get_repo()
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if self.fd:
            self.fd.close()
Beispiel #50
0
        jobLog = open('patch_tested.txt', 'w+')

        if jobLogHistory:
            if len(jobLogHistory) > options.history:
                jobLogHistory = [jobLogHistory[0]] \
                    + jobLogHistory[len(jobLogHistory)
                    - options.history:]
            for jobHistoryRecord in jobLogHistory:
                jobLog.write(jobHistoryRecord + '\n')
        else:
            jobLog.write('TESTED ISSUES\n')
        jobLog.flush()
        rssData = httpGet(options.jiraFilter, False, options.jiraUser,
                          options.jiraPassword)
        tempFile.write(rssData)
        tempFile.flush()
        for (key, attachment) in parseJiraData(tempFile.name).items():
            (project, issue) = key
            if jenkinsUrlOverrides.has_key(project):
                url = jenkinsUrlOverrides[project]
            else:
                url = options.jenkinsUrl

            jenkinsUrlTemplate = url + '/job/' \
                + options.jenkinsJobTemplate \
                + '/buildWithParameters?' + tokenFrag \
                + '&ISSUE_NUM={issue}&ATTACHMENT_ID={attachment}'

            urlArgs = {
                'project': project,
Beispiel #51
0
def main(apiurl, opts, argv):

    repo = argv[0]
    arch = argv[1]
    build_descr = argv[2]
    xp = []
    build_root = None
    cache_dir  = None
    build_uid = ''
    vm_type = config['build-type']
    vm_telnet = None

    build_descr = os.path.abspath(build_descr)
    build_type = os.path.splitext(build_descr)[1][1:]
    if os.path.basename(build_descr) == 'PKGBUILD':
        build_type = 'arch'
    if os.path.basename(build_descr) == 'build.collax':
        build_type = 'collax'
    if build_type not in ['spec', 'dsc', 'kiwi', 'arch', 'collax', 'livebuild']:
        raise oscerr.WrongArgs(
                'Unknown build type: \'%s\'. Build description should end in .spec, .dsc, .kiwi or .livebuild.' \
                        % build_type)
    if not os.path.isfile(build_descr):
        raise oscerr.WrongArgs('Error: build description file named \'%s\' does not exist.' % build_descr)

    buildargs = []
    if not opts.userootforbuild:
        buildargs.append('--norootforbuild')
    if opts.clean:
        buildargs.append('--clean')
    if opts.noinit:
        buildargs.append('--noinit')
    if opts.nochecks:
        buildargs.append('--no-checks')
    if not opts.no_changelog:
        buildargs.append('--changelog')
    if opts.root:
        build_root = opts.root
    if opts.target:
        buildargs.append('--target=%s' % opts.target)
    if opts.threads:
        buildargs.append('--threads=%s' % opts.threads)
    if opts.jobs:
        buildargs.append('--jobs=%s' % opts.jobs)
    elif config['build-jobs'] > 1:
        buildargs.append('--jobs=%s' % config['build-jobs'])
    if opts.icecream or config['icecream'] != '0':
        if opts.icecream:
            num = opts.icecream
        else:
            num = config['icecream']

        if int(num) > 0:
            buildargs.append('--icecream=%s' % num)
            xp.append('icecream')
            xp.append('gcc-c++')
    if opts.ccache:
        buildargs.append('--ccache')
        xp.append('ccache')
    if opts.linksources:
        buildargs.append('--linksources')
    if opts.baselibs:
        buildargs.append('--baselibs')
    if opts.debuginfo:
        buildargs.append('--debug')
    if opts._with:
        for o in opts._with:
            buildargs.append('--with=%s' % o)
    if opts.without:
        for o in opts.without:
            buildargs.append('--without=%s' % o)
    if opts.define:
        for o in opts.define:
            buildargs.append('--define=%s' % o)
    if config['build-uid']:
        build_uid = config['build-uid']
    if opts.build_uid:
        build_uid = opts.build_uid
    if build_uid:
        buildidre = re.compile('^[0-9]{1,5}:[0-9]{1,5}$')
        if build_uid == 'caller':
            buildargs.append('--uid=%s:%s' % (os.getuid(), os.getgid()))
        elif buildidre.match(build_uid):
            buildargs.append('--uid=%s' % build_uid)
        else:
            print('Error: build-uid arg must be 2 colon separated numerics: "uid:gid" or "caller"', file=sys.stderr)
            return 1
    if opts.vm_type:
        vm_type = opts.vm_type
    if opts.vm_telnet:
        vm_telnet = opts.vm_telnet
    if opts.alternative_project:
        prj = opts.alternative_project
        pac = '_repository'
    else:
        prj = store_read_project(os.curdir)
        if opts.local_package:
            pac = '_repository'
        else:
            pac = store_read_package(os.curdir)
    if opts.shell:
        buildargs.append("--shell")

    orig_build_root = config['build-root']
    # make it possible to override configuration of the rc file
    for var in ['OSC_PACKAGECACHEDIR', 'OSC_SU_WRAPPER', 'OSC_BUILD_ROOT']:
        val = os.getenv(var)
        if val:
            if var.startswith('OSC_'): var = var[4:]
            var = var.lower().replace('_', '-')
            if var in config:
                print('Overriding config value for %s=\'%s\' with \'%s\'' % (var, config[var], val))
            config[var] = val

    pacname = pac
    if pacname == '_repository':
        if not opts.local_package:
            try:
                pacname = store_read_package(os.curdir)
            except oscerr.NoWorkingCopy:
                opts.local_package = True
        if opts.local_package:
            pacname = os.path.splitext(build_descr)[0]
    apihost = urlsplit(apiurl)[1]
    if not build_root:
        build_root = config['build-root']
        if build_root == orig_build_root:
            # ENV var was not set
            build_root = config['api_host_options'][apiurl].get('build-root', build_root)
        try:
            build_root = build_root % {'repo': repo, 'arch': arch,
                         'project': prj, 'package': pacname, 'apihost': apihost}
        except:
            pass

    cache_dir = config['packagecachedir'] % {'apihost': apihost}

    extra_pkgs = []
    if not opts.extra_pkgs:
        extra_pkgs = config['extra-pkgs']
    elif opts.extra_pkgs != ['']:
        extra_pkgs = opts.extra_pkgs

    if xp:
        extra_pkgs += xp

    prefer_pkgs = {}
    build_descr_data = open(build_descr).read()

    # XXX: dirty hack but there's no api to provide custom defines
    if opts.without:
        s = ''
        for i in opts.without:
            s += "%%define _without_%s 1\n" % i
        build_descr_data = s + build_descr_data
    if opts._with:
        s = ''
        for i in opts._with:
            s += "%%define _with_%s 1\n" % i
        build_descr_data = s + build_descr_data
    if opts.define:
        s = ''
        for i in opts.define:
            s += "%%define %s\n" % i
        build_descr_data = s + build_descr_data

    cpiodata = None
    servicefile = os.path.join(os.path.dirname(build_descr), "_service")
    if not os.path.isfile(servicefile):
        servicefile = os.path.join(os.path.dirname(build_descr), "_service")
        if not os.path.isfile(servicefile):
            servicefile = None
        else:
            print('Using local _service file')
    buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv." + repo + "." + arch)
    if not os.path.isfile(buildenvfile):
        buildenvfile = os.path.join(os.path.dirname(build_descr), "_buildenv")
        if not os.path.isfile(buildenvfile):
            buildenvfile = None
        else:
            print('Using local buildenv file: %s' % os.path.basename(buildenvfile))
    if buildenvfile or servicefile:
        from .util import cpio
        if not cpiodata:
            cpiodata = cpio.CpioWrite()

    if opts.prefer_pkgs:
        print('Scanning the following dirs for local packages: %s' % ', '.join(opts.prefer_pkgs))
        from .util import cpio
        if not cpiodata:
            cpiodata = cpio.CpioWrite()
        prefer_pkgs = get_prefer_pkgs(opts.prefer_pkgs, arch, build_type, cpiodata)

    if cpiodata:
        cpiodata.add(os.path.basename(build_descr), build_descr_data)
        # buildenv must come last for compatibility reasons...
        if buildenvfile:
            cpiodata.add("buildenv", open(buildenvfile).read())
        if servicefile:
            cpiodata.add("_service", open(servicefile).read())
        build_descr_data = cpiodata.get()

    # special handling for overlay and rsync-src/dest
    specialcmdopts = []
    if opts.rsyncsrc or opts.rsyncdest :
        if not opts.rsyncsrc or not opts.rsyncdest:
            raise oscerr.WrongOptions('When using --rsync-{src,dest} both parameters have to be specified.')
        myrsyncsrc = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.rsyncsrc)))
        if not os.path.isdir(myrsyncsrc):
            raise oscerr.WrongOptions('--rsync-src %s is no valid directory!' % opts.rsyncsrc)
        # can't check destination - its in the target chroot ;) - but we can check for sanity
        myrsyncdest = os.path.expandvars(opts.rsyncdest)
        if not os.path.isabs(myrsyncdest):
            raise oscerr.WrongOptions('--rsync-dest %s is no absolute path (starting with \'/\')!' % opts.rsyncdest)
        specialcmdopts = ['--rsync-src='+myrsyncsrc, '--rsync-dest='+myrsyncdest]
    if opts.overlay:
        myoverlay = os.path.abspath(os.path.expanduser(os.path.expandvars(opts.overlay)))
        if not os.path.isdir(myoverlay):
            raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay)
        specialcmdopts += ['--overlay='+myoverlay]

    bi_file = None
    bc_file = None
    bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
    bc_filename = '_buildconfig-%s-%s' % (repo, arch)
    if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
        bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
        bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
    elif not os.access('.', os.W_OK):
        bi_file = NamedTemporaryFile(prefix=bi_filename)
        bi_filename = bi_file.name
        bc_file = NamedTemporaryFile(prefix=bc_filename)
        bc_filename = bc_file.name
    else:
        bi_filename = os.path.abspath(bi_filename)
        bc_filename = os.path.abspath(bc_filename)

    try:
        if opts.noinit:
            if not os.path.isfile(bi_filename):
                raise oscerr.WrongOptions('--noinit is not possible, no local buildinfo file')
            print('Use local \'%s\' file as buildinfo' % bi_filename)
            if not os.path.isfile(bc_filename):
                raise oscerr.WrongOptions('--noinit is not possible, no local buildconfig file')
            print('Use local \'%s\' file as buildconfig' % bc_filename)
        elif opts.offline:
            if not os.path.isfile(bi_filename):
                raise oscerr.WrongOptions('--offline is not possible, no local buildinfo file')
            print('Use local \'%s\' file as buildinfo' % bi_filename)
            if not os.path.isfile(bc_filename):
                raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file')
        else:
            print('Getting buildinfo from server and store to %s' % bi_filename)
            bi_text = ''.join(get_buildinfo(apiurl,
                                            prj,
                                            pac,
                                            repo,
                                            arch,
                                            specfile=build_descr_data,
                                            addlist=extra_pkgs))
            if not bi_file:
                bi_file = open(bi_filename, 'w')
            # maybe we should check for errors before saving the file
            bi_file.write(bi_text)
            bi_file.flush()
            print('Getting buildconfig from server and store to %s' % bc_filename)
            bc = get_buildconfig(apiurl, prj, repo)
            if not bc_file:
                bc_file = open(bc_filename, 'w')
            bc_file.write(bc)
            bc_file.flush()
    except HTTPError as e:
        if e.code == 404:
            # check what caused the 404
            if meta_exists(metatype='prj', path_args=(quote_plus(prj), ),
                           template_args=None, create_new=False, apiurl=apiurl):
                pkg_meta_e = None
                try:
                    # take care, not to run into double trouble.
                    pkg_meta_e = meta_exists(metatype='pkg', path_args=(quote_plus(prj),
                                        quote_plus(pac)), template_args=None, create_new=False,
                                        apiurl=apiurl)
                except:
                    pass

                if pkg_meta_e:
                    print('ERROR: Either wrong repo/arch as parameter or a parse error of .spec/.dsc/.kiwi file due to syntax error', file=sys.stderr)
                else:
                    print('The package \'%s\' does not exist - please ' \
                                        'rerun with \'--local-package\'' % pac, file=sys.stderr)
            else:
                print('The project \'%s\' does not exist - please ' \
                                    'rerun with \'--alternative-project <alternative_project>\'' % prj, file=sys.stderr)
            sys.exit(1)
        else:
            raise

    bi = Buildinfo(bi_filename, apiurl, build_type, list(prefer_pkgs.keys()))

    if bi.debuginfo and not (opts.disable_debuginfo or '--debug' in buildargs):
        buildargs.append('--debug')

    if opts.release:
        bi.release = opts.release

    if bi.release:
        buildargs.append('--release=%s' % bi.release)

    # real arch of this machine
    # vs.
    # arch we are supposed to build for
    if bi.hostarch != None:
        if hostarch != bi.hostarch and not bi.hostarch in can_also_build.get(hostarch, []):
            print('Error: hostarch \'%s\' is required.' % (bi.hostarch), file=sys.stderr)
            return 1
    elif hostarch != bi.buildarch:
        if not bi.buildarch in can_also_build.get(hostarch, []):
            # OBSOLETE: qemu_can_build should not be needed anymore since OBS 2.3
            if vm_type != "emulator" and not bi.buildarch in qemu_can_build:
                print('Error: hostarch \'%s\' cannot build \'%s\'.' % (hostarch, bi.buildarch), file=sys.stderr)
                return 1
            print('WARNING: It is guessed to build on hostarch \'%s\' for \'%s\' via QEMU.' % (hostarch, bi.buildarch), file=sys.stderr)

    rpmlist_prefers = []
    if prefer_pkgs:
        print('Evaluating preferred packages')
        for name, path in prefer_pkgs.items():
            if bi.has_dep(name):
                # We remove a preferred package from the buildinfo, so that the
                # fetcher doesn't take care about them.
                # Instead, we put it in a list which is appended to the rpmlist later.
                # At the same time, this will make sure that these packages are
                # not verified.
                bi.remove_dep(name)
                rpmlist_prefers.append((name, path))
                print(' - %s (%s)' % (name, path))

    print('Updating cache of required packages')

    urllist = []
    if not opts.download_api_only:
        # transform 'url1, url2, url3' form into a list
        if 'urllist' in config:
            if isinstance(config['urllist'], str):
                re_clist = re.compile('[, ]+')
                urllist = [ i.strip() for i in re_clist.split(config['urllist'].strip()) ]
            else:
                urllist = config['urllist']

        # OBS 1.5 and before has no downloadurl defined in buildinfo
        if bi.downloadurl:
            urllist.append(bi.downloadurl + '/%(extproject)s/%(extrepository)s/%(arch)s/%(filename)s')
    if opts.disable_cpio_bulk_download:
        urllist.append( '%(apiurl)s/build/%(project)s/%(repository)s/%(repoarch)s/%(repopackage)s/%(repofilename)s' )

    fetcher = Fetcher(cache_dir,
                      urllist = urllist,
                      api_host_options = config['api_host_options'],
                      offline = opts.noinit or opts.offline,
                      http_debug = config['http_debug'],
                      enable_cpio = not opts.disable_cpio_bulk_download,
                      cookiejar=cookiejar)

    if not opts.trust_all_projects:
        # implicitly trust the project we are building for
        check_trusted_projects(apiurl, [ i for i in bi.projects.keys() if not i == prj ])

    # now update the package cache
    fetcher.run(bi)

    old_pkg_dir = None
    if opts.oldpackages:
        old_pkg_dir = opts.oldpackages
        if not old_pkg_dir.startswith('/') and not opts.offline:
            data = [ prj, pacname, repo, arch]
            if old_pkg_dir == '_link':
                p = osc.core.findpacs(os.curdir)[0]
                if not p.islink():
                    raise oscerr.WrongOptions('package is not a link')
                data[0] = p.linkinfo.project
                data[1] = p.linkinfo.package
                repos = osc.core.get_repositories_of_project(apiurl, data[0])
                # hack for links to e.g. Factory
                if not data[2] in repos and 'standard' in repos:
                    data[2] = 'standard'
            elif old_pkg_dir != '' and old_pkg_dir != '_self':
                a = old_pkg_dir.split('/')
                for i in range(0, len(a)):
                    data[i] = a[i]

            destdir = os.path.join(cache_dir, data[0], data[2], data[3])
            old_pkg_dir = None
            try:
                print("Downloading previous build from %s ..." % '/'.join(data))
                binaries = get_binarylist(apiurl, data[0], data[2], data[3], package=data[1], verbose=True)
            except Exception as e:
                print("Error: failed to get binaries: %s" % str(e))
                binaries = []

            if binaries:
                class mytmpdir:
                    """ temporary directory that removes itself"""
                    def __init__(self, *args, **kwargs):
                        self.name = mkdtemp(*args, **kwargs)
                    _rmtree = staticmethod(shutil.rmtree)
                    def cleanup(self):
                        self._rmtree(self.name)
                    def __del__(self):
                        self.cleanup()
                    def __exit__(self):
                        self.cleanup()
                    def __str__(self):
                        return self.name

                old_pkg_dir = mytmpdir(prefix='.build.oldpackages', dir=os.path.abspath(os.curdir))
                if not os.path.exists(destdir):
                    os.makedirs(destdir)
            for i in binaries:
                fname = os.path.join(destdir, i.name)
                os.symlink(fname, os.path.join(str(old_pkg_dir), i.name))
                if os.path.exists(fname):
                    st = os.stat(fname)
                    if st.st_mtime == i.mtime and st.st_size == i.size:
                        continue
                get_binary_file(apiurl,
                                data[0],
                                data[2], data[3],
                                i.name,
                                package = data[1],
                                target_filename = fname,
                                target_mtime = i.mtime,
                                progress_meter = True)

        if old_pkg_dir != None:
            buildargs.append('--oldpackages=%s' % old_pkg_dir)

    # Make packages from buildinfo available as repos for kiwi
    if build_type == 'kiwi':
        if os.path.exists('repos'):
            shutil.rmtree('repos')
        os.mkdir('repos')
        for i in bi.deps:
            if not i.extproject:
                # remove
                bi.deps.remove(i)
                continue
            # project
            pdir = str(i.extproject).replace(':/', ':')
            # repo
            rdir = str(i.extrepository).replace(':/', ':')
            # arch
            adir = i.repoarch
            # project/repo
            prdir = "repos/"+pdir+"/"+rdir
            # project/repo/arch
            pradir = prdir+"/"+adir
            # source fullfilename
            sffn = i.fullfilename
            filename = sffn.split("/")[-1]
            # target fullfilename
            tffn = pradir+"/"+filename
            if not os.path.exists(os.path.join(pradir)):
                os.makedirs(os.path.join(pradir))
            if not os.path.exists(tffn):
                print("Using package: "+sffn)
                if opts.linksources:
                    os.link(sffn, tffn)
                else:
                    os.symlink(sffn, tffn)
            if prefer_pkgs:
                for name, path in prefer_pkgs.items():
                    if name == filename:
                        print("Using prefered package: " + path + "/" + filename)
                        os.unlink(tffn)
                        if opts.linksources:
                            os.link(path + "/" + filename, tffn)
                        else:
                            os.symlink(path + "/" + filename, tffn)
        # Is a obsrepositories tag used?
        try:
            tree = ET.parse(build_descr)
        except:
            print('could not parse the kiwi file:', file=sys.stderr)
            print(open(build_descr).read(), file=sys.stderr)
            sys.exit(1)
        root = tree.getroot()
        # product
        for xml in root.findall('instsource'):
            if xml.find('instrepo').find('source').get('path') == 'obsrepositories:/':
                print("obsrepositories:/ for product builds is not yet supported in osc!")
                sys.exit(1)
        # appliance
        expand_obsrepos=None
        for xml in root.findall('repository'):
            if xml.find('source').get('path') == 'obsrepositories:/':
                expand_obsrepos=True
        if expand_obsrepos:
          buildargs.append('--kiwi-parameter')
          buildargs.append('--ignore-repos')
          for xml in root.findall('repository'):
              if xml.find('source').get('path') == 'obsrepositories:/':
                  for path in bi.pathes:
                      if not os.path.isdir("repos/"+path):
                          continue
                      buildargs.append('--kiwi-parameter')
                      buildargs.append('--add-repo')
                      buildargs.append('--kiwi-parameter')
                      buildargs.append("repos/"+path)
                      buildargs.append('--kiwi-parameter')
                      buildargs.append('--add-repotype')
                      buildargs.append('--kiwi-parameter')
                      buildargs.append('rpm-md')
                      if xml.get('priority'):
                          buildargs.append('--kiwi-parameter')
                          buildargs.append('--add-repoprio='+xml.get('priority'))
              else:
                   m = re.match(r"obs://[^/]+/([^/]+)/(\S+)", xml.find('source').get('path'))
                   if not m:
                       # short path without obs instance name
                       m = re.match(r"obs://([^/]+)/(.+)", xml.find('source').get('path'))
                   project=m.group(1).replace(":", ":/")
                   repo=m.group(2)
                   buildargs.append('--kiwi-parameter')
                   buildargs.append('--add-repo')
                   buildargs.append('--kiwi-parameter')
                   buildargs.append("repos/"+project+"/"+repo)
                   buildargs.append('--kiwi-parameter')
                   buildargs.append('--add-repotype')
                   buildargs.append('--kiwi-parameter')
                   buildargs.append('rpm-md')
                   if xml.get('priority'):
                       buildargs.append('--kiwi-parameter')
                       buildargs.append('--add-repopriority='+xml.get('priority'))

    if vm_type == "xen" or vm_type == "kvm" or vm_type == "lxc":
        print('Skipping verification of package signatures due to secure VM build')
    elif bi.pacsuffix == 'rpm':
        if opts.no_verify:
            print('Skipping verification of package signatures')
        else:
            print('Verifying integrity of cached packages')
            verify_pacs(bi)
    elif bi.pacsuffix == 'deb':
        if opts.no_verify or opts.noinit:
            print('Skipping verification of package signatures')
        else:
            print('WARNING: deb packages get not verified, they can compromise your system !')
    else:
        print('WARNING: unknown packages get not verified, they can compromise your system !')

    for i in bi.deps:
        if i.hdrmd5:
            from .util import packagequery
            hdrmd5 = packagequery.PackageQuery.queryhdrmd5(i.fullfilename)
            if not hdrmd5:
                print("Error: cannot get hdrmd5 for %s" % i.fullfilename)
                sys.exit(1)
            if hdrmd5 != i.hdrmd5:
                print("Error: hdrmd5 mismatch for %s: %s != %s" % (i.fullfilename, hdrmd5, i.hdrmd5))
                sys.exit(1)

    print('Writing build configuration')

    if build_type == 'kiwi':
        rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps if not i.noinstall ]
    else:
        rpmlist = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps ]
    rpmlist += [ '%s %s\n' % (i[0], i[1]) for i in rpmlist_prefers ]

    rpmlist.append('preinstall: ' + ' '.join(bi.preinstall_list) + '\n')
    rpmlist.append('vminstall: ' + ' '.join(bi.vminstall_list) + '\n')
    rpmlist.append('runscripts: ' + ' '.join(bi.runscripts_list) + '\n')
    if build_type != 'kiwi' and bi.noinstall_list:
        rpmlist.append('noinstall: ' + ' '.join(bi.noinstall_list) + '\n')
    if build_type != 'kiwi' and bi.installonly_list:
        rpmlist.append('installonly: ' + ' '.join(bi.installonly_list) + '\n')

    rpmlist_file = NamedTemporaryFile(prefix='rpmlist.')
    rpmlist_filename = rpmlist_file.name
    rpmlist_file.writelines(rpmlist)
    rpmlist_file.flush()

    subst = { 'repo': repo, 'arch': arch, 'project' : prj, 'package' : pacname }
    vm_options = []
    # XXX check if build-device present
    my_build_device = ''
    if config['build-device']:
        my_build_device = config['build-device'] % subst
    else:
        # obs worker uses /root here but that collides with the
        # /root directory if the build root was used without vm
        # before
        my_build_device = build_root + '/img'

    need_root = True
    if vm_type:
        if config['build-swap']:
            my_build_swap = config['build-swap'] % subst
        else:
            my_build_swap = build_root + '/swap'

        vm_options = [ '--vm-type=%s' % vm_type ]
        if vm_telnet:
            vm_options += [ '--vm-telnet=' + vm_telnet ]
        if config['build-memory']:
            vm_options += [ '--memory=' + config['build-memory'] ]
        if vm_type != 'lxc':
            vm_options += [ '--vm-disk=' + my_build_device ]
            vm_options += [ '--vm-swap=' + my_build_swap ]
            vm_options += [ '--logfile=%s/.build.log' % build_root ]
            if vm_type == 'kvm':
                if os.access(build_root, os.W_OK) and os.access('/dev/kvm', os.W_OK):
                    # so let's hope there's also an fstab entry
                    need_root = False
                if config['build-kernel']:
                    vm_options += [ '--vm-kernel=' + config['build-kernel'] ]
                if config['build-initrd']:
                    vm_options += [ '--vm-initrd=' + config['build-initrd'] ]

            build_root += '/.mount'

        if config['build-memory']:
            vm_options += [ '--memory=' + config['build-memory'] ]
        if config['build-vmdisk-rootsize']:
            vm_options += [ '--vmdisk-rootsize=' + config['build-vmdisk-rootsize'] ]
        if config['build-vmdisk-swapsize']:
            vm_options += [ '--vmdisk-swapsize=' + config['build-vmdisk-swapsize'] ]
        if config['build-vmdisk-filesystem']:
            vm_options += [ '--vmdisk-filesystem=' + config['build-vmdisk-filesystem'] ]
        if config['build-vm-user']:
            vm_options += [ '--vm-user='******'build-vm-user'] ]


    if opts.preload:
        print("Preload done for selected repo/arch.")
        sys.exit(0)

    print('Running build')
    cmd = [ config['build-cmd'], '--root='+build_root,
                    '--rpmlist='+rpmlist_filename,
                    '--dist='+bc_filename,
                    '--arch='+bi.buildarch ]
    cmd += specialcmdopts + vm_options + buildargs
    cmd += [ build_descr ]

    if need_root:
        sucmd = config['su-wrapper'].split()
        if sucmd[0] == 'su':
            if sucmd[-1] == '-c':
                sucmd.pop()
            cmd = sucmd + ['-s', cmd[0], 'root', '--' ] + cmd[1:]
        else:
            cmd = sucmd + cmd

    # change personality, if needed
    if hostarch != bi.buildarch and bi.buildarch in change_personality:
        cmd = [ change_personality[bi.buildarch] ] + cmd

    try:
        rc = run_external(cmd[0], *cmd[1:])
        if rc:
            print()
            print('The buildroot was:', build_root)
            sys.exit(rc)
    except KeyboardInterrupt as i:
        print("keyboard interrupt, killing build ...")
        cmd.append('--kill')
        run_external(cmd[0], *cmd[1:])
        raise i

    pacdir = os.path.join(build_root, '.build.packages')
    if os.path.islink(pacdir):
        pacdir = os.readlink(pacdir)
        pacdir = os.path.join(build_root, pacdir)

    if os.path.exists(pacdir):
        (s_built, b_built) = get_built_files(pacdir, bi.buildtype)

        print()
        if s_built: print(s_built)
        print()
        print(b_built)

        if opts.keep_pkgs:
            for i in b_built.splitlines() + s_built.splitlines():
                shutil.copy2(i, os.path.join(opts.keep_pkgs, os.path.basename(i)))

    if bi_file:
        bi_file.close()
    if bc_file:
        bc_file.close()
    rpmlist_file.close()
Beispiel #52
0
    def create_router(transport):
        print("\033[1A\033[K" + "Creating shaper for {}".format(transport))
        if transport == "bus":
            routing_type = {
                "weights": {"motorway": 1.5, "trunk": 1.5, "primary": 1.4, "secondary": 1.3, "tertiary": 1.3,
                "unclassified": 1, "residential": 0.6, "living_street": 0.6, "track": 0.3, "service": 0.5},
                "access": ["access", "vehicle", "motor_vehicle", "psv", "bus", "routing:ztm"],
                "name": "bus"
            }
        elif transport == "train":
            routing_type = "train"
        elif transport == "tram":
            routing_type = "tram"
        else:
            raise ValueError("Invalid transport type {} for Shaper".format(transport))

        if transport in {"train", "tram"}:
            request = requests.get("https://mkuran.pl/feed/ztm/ztm-km-rail-shapes.osm", stream=True)

        else:
            # Overpass Query:
            query = "\n".join([
                "[bbox:51.9144,20.4438,52.5007,21.4844][out:xml];"
                "("
                ' way["highway"="motorway"];'
                ' way["highway"="motorway_link"];'
                ' way["highway"="trunk"];'
                ' way["highway"="trunk_link"];'
                ' way["highway"="primary"];'
                ' way["highway"="primary_link"];'
                ' way["highway"="secondary"];'
                ' way["highway"="secondary_link"];'
                ' way["highway"="tertiary"];'
                ' way["highway"="tertiary_link"];'
                ' way["highway"="unclassified"];'
                ' way["highway"="minor"];'
                ' way["highway"="residential"];'
                ' way["highway"="living_street"];'
                ' way["highway"="service"];'
                ');'
                'way._(poly:"52.4455 20.6858 52.376 20.6872 52.3533 20.7868 52.2929 20.726 52.2694 20.6724 52.2740 20.4465 52.2599 20.4438 52.2481 20.5832 52.2538 20.681 52.1865 20.6786 52.1859 20.7129 52.1465 20.7895 52.0966 20.783 52.0632 20.7222 52.0151 20.7617 51.9873 20.9351 51.9269 20.9509 51.9144 21.0226 51.9322 21.1987 51.9569 21.2472 52.0463 21.2368 52.1316 21.4844 52.1429 21.4404 52.2130 21.3814 52.2622 21.3141 52.2652 21.1977 52.3038 21.173 52.3063 21.2925 52.3659 21.3515 52.3829 21.3001 52.4221 21.1929 52.4898 21.1421");'
                '>->.n;'
                '<->.r;'
                '(._;.n;.r;);'
                'out;'
            ])

            request = requests.get(
                "https://overpass-api.de/api/interpreter/",
                params={"data": query},
                stream=True,
            )

        temp_xml = NamedTemporaryFile(delete=False)

        for chunk in request.iter_content(chunk_size=1024):
            temp_xml.write(chunk)

        request.close()
        temp_xml.seek(0)

        router = pyroutelib3.Router(routing_type, temp_xml.name)

        temp_xml.close()

        return router
Beispiel #53
0
class ParallelBlasterTests(TestCase):
    def setUp(self):
        """ """
        self.files_to_remove = []
        self.dirs_to_remove = []

        tmp_dir = get_qiime_temp_dir()
        self.test_out = get_tmp_filename(
            tmp_dir=tmp_dir,
            prefix='qiime_parallel_blaster_tests_',
            suffix='',
            result_constructor=str)
        self.dirs_to_remove.append(self.test_out)
        create_dir(self.test_out)

        self.tmp_seq_filepath = get_tmp_filename(
            tmp_dir=self.test_out,
            prefix='qiime_parallel_blaster_tests_input',
            suffix='.fasta')
        seq_file = open(self.tmp_seq_filepath, 'w')
        seq_file.write(blast_test_seqs)
        seq_file.close()
        self.files_to_remove.append(self.tmp_seq_filepath)

        self.reference_seqs_file = NamedTemporaryFile(
            prefix='qiime_parallel_blaster_tests_ref_seqs',
            suffix='.fasta',
            dir=tmp_dir)
        self.reference_seqs_file.write(blast_ref_seqs)
        self.reference_seqs_file.seek(0)

        initiate_timeout(60)

    def tearDown(self):
        """ """
        disable_timeout()
        remove_files(self.files_to_remove)
        # remove directories last, so we don't get errors
        # trying to remove files which may be in the directories
        for d in self.dirs_to_remove:
            if exists(d):
                rmtree(d)

    def test_parallel_blaster(self):
        """Test ParallelBlaster functions as expected."""
        params = {
            'refseqs_path': self.reference_seqs_file.name,
            'disable_low_complexity_filter': False,
            'e_value': 0.001,
            'num_hits': 1,
            'word_size': 30,
            'suppress_format_blastdb': False,
            'blastmat_dir': None
        }

        app = ParallelBlaster()
        r = app(self.tmp_seq_filepath,
                self.test_out,
                params,
                job_prefix='BLASTTEST',
                poll_directly=True,
                suppress_submit_jobs=False)

        # Basic sanity checks: we should get two blast hits (lines). We ignore
        # all of the comments in the file. Each line should have 12 fields
        # separated by tabs.
        results = [
            line for line in open(
                glob(join(self.test_out, '*_blast_out.txt'))[0], 'U')
            if not line.startswith('#')
        ]
        self.assertEqual(len(results), 2)
        self.assertEqual(len(results[0].split('\t')), 12)
        self.assertEqual(len(results[1].split('\t')), 12)
Beispiel #54
0
    regenerate = '--regenerate-test' in sys.argv[1:]
            
    for dvd in False,True:

        if regenerate:
            f = open(test_template(dvd=dvd), 'w')
        else:
            from tempfile import NamedTemporaryFile
            f = NamedTemporaryFile(mode='w')

        f.write(
            re.sub('(?<!\r)\n', '\r\n',
                   generate(
                    dvd=dvd, 
                    version=r'${NORMALIZED_VERSION}', 
                    human_version=r'%(human_version)s', 
                    sections='%(sections)s',
                    architecture='32'
                    )))
        f.flush()
        
        if not regenerate:
            print subprocess.Popen(
                ['diff', '-wdu', test_template(dvd=dvd), f.name]).communicate()[0]

            if not dvd:
                print
                print '=============================='
                print
    
Beispiel #55
0
    class FileIndex:
        SIGNATURE = b'BOB1'
        CACHE_ENTRY_FMT = '=QQLqLQ20sH'
        CACHE_ENTRY_SIZE = struct.calcsize(CACHE_ENTRY_FMT)

        class Stat:
            def __init__(self):
                self.name = b""
                self.ctime = 0
                self.mtime = 0
                self.dev = 0
                self.ino = 0
                self.mode = 0
                self.size = 0
                self.digest = b''

        def __init__(self, cachePath):
            self.__cachePath = cachePath
            self.__cacheDir = os.path.dirname(cachePath)

        def open(self):
            self.__inPos = 0
            self.__inPosOld = 0
            self.__outFile = None
            try:
                if os.path.exists(self.__cachePath):
                    self.__inFile = open(self.__cachePath, "rb")
                    sig = self.__inFile.read(4)
                    if sig == DirHasher.FileIndex.SIGNATURE:
                        self.__mismatch = False
                        self.__inPos = self.__inPosOld = 4
                    else:
                        logging.getLogger(__name__).info(
                            "Wrong signature at '%s': %s", self.__cachePath,
                            sig)
                        self.__inFile.close()
                        self.__inFile = None
                        self.__mismatch = True
                else:
                    self.__inFile = None
                    self.__mismatch = True
            except OSError as e:
                raise BuildError("Error opening hash cache: " + str(e))
            self.__current = DirHasher.FileIndex.Stat()

        def close(self):
            try:
                if self.__inFile:
                    self.__inFile.close()
                if self.__outFile:
                    self.__outFile.close()
                    os.replace(self.__outFile.name, self.__cachePath)
            except OSError as e:
                raise BuildError("Error closing hash cache: " + str(e))

        def __readEntry(self):
            if not self.__inFile: return False
            raw = self.__inFile.read(DirHasher.FileIndex.CACHE_ENTRY_SIZE)
            if len(raw) < DirHasher.FileIndex.CACHE_ENTRY_SIZE: return False
            e = self.__current
            (e.ctime, e.mtime, e.dev, e.ino, e.mode, e.size, e.digest,
             nameLen) = struct.unpack(DirHasher.FileIndex.CACHE_ENTRY_FMT, raw)
            e.name = self.__inFile.read(nameLen)
            self.__inPosOld = self.__inPos
            self.__inPos += DirHasher.FileIndex.CACHE_ENTRY_SIZE + nameLen
            return True

        def __writeEntry(self, name, st, digest):
            if not self.__outFile:
                self.__outFile = NamedTemporaryFile(mode="wb",
                                                    dir=self.__cacheDir,
                                                    delete=False)
                if self.__inFile:
                    pos = self.__inFile.tell()
                    self.__inFile.seek(0)
                    self.__outFile.write(self.__inFile.read(self.__inPosOld))
                    self.__inFile.seek(pos)
                else:
                    self.__outFile.write(DirHasher.FileIndex.SIGNATURE)
            self.__outFile.write(
                struct.pack(DirHasher.FileIndex.CACHE_ENTRY_FMT,
                            float2ns(st.st_ctime), float2ns(st.st_mtime),
                            st.st_dev, st.st_ino, st.st_mode, st.st_size,
                            digest, len(name)))
            self.__outFile.write(name)

        def __match(self, name, st):
            while self.__current.name < name:
                if not self.__readEntry(): break
            e = self.__current
            res = ((e.name == name) and (e.ctime == float2ns(st.st_ctime))
                   and (e.mtime == float2ns(st.st_mtime))
                   and (e.dev == st.st_dev) and (e.ino == st.st_ino)
                   and (e.mode == st.st_mode) and (e.size == st.st_size))
            #if not res: print("Mismatch", e.name, name)
            return res

        def check(self, prefix, name, st, process):
            if self.__match(name, st):
                digest = self.__current.digest
            else:
                digest = process(os.path.join(prefix, name))
                self.__mismatch = True
            if self.__mismatch:
                self.__writeEntry(name, st, digest)
            return digest
Beispiel #56
0
"""


def sample_run(input_file_path):

    with cp2k.ForceEnvironment(input_file_path, "/dev/null") as fenv:
        print("potential energy: {:e}".format(fenv.potential_energy))
        print("calculating energy..")
        fenv.calc_energy()
        print(".. done!")
        print("potential energy: {:e}".format(fenv.potential_energy))
        print("positions:")
        pos = fenv.positions
        print(pos)
        print("zeroify positions..")
        fenv.positions = np.zeros(pos.shape)
        print("positions:")
        print(fenv.positions)


if __name__ == "__main__":
    cp2k.init()

    tempfile = NamedTemporaryFile(mode="w+")
    tempfile.write(TEST_FILE_CONTENT)
    tempfile.flush()

    sample_run(tempfile.name)

    cp2k.finalize()
Beispiel #57
0
    def _write_local_data_files(self, cursor):
        """
        Takes a cursor, and writes results to a local file.

        :return: A dictionary where keys are filenames to be used as object
            names in GCS, and values are file handles to local files that
            contain the data for the GCS objects.
        """
        schema = list(
            map(lambda schema_tuple: schema_tuple[0], cursor.description))
        col_type_dict = self._get_col_type_dict()
        file_no = 0

        tmp_file_handle = NamedTemporaryFile(delete=True)
        if self.export_format == 'csv':
            file_mime_type = 'text/csv'
        elif self.export_format == 'parquet':
            file_mime_type = 'application/octet-stream'
        else:
            file_mime_type = 'application/json'
        files_to_upload = [{
            'file_name': self.filename.format(file_no),
            'file_handle': tmp_file_handle,
            'file_mime_type': file_mime_type,
        }]
        self.log.info("Current file count: %d", len(files_to_upload))

        if self.export_format == 'csv':
            csv_writer = self._configure_csv_file(tmp_file_handle, schema)
        if self.export_format == 'parquet':
            parquet_schema = self._convert_parquet_schema(cursor)
            parquet_writer = self._configure_parquet_file(
                tmp_file_handle, parquet_schema)

        for row in cursor:
            # Convert datetime objects to utc seconds, and decimals to floats.
            # Convert binary type object to string encoded with base64.
            row = self.convert_types(schema, col_type_dict, row)

            if self.export_format == 'csv':
                if self.null_marker is not None:
                    row = [
                        value if value is not None else self.null_marker
                        for value in row
                    ]
                csv_writer.writerow(row)
            elif self.export_format == 'parquet':
                if self.null_marker is not None:
                    row = [
                        value if value is not None else self.null_marker
                        for value in row
                    ]
                row_pydic = {col: [value] for col, value in zip(schema, row)}
                tbl = pa.Table.from_pydict(row_pydic, parquet_schema)
                parquet_writer.write_table(tbl)
            else:
                row_dict = dict(zip(schema, row))

                tmp_file_handle.write(
                    json.dumps(row_dict, sort_keys=True,
                               ensure_ascii=False).encode("utf-8"))

                # Append newline to make dumps BigQuery compatible.
                tmp_file_handle.write(b'\n')

            # Stop if the file exceeds the file size limit.
            if tmp_file_handle.tell() >= self.approx_max_file_size_bytes:
                file_no += 1

                tmp_file_handle = NamedTemporaryFile(delete=True)
                files_to_upload.append({
                    'file_name':
                    self.filename.format(file_no),
                    'file_handle':
                    tmp_file_handle,
                    'file_mime_type':
                    file_mime_type,
                })
                self.log.info("Current file count: %d", len(files_to_upload))
                if self.export_format == 'csv':
                    csv_writer = self._configure_csv_file(
                        tmp_file_handle, schema)
                if self.export_format == 'parquet':
                    parquet_writer = self._configure_parquet_file(
                        tmp_file_handle, parquet_schema)
        return files_to_upload
Beispiel #58
0
class K8SCopyToPod(K8SCopy):
    """
    Copy files/directory from local filesystem into remote Pod
    """
    def __init__(self, module, client):
        super(K8SCopyToPod, self).__init__(module, client)
        self.files_to_copy = list()

    def run_from_pod(self, command):
        response = stream(self.api_instance.connect_get_namespaced_pod_exec,
                          self.name,
                          self.namespace,
                          command=command,
                          stderr=True,
                          stdin=False,
                          stdout=True,
                          tty=False,
                          _preload_content=False,
                          **self.container_arg)
        errors = []
        while response.is_open():
            response.update(timeout=1)
            if response.peek_stderr():
                errors.append(response.read_stderr())
        response.close()
        err = response.read_channel(ERROR_CHANNEL)
        err = yaml.safe_load(err)
        response.close()
        if err['status'] != 'Success':
            self.module.fail_json(
                msg="Failed to run {0} on Pod.".format(command), errors=errors)

    def is_remote_path_dir(self):
        pod_command = ['test', '-d', self.remote_path]
        response = stream(self.api_instance.connect_get_namespaced_pod_exec,
                          self.name,
                          self.namespace,
                          command=pod_command,
                          stdout=True,
                          stderr=True,
                          stdin=False,
                          tty=False,
                          _preload_content=False,
                          **self.container_arg)
        while response.is_open():
            response.update(timeout=1)
        err = response.read_channel(ERROR_CHANNEL)
        err = yaml.safe_load(err)
        response.close()
        if err['status'] == 'Success':
            return True
        return False

    def close_temp_file(self):
        if self.named_temp_file:
            self.named_temp_file.close()

    def run(self):
        try:
            # remove trailing slash from destination path
            dest_file = self.remote_path.rstrip("/")
            src_file = self.local_path
            self.named_temp_file = None
            if self.content:
                self.named_temp_file = NamedTemporaryFile(mode="w")
                self.named_temp_file.write(self.content)
                self.named_temp_file.flush()
                src_file = self.named_temp_file.name
            else:
                if not os.path.exists(self.local_path):
                    self.module.fail_json(
                        msg="{0} does not exist in local filesystem".format(
                            self.local_path))
                if not os.access(self.local_path, os.R_OK):
                    self.module.fail_json(
                        msg="{0} not readable".format(self.local_path))

            if self.is_remote_path_dir():
                if self.content:
                    self.module.fail_json(
                        msg=
                        "When content is specified, remote path should not be an existing directory"
                    )
                else:
                    dest_file = os.path.join(dest_file,
                                             os.path.basename(src_file))

            if self.no_preserve:
                tar_command = [
                    'tar', '--no-same-permissions', '--no-same-owner', '-xmf',
                    '-'
                ]
            else:
                tar_command = ['tar', '-xmf', '-']

            if dest_file.startswith("/"):
                tar_command.extend(['-C', '/'])

            response = stream(
                self.api_instance.connect_get_namespaced_pod_exec,
                self.name,
                self.namespace,
                command=tar_command,
                stderr=True,
                stdin=True,
                stdout=True,
                tty=False,
                _preload_content=False,
                **self.container_arg)
            with TemporaryFile() as tar_buffer:
                with tarfile.open(fileobj=tar_buffer, mode='w') as tar:
                    tar.add(src_file, dest_file)
                tar_buffer.seek(0)
                commands = []
                # push command in chunk mode
                size = 1024 * 1024
                while True:
                    data = tar_buffer.read(size)
                    if not data:
                        break
                    commands.append(data)

                stderr, stdout = [], []
                while response.is_open():
                    if response.peek_stdout():
                        stdout.append(response.read_stdout().rstrip("\n"))
                    if response.peek_stderr():
                        stderr.append(response.read_stderr().rstrip("\n"))
                    if commands:
                        cmd = commands.pop(0)
                        response.write_stdin(cmd)
                    else:
                        break
                response.close()
                if stderr:
                    self.close_temp_file()
                    self.module.fail_json(
                        command=tar_command,
                        msg=
                        "Failed to copy local file/directory into Pod due to: {0}"
                        .format(''.join(stderr)))
            self.close_temp_file()
            if self.content:
                self.module.exit_json(
                    changed=True,
                    result="Content successfully copied into {0} on remote Pod"
                    .format(self.remote_path))
            self.module.exit_json(
                changed=True,
                result="{0} successfully copied into remote Pod into {1}".
                format(self.local_path, self.remote_path))

        except Exception as e:
            self.module.fail_json(
                msg="Failed to copy local file/directory into Pod due to: {0}".
                format(to_native(e)))
Beispiel #59
0
def run_long_running(params, is_test=False):
    """
    Start the long running server
    :param params: Demisto params
    :param is_test: Indicates whether it's test-module run or regular run
    :return: None
    """
    certificate: str = params.get('certificate', '')
    private_key: str = params.get('key', '')

    certificate_path = str()
    private_key_path = str()

    try:
        port = get_params_port(params)
        ssl_args = dict()

        if (certificate and not private_key) or (private_key
                                                 and not certificate):
            raise DemistoException(
                'If using HTTPS connection, both certificate and private key should be provided.'
            )

        if certificate and private_key:
            certificate_file = NamedTemporaryFile(delete=False)
            certificate_path = certificate_file.name
            certificate_file.write(bytes(certificate, 'utf-8'))
            certificate_file.close()

            private_key_file = NamedTemporaryFile(delete=False)
            private_key_path = private_key_file.name
            private_key_file.write(bytes(private_key, 'utf-8'))
            private_key_file.close()
            context = SSLContext(PROTOCOL_TLSv1_2)
            context.load_cert_chain(certificate_path, private_key_path)
            ssl_args['ssl_context'] = context
            demisto.debug('Starting HTTPS Server')
        else:
            demisto.debug('Starting HTTP Server')

        server = WSGIServer(('0.0.0.0', port),
                            APP,
                            **ssl_args,
                            log=DEMISTO_LOGGER)
        if is_test:
            server_process = Process(target=server.serve_forever)
            server_process.start()
            time.sleep(5)
            server_process.terminate()
        else:
            server.serve_forever()
    except SSLError as e:
        ssl_err_message = f'Failed to validate certificate and/or private key: {str(e)}'
        demisto.error(ssl_err_message)
        raise ValueError(ssl_err_message)
    except Exception as e:
        demisto.error(f'An error occurred in long running loop: {str(e)}')
        raise ValueError(str(e))
    finally:
        if certificate_path:
            os.unlink(certificate_path)
        if private_key_path:
            os.unlink(private_key_path)
Beispiel #60
0
class FileSourceTest(unittest.TestCase):
    def setUp(self):
        dirlist = ["plugins/datagraph"]

        self.loader = PluginsLoader(Application)
        self.loader.load(dirlist)

        self._dataFile = NamedTemporaryFile(mode='w',
                                            delete=False,
                                            encoding='utf8')

    def tearDown(self):
        self.loader.clear()
        self._dataFile.close()
        os.remove(self._dataFile.name)
        self._dataFile = None

    def testFileSource_empty_01(self):
        from datagraph.datasources import FileSource
        data = ''''''

        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0, items)

    def testFileSource_empty_02(self):
        from datagraph.datasources import FileSource
        data = '''
        
        
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0)

    def testFileSource_single_01(self):
        from datagraph.datasources import FileSource
        data = '''123'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 1)
        self.assertEqual(items[0], ['123'])

    def testFileSource_single_02(self):
        from datagraph.datasources import FileSource
        data = '''123    '''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 1)
        self.assertEqual(items[0], ['123'])

    def testFileSource_single_03(self):
        from datagraph.datasources import FileSource
        data = '''   123    '''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 1)
        self.assertEqual(items[0], ['123'])

    def testFileSource_single_04(self):
        from datagraph.datasources import FileSource
        data = '''123
        
        '''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 1)
        self.assertEqual(items[0], ['123'])

    def testFileSource_single_05(self):
        from datagraph.datasources import FileSource
        data = '''123
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 1)
        self.assertEqual(items[0], ['123'])

    def testFileSource_one_col_01(self):
        from datagraph.datasources import FileSource
        data = '''123
234
456'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 3)
        self.assertEqual(items[0], ['123'])
        self.assertEqual(items[1], ['234'])
        self.assertEqual(items[2], ['456'])

    def testFileSource_one_col_02(self):
        from datagraph.datasources import FileSource
        data = '''123
234
456
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 3)
        self.assertEqual(items[0], ['123'])
        self.assertEqual(items[1], ['234'])
        self.assertEqual(items[2], ['456'])

    def testFileSource_one_col_03(self):
        from datagraph.datasources import FileSource
        data = '''123
234
456

'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 3)
        self.assertEqual(items[0], ['123'])
        self.assertEqual(items[1], ['234'])
        self.assertEqual(items[2], ['456'])

    def testFileSource_one_col_04(self):
        from datagraph.datasources import FileSource
        data = '''
    
123
234
4560

1000
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 4)
        self.assertEqual(items[0], ['123'])
        self.assertEqual(items[1], ['234'])
        self.assertEqual(items[2], ['4560'])
        self.assertEqual(items[3], ['1000'])

    def testFileSource_col_01(self):
        from datagraph.datasources import FileSource
        data = '''123    456    789
234    100      111
456    101   99
-10\t55    66
20    30    40    '''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 5)

        self.assertEqual(items[0], ['123', '456', '789'])
        self.assertEqual(items[1], ['234', '100', '111'])
        self.assertEqual(items[2], ['456', '101', '99'])
        self.assertEqual(items[3], ['-10', '55', '66'])
        self.assertEqual(items[4], ['20', '30', '40'])

    def testFileSource_col_02(self):
        from datagraph.datasources import FileSource
        data = '''
123    456    789
234    100      111
456    101   99
-10\t55    66
20    30    40    


'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 5)

        self.assertEqual(items[0], ['123', '456', '789'])
        self.assertEqual(items[1], ['234', '100', '111'])
        self.assertEqual(items[2], ['456', '101', '99'])
        self.assertEqual(items[3], ['-10', '55', '66'])
        self.assertEqual(items[4], ['20', '30', '40'])

    def testFileSource_col_03(self):
        from datagraph.datasources import FileSource
        data = '''123    456    789
234    100      111
456    101
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 2)

        self.assertEqual(items[0], ['123', '456', '789'])
        self.assertEqual(items[1], ['234', '100', '111'])

    def testFileSource_col_04(self):
        from datagraph.datasources import FileSource
        data = '''123    456    789
234    100      111
456    101    99      78
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 2)

        self.assertEqual(items[0], ['123', '456', '789'])
        self.assertEqual(items[1], ['234', '100', '111'])

    def testFileSource_skiprows_01(self):
        from datagraph.datasources import FileSource
        data = '''
123    456    789
234    100      111
456    101    99
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 3)

        self.assertEqual(items[0], ['123', '456', '789'])
        self.assertEqual(items[1], ['234', '100', '111'])

    def testFileSource_skiprows_02(self):
        from datagraph.datasources import FileSource
        data = '''Абырвалг
----
123    456    789
234    100      111
456    101    99
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name, skiprows=2)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 3)

        self.assertEqual(items[0], ['123', '456', '789'])
        self.assertEqual(items[1], ['234', '100', '111'])

    def testFileSource_skiprows_03(self):
        from datagraph.datasources import FileSource
        data = '''Абырвалг Абыр
----
123    456    789
234    100      111
456    101    99
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name, skiprows=0)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 1)

        self.assertEqual(items[0], ['Абырвалг', 'Абыр'])

    def testFileSource_skiprows_04(self):
        from datagraph.datasources import FileSource
        data = '''Абырвалг Абыр
----
123    456    789
234    100      111
456    101    99
'''
        self._dataFile.write(data)
        self._dataFile.flush()
        source = FileSource(self._dataFile.name, skiprows=5)

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0)

    def testFileSource_invalid_01(self):
        from datagraph.datasources import FileSource
        source = FileSource('invalid_fname.txt')

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0)

    def testFileSource_invalid_02(self):
        from datagraph.datasources import FileSource
        source = FileSource('testdata/samplefiles/image.png')

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0)

    def testFileSource_invalid_03(self):
        from datagraph.datasources import FileSource
        source = FileSource('testdata/samplefiles/invalid.exe')

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0)

    def testFileSource_invalid_04(self):
        from datagraph.datasources import FileSource
        source = FileSource('testdata/samplefiles/text_1251.txt')

        items = list(source.getRowsIterator())
        self.assertEqual(len(items), 0)