def test_consistent_output_PE(self): test_primary_specific_outfile = io.StringIO() test_secondary_specific_outfile = io.StringIO() test_primary_multi_outfile = io.StringIO() test_secondary_multi_outfile = io.StringIO() test_unassigned_outfile = io.StringIO() test_unresolved_outfile = io.StringIO() sam1 = io.TextIOWrapper(resource_stream(__name__, 'data/paired_end_testdata_human.sam')) sam2 = io.TextIOWrapper(resource_stream(__name__, 'data/paired_end_testdata_mouse.sam')) process_headers(sam1,sam2,primary_specific=test_primary_specific_outfile, secondary_specific=test_secondary_specific_outfile) cat_counts = main_paired_end(getReadPairs(sam1,sam2), primary_specific=test_primary_specific_outfile, secondary_specific=test_secondary_specific_outfile, primary_multi=test_primary_multi_outfile, secondary_multi=test_secondary_multi_outfile, unresolved=test_unresolved_outfile, unassigned=test_unassigned_outfile, ) self.assertEqual(sum([cat_counts[x] for x in cat_counts if 'primary_specific' in x])*2, len(test_primary_specific_outfile.getvalue().split('\n'))-30) #29 lines of header in this file self.assertEqual(sum([cat_counts[x] for x in cat_counts if 'secondary_specific' in x and not 'primary_specific' in x])*2, len(test_secondary_specific_outfile.getvalue().split('\n'))-27) #26 lines of header in this file self.assertEqual(sum([cat_counts[x] for x in cat_counts if 'primary_multi' in x and not 'primary_specific' in x and not 'secondary_specific' in x])*2, len(test_primary_multi_outfile.getvalue().split('\n'))-1) self.assertEqual(sum([cat_counts[x] for x in cat_counts if 'secondary_multi' in x \ and not 'primary_multi' in x and not 'primary_specific' in x and not 'secondary_specific' in x])*2, len(test_secondary_multi_outfile.getvalue().split('\n'))-1) self.assertEqual(hashlib.sha224(test_primary_specific_outfile.getvalue().encode('latin-1')).hexdigest(),'ecba2de3e3af9c7405a84ad2a4ebaf194ebfb4df76f45c311c0f681d') sam1.close() sam2.close() pass
def build_program(main_file_data, library_header, shared_library, gcc_prefix, cflags, package=__name__): with tempfile.TemporaryDirectory() as tempdir: shead_name = os.path.basename(library_header) shlib_name = os.path.basename(shared_library) assert shlib_name.startswith("lib") and shlib_name.endswith(".so") and shead_name.endswith(".h") shlib_short = shlib_name[3:-3] # strip "lib" and ".so" shared_lib_path = os.path.join(tempdir, shlib_name) lib_header_path = os.path.join(tempdir, shead_name) main_file_path = os.path.join(tempdir, "themis_main.c") main_output_path = os.path.join(tempdir, "themis_main") with open(shared_lib_path, "wb") as fout: with pkg_resources.resource_stream(package, shared_library) as fin: shutil.copyfileobj(fin, fout) with open(lib_header_path, "wb") as fout: with pkg_resources.resource_stream(package, library_header) as fin: shutil.copyfileobj(fin, fout) with open(main_file_path, "w") as fout: fout.write(main_file_data) subprocess.check_call([gcc_prefix + "gcc", *cflags.split(), "-I", tempdir, "-L", tempdir, "-l", shlib_short, main_file_path, "-o", main_output_path]) subprocess.check_call([gcc_prefix + "strip", main_output_path]) with open(main_output_path, "rb") as fin: main_output_data = fin.read() return main_output_data
def test_consistent_output_conservative_PE(self): test_primary_specific_outfile = io.StringIO() test_secondary_specific_outfile = io.StringIO() test_primary_multi_outfile = io.StringIO() test_secondary_multi_outfile = io.StringIO() test_unassigned_outfile = io.StringIO() test_unresolved_outfile = io.StringIO() sam1 = io.TextIOWrapper(resource_stream(__name__, 'data/paired_end_testdata_human.sam')) sam2 = io.TextIOWrapper(resource_stream(__name__, 'data/paired_end_testdata_mouse.sam')) process_headers(sam1,sam2,primary_specific=test_primary_specific_outfile, secondary_specific=test_secondary_specific_outfile) cat_counts = conservative_main_paired_end(getReadPairs(sam1,sam2), primary_specific=test_primary_specific_outfile, secondary_specific=test_secondary_specific_outfile, primary_multi=test_primary_multi_outfile, secondary_multi=test_secondary_multi_outfile, unresolved=test_unresolved_outfile, unassigned=test_unassigned_outfile, ) self.assertEqual(cat_counts[('primary_specific', 'secondary_specific')]*4 +\ cat_counts[('unresolved', 'unresolved')]*4, len(test_unresolved_outfile.getvalue().split('\n'))-1) #this test is not exhastive. Only states in test data self.assertEqual(cat_counts[('primary_multi', 'primary_multi')]*2, len(test_primary_multi_outfile.getvalue().split('\n'))-1) self.assertEqual(cat_counts[('secondary_multi', 'secondary_multi')]*2, len(test_secondary_multi_outfile.getvalue().split('\n'))-1) self.assertEqual(cat_counts[('primary_specific', 'primary_specific')]*2 + cat_counts[('primary_specific', 'primary_multi')]*2 + cat_counts[('primary_multi', 'primary_specific')]*2, len(test_primary_specific_outfile.getvalue().split('\n'))-30) #29 lines of header in this file self.assertEqual(cat_counts[('secondary_specific', 'secondary_specific')]*2 + cat_counts[('secondary_specific', 'secondary_multi')]*2 + cat_counts[('secondary_multi', 'secondary_specific')]*2, len(test_secondary_specific_outfile.getvalue().split('\n'))-27) #26 lines of header in this file self.assertEqual(cat_counts[('unassigned', 'unassigned')]*2, len(test_unassigned_outfile.getvalue().split('\n'))-1) self.assertEqual(hashlib.sha224(test_primary_specific_outfile.getvalue().encode('latin-1')).hexdigest(),'8f5349ac96f194a4600bf0542cb1a6ebf71ada14b8ee0986598d7f58') sam1.close() sam2.close() pass
def test_process_headers(self): test_primary_specific_outfile = io.StringIO() test_secondary_specific_outfile = io.StringIO() test_primary_multi_outfile = io.StringIO() test_secondary_multi_outfile = io.StringIO() test_unassigned_outfile = io.StringIO() test_unresolved_outfile = io.StringIO() sam1 = io.TextIOWrapper(resource_stream(__name__, 'data/paired_end_testdata_human.sam')) sam2 = io.TextIOWrapper(resource_stream(__name__, 'data/paired_end_testdata_mouse.sam')) process_headers(sam1,sam2, primary_specific=test_primary_specific_outfile, secondary_specific=test_secondary_specific_outfile, primary_multi=test_primary_multi_outfile, secondary_multi=test_secondary_multi_outfile, unresolved=test_unresolved_outfile, unassigned=test_unassigned_outfile, ) self.assertEqual(len(test_primary_specific_outfile.getvalue()),695) self.assertEqual(len(test_secondary_specific_outfile.getvalue()),629) self.assertEqual(len(test_primary_multi_outfile.getvalue()),708) self.assertEqual(len(test_secondary_multi_outfile.getvalue()),642) self.assertEqual(len(test_unassigned_outfile.getvalue()),705) self.assertEqual(len(test_unresolved_outfile.getvalue()),705) sam1.close() sam2.close() pass
def cdr_gff3(out_fp, fname='ighv_aligned.fasta', gffname='ighv_aligned.gff3'): """ Calculate cysteine position in each sequence of the ighv alignment """ from .. import gff3 with resource_stream(_PKG, fname) as fp, resource_stream(_PKG, gffname) as gff_fp: sequences = ((name.split('|')[1], seq) for name, seq, _ in util.readfq(fp)) records = list(gff3.parse(gff_fp)) out_fp.write('##gff-version\t3\n') w = csv.writer(out_fp, delimiter='\t', quoting=csv.QUOTE_NONE, lineterminator='\n') for name, s in sequences: pl = _position_lookup(s) for feature in records: if feature.start0 not in pl and feature.end - 1 not in pl: continue start0 = pl.get(feature.start0, min(q for r, q in pl.items() if r >= feature.start0)) end = pl.get(feature.end - 1, max(q for r, q in pl.items() if r <= feature.end - 1)) f = feature._replace(seqid=name, start=start0 + 1, end=end + 1) w.writerow(f.update_attributes(ID=f.attribute_dict()['Name'] + '_' + name))
def process_saml_md_about_sps(saml_md: bytes): saml_md_tree = XML(saml_md) parser = XMLParser( remove_blank_text=True, resolve_entities=False, remove_comments=False) with resource_stream(__name__, REMOVE_NAMESPACE_PREFIXES_XSL_FILE_PATH) as \ xslt_root1_file: xslt_root1 = parse(xslt_root1_file, parser=parser) transform1 = XSLT(xslt_root1) saml_md_tree_1 = transform1(saml_md_tree) with resource_stream(__name__, REMOVE_KEY_WHITESPACE_XSL_FILE_PATH) as \ xslt_root2_file: xslt_root2 = parse(xslt_root2_file, parser=parser) transform2 = XSLT(xslt_root2) saml_md_2 = transform2(saml_md_tree_1) canonicalized_saml_md_2 = BytesIO() saml_md_2.write_c14n( canonicalized_saml_md_2, exclusive=True, with_comments=False) parser = XMLParser( remove_blank_text=True, resolve_entities=False, remove_comments=False) saml_md_tree_3 = XML(canonicalized_saml_md_2.getvalue(), parser).getroottree() return saml_md_tree_3
def main(): try_import("scipy") try_import("scipy.interpolate.rbf") try_import("Crypto") try_import("serial") try_import("PIL") try_import("numpy") try_import("zipimport") try_import("fluxclient") try_import("fluxclient.fcode") try_import("fluxclient.hw_profile") try_import("fluxclient.laser") try_import("fluxclient.printer") try_import("fluxclient.printer._printer") try_import("fluxclient.robot") try_import("fluxclient.scanner") try_import("fluxclient.scanner._scanner") try_import("fluxclient.upnp") sys.stdout.write("Open resource fluxclient::assets/flux3dp-icon.png ... ") sys.stdout.flush() try: import pkg_resources pkg_resources.resource_stream("fluxclient", "assets/flux3dp-icon.png") sys.stdout.write("OK\n") except Exception as e: sys.stdout.write("ERROR: %s" % e) sys.stdout.flush()
def testscheme_open(self, req): try: selector = req.get_selector() if selector == u'/ws_newcompass.asmx?WSDL': return urllib.addinfourl( pkg_resources.resource_stream(__name__, 'tests/testdata/wsdl.xml'), httplib.HTTPMessage(open('/dev/null')), req.get_full_url(), 200 ) elif selector == u'/ws_newcompass.asmx': soapResponse = urlparse.urlparse(req.get_header('Soapaction')).path.strip('"').split('/')[-1] + '.xml' return urllib.addinfourl( pkg_resources.resource_stream(__name__, 'tests/testdata/' + soapResponse), httplib.HTTPMessage(open('/dev/null')), req.get_full_url(), 200 ) elif selector == u'/biomuta.tsv': return urllib2.addinfourl( pkg_resources.resource_stream(__name__, 'tests/testdata/Biomuta.tsv'), httplib.HTTPMessage(open('/dev/null')), req.get_full_url(), 200 ) else: raise urllib2.URLError('Not found') except Exception: raise urllib2.URLError('Not found')
def main(): xls_file = resource_string('rdfconverters.resources', 'gaaps/xebrv7.xls') book = xlrd.open_workbook(file_contents=xls_file) xebr_concepts = XEBRV7Concepts(book) # RDF output graph xebr2xbrl = Graph() for key, ns in NS.items(): xebr2xbrl.bind(key, ns) rdf_converter = RDFConverter(xebr2xbrl, xebr_concepts) def map_gaap(clazz_mapper, f, clazz_labels, base): mapper = clazz_mapper(book) graph = Graph().parse(f, format='n3') labels = clazz_labels(graph, mapper).get_labels() rdf_converter.add_mappings_to_graph(mapper, labels, base) f = resource_stream('rdfconverters.resources', 'gaaps/it.n3') map_gaap(ITMapper, f, ITGaapLabels, 'http://www.dfki.de/lt/xbrl_it.owl#') f = resource_stream('rdfconverters.resources', 'gaaps/pgc.n3') map_gaap(ESMapper, f, ESGaapLabels, 'http://www.dfki.de/lt/xbrl_es.owl#') f = resource_stream('rdfconverters.resources', 'gaaps/be.n3') map_gaap(BEMapper, f, BEGaapLabels, 'http://www.dfki.de/lt/xbrl_be.owl#') util.write_graph(xebr2xbrl, '/tmp/xebr2xbrl.n3')
def test_AmpliconData_align_to_reference(self): forward_file = resource_stream(__name__, 'data/testdata_R1.fastq') reverse_file = resource_stream(__name__, 'data/testdata_R2.fastq') manifest = io.TextIOWrapper(resource_stream(__name__, 'data/testdatamanifest.txt')) self.amplicons.process_twofile_readpairs(forward_file, reverse_file) self.amplicons.add_references_from_manifest(manifest) self.amplicons.merge_overlaps() self.amplicons.reference = {'002b0ade8cda6a7bdc15f09ed5812126':('BRCA1_Exon9_UserDefined_(9825051)_7473614_chr17_41243841_41244065', 'chr17', '41243841', '41244065', '-'), 'randomfoo': ('MadeUp','chr99','1','100','+'),} self.amplicons.match_to_reference(min_score = 10, trim_primers=0, global_align=True) self.amplicons.align_to_reference() #print(str(sorted(list(self.amplicons.aligned.items())))) #print(str(sorted(list(self.amplicons.location.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.aligned.items())))).hexdigest(),'ffb3aa886476f377b485c9d967a844c7') self.assertEqual(md5(str(sorted(list(self.amplicons.location.items())))).hexdigest(),'cd58f75e64192227c8c2c6beca7564cf') self.amplicons.aligned = {} self.amplicons.location = {} self.amplicons.align_to_reference(global_align=False) #print(str(sorted(list(self.amplicons.aligned.items())))) #print(str(sorted(list(self.amplicons.location.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.aligned.items())))).hexdigest(),'0bec5d7ad6ca3e6f0a5f3bd366ab1c66') self.assertEqual(md5(str(sorted(list(self.amplicons.location.items())))).hexdigest(),'1058c4388032a963aec5a03679d5a899') pass
def get_schema(version): # type: (str) -> Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[unicode,Any], Loader] if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] cache = {} version = version.split("#")[-1].split(".")[0] for f in cwl_files: try: res = resource_stream(__name__, 'schemas/%s/%s' % (version, f)) cache["https://w3id.org/cwl/" + f] = res.read() res.close() except IOError: pass for f in salad_files: try: res = resource_stream( __name__, 'schemas/%s/salad/schema_salad/metaschema/%s' % (version, f)) cache["https://w3id.org/cwl/salad/schema_salad/metaschema/" + f] = res.read() res.close() except IOError: pass SCHEMA_CACHE[version] = schema_salad.schema.load_schema( "https://w3id.org/cwl/CommonWorkflowLanguage.yml", cache=cache) global SCHEMA_FILE, SCHEMA_ANY # pylint: disable=global-statement SCHEMA_FILE = SCHEMA_CACHE[version][3].idx["https://w3id.org/cwl/cwl#File"] SCHEMA_ANY = SCHEMA_CACHE[version][3].idx["https://w3id.org/cwl/salad#Any"] return SCHEMA_CACHE[version]
def read_table(self): self.tables = [] for i in range(1,6): cols = [1,2,3,4,5,6] if i==4: cols = [1,2,3,4,5] stream = resource_stream(__name__, self.basefilename + str(i) + '.dat') table = numpy.loadtxt(stream, #converters={0: self._CaLa_iso_converter}, usecols=cols ) stream.close() if i==4: table = numpy.hstack((table, numpy.nan * numpy.ones((len(table),1)))) self.tables.append(table) #print "Table " + str(i) #print table.shape #import newio #table = newio.loadtxt(self.basefilename + '6.dat') stream = resource_stream(__name__, self.basefilename + '6.dat') table = numpy.loadtxt(stream) stream.close() self.tables.append(table) self.data = numpy.vstack(self.tables[0:5]) self.remnant_data = self.tables[5] return self.tables
def _zone_apl(depcom): """Retrouve la zone APL (aide personnalisée au logement) de la commune en fonction du depcom (code INSEE).""" global zone_apl_by_depcom if zone_apl_by_depcom is None: with pkg_resources.resource_stream( openfisca_france.__name__, 'assets/apl/20110914_zonage.csv', ) as csv_file: csv_reader = csv.DictReader(csv_file) zone_apl_by_depcom = { # Keep only first char of Zonage column because of 1bis value considered equivalent to 1. row['CODGEO']: int(row['Zonage'][0]) for row in csv_reader } # Add subcommunes (arrondissements and communes associées), use the same value as their parent commune. with pkg_resources.resource_stream( openfisca_france.__name__, 'assets/apl/commune_depcom_by_subcommune_depcom.json', ) as json_file: commune_depcom_by_subcommune_depcom = json.load(json_file) for subcommune_depcom, commune_depcom in commune_depcom_by_subcommune_depcom.iteritems(): zone_apl_by_depcom[subcommune_depcom] = zone_apl_by_depcom[commune_depcom] default_value = 2 return fromiter( ( zone_apl_by_depcom.get(depcom_cell, default_value) for depcom_cell in depcom ), dtype = int16, )
def loadSecretome(portal): if portal._p_jar is not None and isinstance(portal._p_jar.db()._storage, DemoStorage): # Don't bother if we're just testing return try: resources = portal['resources'] except KeyError: resources = portal[portal.invokeFactory('Folder', 'resrouces')] resources.title = u'Resources' try: secretome = resources['secretome'] except KeyError: secretome = createContentInContainer(resources, 'eke.secretome.secretomefolder', title=u'Secretome') ids = secretome.keys() if len(ids) > 0: secretome.manage_delObjects(list(ids)) with pkg_resources.resource_stream(__name__, 'data/uniqueIDs.csv') as infile: rows = csv.DictReader(infile) for row in rows: probesetID, hgnc, timesMapped = row['hgu133plus2ID'], row['HGNC.symbol'], int(row['times.mapped.to']) databases = row['databases.foundin'].split(u'|') _logger.info('Creating probeset %s', probesetID) createContentInContainer( secretome, 'eke.secretome.probeset', title=probesetID, hgncSymbol=hgnc, databaseNames=databases, timesMapped=timesMapped ) transaction.commit() with pkg_resources.resource_stream(__name__, 'data/mappedIDs.csv') as infile: rows = csv.DictReader(infile) mappings = {} class Mapping(object): def __init__(self): self.databases, self.probesets = set(), set() for row in rows: database, gene, probesets = row['database'], row['beforemapping'], row['hgu133plus2.ID'] if probesets == 'none': probesets = set() else: probesets = set(probesets.split(u'|')) mapping = mappings.get(gene, Mapping()) mapping.databases.add(database) mapping.probesets = mapping.probesets | probesets mappings[gene] = mapping for gene, mapping in mappings.iteritems(): _logger.info('Creating gene/protein %s', gene) createContentInContainer( secretome, 'eke.secretome.geneprotein', title=gene, databaseNames=list(mapping.databases), probesetMappings=list(mapping.probesets) ) transaction.commit() _logger.info('Publishing everything') publish(resources) transaction.commit()
def get(self, filename=None): """return a static file""" if self.module is not None: try: fp = pkg_resources.resource_stream(self.module.import_name, os.path.join(self.module.config.static_folder, filename)) except IOError: raise werkzeug.exceptions.NotFound() config = self.module.config else: try: fp = pkg_resources.resource_stream(self.app.import_name, os.path.join(self.app.config.static_folder, filename)) except IOError: raise werkzeug.exceptions.NotFound() config = self.app.config mimetype = mimetypes.guess_type(filename)[0] if mimetype is None: mimetype = 'application/octet-stream' headers = Headers() data = wrap_file(self.request.environ, fp) rv = self.app.response_class(data, mimetype=mimetype, headers=headers, direct_passthrough=True) rv.cache_control.public = True cache_timeout = self.config.static_cache_timeout if cache_timeout is not None: rv.cache_control.max_age = cache_timeout rv.expires = int(time.time() + cache_timeout) return rv
def setUp(self): """Setup tests.""" from inspire.modules.workflows.receivers import precache_holdingpen_row from invenio_workflows.receivers import index_holdingpen_record from invenio_workflows.signals import workflow_halted, workflow_object_saved # Disable the holdingpen caching receiver. workflow_halted.disconnect(precache_holdingpen_row) workflow_object_saved.disconnect(index_holdingpen_record) self.create_registries() self.record_oai_arxiv_plots = pkg_resources.resource_string( "inspire.testsuite", os.path.join("workflows", "fixtures", "oai_arxiv_record_with_plots.xml") ) self.record_oai_arxiv_accept = pkg_resources.resource_string( "inspire.testsuite", os.path.join("workflows", "fixtures", "oai_arxiv_record_to_accept.xml") ) self.some_record = pkg_resources.resource_string( "inspire.testsuite", os.path.join("workflows", "fixtures", "some_record.xml") ) self.arxiv_tarball = pkg_resources.resource_stream( "inspire.testsuite", os.path.join("workflows", "fixtures", "1407.7587v1") ) self.arxiv_pdf = pkg_resources.resource_stream( "inspire.testsuite", os.path.join("workflows", "fixtures", "1407.7587v1.pdf") ) self.arxiv_tarball_accept = pkg_resources.resource_stream( "inspire.testsuite", os.path.join("workflows", "fixtures", "1511.01097") ) self.arxiv_pdf_accept = pkg_resources.resource_stream( "inspire.testsuite", os.path.join("workflows", "fixtures", "1511.01097v1.pdf") )
def get_schema(version): # type: (Text) -> Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[Text,Any], Loader] if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] cache = {} version = version.split("#")[-1] if '.dev' in version: version = ".".join(version.split(".")[:-1]) for f in cwl_files: try: res = resource_stream(__name__, 'schemas/%s/%s' % (version, f)) cache["https://w3id.org/cwl/" + f] = res.read() res.close() except IOError: pass for f in salad_files: try: res = resource_stream( __name__, 'schemas/%s/salad/schema_salad/metaschema/%s' % (version, f)) cache["https://w3id.org/cwl/salad/schema_salad/metaschema/" + f] = res.read() res.close() except IOError: pass SCHEMA_CACHE[version] = schema_salad.schema.load_schema( "https://w3id.org/cwl/CommonWorkflowLanguage.yml", cache=cache) return SCHEMA_CACHE[version]
def test_AmpliconData_match_to_reference(self): forward_file = resource_stream(__name__, 'data/testdata_R1.fastq') reverse_file = resource_stream(__name__, 'data/testdata_R2.fastq') manifest = io.TextIOWrapper(resource_stream(__name__, 'data/testdatamanifest.txt')) self.amplicons.process_twofile_readpairs(forward_file, reverse_file) self.amplicons.add_references_from_manifest(manifest) self.amplicons.merge_overlaps() self.amplicons.match_to_reference(min_score = 0.1, trim_primers=0, global_align=True) #print(str(sorted(list(self.amplicons.reference.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.reference.items())))).hexdigest(),'e08f4ac8d71a067547f43746972e86dc') self.amplicons.reference = {} self.amplicons.match_to_reference(min_score = 0.1, trim_primers=0, global_align=False) #print(str(sorted(list(self.amplicons.reference.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.reference.items())))).hexdigest(),'e08f4ac8d71a067547f43746972e86dc') self.amplicons.reference = {} self.amplicons.match_to_reference(min_score = 0.1, trim_primers=10, global_align=False) #print(str(sorted(list(self.amplicons.reference.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.reference.items())))).hexdigest(),'e08f4ac8d71a067547f43746972e86dc') self.amplicons.reference = {} self.amplicons.match_to_reference(min_score = 10, trim_primers=50, global_align=True) #print(str(sorted(list(self.amplicons.reference.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.reference.items())))).hexdigest(),'fbfdb58228dcdabe280e742a56127b91') self.amplicons.reference = {'002b0ade8cda6a7bdc15f09ed5812126':('BRCA1_Exon9_UserDefined_(9825051)_7473614_chr17_41243841_41244065', 'chr17', '41243841', '41244065', '-'), 'randomfoo': ('MadeUp','chr99','1','100','+'),} self.amplicons.match_to_reference(min_score = 10, trim_primers=0, global_align=True) #print(str(sorted(list(self.amplicons.reference.items())))) self.assertEqual(md5(str(sorted(list(self.amplicons.reference.items())))).hexdigest(),'a963bb251be3c27f4780b8292cdc3e13') pass
def testUncompressed(self): khepburn = self.view.findPath('//CineGuide/KHepburn') self.assert_(khepburn is not None) input = resource_stream('tests', 'data/khepltr.jpg') binary = khepburn.getAttributeAspect('picture', 'type').makeValue(None, mimetype='image/jpg') outputStream = binary.getOutputStream(compression=None) while True: data = input.read(1048576) if len(data) > 0: outputStream.write(data) else: break input.close() outputStream.close() khepburn.picture = binary self._reopenRepository() khepburn = self.view.findPath('//CineGuide/KHepburn') self.assert_(khepburn is not None) input = resource_stream('tests', 'data/khepltr.jpg') inputStream = khepburn.picture.getInputStream() data = input.read() picture = inputStream.read() input.close() inputStream.close() self.assert_(data == picture)
def test_save_hash_table(self): from ambivert import ambivert forward_file = resource_stream(__name__, 'data/testdata_R1.fastq') reverse_file = resource_stream(__name__, 'data/testdata_R2.fastq') manifest = io.TextIOWrapper(resource_stream(__name__, 'data/testdatamanifest.txt')) amplicons = ambivert.process_amplicon_data(forward_file, reverse_file, manifest=manifest, fasta=None, threshold=50, overlap=20, savehashtable=None, hashtable=None, ) outfile = NamedTemporaryFile(delete=False,mode='wb') outfilename = outfile.name amplicons.save_hash_table(outfile) amplicons.reference = {} amplicons.load_hash_table(open(outfilename,mode='rb')) self.assertEqual(md5(str(sorted(amplicons.reference))).hexdigest(),'c5e174ffd7ee8cfbd47e162667c83796') outfile.close() #test case where references dont match the hash table amplicons.reference_sequences = {} amplicons.load_hash_table(open(outfilename,mode='rb')) self.assertWarns(UserWarning) outfile.close() os.unlink(outfile.name) pass
def test_AmpliconData_print_to_sam(self): from ambivert import ambivert forward_file = resource_stream(__name__, 'data/testdata_R1.fastq') reverse_file = resource_stream(__name__, 'data/testdata_R2.fastq') manifest = io.TextIOWrapper(resource_stream(__name__, 'data/testdatamanifest.txt')) amplicons = ambivert.process_amplicon_data(forward_file, reverse_file, manifest=manifest, fasta=None, threshold=50, overlap=20, savehashtable=None, hashtable=None, ) outfile = io.StringIO() amplicons.aligned = {} amplicons.print_to_sam('32ff3ea55601305b5e8b3bd266c4080a',samfile=outfile) #print(outfile.getvalue()) self.assertEqual(md5(outfile.getvalue()).hexdigest(),'275aab1624c01fea59326e7889370fbd') outfile.close() outfile = io.StringIO() amplicons.printall_to_sam(samfile=outfile) #print(outfile.getvalue().split('\n')[:27]) #print(outfile.getvalue().split('\n')[27]) #print(outfile.getvalue().split('\n')[28:]) self.assertEqual(md5(repr(outfile.getvalue().split('\n')[:27])).hexdigest(),'5c5ab03009ec9d805ef09979e5bae3c4') self.assertEqual(outfile.getvalue().split('\n')[27],'@PG\tID:AmBiVErT\tVN:{0}'.format(__version__)) self.assertEqual(md5(repr(outfile.getvalue().split('\n')[28:])).hexdigest(),'462c330929084d6a88d51679318e37e2') #self.assertEqual(md5(outfile.getvalue()).hexdigest(),'0a879b5f8d996617f3f48da47ed18362') outfile.close()
def _get_tld_extractor(self): if self._extractor: return self._extractor cached_file = self.cache_file try: if self._extractor is None: with open(cached_file, "r") as f: jsonfile = f.readlines() self._extractor = _PublicSuffixListTLDExtractor(json.loads(jsonfile[0])) return self._extractor except IOError as ioe: file_not_found = ioe.errno == errno.ENOENT if not file_not_found: LOG.error("error reading TLD cache file %s: %s", cached_file, ioe) except Exception as ex: LOG.error("error reading TLD cache file %s: %s", cached_file, ex) if self.fetch: tld_sources = (_PublicSuffixListSource,) tlds = [tld for tld_source in tld_sources for tld in tld_source()] if not tlds: with pkg_resources.resource_stream(__name__, ".tld_set_snapshot") as snapshot_file: self._extractor = _PublicSuffixListTLDExtractor(pickle.load(snapshot_file)) return self._extractor LOG.info("computed TLDs: [%s, ...]", ", ".join(list(tlds)[:10])) if LOG.isEnabledFor(logging.DEBUG): import difflib with pkg_resources.resource_stream(__name__, ".tld_set_snapshot.json") as snapshot_file: snapshot = sorted(json.load(snapshot_file)) new = sorted(tlds) for line in difflib.unified_diff(snapshot, new, fromfile=".tld_set_snapshot.json", tofile=cached_file): print(line) try: with open(cached_file, "w") as f: data_to_dump = {} for tld in tlds: if tld.find(".") != -1: token = tld.split(".") tld_info = token[len(token) - 1] if tld_info in data_to_dump: list_tld = data_to_dump[tld_info] list_tld.append(tld) else: data_to_dump[tld_info] = [tld] else: data_to_dump[tld] = [tld] if sys.version.split(".")[0].split(".")[0] == "2": json.dump(data_to_dump, f, encoding="utf-8", ensure_ascii=False) if sys.version.split(".")[0].split(".")[0] == "3": json.dump(data_to_dump, f) except IOError as e: LOG.warn("unable to cache TLDs in file %s: %s", cached_file, e) self._extractor = _PublicSuffixListTLDExtractor(data_to_dump) return self._extractor
def test_AmpliconData_process_twofile_readpairs(self): forward_file = resource_stream(__name__, 'data/testdata_R1.fastq') reverse_file = resource_stream(__name__, 'data/testdata_R2.fastq') self.amplicons.process_twofile_readpairs(forward_file, reverse_file) #print(str(self.amplicons.data)) self.assertEqual(md5(str(sorted(self.amplicons.data))).hexdigest(),'37e10d52aa874a8fbdb40330e1294f02') pass
def _get_tld_extractor(self): if self._extractor: return self._extractor if self.cache_file: try: with open(self.cache_file, 'rb') as f: self._extractor = _PublicSuffixListTLDExtractor( pickle.load(f)) return self._extractor except IOError as ioe: file_not_found = ioe.errno == errno.ENOENT if not file_not_found: LOG.error( "error reading TLD cache file %s: %s", self.cache_file, ioe) except Exception as ex: LOG.error( "error reading TLD cache file %s: %s", self.cache_file, ex) tlds = frozenset() if self.suffix_list_urls: raw_suffix_list_data = fetch_file(self.suffix_list_urls) tlds = get_tlds_from_raw_suffix_list_data( raw_suffix_list_data, self.include_psl_private_domains) if not tlds: if self.fallback_to_snapshot: with closing(pkg_resources.resource_stream(__name__, '.tld_set_snapshot')) as snapshot_file: self._extractor = _PublicSuffixListTLDExtractor( pickle.load(snapshot_file)) return self._extractor else: raise Exception("tlds is empty, but fallback_to_snapshot is set" " to false. Cannot proceed without tlds.") LOG.info("computed TLDs: [%s, ...]", ', '.join(list(tlds)[:10])) if LOG.isEnabledFor(logging.DEBUG): import difflib with closing(pkg_resources.resource_stream(__name__, '.tld_set_snapshot')) as snapshot_file: snapshot = sorted(pickle.load(snapshot_file)) new = sorted(tlds) for line in difflib.unified_diff(snapshot, new, fromfile=".tld_set_snapshot", tofile=self.cache_file): if sys.version_info < (3,): sys.stderr.write(line.encode('utf-8') + "\n") else: sys.stderr.write(line + "\n") if self.cache_file: try: with open(self.cache_file, 'wb') as f: pickle.dump(tlds, f) except IOError as e: LOG.warn( "unable to cache TLDs in file %s: %s", self.cache_file, e) self._extractor = _PublicSuffixListTLDExtractor(tlds) return self._extractor
def findEntryPointPlugins(allPlugins): # look for plugins enabled via setuptools `entry_points` for entry_point in iter_entry_points(group='girder.plugin'): # set defaults allPlugins[entry_point.name] = { 'name': entry_point.name, 'description': '', 'version': '', 'dependencies': set() } configJson = os.path.join('girder', 'plugin.json') configYml = os.path.join('girder', 'plugin.yml') data = {} try: if pkg_resources.resource_exists(entry_point.name, configJson): with pkg_resources.resource_stream( entry_point.name, configJson) as conf: try: data = json.load(codecs.getreader('utf8')(conf)) except ValueError: _recordPluginFailureInfo( plugin=entry_point.name, traceback=traceback.format_exc()) logprint.exception( 'ERROR: Plugin "%s": plugin.json is not valid ' 'JSON.' % entry_point.name) elif pkg_resources.resource_exists(entry_point.name, configYml): with pkg_resources.resource_stream( entry_point.name, configYml) as conf: try: data = yaml.safe_load(conf) except yaml.YAMLError: _recordPluginFailureInfo( plugin=entry_point.name, traceback=traceback.format_exc()) logprint.exception( 'ERROR: Plugin "%s": plugin.yml is not valid ' 'YAML.' % entry_point.name) except (ImportError, SystemError): # Fall through and just try to load the entry point below. If # there is still an error, we'll log it there. pass if data == {}: try: data = getattr(entry_point.load(), 'config', {}) except (ImportError, SystemError): # If the plugin failed to load via entrypoint, but is in the # plugins directory, it may still load. We mark and report the # failure; if it loads later, the failure will be cleared, but # the report is still desired. _recordPluginFailureInfo( plugin=entry_point.name, traceback=traceback.format_exc()) logprint.exception( 'ERROR: Plugin "%s": could not be loaded by entrypoint.' % entry_point.name) continue allPlugins[entry_point.name].update(data) allPlugins[entry_point.name]['dependencies'] = set( allPlugins[entry_point.name]['dependencies'])
def create_binaries( self, binaries=[], zipmode=zipfile.ZIP_STORED, patch=None, compile_python=True, ): """Create binaries passed as argument (or all found in packages) :param binaries: a list of binaries to create or an empty list to create all binaries found in the packages. :param zipmode: a valid zipmode for the binary payload. By default use ZIP_STORED. :param patch: a file name where monkey patching for libraries live :param compile_python: a boolean which indicates if .py files should be compiled and added to the binary payload. """ with directory() as tmp: # first install the required package in temporary directory self.install_packages(tmp) # extract base library with pkg_resources.resource_stream( 'pyload.resources', 'xlib.zip' ) as src_stream: with zipfile.ZipFile(src_stream) as z: z.extractall(tmp) # for any script defined in project(s) for mod, fun, name in self.scripts(tmp, binaries): out_file = os.path.join(self.out_dir, name) if not os.path.exists(self.out_dir): os.makedirs(self.out_dir) with tempfile.TemporaryFile() as f: with zipfile.PyZipFile(f, 'a', zipmode) as lib: self.add_libraries(tmp, lib, compile_python) self.add_mainpy(lib, mod, fun, patch=patch) # rewind stream to read from the beginning f.seek(0) # add pyload core with pkg_resources.resource_stream( 'pyload.resources', 'pyload' ) as src_stream: with open(out_file, 'wb') as target_stream: shutil.copyfileobj(src_stream, target_stream) # fill binary with zip payload with open(out_file, 'ab') as d: d.write(f.read()) os.chmod(out_file, 511) # 0o777 print("Successfully create binary %s" % (out_file,))
def __init__(self): QDialog.__init__(self) self.__mapWidgetUi = Ui_MapWidget() self.__mapWidgetUi.setupUi(self) self.__view = self.__mapWidgetUi.graphicsView self.__view.setObjectName("ACS_Map_View") self.__scene = QGraphicsScene() self.__view.setScene(self.__scene) self.__current_lat = 35.720428 self.__current_lon = -120.769924 self.__current_ground_width = 41000000. #meters(start w/ view of whole earth) #TODO: don't hard code location self.__tiler = acs_map_tiler.ACS_MapTiler(35.720428, -120.769924) self.__current_detail_layer = 0 #corresponds to "zoom" in map_tiler module self.__detail_layers = [] self.__rect_tiles = OrderedDict() #see rectKey method for how key works #detail layers are various levels of raster detail for the map. #0 is lowest level of detail, 20 highest. 0 loads fast and the #entire world can fit on a single tile. 20 loads slow, and it is unwise #to try to show the entire world due to the number of tiles required #(higher numbered detail layers are intended for "zooming in") self.setupDetailLayers() self.__plane_layer = QGraphicsItemGroup() self.__scene.addItem(self.__plane_layer) self.__plane_icons = {} self.__plane_labels = {} self.__plane_icon_pixmaps = {} img_bytes = pkg_resources.resource_stream("acs_dashboards", "data/images/flyingWingTiny.png").read() self.__plane_icon_pixmaps[0] = QPixmap() self.__plane_icon_pixmaps[0].loadFromData(img_bytes) img_bytes2 = pkg_resources.resource_stream("acs_dashboards", "data/images/flyingWingTiny2.png").read() self.__plane_icon_pixmaps[1] = QPixmap() self.__plane_icon_pixmaps[1].loadFromData(img_bytes2) #for drawing waypoints: self.__mission_layer = QGraphicsItemGroup() self.__scene.addItem(self.__mission_layer) self.__wp_diameter = 0.0001 self.__prev_drawn_nav_wp = None #needed when drawing lines between wps self.__wp_loiter_radius = None #NOTE: this is in meters; the map itself #is in degrees #for drawing fence: self.__fence_layer = QGraphicsItemGroup() self.__scene.addItem(self.__fence_layer) #slots self.__view.just_zoomed.connect(self.onZoom) self.__mapWidgetUi.zoom_sb.valueChanged.connect(self.onZoomSBValueChanged) self.__view.just_panned.connect(self.onPan)
def test_add_references_from_manifest_and_fasta(self): manifest = io.TextIOWrapper(resource_stream(__name__, 'data/testdatamanifest.txt')) self.amplicons.add_references_from_manifest(manifest) self.assertEqual(md5(str(sorted(self.amplicons.reference_sequences.items()))).hexdigest(),'c899019b1ac4ccdb6e3ebc19f40add63') fasta = io.TextIOWrapper(resource_stream(__name__, 'data/testdatareferences.fasta')) self.amplicons.reference_sequences = {} self.amplicons.add_references_from_fasta(fasta) self.assertEqual(md5(str(sorted(self.amplicons.reference_sequences.items()))).hexdigest(),'fc3c6701032dbd84c6f5731d344df060') pass
def get_test_case(name): """Returns a test case file object by name. This works even if we're in a zip file! """ expected_name = re.sub(r"\.txt", ".expected.txt", name) case = pkg_resources.resource_stream("tbget.tests", name) expected = pkg_resources.resource_stream("tbget.tests", expected_name) return case, expected
def setUp(self): self.keys = {} for key, cert in KEY_FILE_PAIRS: cdict = rsa_pem.parse(pkg_resources.resource_stream(__name__, key).read()) t = rsa_pem.dict_to_tuple(cdict) self.keys[key] = RSA.construct(t) cdict = x509_pem.parse(pkg_resources.resource_stream(__name__, cert).read()) t = x509_pem.dict_to_tuple(cdict) self.keys[cert] = RSA.construct(t)
def load_sql(file): return ''.join(x for x in TextIOWrapper(resource_stream('CGRdb.sql', file)) if not x.startswith(('#', '/*', '*/', '\n'))).replace( '$', '$$')
from base64 import encodestring from pkg_resources import resource_stream from itertools import product import numpy as np from IPython.core.display import HTML from probability import WeightedBox, RandomVariable height, width = 120, 120 host_html = ('<img height="%d" width="%d" src="data:image/png;base64,%s">' % (height, width, encodestring(resource_stream('stats_lectures', \ 'data/host.png').read()).decode('ascii'))) student_html = ('<img height="%d" width="%d" src="data:image/png;base64,%s"/>' % (height, width, encodestring(resource_stream('stats_lectures', \ 'data/user-student.png').read()).decode('ascii'))) student_win_html = ('<img height="%d" width="%d" src="data:image/png;base64,%s"/>' % (height, width, encodestring(resource_stream('stats_lectures', \ 'data/student_win.png').read()).decode('ascii'))) student_lose_html = ('<img height="%d" width="%d" src="data:image/png;base64,%s"/>' % (height, width,
def qc_fasta(arg_sequence, check_with_mimimap2=True): log.debug("Starting qc_fasta") schema_resource = pkg_resources.resource_stream(__name__, "validation/formats") with tempfile.NamedTemporaryFile() as tmp: tmp.write(schema_resource.read()) tmp.flush() val = magic.Magic(magic_file=tmp.name, uncompress=False, mime=True) gz = "" if arg_sequence.name.endswith(".gz"): sequence = gzip.GzipFile(fileobj=arg_sequence, mode='rb') gz = ".gz" else: sequence = arg_sequence sequence = io.TextIOWrapper(sequence) r = sequence.read(4096) sequence.seek(0) seqlabel = r[1:r.index("\n")] seq_type = val.from_buffer(r).lower() if seq_type == "text/fasta": # ensure that contains only one entry submitlabel, submitseq = read_fasta(sequence) sequence.seek(0) sequence.detach() if check_with_mimimap2: with tempfile.NamedTemporaryFile() as tmp1: with tempfile.NamedTemporaryFile() as tmp2: refstring = pkg_resources.resource_string(__name__, "SARS-CoV-2-reference.fasta") tmp1.write(refstring) tmp1.flush() tmp2.write(submitlabel.encode("utf8")) tmp2.write(("".join(submitseq)).encode("utf8")) tmp2.flush() similarity = 0 try: log.debug("Trying to run minimap2") cmd = ["minimap2", "-c", "-x", "asm20", tmp1.name, tmp2.name] logging.info("QC checking similarity to reference") logging.info(" ".join(cmd)) result = subprocess.run(cmd, stdout=subprocess.PIPE) result.check_returncode() res = result.stdout.decode("utf-8") mm = res.split("\t") if len(mm) >= 10: # divide Number of matching bases in the mapping / Target sequence length similarity = (float(mm[9]) / float(mm[6])) * 100.0 else: similarity = 0 except Exception as e: logging.warn("QC against reference sequence using 'minimap2': %s", e, exc_info=e) if similarity < 70.0: raise ValueError( f"QC fail for {seqlabel}: alignment to reference was less than 70% (was {similarity})") return "sequence.fasta" + gz, seqlabel, seq_type elif seq_type == "text/fastq": sequence.seek(0) sequence.detach() return "reads.fastq" + gz, seqlabel, seq_type else: log.debug(seqlabel) log.debug(seq_type) raise ValueError("Sequence file ({}) does not look like a DNA FASTA or FASTQ".format(arg_sequence))
def get_schematrons(cls): with resource_stream( __name__, "xml/gemini2/gemini2-schematron-20110906-v1.2.sch") as schema: return [cls.schematron(schema)]
__author__ = 'marcusmorgenstern' __mail__ = '' import pickle import re from abc import ABCMeta, abstractmethod from copy import deepcopy, copy from numbers import Number from numpy import sqrt from pkg_resources import resource_stream from pyfluka.base import IllegalArgumentError from pyfluka.utils import ureg f = resource_stream(__name__, "../data/periodic_table.p") _periodic_table = pickle.load(f) f.close() class AbsPhysicsQuantity(object): __metaclass__ = ABCMeta @abstractmethod def __init__(self, val, unc, unit, symbol=None): if issubclass(type(val), AbsPhysicsQuantity): self.val = copy(val.val) self.unc = copy(val.unc) elif isinstance(val, str) and unc == 0: self.val = ureg.Quantity(val) self.unc = ureg.Quantity(0., self.val.units) elif not isinstance(val, ureg.Quantity): self.val = val * ureg.Quantity(unit)
def test_x509_parse(self): self.assertEqual(top.x509_parse, x509_pem.parse) data = pkg_resources.resource_stream(__name__, KEY_FILE_PAIRS[0][1]).read() x509_dict = top.parse(data) self.assertTrue(x509_dict)
def data_stream(name): return resource_stream(__name__, 'test_weblogo/data/' + name)
unit_type_dict = {} translations = {} resource_package = __name__ directory = os.fsencode( pkg_resources.resource_filename(resource_package, "dicts")) for f in os.listdir(directory): file_name = "dicts/" + os.fsdecode(f) new_dict, f_type = read_dictionary(file_name) if f_type == "translation_file": translations.update(new_dict) elif f_type == "mapping_file": unit_type_dict.update(new_dict) elif f_type == "definitions_file": try: with closing(pkg_resources.resource_stream(__name__, file_name)) as f: rbytes = f.read() load_user_definitions(StringIO(rbytes.decode('utf-8'))) except Exception as e: msg = getattr(e, 'message', '') or str(e) raise ValueError("While opening {}\n{}".format(f, msg)) def convert_attr_to_units(attr): """ Calculate unit conversions if passed a convertible attribute.""" # get all preferred units that have not yet been derived from this attribute to_units_URIs = attr_incomplete_preferred_units(attr) if not to_units_URIs: return None # pull important values off attribute meas_type = attr_type_URI(attr)
def test_rsa_parse(self): self.assertEqual(top.rsa_parse, rsa_pem.parse) data = pkg_resources.resource_stream(__name__, KEY_FILE_PAIRS[0][0]).read() rsa_dict = top.parse(data) self.assertTrue(rsa_dict)
def datafile(name, sep='\t'): "Read key,value pairs from file." fp = resource_stream(__name__, join('data', name)) for line in fp: yield line.decode(encoding='latin-1').strip().split(sep)
def main(argv=None): if argv is None: argv = sys.argv logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser( description= """Read ham log and output GIS data for callsigns worked. Output files will be prefixed with output path. E.g. given "foo/bar", the following files will be created: "foo/bar_points.geojson", "foo/bar_lines.geojson", and "foo/bar.kml" """) parser.add_argument('infile', type=str, help='Input log file (ADIF or Cabrillo)') parser.add_argument('outpath', type=str, help='Output path prefix') parser.add_argument('-c', '--cfg', type=str, help='Config file path', default=os.path.join(os.environ['HOME'], '.geologrc')) parser.add_argument('-v', '--verbose', type=bool, help='Turn on additional output', default=False) args = parser.parse_args(argv[1:]) cfg = ConfigParser.SafeConfigParser() cfg.read(args.cfg) try: un = cfg.get('qrz', 'username') except ConfigParser.Error: un = raw_input("QRZ.com user name:") try: pw = cfg.get('qrz', 'password') except ConfigParser.Error: pw = raw_input("QRZ.com password (not stored):") try: cachepath = cfg.get('qrz', 'cachepath') except ConfigParser.Error: cachepath = CACHEPATH try: cachepath = cfg.get('qrz', 'cachepath') except ConfigParser.Error: cachepath = CACHEPATH try: ctydatpath = cfg.get('geolog', 'cachepath') ctydatflo = open(ctydatpath) except ConfigParser.Error: ctydatflo = resource_stream(__name__, "ctydat/cty.dat") log.info("QRZ cache: %s" % cachepath) geolog(args.infile, args.outpath, un, pw, cachepath, ctydatflo) return 0
def get_stream(self, resource_name): path = self.get_path(resource_name) if pkg_resources.resource_exists(self.pkg_name, path): return pkg_resources.resource_stream(self.pkg_name, path)
def main( argsl=None, # type: List[str] args=None, # type: argparse.Namespace executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]] makeTool=workflow.defaultMakeTool, # type: Callable[..., Process] selectResources=None, # type: Callable[[Dict[Text, int]], Dict[Text, int]] stdin=sys.stdin, # type: IO[Any] stdout=sys.stdout, # type: IO[Any] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] job_order_object=None, # type: MutableMapping[Text, Any] make_fs_access=StdFsAccess, # type: Callable[[Text], StdFsAccess] fetcher_constructor=None, # type: FetcherConstructorType resolver=tool_resolver, logger_handler=None, custom_schema_callback=None # type: Callable[[], None] ): # type: (...) -> int _logger.removeHandler(defaultStreamHandler) if logger_handler: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) try: if args is None: if argsl is None: argsl = sys.argv[1:] args = arg_parser().parse_args(argsl) # If On windows platform, A default Docker Container is Used if not explicitely provided by user if onWindows() and not args.default_container: # This docker image is a minimal alpine image with bash installed(size 6 mb). source: https://github.com/frol/docker-alpine-bash args.default_container = windows_default_container_id # If caller provided custom arguments, it may be not every expected # option is set, so fill in no-op defaults to avoid crashing when # dereferencing them in args. for k, v in six.iteritems({ 'print_deps': False, 'print_pre': False, 'print_rdf': False, 'print_dot': False, 'relative_deps': False, 'tmp_outdir_prefix': 'tmp', 'tmpdir_prefix': 'tmp', 'print_input_deps': False, 'cachedir': None, 'quiet': False, 'debug': False, 'timestamps': False, 'js_console': False, 'version': False, 'enable_dev': False, 'enable_ext': False, 'strict': True, 'skip_schemas': False, 'rdf_serializer': None, 'basedir': None, 'tool_help': False, 'workflow': None, 'job_order': None, 'pack': False, 'on_error': 'continue', 'relax_path_checks': False, 'validate': False, 'enable_ga4gh_tool_registry': False, 'ga4gh_tool_registries': [], 'find_default_container': None, 'make_template': False, 'overrides': None }): if not hasattr(args, k): setattr(args, k, v) if args.quiet: _logger.setLevel(logging.WARN) if args.debug: _logger.setLevel(logging.DEBUG) if args.timestamps: formatter = logging.Formatter("[%(asctime)s] %(message)s", "%Y-%m-%d %H:%M:%S") stderr_handler.setFormatter(formatter) if args.version: print(versionfunc()) return 0 else: _logger.info(versionfunc()) if args.print_supported_versions: print("\n".join(supportedCWLversions(args.enable_dev))) return 0 if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error( "CWL document required, no input file was provided") arg_parser().print_help() return 1 if args.relax_path_checks: command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries if not args.enable_ga4gh_tool_registry: del ga4gh_tool_registries[:] if custom_schema_callback: custom_schema_callback() elif args.enable_ext: res = pkg_resources.resource_stream(__name__, 'extensions.yml') use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) res.close() else: use_standard_schema("v1.0") uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=resolver, fetcher_constructor=fetcher_constructor) overrides = [] # type: List[Dict[Text, Any]] try: job_order_object, input_basedir, jobloader = load_job_order( args, stdin, fetcher_constructor, overrides, tool_file_uri) except Exception as e: _logger.error(Text(e), exc_info=args.debug) if args.overrides: overrides.extend( load_overrides(file_uri(os.path.abspath(args.overrides)), tool_file_uri)) try: document_loader, workflowobj, uri = fetch_document( uri, resolver=resolver, fetcher_constructor=fetcher_constructor) if args.print_deps: printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri) return 0 document_loader, avsc_names, processobj, metadata, uri \ = validate_document(document_loader, workflowobj, uri, enable_dev=args.enable_dev, strict=args.strict, preprocess_only=args.print_pre or args.pack, fetcher_constructor=fetcher_constructor, skip_schemas=args.skip_schemas, overrides=overrides) if args.print_pre: stdout.write(json.dumps(processobj, indent=4)) return 0 overrides.extend(metadata.get("cwltool:overrides", [])) conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) # Text use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # Text make_tool_kwds = vars(args) job_script_provider = None # type: Callable[[Any, List[str]], Text] if conf_file or use_conda_dependencies: dependencies_configuration = DependenciesConfiguration( args) # type: DependenciesConfiguration make_tool_kwds[ "job_script_provider"] = dependencies_configuration make_tool_kwds["find_default_container"] = functools.partial( find_default_container, args) make_tool_kwds["overrides"] = overrides tool = make_tool(document_loader, avsc_names, metadata, uri, makeTool, make_tool_kwds) if args.make_template: yaml.safe_dump(generate_input_template(tool), sys.stdout, default_flow_style=False, indent=4, block_seq_indent=2) return 0 if args.validate: _logger.info("Tool definition is valid") return 0 if args.pack: stdout.write( print_pack(document_loader, processobj, uri, metadata)) return 0 if args.print_rdf: stdout.write( printrdf(tool, document_loader.ctx, args.rdf_serializer)) return 0 if args.print_dot: printdot(tool, document_loader.ctx, stdout) return 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) return 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s", ", try again with --debug for more information.\nThe error was: " "%s" % exc if not args.debug else ". The error was:", exc_info=args.debug) return 1 if isinstance(tool, int): return tool # If on MacOS platform, TMPDIR must be set to be under one of the shared volumes in Docker for Mac # More info: https://dockstore.org/docs/faq if sys.platform == "darwin": tmp_prefix = "tmp_outdir_prefix" default_mac_path = "/private/tmp/docker_tmp" if getattr(args, tmp_prefix) and getattr( args, tmp_prefix) == DEFAULT_TMP_PREFIX: setattr(args, tmp_prefix, default_mac_path) for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(args, dirprefix) and getattr( args, dirprefix) != DEFAULT_TMP_PREFIX: sl = "/" if getattr(args, dirprefix).endswith( "/") or dirprefix == "cachedir" else "" setattr(args, dirprefix, os.path.abspath(getattr(args, dirprefix)) + sl) if not os.path.exists(os.path.dirname(getattr(args, dirprefix))): try: os.makedirs(os.path.dirname(getattr(args, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) return 1 if args.cachedir: if args.move_outputs == "move": setattr(args, 'move_outputs', "copy") setattr(args, "tmp_outdir_prefix", args.cachedir) secret_store = SecretStore() try: job_order_object = init_job_order( job_order_object, args, tool, print_input_deps=args.print_input_deps, relative_deps=args.relative_deps, stdout=stdout, make_fs_access=make_fs_access, loader=jobloader, input_basedir=input_basedir, secret_store=secret_store) except SystemExit as e: return e.code if not executor: if args.parallel: executor = MultithreadedJobExecutor() else: executor = SingleJobExecutor() if isinstance(job_order_object, int): return job_order_object try: setattr(args, 'basedir', input_basedir) del args.workflow del args.job_order (out, status) = executor(tool, job_order_object, logger=_logger, makeTool=makeTool, select_resources=selectResources, make_fs_access=make_fs_access, secret_store=secret_store, **vars(args)) # This is the workflow output, it needs to be written if out is not None: def locToPath(p): for field in ("path", "nameext", "nameroot", "dirname"): if field in p: del p[field] if p["location"].startswith("file://"): p["path"] = uri_file_path(p["location"]) visit_class(out, ("File", "Directory"), locToPath) # Unsetting the Generation fron final output object visit_class(out, ("File", ), MutationManager().unset_generation) if isinstance(out, six.string_types): stdout.write(out) else: stdout.write(json.dumps(out, indent=4)) stdout.write("\n") stdout.flush() if status != "success": _logger.warning(u"Final process status is %s", status) return 1 else: _logger.info(u"Final process status is %s", status) return 0 except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) return 1 except UnsupportedRequirement as exc: _logger.error(u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) return 33 except WorkflowException as exc: _logger.error( u"Workflow error, try again with --debug for more " "information:\n%s", strip_dup_lineno(six.text_type(exc)), exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"Unhandled error, try again with --debug for more information:\n" " %s", exc, exc_info=args.debug) return 1 finally: _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler)
def serve_openapi_yaml(): stream = pkg_resources.resource_stream('slivka', 'data/openapi-docs/openapi.yaml') return flask.send_file(stream, 'application/yaml', as_attachment=False)
def load_resource(filename): with pkg_resources.resource_stream(__name__, filename) as fd: return fd.read().decode("utf-8")
def test_parse_steria_cert(self): haka = pkg_resources.resource_stream(__name__, 'keys/steria.pem').read() self.assertTrue(haka) cdict = x509_pem.parse(haka) self.assertTrue(cdict)
def main(args=None): """ This is the main function called by the `ivelplot` script. """ from astropy.utils.compat import argparse from astropy.extern.configobj import configobj, validate from pkg_resources import resource_stream parser = argparse.ArgumentParser( description='An interactive environment for absorption line ' 'identification and Voigt profile \nfitting with VPFIT.\n' '\nTo dump a default configuration file: ivelplot -d' '\nTo dump an extended default configuration file: ' 'ivelplot -dd', formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('config', help='path to the configuration file') parser.add_argument('-z', '--redshift', help='redshift') parser.add_argument('--search', action='store_true', help='display a general search list of ions') parser.add_argument('--lyman', action='store_true', help='display the Lyman series transitions') parser.add_argument('--galactic', action='store_true', help='display the common Galactic transitions') parser.add_argument('--agn', action='store_true', help='display the common AGN associated transitions') config = resource_stream(__name__, '/config/ivelplot.cfg') config_extended = resource_stream( __name__, '/config/ivelplot_extended.cfg') spec = resource_stream(__name__, '/config/ivelplot_specification.cfg') if len(sys.argv) > 1: if sys.argv[1] == '-d': cfg = configobj.ConfigObj(config) cfg.filename = '{0}/ivelplot.cfg'.format(os.getcwd()) cfg.write() return elif sys.argv[1] == '-dd': cfg = configobj.ConfigObj(config_extended) cfg.filename = '{0}/ivelplot.cfg'.format(os.getcwd()) cfg.write() return args = parser.parse_args(args) try: cfg = configobj.ConfigObj(args.config, configspec=spec) validator = validate.Validator() cfg.validate(validator) except: raise IOError('Configuration file could not be read') fname = cfg['WINDOW'].pop('transitions') if args.search: fh = resource_stream(__name__, '/data/search.dat') transitions = list(fh) fh.close() elif args.lyman: fh = resource_stream(__name__, '/data/lyman.dat') transitions = list(fh) fh.close() elif args.galactic: fh = resource_stream(__name__, '/data/galactic.dat') transitions = list(fh) fh.close() elif args.agn: fh = resource_stream(__name__, '/data/agn.dat') transitions = list(fh) fh.close() else: print('Reading transitions from ', fname) fh = open(fname) transitions = list(fh) fh.close() transitions = [transition for transition in transitions if not transition.startswith('#')] fname = cfg['DATA'].pop('filename') if not fname: raise IOError('no data to plot!') spectrum = Table.read(fname) if fname[-4:] == 'fits' else ascii.read(fname) wavelength = spectrum[cfg['DATA'].pop('wavelength_column')] flux = spectrum[cfg['DATA'].pop('flux_column')] error = spectrum[cfg['DATA'].pop('error_column')] continuum = spectrum[cfg['DATA'].pop('continuum_column')] redshift = float(args.redshift) if args.redshift is not None else 0 cfg['MODEL']['system_width'] = (cfg['WINDOW']['vmax'] - cfg['WINDOW']['vmin']) cfg['MODEL']['absorbers'] = None print(info) app = QApplication(sys.argv) app.aboutToQuit.connect(app.deleteLater) desktop = app.desktop() screen = desktop.screenGeometry() width = screen.width() / desktop.physicalDpiX() * 0.88 fontsize = 0.7 * width label_fontsize = 0.6 * width cfg['WINDOW']['width'] = width cfg['WINDOW']['fontsize'] = fontsize cfg['WINDOW']['label_fontsize'] = label_fontsize velocity_plot = InteractiveVelocityPlot( fname, transitions, wavelength, flux, error, continuum, redshift, **cfg) velocity_plot.window.show() output_stream = OutputStream() output_stream.text_written.connect(velocity_plot.on_output) sys.stdout = output_stream sys.exit(app.exec_())
def new_js_proc(): # type: () -> subprocess.Popen res = resource_stream(__name__, 'cwlNodeEngine.js') nodecode = res.read() nodejs = None trynodes = ("nodejs", "node") for n in trynodes: try: if subprocess.check_output( [n, "--eval", "process.stdout.write('t')"]) != "t": continue nodejs = subprocess.Popen([n, "--eval", nodecode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) break except subprocess.CalledProcessError: pass except OSError as e: if e.errno == errno.ENOENT: pass else: raise if nodejs is None: try: nodeimg = "node:slim" global have_node_slim if not have_node_slim: dockerimgs = subprocess.check_output( ["docker", "images", nodeimg]) if len(dockerimgs.split("\n")) <= 1: nodejsimg = subprocess.check_output( ["docker", "pull", nodeimg]) _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg) have_node_slim = True nodejs = subprocess.Popen([ "docker", "run", "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", "--sig-proxy=true", "--interactive", "--rm", nodeimg, "node", "--eval", nodecode ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError as e: if e.errno == errno.ENOENT: pass else: raise except subprocess.CalledProcessError: pass if nodejs is None: raise JavascriptException( u"cwltool requires Node.js engine to evaluate Javascript " "expressions, but couldn't find it. Tried %s, docker run " "node:slim" % u", ".join(trynodes)) return nodejs
def load(): path = 'etc/hook.yaml' with resource_stream(catchbot.__name__, path) as f: return yaml.load(f)
def main( argsl=None, # type: List[str] args=None, # type: argparse.Namespace job_order_object=None, # type: MutableMapping[Text, Any] stdin=sys.stdin, # type: IO[Any] stdout=None, # type: Union[TextIO, codecs.StreamWriter] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] logger_handler=None, # custom_schema_callback=None, # type: Callable[[], None] executor=None, # type: Callable[..., Tuple[Dict[Text, Any], Text]] loadingContext=None, # type: LoadingContext runtimeContext=None # type: RuntimeContext ): # type: (...) -> int if not stdout: # force UTF-8 even if the console is configured differently if (hasattr(sys.stdout, "encoding") # type: ignore and sys.stdout.encoding != 'UTF-8'): # type: ignore if six.PY3 and hasattr(sys.stdout, "detach"): stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') else: stdout = codecs.getwriter('utf-8')(sys.stdout) # type: ignore else: stdout = cast(TextIO, sys.stdout) # type: ignore _logger.removeHandler(defaultStreamHandler) if logger_handler: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) # pre-declared for finally block workflowobj = None input_for_prov = None try: if args is None: if argsl is None: argsl = sys.argv[1:] args = arg_parser().parse_args(argsl) if runtimeContext is None: runtimeContext = RuntimeContext(vars(args)) else: runtimeContext = runtimeContext.copy() # If on Windows platform, a default Docker Container is used if not # explicitely provided by user if onWindows() and not runtimeContext.default_container: # This docker image is a minimal alpine image with bash installed # (size 6 mb). source: https://github.com/frol/docker-alpine-bash runtimeContext.default_container = windows_default_container_id # If caller parsed its own arguments, it may not include every # cwltool option, so fill in defaults to avoid crashing when # dereferencing them in args. for key, val in six.iteritems(get_default_args()): if not hasattr(args, key): setattr(args, key, val) rdflib_logger = logging.getLogger("rdflib.term") rdflib_logger.addHandler(stderr_handler) rdflib_logger.setLevel(logging.ERROR) if args.quiet: _logger.setLevel(logging.WARN) if runtimeContext.debug: _logger.setLevel(logging.DEBUG) rdflib_logger.setLevel(logging.DEBUG) if args.timestamps: formatter = logging.Formatter("[%(asctime)s] %(message)s", "%Y-%m-%d %H:%M:%S") stderr_handler.setFormatter(formatter) if args.version: print(versionfunc()) return 0 else: _logger.info(versionfunc()) if args.print_supported_versions: print("\n".join(supportedCWLversions(args.enable_dev))) return 0 if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error( "CWL document required, no input file was provided") arg_parser().print_help() return 1 if args.relax_path_checks: command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE if args.ga4gh_tool_registries: ga4gh_tool_registries[:] = args.ga4gh_tool_registries if not args.enable_ga4gh_tool_registry: del ga4gh_tool_registries[:] if custom_schema_callback: custom_schema_callback() elif args.enable_ext: res = pkg_resources.resource_stream(__name__, 'extensions.yml') use_custom_schema("v1.0", "http://commonwl.org/cwltool", res.read()) res.close() else: use_standard_schema("v1.0") #call function from provenance.py if the provenance flag is enabled. if args.provenance: if not args.compute_checksum: _logger.error( "--provenance incompatible with --no-compute-checksum") return 1 runtimeContext.research_obj = ResearchObject( temp_prefix_ro=args.tmpdir_prefix, # Optionals, might be None orcid=args.orcid, full_name=args.cwl_full_name) if loadingContext is None: loadingContext = LoadingContext(vars(args)) else: loadingContext = loadingContext.copy() loadingContext.research_obj = runtimeContext.research_obj loadingContext.disable_js_validation = \ args.disable_js_validation or (not args.do_validate) loadingContext.construct_tool_object = getdefault( loadingContext.construct_tool_object, workflow.default_make_tool) loadingContext.resolver = getdefault(loadingContext.resolver, tool_resolver) uri, tool_file_uri = resolve_tool_uri( args.workflow, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) try_again_msg = "" if args.debug else ", try again with --debug for more information" try: job_order_object, input_basedir, jobloader = load_job_order( args, stdin, loadingContext.fetcher_constructor, loadingContext.overrides_list, tool_file_uri) if args.overrides: loadingContext.overrides_list.extend( load_overrides(file_uri(os.path.abspath(args.overrides)), tool_file_uri)) document_loader, workflowobj, uri = fetch_document( uri, resolver=loadingContext.resolver, fetcher_constructor=loadingContext.fetcher_constructor) if args.print_deps: printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri) return 0 document_loader, avsc_names, processobj, metadata, uri \ = validate_document(document_loader, workflowobj, uri, enable_dev=loadingContext.enable_dev, strict=loadingContext.strict, preprocess_only=(args.print_pre or args.pack), fetcher_constructor=loadingContext.fetcher_constructor, skip_schemas=args.skip_schemas, overrides=loadingContext.overrides_list, do_validate=loadingContext.do_validate) if args.pack: stdout.write( print_pack(document_loader, processobj, uri, metadata)) return 0 if args.provenance and runtimeContext.research_obj: # Can't really be combined with args.pack at same time runtimeContext.research_obj.packed_workflow( print_pack(document_loader, processobj, uri, metadata)) if args.print_pre: stdout.write(json_dumps(processobj, indent=4)) return 0 loadingContext.overrides_list.extend( metadata.get("cwltool:overrides", [])) tool = make_tool(document_loader, avsc_names, metadata, uri, loadingContext) if args.make_template: yaml.safe_dump(generate_input_template(tool), sys.stdout, default_flow_style=False, indent=4, block_seq_indent=2) return 0 if args.validate: _logger.info("Tool definition is valid") return 0 if args.print_rdf: stdout.write( printrdf(tool, document_loader.ctx, args.rdf_serializer)) return 0 if args.print_dot: printdot(tool, document_loader.ctx, stdout) return 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) return 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s.\nThe error was: %s", try_again_msg, exc if not args.debug else "", exc_info=args.debug) return 1 if isinstance(tool, int): return tool # If on MacOS platform, TMPDIR must be set to be under one of the # shared volumes in Docker for Mac # More info: https://dockstore.org/docs/faq if sys.platform == "darwin": default_mac_path = "/private/tmp/docker_tmp" if runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: runtimeContext.tmp_outdir_prefix = default_mac_path for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(runtimeContext, dirprefix) and getattr( runtimeContext, dirprefix) != DEFAULT_TMP_PREFIX: sl = "/" if getattr(runtimeContext, dirprefix).endswith("/") or dirprefix == "cachedir" \ else "" setattr( runtimeContext, dirprefix, os.path.abspath(getattr(runtimeContext, dirprefix)) + sl) if not os.path.exists( os.path.dirname(getattr(runtimeContext, dirprefix))): try: os.makedirs( os.path.dirname(getattr(runtimeContext, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) return 1 if args.cachedir: if args.move_outputs == "move": runtimeContext.move_outputs = "copy" runtimeContext.tmp_outdir_prefix = args.cachedir runtimeContext.secret_store = getdefault(runtimeContext.secret_store, SecretStore()) try: initialized_job_order_object, input_for_prov = init_job_order( job_order_object, args, tool, jobloader, stdout, print_input_deps=args.print_input_deps, provArgs=runtimeContext.research_obj, relative_deps=args.relative_deps, input_basedir=input_basedir, secret_store=runtimeContext.secret_store) except SystemExit as err: return err.code if not executor: if args.parallel: executor = MultithreadedJobExecutor() else: executor = SingleJobExecutor() assert executor is not None try: runtimeContext.basedir = input_basedir del args.workflow del args.job_order conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) # Text use_conda_dependencies = getattr(args, "beta_conda_dependencies", None) # Text job_script_provider = None # type: Optional[DependenciesConfiguration] if conf_file or use_conda_dependencies: runtimeContext.job_script_provider = DependenciesConfiguration( args) runtimeContext.find_default_container = \ functools.partial(find_default_container, args) runtimeContext.make_fs_access = getdefault( runtimeContext.make_fs_access, StdFsAccess) (out, status) = executor(tool, initialized_job_order_object, runtimeContext, logger=_logger) if out is not None: def loc_to_path(obj): for field in ("path", "nameext", "nameroot", "dirname"): if field in obj: del obj[field] if obj["location"].startswith("file://"): obj["path"] = uri_file_path(obj["location"]) visit_class(out, ("File", "Directory"), loc_to_path) # Unsetting the Generation from final output object visit_class(out, ("File", ), MutationManager().unset_generation) if isinstance(out, string_types): stdout.write(out) else: stdout.write( json_dumps( out, indent=4, # type: ignore ensure_ascii=False)) stdout.write("\n") if hasattr(stdout, "flush"): stdout.flush() # type: ignore if status != "success": _logger.warning(u"Final process status is %s", status) return 1 _logger.info(u"Final process status is %s", status) return 0 except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) return 1 except UnsupportedRequirement as exc: _logger.error(u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) return 33 except WorkflowException as exc: _logger.error(u"Workflow error%s:\n%s", try_again_msg, strip_dup_lineno(six.text_type(exc)), exc_info=args.debug) return 1 except Exception as exc: _logger.error(u"Unhandled error%s:\n %s", try_again_msg, exc, exc_info=args.debug) return 1 finally: if args and runtimeContext and runtimeContext.research_obj \ and args.rm_tmpdir and workflowobj: #adding all related cwl files to RO prov_dependencies = printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri, runtimeContext.research_obj) prov_dep = prov_dependencies[1] assert prov_dep runtimeContext.research_obj.generate_snapshot(prov_dep) #for input file dependencies if input_for_prov: runtimeContext.research_obj.generate_snapshot(input_for_prov) #NOTE: keep these commented out lines to evaluate tests later #if job_order_object: #runtimeContext.research_obj.generate_snapshot(job_order_object) runtimeContext.research_obj.close(args.provenance) _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler)
from sylvia import * import pkg_resources _debug = False # get arguments if len(sys.argv) < 2: print "Usage: python rhymeColors.py <input_file>" sys.exit(2) inFile = sys.argv[1] # instantiate phonetic dictionary class from sylvia pd = PhoneticDictionary( binFile=pkg_resources.resource_stream("sylvia", "cmudict.sylviabin")) colors = ['red', 'green', 'yellow', 'blue', 'cyan', 'white', 'grey'] # define list of colors omit = ["the", "and", "as", "of"] # filler words to omit output = "" # initialized output text word_list = [] # list of words from input w/o filler words inp = [] # list of words from input w/ filler words r_matrix = [] # list of lists of rhymes # the function that does it all def get_dem_rhymes(word_list, file): # get rhymes for each word for word in word_list:
from enum import Enum from functools import wraps from reasoner_pydantic import Query as PDQuery, AsyncQuery as PDAsyncQuery, Response as PDResponse from pydantic import BaseModel from src.util import create_log_entry from fastapi import Body, FastAPI, BackgroundTasks from fastapi.responses import JSONResponse from fastapi.openapi.utils import get_openapi from fastapi.middleware.cors import CORSMiddleware from src.service_aggregator import entry # set the app version APP_VERSION = '2.0.23' # Set up default logger. with pkg_resources.resource_stream('src', 'logging.yml') as f: config = yaml.safe_load(f.read()) # declare the log directory log_dir = './logs' # make the directory if it does not exist if not os.path.exists(log_dir): os.makedirs(log_dir) # create a configuration for the log file config['handlers']['file']['filename'] = os.path.join(log_dir, 'aragorn.log') # load the log config logging.config.dictConfig(config)
def get_schematrons(cls): with resource_stream(__name__, "xml/gemini2/Gemini2_R1r3.sch") as schema: return [cls.schematron(schema)]
endpoint_override_path) try: #Modification for compilation by py2exe: if _is_from_exe(): #Load endpoints.json from .exe directory, i.e. C:\Program Files\Amazon\cfn-bootstrap log.debug('Loading py2exe endpoints.json file') with open( os.path.join(os.path.dirname(sys.executable), 'endpoints.json'), 'r') as f: _endpoint_data = json.load(f) # end py2exe else: _endpoint_data = json.load( pkg_resources.resource_stream("cfnbootstrap.resources.documents", "endpoints.json")) except ValueError: log.exception("Failed to load endpoints.json") raise try: #Apply endpoint_override, if present if os.path.isfile(endpoint_override_path): log.debug('Loading existing endpoints override file %s' % (endpoint_override_path, )) with open(endpoint_override_path, 'r') as f: _endpoint_data = json.load(f) except ValueError: log.exception("Failed to load endpoints override file: %s" % (endpoint_override_path, ))
def get_schematrons(cls): with resource_stream( __name__, "xml/medin/ISOTS19139A1Constraints_v1.4.sch") as schema: return [cls.schematron(schema)]
def get_cli_spec(device, version): # pylint: disable-msg=E1101 return pkg_resources.resource_stream( __name__, '%s-%s.xml' % device_type_alias(device, version))
def get_test_image(): return pkg_resources.resource_stream('sw.allotmentclub.browser.tests', 'assyrian.gif').read()
def from_resource(cls, resource_name, output_dir=None, root_dir=None, requirement=REQ, defaults=None, printer=logger): conf = pkg_resources.resource_stream(requirement, resource_name) merger = cls(output_dir, root_dir, defaults=defaults, printer=printer) merger.readfp(conf) return merger
def activate(self): InfiniteGlass.DEBUG("ghost", "SHADOW ACTIVATE %s\n" % (self, )) sys.stderr.flush() if self.properties.get("IG_GHOSTS_DISABLED", 0): return for name, value in self.properties.items(): InfiniteGlass.DEBUG("ghost.properties", "%s=%s\n" % (name, str(value)[:100])) sys.stderr.flush() self.window = self.manager.display.root.create_window( map=False, **self.properties.get("__attributes__", {})) self.window["IG_GHOST"] = "IG_GHOST" with pkg_resources.resource_stream("glass_ghosts", "ghost.svg") as f: ghost_image = f.read() for name, value in self.properties.items(): pattern, value = self.format_pair(name, value) if pattern in ghost_image: ghost_image = ghost_image.replace(pattern, value) pattern, value = self.format_pair("key", self.key()) if pattern in ghost_image: ghost_image = ghost_image.replace(pattern, value) if "SM_CLIENT_ID" in self.properties: if self.properties["SM_CLIENT_ID"] in self.manager.clients: for name, value in self.manager.clients[ self.properties["SM_CLIENT_ID"]].properties.items(): pattern, value = self.format_pair(name, value[1], b" ") if pattern in ghost_image: ghost_image = ghost_image.replace(pattern, value) else: pattern, value = self.format_pair("SM_CLIENT_ID", "No state saved") if pattern in ghost_image: ghost_image = ghost_image.replace(pattern, value) if "WM_COMMAND" in self.properties: pattern, value = self.format_pair( "RestartCommand", self.properties["WM_COMMAND"]) if pattern in ghost_image: ghost_image = ghost_image.replace(pattern, value) self.window["IG_CONTENT"] = ("IG_SVG", ghost_image) self.window["WM_PROTOCOLS"] = ["WM_DELETE_WINDOW"] self.apply(self.window, action_type="ghost_set") @self.window.on(mask="StructureNotifyMask") def DestroyNotify(win, event): InfiniteGlass.DEBUG("ghost", "GHOST DESTROY %s\n" % (self, )) sys.stderr.flush() self.destroy() self.DestroyNotify = DestroyNotify @self.window.on(mask="StructureNotifyMask", client_type="IG_CLOSE") def ClientMessage(win, event): InfiniteGlass.DEBUG("ghost", "GHOST CLOSE %s\n" % (self, )) sys.stderr.flush() win.destroy() self.CloseMessage = ClientMessage @self.window.on(mask="StructureNotifyMask", client_type="IG_DELETE") def ClientMessage(win, event): InfiniteGlass.DEBUG("ghost", "GHOST DELETE %s\n" % (self, )) sys.stderr.flush() win.destroy() self.DeleteMessage = ClientMessage @self.window.on(mask="NoEventMask", client_type="WM_PROTOCOLS") def ClientMessage(win, event): InfiniteGlass.DEBUG("ghost", "GHOST WM_DELETE_WINDOW %s\n" % (self, )) sys.stderr.flush() if event.parse("ATOM")[0] == "WM_DELETE_WINDOW": win.destroy() else: InfiniteGlass.DEBUG( "ghost", "%s: Unknown WM_PROTOCOLS message: %s\n" % (self, event)) sys.stderr.flush() self.WMDelete = ClientMessage @self.window.on() def PropertyNotify(win, event): name = self.manager.display.get_atom_name(event.atom) if name not in self.manager.config["ghost_update"]: return try: self.properties.update( glass_ghosts.helpers.expand_property(win, name)) InfiniteGlass.DEBUG( "ghost.update.property", "%s.%s=<%s>%s from %s\n" % (self, name, type( self.properties[name]), self.properties[name], win)) sys.stderr.flush() except: pass else: self.update_key() self.PropertyNotify = PropertyNotify @self.window.on() def ButtonPress(win, event): self.restart() self.ButtonPress = ButtonPress @self.window.on(mask="StructureNotifyMask", client_type="IG_RESTART") def ClientMessage(win, event): self.restart() self.RestartMessage = ClientMessage @self.window.on() def Expose(win, event): self.redraw() self.Expose = Expose self.window.map() self.redraw()
from pkg_resources import resource_stream import pandas as pd print(__name__) stream = resource_stream(__name__, "data/df_players.pkl") df_players = pd.read_pickle(stream, compression=None) df_players = df_players.dropna(subset=["goal_difference"]) stream = resource_stream(__name__, "data/df_matches.pkl") df_matches = pd.read_pickle(stream, compression=None)