def test_retrieve_record(self): """ SCENARIO: We have a URL for a landing page. EXPECTED RESULT: The identifier is retrieved. """ url = ("https://www.northwestknowledge.net" "/data/0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0/") # External I/O # # 1st: landing page # 2nd: XML metadata document package = 'tests.data.nkn.0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0' contents = [ ir.read_binary(package, 'index.html'), ir.read_binary(package, 'metadata.xml') ] harvester = NKNHarvester() with self.assertLogs(logger=harvester.logger, level='INFO'): with aioresponses() as m: m.get(self.regex, body=contents[0]) m.get(self.regex, body=contents[1]) awaitable = harvester.retrieve_record(url) sid, _, lastmod, doc = asyncio.run(awaitable) expected = '0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0' self.assertEqual(sid, expected) self.assertIsNone(lastmod)
def test_importlib_resources(self): # App ZIP pkg = "android1" names = ["subdir", "__init__.py", "a.txt", "b.so", "mod1.py"] self.assertCountEqual(names, resources.contents(pkg)) for name in names: with self.subTest(name=name): self.assertEqual(resources.is_resource(pkg, name), name != "subdir") self.check_ir_resource(APP_ZIP, pkg, "__init__.py", b"# This package is") self.check_ir_resource(APP_ZIP, pkg, "a.txt", b"alpha\n") self.check_ir_resource(APP_ZIP, pkg, "b.so", b"bravo\n") self.assertFalse(resources.is_resource(pkg, "invalid.py")) with self.assertRaisesRegex(FileNotFoundError, "invalid.py"): resources.read_binary(pkg, "invalid.py") with self.assertRaisesRegex(FileNotFoundError, "invalid.py"): with resources.path(pkg, "invalid.py"): pass # Requirements ZIP self.reset_package("murmurhash") self.assertCountEqual([ "include", "tests", "__init__.pxd", "__init__.pyc", "about.pyc", "mrmr.pxd", "mrmr.pyx", "mrmr.so" ], resources.contents("murmurhash")) self.check_ir_resource(REQS_COMMON_ZIP, "murmurhash", "__init__.pyc", MAGIC_NUMBER) self.check_ir_resource(REQS_COMMON_ZIP, "murmurhash", "mrmr.pxd", b"from libc.stdint") self.check_ir_resource(REQS_ABI_ZIP, "murmurhash", "mrmr.so", b"\x7fELF")
def test_unsupported_xml_format(self, mock_logger): """ SCENARIO: Sometimes the XML format is not supported. We have tests for this in other places, but NKN is a special case because we have to extract the identifier from the XML metadata rather than the JSON-LD. This particular file is some sort of ESRI XML file. EXPECTED RESULT: XMLValidationError """ url = ("https://www.northwestknowledge.net" "/data/A62BEE88-8F92-4649-BC8D-BC56CE96AE2B") package = 'tests.data.nkn.A62BEE88-8F92-4649-BC8D-BC56CE96AE2B' contents = [ ir.read_binary(package, 'metadata.xml'), ir.read_binary(package, 'index.html'), ] harvester = NKNHarvester() with aioresponses() as m: m.get(self.regex, body=contents[0]) m.get(self.regex, body=contents[1]) with self.assertRaises(XMLValidationError): asyncio.run(harvester.retrieve_record(url))
def __load_labels(cls, label_id: int): """ Load labels if needed ===================== Parameters ---------- label_id : int Label storage to load. - 0 to load all supported DICOM tags. - CortxDicom.FILTER_HIPAA to load HIPAA protected tags - CortxDicom.FILTER_GDPR to load GDPR protected tags Raises ------ ValueError When a not supported label_id is given. """ if label_id == 0: if 0 not in cls.__labels.keys(): cls.__labels[0] = pickle.loads( read_binary(dicomfields, 'dicom_tags.dict')) elif label_id == CortxDicom.FILTER_HIPAA: if CortxDicom.FILTER_HIPAA not in cls.__labels.keys(): cls.__labels[CortxDicom.FILTER_HIPAA] = pickle.loads( read_binary(dicomfields, 'phi_hipaa.list')) elif label_id == CortxDicom.FILTER_GDPR: if CortxDicom.FILTER_GDPR not in cls.__labels.keys(): cls.__labels[CortxDicom.FILTER_GDPR] = pickle.loads( read_binary(dicomfields, 'phi_gdpr.list')) else: raise ValueError('CortxDicom.__load_labels(): unsupported label ' + 'source.')
def test_limit_number_of_documents(self, aioresp_mocker): """ SCENARIO: We do not wish to go through the entire list of documents, so a limit is specified. There are 3 records in the sitemap, only 2 are to be processed. EXPECTED RESULT: Three records are detected in the sitemap, but the log shows only two were processed. """ # External calls to read the: # # 1) sitemap # 2) HTML document for record 1 # 3) XML document for record 1 # 3) HTML document for record 2 # 4) XML document for record 2 # contents = [ ir.read_binary('tests.data.arm', 'sitemap3.xml'), ir.read_binary('tests.data.arm', 'nsaqcrad1longC2.c2.fixed.html'), ir.read_binary('tests.data.arm', 'nsaqcrad1longC2.c2.fixed.xml'), ir.read_binary('tests.data.arm', 'nsasondewnpnS01.b1.fixed.html'), ir.read_binary('tests.data.arm', 'nsasondewnpnS01.b1.fixed.xml'), ] headers = [ { 'Content-Type': 'text/xml' }, { 'Content-Type': 'text/html' }, { 'Content-Type': 'text/xml' }, { 'Content-Type': 'text/html' }, { 'Content-Type': 'text/xml' }, ] for content, headers in zip(contents, headers): aioresp_mocker.get(self.pattern, body=content, headers=headers) url = 'https://www.archive.arm.gov/metadata/adc/sitemap.xml' obj = D1CheckSitemap(sitemap_url=url, num_documents=2) with self.assertLogs(logger=obj.logger, level='DEBUG') as cm: asyncio.run(obj.run()) # Only two records processed. self.assertSuccessfulDebugIngest(cm.output, n=2) # And just to show, there are 3 URLs in the sitemap. doc = lxml.etree.parse(io.BytesIO(contents[0])) urls = doc.xpath('.//sm:loc/text()', namespaces=schema_org.core.SITEMAP_NS) self.assertEqual(len(urls), 3)
def test__harvest__but_landing_page_has_no_json_ld(self, harvest_time_mocker): """ SCENARIO: We have a valid sitemap but one of the landing pages does not have any JSON-LD. EXPECTED RESULT: A SkipError is issued and caught. """ harvest_time_mocker.return_value = '1900-01-01T00:00:00Z' host, port = 'www.hydroshare.org', 443 harvester = CUAHSIHarvester(host=host, port=port) # External calls to read the: # # 1) sitemap # 2) HTML document for record 1 # contents = [ ir.read_binary('tests.data.cuahsi', 'sitemap.81e947faccf04de59392dddaac77bc75.xml'), ir.read_binary( 'tests.data.cuahsi.81e947faccf04de59392dddaac77bc75', # noqa : E501 'landing_page.no_json_ld.html') ] status_codes = [200, 200] headers = [ { 'Content-Type': 'application/xml' }, { 'Content-Type': 'application/html' }, ] z = zip(contents, status_codes, headers) with aioresponses() as m: for content, status_code, headers in z: m.get(self.regex, body=content, status=status_code, headers=headers) with self.assertLogs(logger=harvester.logger, level='DEBUG') as cm: asyncio.run(harvester.run()) expected = "Successfully processed 0 records." self.assertInfoLogMessage(cm.output, expected) expected = "No JSON-LD <SCRIPT> elements were located" self.assertWarningLogMessage(cm.output, expected) expected = "SkipError" self.assertErrorLogMessage(cm.output, expected)
def test_document_already_harvested_but_can_be_successfully_updated( self, mock_check_if_identifier_exists, mock_update_science_metadata ): """ SCENARIO: The GMN existance check shows that the document has already been harvested. It has been updated since the last harvest time, and the update succeeds. The document in question has a DOI of doi:10.15784/600121 EXPECTED RESULT: The event is logged at the info level. The update count increases by one. The update_science_metadata routine should have been called with specific parameters. """ harvester = IEDAHarvester(host=self.host, port=self.port) # This is the existing document in the MN. It is marked as complete. existing_content = ir.read_binary('tests.data.ieda', '600121iso.xml') record_date = dt.datetime.now() mock_check_if_identifier_exists.return_value = { 'outcome': 'yes', 'record_date': record_date - dt.timedelta(days=1), 'current_version_id': 1, } mock_update_science_metadata.return_value = True harvester = IEDAHarvester(host=self.host, port=self.port) update_count = harvester.updated_count # This is the "update" document, same as the existing document. It is # already marked as "complete". Bump the timestamp to just a bit later # to make ok to proceed. docbytes = ir.read_binary('tests.data.ieda', '600121iso-later.xml') doc = lxml.etree.parse(io.BytesIO(docbytes)) doi = 'doi:10.15784/600121' pid = 'https://get.iedadata.org/metadata/iso/600121' regex = re.compile('https://ieda.mn.org:443/') with self.assertLogs(logger=harvester.logger, level='DEBUG') as cm: with aioresponses() as m: m.get(regex, body=existing_content) asyncio.run(harvester.harvest_document(doi, pid, doc, record_date)) # Did we see an informational message? log_msg_count = self.logLevelCallCount(cm.output, level='INFO') self.assertTrue(log_msg_count > 0) # Did we increase the update count? self.assertEqual(harvester.updated_count, update_count + 1)
def test__retrieve_record(self): """ SCENARIO: We have a valid landing page URL and JSON-LD document for ARM. EXPECTED RESULT: The expected SID and PID are returned. """ landing_page_url = ( 'https://www.archive.arm.gov/metadata/adc/html/wsacrcrcal.html' ) # External calls to read the: # # 2) HTML document for the landing page # 3) XML document associated with the landing page # contents = [ ir.read_binary('tests.data.arm', 'wsacrcrcal.html'), ir.read_binary('tests.data.arm', 'wsacrcrcal.xml'), ] status_codes = [200, 200, 200] headers = [ {'Content-Type': 'text/html'}, {'Content-Type': 'application/xml'}, ] z = zip(contents, status_codes, headers) with aioresponses() as m: for content, status_code, headers in z: m.get(self.regex, body=content, status=status_code, headers=headers) harvester = SchemaDotOrgHarvester() with self.assertLogs(logger=harvester.logger, level='DEBUG'): awaitable = harvester.retrieve_record(landing_page_url) sid, pid, dateMod, doc = asyncio.run(awaitable) self.assertEqual(sid, "http://dx.doi.org/10.5439/1150280") self.assertIsNone(pid) expected = dt.datetime(2019, 11, 25, 16, 0, 21, 746316, tzinfo=dt.timezone.utc) self.assertEqual(dateMod, expected) actual = lxml.etree.tostring(doc) bf = io.BytesIO(contents[1]) expected = lxml.etree.tostring(lxml.etree.parse(bf)) self.assertEqual(actual, expected)
def test_document_already_harvested__followup_record_is_complete( self, mock_logger, mock_check_if_identifier_exists, mock_update_science_metadata ): """ SCENARIO: The GMN existance check shows that the document has already been harvested but has been marked as "ongoing". It has been updated since the last harvest time, and the new record is marked as "complete". The update succeeds. EXPECTED RESULT: The updated count goes up by one and the event is logged at the info level. """ # This is the existing document in the MN. It is requested by the # update check, and it is marked as ongoing. existing_content = ir.read_binary('tests.data.ieda', '600121iso-ongoing.xml') # This is the proposed update document that is the same except it is # marked as complete. update_doc_bytes = ir.read_binary('tests.data.ieda', '600121iso.xml') doc = lxml.etree.parse(io.BytesIO(update_doc_bytes)) record_date = dt.datetime.now() mock_check_if_identifier_exists.return_value = { 'outcome': 'yes', 'record_date': record_date - dt.timedelta(days=1), 'current_version_id': 1, } mock_update_science_metadata.return_value = True host, port = 'ieda.mn.org', 443 harvester = SchemaDotOrgHarvester(host=host, port=port) initial_updated_count = harvester.updated_count doi = 'doi.10000/abcde' regex = re.compile('https://ieda.mn.org:443/') with aioresponses() as m: m.get(regex, body=existing_content) asyncio.run(harvester.harvest_document(doi, '1', doc, record_date)) # Did we increase the failure count? self.assertEqual(harvester.updated_count, initial_updated_count + 1) # Did we see a warning? self.assertTrue(harvester.logger.info.call_count > 0)
def test_retrieve_record__404_error(self): """ SCENARIO: We have a URL for a landing page. However, the directory on the remote end does not have a metadata.xml document. EXPECTED RESULT: A SkipError is raised. For other clients, this is NOT a SkipError. """ url = ("https://www.northwestknowledge.net" "/data/94E2D569-200F-44F7-8937-AB4BD0862C91") # External I/O # # 1st: landing page # 2nd: XML metadata document (raises 404) package = 'tests.data.nkn.0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0' contents = [ ir.read_binary(package, 'index.html'), ] harvester = NKNHarvester() with self.assertLogs(logger=harvester.logger, level='INFO'): with aioresponses() as m: m.get(self.regex, body=contents[0]) m.get(self.regex, status=404) with self.assertRaises(SkipError): asyncio.run(harvester.retrieve_record(url))
def test_retrieve_record__500_error(self): """ SCENARIO: We have a URL for a landing page. The response is a server error, however. This is complimentary to the _404_error test below. EXPECTED RESULT: An Exception is raised (but not a SkipError). """ url = ("https://www.northwestknowledge.net" "/data/94E2D569-200F-44F7-8937-AB4BD0862C91") # External I/O # # 1st: landing page # 2nd: XML metadata document (raises 404) package = 'tests.data.nkn.0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0' contents = [ ir.read_binary(package, 'index.html'), ] harvester = NKNHarvester() with self.assertLogs(logger=harvester.logger, level='INFO'): with aioresponses() as m: m.get(self.regex, body=contents[0]) m.get(self.regex, status=500) with self.assertRaises(aiohttp.ClientResponseError): asyncio.run(harvester.retrieve_record(url))
def _bootstrap(*, root=None, upgrade=False, user=False, altinstall=False, default_pip=False, verbosity=0): """ Bootstrap pip into the current Python installation (or the given root directory). Returns pip command status code. Note that calling this function will alter both sys.path and os.environ. """ if altinstall and default_pip: raise ValueError("Cannot use altinstall and default_pip together") sys.audit("ensurepip.bootstrap", root) _disable_pip_configuration_settings() # By default, installing pip and setuptools installs all of the # following scripts (X.Y == running Python version): # # pip, pipX, pipX.Y, easy_install, easy_install-X.Y # # pip 1.5+ allows ensurepip to request that some of those be left out if altinstall: # omit pip, pipX and easy_install os.environ["ENSUREPIP_OPTIONS"] = "altinstall" elif not default_pip: # omit pip and easy_install os.environ["ENSUREPIP_OPTIONS"] = "install" with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] for project, version in _PROJECTS: wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) whl = resources.read_binary( _bundled, wheel_name, ) with open(os.path.join(tmpdir, wheel_name), "wb") as fp: fp.write(whl) additional_paths.append(os.path.join(tmpdir, wheel_name)) # Construct the arguments to be passed to the pip command args = ["install", "--no-index", "--find-links", tmpdir] if root: args += ["--root", root] if upgrade: args += ["--upgrade"] if user: args += ["--user"] if verbosity: args += ["-" + "v" * verbosity] return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
def test_update_science_metadata(self, mock_client): """ SCENARIO: A new science metadata record is successfully loaded. EXPECTED RESULT: True """ sci_metadata_bytes = ir.read_binary('tests.data.ieda', '600121iso.xml') sid = 'i_am_a_sid' record_date = dt.datetime.now().isoformat() old_version_pid = 'b645a195302ca652ec39f1bf3b908dbf' client_mgr = D1ClientManager( self.gmn_base_url, self.auth_cert, self.auth_cert_key, self.mock_logger ) system_metadata = self._generate_system_metadata(sci_metadata_bytes, sid, old_version_pid, record_date) actual = client_mgr.update_science_metadata( sci_metadata_bytes=sci_metadata_bytes, native_identifier_sid=sid, record_date=record_date, old_version_pid=old_version_pid, system_metadata=system_metadata ) self.assertTrue(actual)
def anyascii(string): # type: (str) -> str """Transliterate a string into ASCII.""" try: if string.isascii(): return string except AttributeError: pass result = [] for char in string: codepoint = ord(char) if codepoint <= 0x7f: result.append(char) continue blocknum = codepoint >> 8 lo = codepoint & 0xff try: block = _blocks[blocknum] except KeyError: try: b = read_binary('anyascii._data', '%03x' % blocknum) s = decompress(b, -MAX_WBITS).decode('ascii') block = tuple(map(intern, s.split('\t'))) except FileNotFoundError: block = () _blocks[blocknum] = block if len(block) > lo: result.append(block[lo]) return ''.join(result)
def test_document_is_unrecognized_but_successfully_harvested( self, mock_check_if_identifier_exists, mock_load_science_metadata ): """ SCENARIO: The GMN has not seen the document before and is successfully harvested. EXPECTED RESULT: The event is logged at the info level. The "created_count" goes up by one. """ mock_check_if_identifier_exists.return_value = {'outcome': 'no'} mock_load_science_metadata.return_value = True docbytes = ir.read_binary('tests.data.ieda', '600121iso.xml') doc = lxml.etree.parse(io.BytesIO(docbytes)) harvester = SchemaDotOrgHarvester() with self.assertLogs(logger=harvester.logger, level='DEBUG') as cm: pargs = ('doi.10000/abcde', '1', doc, dt.datetime.now()) awaitable = harvester.harvest_document(*pargs) asyncio.run(awaitable) self.assertTrue(self.logLevelCallCount(cm.output, 'INFO') > 0) self.assertEqual(harvester.created_count, 1)
def test_document_already_harvested_at_same_date( self, mock_check_if_identifier_exists, mock_update_science_metadata ): """ SCENARIO: The GMN existance check shows that the document has already been harvested and has not changed since the last time. We don't want to attempt to harvest in this case. EXPECTED RESULT: A SkipError is raised. """ record_date = dt.datetime.now() mock_check_if_identifier_exists.return_value = { 'outcome': 'yes', 'record_date': record_date, } mock_update_science_metadata.return_value = False harvester = SchemaDotOrgHarvester() docbytes = ir.read_binary('tests.data.ieda', '600121iso.xml') doc = lxml.etree.parse(io.BytesIO(docbytes)) doi = 'doi.10000/abcde' with self.assertRaises(schema_org.core.SkipError): asyncio.run(harvester.harvest_document(doi, '1', doc, record_date))
def test_update_science_metadata_fails(self, mock_client): """ SCENARIO: A new science metadata record is not successfully updated because the update routine errors out for some reason. EXPECTED RESULT: False, and the failure is logged at the error level. """ mock_client.return_value.update.side_effect = RuntimeError( 'something bad happened') # noqa: E501 sci_metadata_bytes = ir.read_binary('tests.data.ieda', '600121iso.xml') sid = 'i_am_a_sid' record_date = dt.datetime.now().isoformat() old_version_pid = 'b645a195302ca652ec39f1bf3b908dbf' client_mgr = D1ClientManager(self.gmn_base_url, self.auth_cert, self.auth_cert_key, self.mock_logger) system_metadata = self._generate_system_metadata( sci_metadata_bytes, sid, old_version_pid, record_date) actual = client_mgr.update_science_metadata( sci_metadata_bytes=sci_metadata_bytes, native_identifier_sid=sid, record_date=record_date, old_version_pid=old_version_pid, system_metadata=system_metadata) self.assertFalse(actual) self.assertEqual(self.mock_logger.error.call_count, 1)
def path_for_font(name): if name not in font_path_cache: with open(os.path.join(self.tdir, name), 'wb') as f: font_path_cache[name] = f.name data = read_binary(__name__.rpartition('.')[0], name) f.write(data) return font_path_cache[name]
def _load_icons(self): _icons = (('gimp-flip-horizontal.png', 'mcomix-flip-horizontal'), ('gimp-flip-vertical.png', 'mcomix-flip-vertical'), ('gimp-rotate-180.png', 'mcomix-rotate-180'), ('gimp-rotate-270.png', 'mcomix-rotate-270'), ('gimp-rotate-90.png', 'mcomix-rotate-90'), ('gimp-transform.png', 'mcomix-transform'), ('tango-enhance-image.png', 'mcomix-enhance-image'), ('tango-add-bookmark.png', 'mcomix-add-bookmark'), ('tango-archive.png', 'mcomix-archive'), ('tango-image.png', 'mcomix-image'), ('zoom.png', 'mcomix-zoom'), ('lens.png', 'mcomix-lens')) # Load window title icons. Gtk.Window.set_default_icon_list(self.__mcomix_icons) # Load application icons. factory = Gtk.IconFactory() for filename, stockid in _icons: try: icon_data = resources.read_binary('mcomix.images', filename) pixbuf = ImageTools.load_pixbuf_data(icon_data) iconset = Gtk.IconSet.new_from_pixbuf(pixbuf) factory.add(stockid, iconset) except FileNotFoundError: logger.warning(f'Could not load icon: \'{filename}\'') factory.add_default()
def test_handle_create_snews_event(mock_create_event, mock_get_event, mock_upload, mock_json): text = read_binary(data, 'snews_gcn.xml') external_triggers.handle_snews_gcn(payload=text) mock_create_event.assert_called_once_with(filecontents=text, search='Supernova', pipeline='SNEWS', group='External') calls = [ call('"dqrjson"', 'gwcelerydetcharcheckvectors-E1.json', 'E1', 'DQR-compatible json generated from check_vectors results'), call(None, None, 'E1', ('Detector state for active instruments is unknown.\n{}' 'Check looked within -10/+10 seconds of superevent. ').format( detchar.generate_table('Data quality bits', [], [], [ 'H1:NO_OMC_DCPD_ADC_OVERFLOW', 'H1:NO_DMT-ETMY_ESD_DAC_OVERFLOW', 'L1:NO_OMC_DCPD_ADC_OVERFLOW', 'L1:NO_DMT-ETMY_ESD_DAC_OVERFLOW', 'H1:HOFT_OK', 'H1:OBSERVATION_INTENT', 'L1:HOFT_OK', 'L1:OBSERVATION_INTENT', 'V1:HOFT_OK', 'V1:OBSERVATION_INTENT', 'V1:GOOD_DATA_QUALITY_CAT1' ])), ['data_quality']) ] mock_upload.assert_has_calls(calls, any_order=True)
def add_static_paths(app): """Ensure CSS/JS is loaded.""" app.env.book_theme_resources_changed = False output_static_folder = Path(app.outdir) / "_static" theme_static_files = resources.contents(theme_static) if (app.config.html_theme_options.get("theme_dev_mode", False) and output_static_folder.exists()): # during development, the JS/CSS may change, if this is the case, # we want to remove the old files and ensure that the new files are loaded for path in output_static_folder.glob("sphinx-book-theme*"): if path.name not in theme_static_files: app.env.book_theme_resources_changed = True path.unlink() # note sphinx treats theme css different to regular css # (it is specified in theme.conf), so we don't directly use app.add_css_file for fname in resources.contents(theme_static): if fname.endswith(".css"): if not (output_static_folder / fname).exists(): (output_static_folder / fname).write_bytes( resources.read_binary(theme_static, fname)) app.env.book_theme_resources_changed = True # add javascript for fname in resources.contents(theme_static): if fname.endswith(".js"): app.add_js_file(fname)
def read_ssl_certificate(): """Get the CA certificate that should be used to verify a TLS connection to MinKNOW. If the ``$MINKNOW_TRUSTED_CA`` environment variable is set to the path to an existing file, its contents will be used. Otherwise, an internal copy of MinKNOW Core's default CA will be used. """ try: with open(os.environ["MINKNOW_TRUSTED_CA"], "rb") as f: return f.read() except KeyError: pass except OSError: logger.warning( "$MINKNOW_TRUSTED_CA set but failed to read '%s'", os.environ.get("MINKNOW_TRUSTED_CA"), ) try: # python 3.7+ import importlib.resources as importlib_resources except ImportError: # python 3.5/3.6 import importlib_resources # using the syntax that works with python 3.7 return importlib_resources.read_binary("minknow_api", "ca.crt")
def fake_gcn(notice_type): # Check the real GCN notice, which is valid. payload = resources.read_binary(data, 'G298048-1-Initial.xml') root = lxml.etree.fromstring(payload) notice_type = str(int(notice_type)) root.find(".//Param[@name='Packet_Type']").attrib['value'] = notice_type return xml_document(root)
def test__read_record__invalid_jsonld(self): """ SCENARIO: A landing page is properly retrieved, but has invalid JSON-LD. EXPECTED RESULT: JSON-LD error """ url = ( 'https://www.archive.arm.gov' '/metadata/adc/html/nsaqcrad1longC2.c2.invalid_jsonld.html' ) harvester = ARMHarvester() contents = ir.read_binary('tests.data.arm', 'nsaqcrad1longC2.c2.no_dataset_in_jsonld.html') # noqa : E501 status_code = 200 headers = {'Content-Type': 'text/html'} regex = re.compile('https://www.archive.arm.gov/metadata/adc') with aioresponses() as m: m.get(regex, body=contents, status=status_code, headers=headers) with self.assertRaises(JsonLdError): with self.assertLogs(logger=harvester.logger, level='DEBUG'): asyncio.run(harvester.retrieve_record(url))
def test_calc(): fp = resources.read_binary("tests.data.sample_1", "take-1.log") fpcalc = mock.MagicMock(spec=CompletedProcess, stdout=fp) with mock.patch("subprocess.run", return_value=fpcalc) as fpcalc_mock: fp = fingerprints.calc("/path/to/audio", length=120) assert fp == _load_fp("sample_1/take-1.log") assert fpcalc_mock.called
def setUp(self): super(TestSuiteHiRISE, self).setUp() # Hand-create the boxes needed for HiRISE. the_uuid = uuid.UUID('2b0d7e97-aa2e-317d-9a33-e53161a2f7d0') ulst = glymur.jp2box.UUIDListBox([the_uuid]) version = 0 flag = [0, 0, 0] url = 'ESP_032436_1755_COLOR.LBL' debox = glymur.jp2box.DataEntryURLBox(version, flag, url) uuidinfo = glymur.jp2box.UUIDInfoBox([ulst, debox]) uuid_data = ir.read_binary(data, 'degenerate_geotiff.tif') the_uuid = uuid.UUID('b14bf8bd-083d-4b43-a5ae-8cd7d5a6ce03') geotiff_uuid = glymur.jp2box.UUIDBox(the_uuid, uuid_data) # Fabricate a new JP2 file out of the signature, file type, header, # and codestream out of nemo.jp2, but add in the UUIDInfo and UUID # box from HiRISE. jp2 = Jp2k(self.jp2file) boxes = [ jp2.box[0], jp2.box[1], jp2.box[2], uuidinfo, geotiff_uuid, jp2.box[-1] ] self.hirise_jp2file_name = self.test_dir_path / 'hirise.jp2' jp2.wrap(self.hirise_jp2file_name, boxes=boxes)
def read_image(name: Text): encoded_image = resources.read_binary(test_data, name) pil_image = (PIL.Image.open(io.BytesIO(encoded_image)) ) # type: PIL.Image.Image pil_image = pil_image.convert("RGB").resize( (192, 128), resample=PIL.Image.BICUBIC) image = t.as_tensor(np.array(pil_image)).flip(0).contiguous() return image, encoded_image
def instantiate_template(self, replacements, template, creator): # read content as binary to avoid platform specific line normalization (\n -> \r\n) binary = read_binary(self.__module__, str(template)) text = binary.decode("utf-8", errors="strict") for key, value in replacements.items(): value = self._repr_unicode(creator, value) text = text.replace(key, value) return text
def mock_download(filename, graceid, *args, **kwargs): assert graceid == 'T250822' filenames = { 'coinc.xml': 'coinc.xml', 'psd.xml.gz': 'psd.xml.gz', 'ranking_data.xml.gz': 'ranking_data_G322589.xml.gz' } return resources.read_binary(data, filenames[filename])
def test_old_world(self, runner): with runner.isolated_filesystem(): LEVEL_DATA_PATH.write_bytes( read_binary('tests.files', 'old_level.dat')) result = runner.invoke(ender, ['init']) assert result.exit_code == 1 assert 'not compatible with endermite' in result.output
def test_read_binary(self): result = resources.read_binary(self.data, 'binary.file') self.assertEqual(result, b'\0\1\2\3')
def execute(self, package, path): resources.read_binary(package, path)