def test_retrieve_record(self):
"""
SCENARIO: We have a URL for a landing page.
EXPECTED RESULT: The identifier is retrieved.
"""
url = ("https://www.northwestknowledge.net"
"/data/0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0/")
# External I/O
#
# 1st: landing page
# 2nd: XML metadata document
package = 'tests.data.nkn.0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0'
contents = [
ir.read_binary(package, 'index.html'),
ir.read_binary(package, 'metadata.xml')
]
harvester = NKNHarvester()
with self.assertLogs(logger=harvester.logger, level='INFO'):
with aioresponses() as m:
m.get(self.regex, body=contents[0])
m.get(self.regex, body=contents[1])
awaitable = harvester.retrieve_record(url)
sid, _, lastmod, doc = asyncio.run(awaitable)
expected = '0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0'
self.assertEqual(sid, expected)
self.assertIsNone(lastmod)
def test_importlib_resources(self):
# App ZIP
pkg = "android1"
names = ["subdir", "__init__.py", "a.txt", "b.so", "mod1.py"]
self.assertCountEqual(names, resources.contents(pkg))
for name in names:
with self.subTest(name=name):
self.assertEqual(resources.is_resource(pkg, name),
name != "subdir")
self.check_ir_resource(APP_ZIP, pkg, "__init__.py",
b"# This package is")
self.check_ir_resource(APP_ZIP, pkg, "a.txt", b"alpha\n")
self.check_ir_resource(APP_ZIP, pkg, "b.so", b"bravo\n")
self.assertFalse(resources.is_resource(pkg, "invalid.py"))
with self.assertRaisesRegex(FileNotFoundError, "invalid.py"):
resources.read_binary(pkg, "invalid.py")
with self.assertRaisesRegex(FileNotFoundError, "invalid.py"):
with resources.path(pkg, "invalid.py"):
pass
# Requirements ZIP
self.reset_package("murmurhash")
self.assertCountEqual([
"include", "tests", "__init__.pxd", "__init__.pyc", "about.pyc",
"mrmr.pxd", "mrmr.pyx", "mrmr.so"
], resources.contents("murmurhash"))
self.check_ir_resource(REQS_COMMON_ZIP, "murmurhash", "__init__.pyc",
MAGIC_NUMBER)
self.check_ir_resource(REQS_COMMON_ZIP, "murmurhash", "mrmr.pxd",
b"from libc.stdint")
self.check_ir_resource(REQS_ABI_ZIP, "murmurhash", "mrmr.so",
b"\x7fELF")
def test_unsupported_xml_format(self, mock_logger):
"""
SCENARIO: Sometimes the XML format is not supported. We have tests
for this in other places, but NKN is a special case because we have to
extract the identifier from the XML metadata rather than the JSON-LD.
This particular file is some sort of ESRI XML file.
EXPECTED RESULT: XMLValidationError
"""
url = ("https://www.northwestknowledge.net"
"/data/A62BEE88-8F92-4649-BC8D-BC56CE96AE2B")
package = 'tests.data.nkn.A62BEE88-8F92-4649-BC8D-BC56CE96AE2B'
contents = [
ir.read_binary(package, 'metadata.xml'),
ir.read_binary(package, 'index.html'),
]
harvester = NKNHarvester()
with aioresponses() as m:
m.get(self.regex, body=contents[0])
m.get(self.regex, body=contents[1])
with self.assertRaises(XMLValidationError):
asyncio.run(harvester.retrieve_record(url))
def __load_labels(cls, label_id: int):
"""
Load labels if needed
=====================
Parameters
----------
label_id : int
Label storage to load.
- 0 to load all supported DICOM tags.
- CortxDicom.FILTER_HIPAA to load HIPAA protected tags
- CortxDicom.FILTER_GDPR to load GDPR protected tags
Raises
------
ValueError
When a not supported label_id is given.
"""
if label_id == 0:
if 0 not in cls.__labels.keys():
cls.__labels[0] = pickle.loads(
read_binary(dicomfields, 'dicom_tags.dict'))
elif label_id == CortxDicom.FILTER_HIPAA:
if CortxDicom.FILTER_HIPAA not in cls.__labels.keys():
cls.__labels[CortxDicom.FILTER_HIPAA] = pickle.loads(
read_binary(dicomfields, 'phi_hipaa.list'))
elif label_id == CortxDicom.FILTER_GDPR:
if CortxDicom.FILTER_GDPR not in cls.__labels.keys():
cls.__labels[CortxDicom.FILTER_GDPR] = pickle.loads(
read_binary(dicomfields, 'phi_gdpr.list'))
else:
raise ValueError('CortxDicom.__load_labels(): unsupported label ' +
'source.')
def test_limit_number_of_documents(self, aioresp_mocker):
"""
SCENARIO: We do not wish to go through the entire list of documents,
so a limit is specified. There are 3 records in the sitemap, only 2
are to be processed.
EXPECTED RESULT: Three records are detected in the sitemap, but the
log shows only two were processed.
"""
# External calls to read the:
#
# 1) sitemap
# 2) HTML document for record 1
# 3) XML document for record 1
# 3) HTML document for record 2
# 4) XML document for record 2
#
contents = [
ir.read_binary('tests.data.arm', 'sitemap3.xml'),
ir.read_binary('tests.data.arm', 'nsaqcrad1longC2.c2.fixed.html'),
ir.read_binary('tests.data.arm', 'nsaqcrad1longC2.c2.fixed.xml'),
ir.read_binary('tests.data.arm', 'nsasondewnpnS01.b1.fixed.html'),
ir.read_binary('tests.data.arm', 'nsasondewnpnS01.b1.fixed.xml'),
]
headers = [
{
'Content-Type': 'text/xml'
},
{
'Content-Type': 'text/html'
},
{
'Content-Type': 'text/xml'
},
{
'Content-Type': 'text/html'
},
{
'Content-Type': 'text/xml'
},
]
for content, headers in zip(contents, headers):
aioresp_mocker.get(self.pattern, body=content, headers=headers)
url = 'https://www.archive.arm.gov/metadata/adc/sitemap.xml'
obj = D1CheckSitemap(sitemap_url=url, num_documents=2)
with self.assertLogs(logger=obj.logger, level='DEBUG') as cm:
asyncio.run(obj.run())
# Only two records processed.
self.assertSuccessfulDebugIngest(cm.output, n=2)
# And just to show, there are 3 URLs in the sitemap.
doc = lxml.etree.parse(io.BytesIO(contents[0]))
urls = doc.xpath('.//sm:loc/text()',
namespaces=schema_org.core.SITEMAP_NS)
self.assertEqual(len(urls), 3)
def test__harvest__but_landing_page_has_no_json_ld(self,
harvest_time_mocker):
"""
SCENARIO: We have a valid sitemap but one of the landing pages does
not have any JSON-LD.
EXPECTED RESULT: A SkipError is issued and caught.
"""
harvest_time_mocker.return_value = '1900-01-01T00:00:00Z'
host, port = 'www.hydroshare.org', 443
harvester = CUAHSIHarvester(host=host, port=port)
# External calls to read the:
#
# 1) sitemap
# 2) HTML document for record 1
#
contents = [
ir.read_binary('tests.data.cuahsi',
'sitemap.81e947faccf04de59392dddaac77bc75.xml'),
ir.read_binary(
'tests.data.cuahsi.81e947faccf04de59392dddaac77bc75', # noqa : E501
'landing_page.no_json_ld.html')
]
status_codes = [200, 200]
headers = [
{
'Content-Type': 'application/xml'
},
{
'Content-Type': 'application/html'
},
]
z = zip(contents, status_codes, headers)
with aioresponses() as m:
for content, status_code, headers in z:
m.get(self.regex,
body=content,
status=status_code,
headers=headers)
with self.assertLogs(logger=harvester.logger, level='DEBUG') as cm:
asyncio.run(harvester.run())
expected = "Successfully processed 0 records."
self.assertInfoLogMessage(cm.output, expected)
expected = "No JSON-LD <SCRIPT> elements were located"
self.assertWarningLogMessage(cm.output, expected)
expected = "SkipError"
self.assertErrorLogMessage(cm.output, expected)
def test_document_already_harvested_but_can_be_successfully_updated(
self,
mock_check_if_identifier_exists,
mock_update_science_metadata
):
"""
SCENARIO: The GMN existance check shows that the document has already
been harvested. It has been updated since the last harvest time, and
the update succeeds.
The document in question has a DOI of doi:10.15784/600121
EXPECTED RESULT: The event is logged at the info level. The update
count increases by one. The update_science_metadata routine should
have been called with specific parameters.
"""
harvester = IEDAHarvester(host=self.host, port=self.port)
# This is the existing document in the MN. It is marked as complete.
existing_content = ir.read_binary('tests.data.ieda', '600121iso.xml')
record_date = dt.datetime.now()
mock_check_if_identifier_exists.return_value = {
'outcome': 'yes',
'record_date': record_date - dt.timedelta(days=1),
'current_version_id': 1,
}
mock_update_science_metadata.return_value = True
harvester = IEDAHarvester(host=self.host, port=self.port)
update_count = harvester.updated_count
# This is the "update" document, same as the existing document. It is
# already marked as "complete". Bump the timestamp to just a bit later
# to make ok to proceed.
docbytes = ir.read_binary('tests.data.ieda', '600121iso-later.xml')
doc = lxml.etree.parse(io.BytesIO(docbytes))
doi = 'doi:10.15784/600121'
pid = 'https://get.iedadata.org/metadata/iso/600121'
regex = re.compile('https://ieda.mn.org:443/')
with self.assertLogs(logger=harvester.logger, level='DEBUG') as cm:
with aioresponses() as m:
m.get(regex, body=existing_content)
asyncio.run(harvester.harvest_document(doi, pid, doc,
record_date))
# Did we see an informational message?
log_msg_count = self.logLevelCallCount(cm.output, level='INFO')
self.assertTrue(log_msg_count > 0)
# Did we increase the update count?
self.assertEqual(harvester.updated_count, update_count + 1)
def test__retrieve_record(self):
"""
SCENARIO: We have a valid landing page URL and JSON-LD document for
ARM.
EXPECTED RESULT: The expected SID and PID are returned.
"""
landing_page_url = (
'https://www.archive.arm.gov/metadata/adc/html/wsacrcrcal.html'
)
# External calls to read the:
#
# 2) HTML document for the landing page
# 3) XML document associated with the landing page
#
contents = [
ir.read_binary('tests.data.arm', 'wsacrcrcal.html'),
ir.read_binary('tests.data.arm', 'wsacrcrcal.xml'),
]
status_codes = [200, 200, 200]
headers = [
{'Content-Type': 'text/html'},
{'Content-Type': 'application/xml'},
]
z = zip(contents, status_codes, headers)
with aioresponses() as m:
for content, status_code, headers in z:
m.get(self.regex,
body=content, status=status_code, headers=headers)
harvester = SchemaDotOrgHarvester()
with self.assertLogs(logger=harvester.logger, level='DEBUG'):
awaitable = harvester.retrieve_record(landing_page_url)
sid, pid, dateMod, doc = asyncio.run(awaitable)
self.assertEqual(sid, "http://dx.doi.org/10.5439/1150280")
self.assertIsNone(pid)
expected = dt.datetime(2019, 11, 25, 16, 0, 21, 746316,
tzinfo=dt.timezone.utc)
self.assertEqual(dateMod, expected)
actual = lxml.etree.tostring(doc)
bf = io.BytesIO(contents[1])
expected = lxml.etree.tostring(lxml.etree.parse(bf))
self.assertEqual(actual, expected)
def test_document_already_harvested__followup_record_is_complete(
self,
mock_logger,
mock_check_if_identifier_exists,
mock_update_science_metadata
):
"""
SCENARIO: The GMN existance check shows that the document has already
been harvested but has been marked as "ongoing". It has been updated
since the last harvest time, and the new record is marked as
"complete". The update succeeds.
EXPECTED RESULT: The updated count goes up by one and the event is
logged at the info level.
"""
# This is the existing document in the MN. It is requested by the
# update check, and it is marked as ongoing.
existing_content = ir.read_binary('tests.data.ieda',
'600121iso-ongoing.xml')
# This is the proposed update document that is the same except it is
# marked as complete.
update_doc_bytes = ir.read_binary('tests.data.ieda', '600121iso.xml')
doc = lxml.etree.parse(io.BytesIO(update_doc_bytes))
record_date = dt.datetime.now()
mock_check_if_identifier_exists.return_value = {
'outcome': 'yes',
'record_date': record_date - dt.timedelta(days=1),
'current_version_id': 1,
}
mock_update_science_metadata.return_value = True
host, port = 'ieda.mn.org', 443
harvester = SchemaDotOrgHarvester(host=host, port=port)
initial_updated_count = harvester.updated_count
doi = 'doi.10000/abcde'
regex = re.compile('https://ieda.mn.org:443/')
with aioresponses() as m:
m.get(regex, body=existing_content)
asyncio.run(harvester.harvest_document(doi, '1', doc, record_date))
# Did we increase the failure count?
self.assertEqual(harvester.updated_count, initial_updated_count + 1)
# Did we see a warning?
self.assertTrue(harvester.logger.info.call_count > 0)
def test_retrieve_record__404_error(self):
"""
SCENARIO: We have a URL for a landing page. However, the directory
on the remote end does not have a metadata.xml document.
EXPECTED RESULT: A SkipError is raised. For other clients, this is
NOT a SkipError.
"""
url = ("https://www.northwestknowledge.net"
"/data/94E2D569-200F-44F7-8937-AB4BD0862C91")
# External I/O
#
# 1st: landing page
# 2nd: XML metadata document (raises 404)
package = 'tests.data.nkn.0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0'
contents = [
ir.read_binary(package, 'index.html'),
]
harvester = NKNHarvester()
with self.assertLogs(logger=harvester.logger, level='INFO'):
with aioresponses() as m:
m.get(self.regex, body=contents[0])
m.get(self.regex, status=404)
with self.assertRaises(SkipError):
asyncio.run(harvester.retrieve_record(url))
def test_retrieve_record__500_error(self):
"""
SCENARIO: We have a URL for a landing page. The response is a server
error, however. This is complimentary to the _404_error test below.
EXPECTED RESULT: An Exception is raised (but not a SkipError).
"""
url = ("https://www.northwestknowledge.net"
"/data/94E2D569-200F-44F7-8937-AB4BD0862C91")
# External I/O
#
# 1st: landing page
# 2nd: XML metadata document (raises 404)
package = 'tests.data.nkn.0a42d2bc-700a-4cf2-a7ac-ad6b892da7f0'
contents = [
ir.read_binary(package, 'index.html'),
]
harvester = NKNHarvester()
with self.assertLogs(logger=harvester.logger, level='INFO'):
with aioresponses() as m:
m.get(self.regex, body=contents[0])
m.get(self.regex, status=500)
with self.assertRaises(aiohttp.ClientResponseError):
asyncio.run(harvester.retrieve_record(url))
def _bootstrap(*,
root=None,
upgrade=False,
user=False,
altinstall=False,
default_pip=False,
verbosity=0):
"""
Bootstrap pip into the current Python installation (or the given root
directory). Returns pip command status code.
Note that calling this function will alter both sys.path and os.environ.
"""
if altinstall and default_pip:
raise ValueError("Cannot use altinstall and default_pip together")
sys.audit("ensurepip.bootstrap", root)
_disable_pip_configuration_settings()
# By default, installing pip and setuptools installs all of the
# following scripts (X.Y == running Python version):
#
# pip, pipX, pipX.Y, easy_install, easy_install-X.Y
#
# pip 1.5+ allows ensurepip to request that some of those be left out
if altinstall:
# omit pip, pipX and easy_install
os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
elif not default_pip:
# omit pip and easy_install
os.environ["ENSUREPIP_OPTIONS"] = "install"
with tempfile.TemporaryDirectory() as tmpdir:
# Put our bundled wheels into a temporary directory and construct the
# additional paths that need added to sys.path
additional_paths = []
for project, version in _PROJECTS:
wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
whl = resources.read_binary(
_bundled,
wheel_name,
)
with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
fp.write(whl)
additional_paths.append(os.path.join(tmpdir, wheel_name))
# Construct the arguments to be passed to the pip command
args = ["install", "--no-index", "--find-links", tmpdir]
if root:
args += ["--root", root]
if upgrade:
args += ["--upgrade"]
if user:
args += ["--user"]
if verbosity:
args += ["-" + "v" * verbosity]
return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
def test_update_science_metadata(self, mock_client):
"""
SCENARIO: A new science metadata record is successfully loaded.
EXPECTED RESULT: True
"""
sci_metadata_bytes = ir.read_binary('tests.data.ieda', '600121iso.xml')
sid = 'i_am_a_sid'
record_date = dt.datetime.now().isoformat()
old_version_pid = 'b645a195302ca652ec39f1bf3b908dbf'
client_mgr = D1ClientManager(
self.gmn_base_url, self.auth_cert, self.auth_cert_key,
self.mock_logger
)
system_metadata = self._generate_system_metadata(sci_metadata_bytes,
sid,
old_version_pid,
record_date)
actual = client_mgr.update_science_metadata(
sci_metadata_bytes=sci_metadata_bytes,
native_identifier_sid=sid,
record_date=record_date,
old_version_pid=old_version_pid,
system_metadata=system_metadata
)
self.assertTrue(actual)
def anyascii(string):
# type: (str) -> str
"""Transliterate a string into ASCII."""
try:
if string.isascii():
return string
except AttributeError:
pass
result = []
for char in string:
codepoint = ord(char)
if codepoint <= 0x7f:
result.append(char)
continue
blocknum = codepoint >> 8
lo = codepoint & 0xff
try:
block = _blocks[blocknum]
except KeyError:
try:
b = read_binary('anyascii._data', '%03x' % blocknum)
s = decompress(b, -MAX_WBITS).decode('ascii')
block = tuple(map(intern, s.split('\t')))
except FileNotFoundError:
block = ()
_blocks[blocknum] = block
if len(block) > lo:
result.append(block[lo])
return ''.join(result)
def test_document_is_unrecognized_but_successfully_harvested(
self,
mock_check_if_identifier_exists,
mock_load_science_metadata
):
"""
SCENARIO: The GMN has not seen the document before and is successfully
harvested.
EXPECTED RESULT: The event is logged at the info level. The
"created_count" goes up by one.
"""
mock_check_if_identifier_exists.return_value = {'outcome': 'no'}
mock_load_science_metadata.return_value = True
docbytes = ir.read_binary('tests.data.ieda', '600121iso.xml')
doc = lxml.etree.parse(io.BytesIO(docbytes))
harvester = SchemaDotOrgHarvester()
with self.assertLogs(logger=harvester.logger, level='DEBUG') as cm:
pargs = ('doi.10000/abcde', '1', doc, dt.datetime.now())
awaitable = harvester.harvest_document(*pargs)
asyncio.run(awaitable)
self.assertTrue(self.logLevelCallCount(cm.output, 'INFO') > 0)
self.assertEqual(harvester.created_count, 1)
def test_document_already_harvested_at_same_date(
self,
mock_check_if_identifier_exists,
mock_update_science_metadata
):
"""
SCENARIO: The GMN existance check shows that the document has already
been harvested and has not changed since the last time. We don't want
to attempt to harvest in this case.
EXPECTED RESULT: A SkipError is raised.
"""
record_date = dt.datetime.now()
mock_check_if_identifier_exists.return_value = {
'outcome': 'yes',
'record_date': record_date,
}
mock_update_science_metadata.return_value = False
harvester = SchemaDotOrgHarvester()
docbytes = ir.read_binary('tests.data.ieda', '600121iso.xml')
doc = lxml.etree.parse(io.BytesIO(docbytes))
doi = 'doi.10000/abcde'
with self.assertRaises(schema_org.core.SkipError):
asyncio.run(harvester.harvest_document(doi, '1', doc, record_date))
def test_update_science_metadata_fails(self, mock_client):
"""
SCENARIO: A new science metadata record is not successfully updated
because the update routine errors out for some reason.
EXPECTED RESULT: False, and the failure is logged at the error level.
"""
mock_client.return_value.update.side_effect = RuntimeError(
'something bad happened') # noqa: E501
sci_metadata_bytes = ir.read_binary('tests.data.ieda', '600121iso.xml')
sid = 'i_am_a_sid'
record_date = dt.datetime.now().isoformat()
old_version_pid = 'b645a195302ca652ec39f1bf3b908dbf'
client_mgr = D1ClientManager(self.gmn_base_url, self.auth_cert,
self.auth_cert_key, self.mock_logger)
system_metadata = self._generate_system_metadata(
sci_metadata_bytes, sid, old_version_pid, record_date)
actual = client_mgr.update_science_metadata(
sci_metadata_bytes=sci_metadata_bytes,
native_identifier_sid=sid,
record_date=record_date,
old_version_pid=old_version_pid,
system_metadata=system_metadata)
self.assertFalse(actual)
self.assertEqual(self.mock_logger.error.call_count, 1)
def path_for_font(name):
if name not in font_path_cache:
with open(os.path.join(self.tdir, name), 'wb') as f:
font_path_cache[name] = f.name
data = read_binary(__name__.rpartition('.')[0], name)
f.write(data)
return font_path_cache[name]
def _load_icons(self):
_icons = (('gimp-flip-horizontal.png', 'mcomix-flip-horizontal'),
('gimp-flip-vertical.png', 'mcomix-flip-vertical'),
('gimp-rotate-180.png', 'mcomix-rotate-180'),
('gimp-rotate-270.png', 'mcomix-rotate-270'),
('gimp-rotate-90.png', 'mcomix-rotate-90'),
('gimp-transform.png', 'mcomix-transform'),
('tango-enhance-image.png', 'mcomix-enhance-image'),
('tango-add-bookmark.png', 'mcomix-add-bookmark'),
('tango-archive.png', 'mcomix-archive'),
('tango-image.png', 'mcomix-image'),
('zoom.png', 'mcomix-zoom'),
('lens.png', 'mcomix-lens'))
# Load window title icons.
Gtk.Window.set_default_icon_list(self.__mcomix_icons)
# Load application icons.
factory = Gtk.IconFactory()
for filename, stockid in _icons:
try:
icon_data = resources.read_binary('mcomix.images', filename)
pixbuf = ImageTools.load_pixbuf_data(icon_data)
iconset = Gtk.IconSet.new_from_pixbuf(pixbuf)
factory.add(stockid, iconset)
except FileNotFoundError:
logger.warning(f'Could not load icon: \'{filename}\'')
factory.add_default()
def test_handle_create_snews_event(mock_create_event, mock_get_event,
mock_upload, mock_json):
text = read_binary(data, 'snews_gcn.xml')
external_triggers.handle_snews_gcn(payload=text)
mock_create_event.assert_called_once_with(filecontents=text,
search='Supernova',
pipeline='SNEWS',
group='External')
calls = [
call('"dqrjson"', 'gwcelerydetcharcheckvectors-E1.json', 'E1',
'DQR-compatible json generated from check_vectors results'),
call(None, None, 'E1',
('Detector state for active instruments is unknown.\n{}'
'Check looked within -10/+10 seconds of superevent. ').format(
detchar.generate_table('Data quality bits', [], [], [
'H1:NO_OMC_DCPD_ADC_OVERFLOW',
'H1:NO_DMT-ETMY_ESD_DAC_OVERFLOW',
'L1:NO_OMC_DCPD_ADC_OVERFLOW',
'L1:NO_DMT-ETMY_ESD_DAC_OVERFLOW', 'H1:HOFT_OK',
'H1:OBSERVATION_INTENT', 'L1:HOFT_OK',
'L1:OBSERVATION_INTENT', 'V1:HOFT_OK',
'V1:OBSERVATION_INTENT', 'V1:GOOD_DATA_QUALITY_CAT1'
])), ['data_quality'])
]
mock_upload.assert_has_calls(calls, any_order=True)
def add_static_paths(app):
"""Ensure CSS/JS is loaded."""
app.env.book_theme_resources_changed = False
output_static_folder = Path(app.outdir) / "_static"
theme_static_files = resources.contents(theme_static)
if (app.config.html_theme_options.get("theme_dev_mode", False)
and output_static_folder.exists()):
# during development, the JS/CSS may change, if this is the case,
# we want to remove the old files and ensure that the new files are loaded
for path in output_static_folder.glob("sphinx-book-theme*"):
if path.name not in theme_static_files:
app.env.book_theme_resources_changed = True
path.unlink()
# note sphinx treats theme css different to regular css
# (it is specified in theme.conf), so we don't directly use app.add_css_file
for fname in resources.contents(theme_static):
if fname.endswith(".css"):
if not (output_static_folder / fname).exists():
(output_static_folder / fname).write_bytes(
resources.read_binary(theme_static, fname))
app.env.book_theme_resources_changed = True
# add javascript
for fname in resources.contents(theme_static):
if fname.endswith(".js"):
app.add_js_file(fname)
def read_ssl_certificate():
"""Get the CA certificate that should be used to verify a TLS connection to MinKNOW.
If the ``$MINKNOW_TRUSTED_CA`` environment variable is set to the path to an
existing file, its contents will be used. Otherwise, an internal copy of MinKNOW
Core's default CA will be used.
"""
try:
with open(os.environ["MINKNOW_TRUSTED_CA"], "rb") as f:
return f.read()
except KeyError:
pass
except OSError:
logger.warning(
"$MINKNOW_TRUSTED_CA set but failed to read '%s'",
os.environ.get("MINKNOW_TRUSTED_CA"),
)
try:
# python 3.7+
import importlib.resources as importlib_resources
except ImportError:
# python 3.5/3.6
import importlib_resources
# using the syntax that works with python 3.7
return importlib_resources.read_binary("minknow_api", "ca.crt")
def fake_gcn(notice_type):
# Check the real GCN notice, which is valid.
payload = resources.read_binary(data, 'G298048-1-Initial.xml')
root = lxml.etree.fromstring(payload)
notice_type = str(int(notice_type))
root.find(".//Param[@name='Packet_Type']").attrib['value'] = notice_type
return xml_document(root)
def test__read_record__invalid_jsonld(self):
"""
SCENARIO: A landing page is properly retrieved, but has invalid
JSON-LD.
EXPECTED RESULT: JSON-LD error
"""
url = (
'https://www.archive.arm.gov'
'/metadata/adc/html/nsaqcrad1longC2.c2.invalid_jsonld.html'
)
harvester = ARMHarvester()
contents = ir.read_binary('tests.data.arm', 'nsaqcrad1longC2.c2.no_dataset_in_jsonld.html') # noqa : E501
status_code = 200
headers = {'Content-Type': 'text/html'}
regex = re.compile('https://www.archive.arm.gov/metadata/adc')
with aioresponses() as m:
m.get(regex, body=contents, status=status_code, headers=headers)
with self.assertRaises(JsonLdError):
with self.assertLogs(logger=harvester.logger, level='DEBUG'):
asyncio.run(harvester.retrieve_record(url))
def test_calc():
fp = resources.read_binary("tests.data.sample_1", "take-1.log")
fpcalc = mock.MagicMock(spec=CompletedProcess, stdout=fp)
with mock.patch("subprocess.run", return_value=fpcalc) as fpcalc_mock:
fp = fingerprints.calc("/path/to/audio", length=120)
assert fp == _load_fp("sample_1/take-1.log")
assert fpcalc_mock.called
def setUp(self):
super(TestSuiteHiRISE, self).setUp()
# Hand-create the boxes needed for HiRISE.
the_uuid = uuid.UUID('2b0d7e97-aa2e-317d-9a33-e53161a2f7d0')
ulst = glymur.jp2box.UUIDListBox([the_uuid])
version = 0
flag = [0, 0, 0]
url = 'ESP_032436_1755_COLOR.LBL'
debox = glymur.jp2box.DataEntryURLBox(version, flag, url)
uuidinfo = glymur.jp2box.UUIDInfoBox([ulst, debox])
uuid_data = ir.read_binary(data, 'degenerate_geotiff.tif')
the_uuid = uuid.UUID('b14bf8bd-083d-4b43-a5ae-8cd7d5a6ce03')
geotiff_uuid = glymur.jp2box.UUIDBox(the_uuid, uuid_data)
# Fabricate a new JP2 file out of the signature, file type, header,
# and codestream out of nemo.jp2, but add in the UUIDInfo and UUID
# box from HiRISE.
jp2 = Jp2k(self.jp2file)
boxes = [
jp2.box[0], jp2.box[1], jp2.box[2], uuidinfo, geotiff_uuid,
jp2.box[-1]
]
self.hirise_jp2file_name = self.test_dir_path / 'hirise.jp2'
jp2.wrap(self.hirise_jp2file_name, boxes=boxes)
def read_image(name: Text):
encoded_image = resources.read_binary(test_data, name)
pil_image = (PIL.Image.open(io.BytesIO(encoded_image))
) # type: PIL.Image.Image
pil_image = pil_image.convert("RGB").resize(
(192, 128), resample=PIL.Image.BICUBIC)
image = t.as_tensor(np.array(pil_image)).flip(0).contiguous()
return image, encoded_image
def instantiate_template(self, replacements, template, creator):
# read content as binary to avoid platform specific line normalization (\n -> \r\n)
binary = read_binary(self.__module__, str(template))
text = binary.decode("utf-8", errors="strict")
for key, value in replacements.items():
value = self._repr_unicode(creator, value)
text = text.replace(key, value)
return text
def mock_download(filename, graceid, *args, **kwargs):
assert graceid == 'T250822'
filenames = {
'coinc.xml': 'coinc.xml',
'psd.xml.gz': 'psd.xml.gz',
'ranking_data.xml.gz': 'ranking_data_G322589.xml.gz'
}
return resources.read_binary(data, filenames[filename])
def test_old_world(self, runner):
with runner.isolated_filesystem():
LEVEL_DATA_PATH.write_bytes(
read_binary('tests.files', 'old_level.dat'))
result = runner.invoke(ender, ['init'])
assert result.exit_code == 1
assert 'not compatible with endermite' in result.output
def test_read_binary(self):
result = resources.read_binary(self.data, 'binary.file')
self.assertEqual(result, b'\0\1\2\3')
def execute(self, package, path):
resources.read_binary(package, path)
