def get_data(filename: str):
fasta_gz = pkg_resources.read_binary(dcp.test, f"{filename}.fasta.gz")
fasta = gzip.decompress(fasta_gz).decode()
json_gz = pkg_resources.read_binary(dcp.test, f"{filename}.json.gz")
data = json.loads(gzip.decompress(json_gz))
desired = {}
for d in data:
key = (d["multiple_hits"], d["hmmer3_compat"], d["target"],
d["profile"])
desired[key] = {
"alt_loglik": d["alt_loglik"],
"alt_path": d["alt_path"],
"alt_codon_stream": d["alt_codon_stream"],
"alt_amino_stream": d["alt_amino_stream"],
"null_loglik": d["null_loglik"],
"null_path": d["null_path"],
"null_codon_stream": d["null_codon_stream"],
"null_amino_stream": d["null_amino_stream"],
}
return {
"hmm": dcp.example.get(f"{filename}.hmm"),
"targets": fr.read_fasta(StringIO(fasta)).read_items(),
"desired": desired,
}
def load_resource(self, package, resource):
"""
Load environment with distance map from a binary
"""
from importlib_resources import read_binary
load_data = read_binary(package, resource)
self.set_full_state_msg(load_data)
def generate_pdfa_ps(target_filename: Path, icc: str = 'sRGB'):
"""Create a Postscript PDFMARK file for Ghostscript PDF/A conversion
pdfmark is an extension to the Postscript language that describes some PDF
features like bookmarks and annotations. It was originally specified Adobe
Distiller, for Postscript to PDF conversion.
Ghostscript uses pdfmark for PDF to PDF/A conversion as well. To use Ghostscript
to create a PDF/A, we need to create a pdfmark file with the necessary metadata.
This function takes care of the many version-specific bugs and pecularities in
Ghostscript's handling of pdfmark.
The only information we put in specifies that we want the file to be a
PDF/A, and we want to Ghostscript to convert objects to the sRGB colorspace
if it runs into any object that it decides must be converted.
Arguments:
target_filename: filename to save
icc: ICC identifier such as 'sRGB'
References:
Adobe PDFMARK Reference: https://www.adobe.com/content/dam/acom/en/devnet/acrobat/pdfs/pdfmark_reference.pdf
"""
if icc != 'sRGB':
raise NotImplementedError("Only supporting sRGB")
bytes_icc_profile = read_binary(
'ocrmypdf.data', SRGB_ICC_PROFILE_NAME
)
ps = '\n'.join(_make_postscript(icc, bytes_icc_profile, 3))
# We should have encoded everything to pure ASCII by this point, and
# to be safe, only allow ASCII in PostScript
Path(target_filename).write_text(ps, encoding='ascii')
return target_filename
def read_resource_binary(*path):
"""Return the binary contents of the resource."""
if len(path) == 0:
raise TypeError("must provide a path")
file_name = path[-1]
package = ".".join([RESOURCE_MODULE] + list(path[:-1]))
return importlib_resources.read_binary(package, file_name)
def setUp(self) -> None:
super().setUp()
self.metadata_conn = sqlite3.Connection(":memory:", isolation_level=None)
self.metadata_pool = lambda: nullcontext(self.metadata_conn)
self.scraper = scrape.MetadataTipScraper(
api=self.api,
metadata_pool=self.metadata_pool,
user_access=site.UserAccess(auth=TEST_AUTH),
)
# Set up mock response for feed
feed_content = importlib_resources.read_binary(
"btn_cache.tests", "test_feed.xml"
)
# Don't need full query string for match
self.requests_mocker.get(
"https://broadcasthe.net/feeds.php",
content=feed_content,
headers={"Content-Type": "application/xml"},
)
self.torrents: Dict[str, api_types.TorrentEntry] = {}
for i in range(1, 6):
entry = TEST_ENTRY.copy()
entry["TorrentID"] = str(i)
entry["Time"] = str(i)
self.torrents[str(i)] = entry
result = api_types.GetTorrentsResult(results="5", torrents=self.torrents)
self.api_mock = self.mock_api_request(
"getTorrents", [self.key, {}, 2 ** 31, 0], result
)
def load_env_dict(cls, filename, load_from_package=None):
if load_from_package is not None:
from importlib_resources import read_binary
load_data = read_binary(load_from_package, filename)
else:
with open(filename, "rb") as file_in:
load_data = file_in.read()
if filename.endswith("mpk"):
env_dict = msgpack.unpackb(load_data,
use_list=False,
encoding="utf-8")
elif filename.endswith("pkl"):
env_dict = pickle.loads(load_data)
else:
print(f"filename {filename} must end with either pkl or mpk")
env_dict = {}
# Replace the agents tuple with EnvAgent objects
if "agents_static" in env_dict:
env_dict["agents"] = EnvAgent.load_legacy_static_agent(
env_dict["agents_static"])
# remove the legacy key
del env_dict["agents_static"]
elif "agents" in env_dict:
env_dict["agents"] = [
EnvAgent(*d[0:12]) for d in env_dict["agents"]
]
return env_dict
def test_hold_chain_charset(self):
# Issue #144 - UnicodeEncodeError in the hold chain.
self._mlist.admin_immed_notify = True
self._mlist.respond_to_post_requests = False
bart = self._user_manager.create_user('*****@*****.**', 'Bart User')
address = set_preferred(bart)
self._mlist.subscribe(address, MemberRole.moderator)
msg = mfb(read_binary('mailman.chains.tests', 'issue144.eml'))
msg.sender = '*****@*****.**'
process_chain(self._mlist, msg, {}, start_chain='hold')
# The postauth.txt message is now in the virgin queue awaiting
# delivery to the moderators.
items = get_queue_messages('virgin', expected_count=1)
msgdata = items[0].msgdata
# Should get sent to -owner address.
self.assertEqual(msgdata['recipients'], {'*****@*****.**'})
# Ensure that the subject looks correct in the postauth.txt.
msg = items[0].msg
value = None
for line in msg.get_payload(0).get_payload().splitlines():
if line.strip().startswith('Subject:'):
header, colon, value = line.partition(':')
break
self.assertEqual(value.lstrip(), 'Vi?enamjenski pi?tolj za vodu 8/1')
self.assertEqual(
msg['Subject'],
'[email protected] post from [email protected] requires approval')
def generator(rail: GridTransitionMap,
num_agents: int,
hints: Any = None,
num_resets: int = 0) -> Schedule:
if load_from_package is not None:
from importlib_resources import read_binary
load_data = read_binary(load_from_package, filename)
else:
with open(filename, "rb") as file_in:
load_data = file_in.read()
data = msgpack.unpackb(load_data, use_list=False, encoding='utf-8')
if "agents_static" in data:
agents = EnvAgent.load_legacy_static_agent(data["agents_static"])
else:
agents = [EnvAgent(*d[0:12]) for d in data["agents"]]
# setup with loaded data
agents_position = [a.initial_position for a in agents]
agents_direction = [a.direction for a in agents]
agents_target = [a.target for a in agents]
agents_speed = [a.speed_data['speed'] for a in agents]
agents_malfunction = [
a.malfunction_data['malfunction_rate'] for a in agents
]
return Schedule(agent_positions=agents_position,
agent_directions=agents_direction,
agent_targets=agents_target,
agent_speeds=agents_speed,
agent_malfunction_rates=None)
def load_data_file(name, load_json=True, as_binary=False):
"""Load a data file."""
if as_binary:
return importlib_resources.read_binary(data, name)
string = importlib_resources.read_text(data, name)
if load_json and name.endswith(".json"):
return json.loads(string)
return string
def _handle_authentication_response(self):
has_error = flask.request.args.get('error', False, lambda x: bool(int(x)))
if has_error:
if 'error' in flask.session:
return self._show_error_response(flask.session.pop('error'))
return 'Something went wrong.'
try:
session = UserSession(flask.session)
except UninitialisedSession:
return self._handle_error_response({'error': 'unsolicited_response', 'error_description': 'No initialised user session.'})
if 'auth_request' not in flask.session:
return self._handle_error_response({'error': 'unsolicited_response', 'error_description': 'No authentication request stored.'})
auth_request = AuthorizationRequest().from_json(flask.session.pop('auth_request'))
if flask.session.pop('fragment_encoded_response', False):
return importlib_resources.read_binary('flask_pyoidc', 'parse_fragment.html').decode('utf-8')
is_processing_fragment_encoded_response = flask.request.method == 'POST'
if is_processing_fragment_encoded_response:
auth_resp = flask.request.form
else:
auth_resp = flask.request.args
client = self.clients[session.current_provider]
authn_resp = client.parse_authentication_response(auth_resp)
logger.debug('received authentication response: %s', authn_resp.to_json())
try:
result = AuthResponseHandler(client).process_auth_response(authn_resp, auth_request)
except AuthResponseErrorResponseError as e:
return self._handle_error_response(e.error_response, is_processing_fragment_encoded_response)
except AuthResponseProcessError as e:
return self._handle_error_response({'error': 'unexpected_error', 'error_description': str(e)},
is_processing_fragment_encoded_response)
if current_app.config.get('OIDC_SESSION_PERMANENT', True):
flask.session.permanent = True
UserSession(flask.session).update(access_token=result.access_token,
expires_in=result.expires_in,
id_token=result.id_token_claims,
id_token_jwt=result.id_token_jwt,
userinfo=result.userinfo_claims,
refresh_token=result.refresh_token)
destination = flask.session.pop('destination')
if is_processing_fragment_encoded_response:
# if the POST request was from the JS page handling fragment encoded responses we need to return
# the destination URL as the response body
return destination
return redirect(destination)
def gh_and_response(self, payload_filename):
payload = importlib_resources.read_binary(graphql_samples,
payload_filename)
status_code_match = re.match(r"^.+-(\d+)\.json$", payload_filename)
status_code = int(status_code_match.group(1))
return (
MockGitHubAPI(status_code, body=payload,
oauth_token="oauth-token"),
json.loads(payload.decode("utf-8")),
)
def _write_file(path, filename):
import importlib_resources as pkg_resources
import fasta_reader
content = pkg_resources.read_binary(fasta_reader.test, filename)
with open(path / filename, "wb") as f:
f.write(content)
return path / filename
def ensure_current_suffix_list():
# Read and parse the organizational domain suffix list. First look in the
# cached directory to see if we already have a valid copy of it.
cached_copy_path = os.path.join(config.VAR_DIR, LOCAL_FILE_NAME)
lifetime = as_timedelta(config.dmarc.cache_lifetime)
download = False
try:
mtime = os.stat(cached_copy_path).st_mtime
except FileNotFoundError:
vlog.info('No cached copy of the public suffix list found')
download = True
cache_found = False
else:
cache_found = True
# Is the cached copy out-of-date? Note that when we write a new cache
# version we explicitly set its mtime to the time in the future when
# the cache will expire.
if mtime < now().timestamp():
download = True
vlog.info('Cached copy of public suffix list is out of date')
if download:
try:
content = get(config.dmarc.org_domain_data_url)
except (URLError, HTTPError) as error:
elog.error('Unable to retrieve public suffix list from %s: %s',
config.dmarc.org_domain_data_url,
getattr(error, 'reason', str(error)))
if cache_found:
vlog.info('Using out of date public suffix list')
content = None
else:
# We couldn't access the URL and didn't even have an out of
# date suffix list cached. Use the shipped version.
content = read_binary('mailman.rules.data', LOCAL_FILE_NAME)
if content is not None:
# Content is either a string or UTF-8 encoded bytes.
if isinstance(content, bytes):
content = content.decode('utf-8')
# Write the cache atomically.
new_path = cached_copy_path + '.new'
with open(new_path, 'w', encoding='utf-8') as fp:
fp.write(content)
# Set the expiry time to the future.
mtime = (now() + lifetime).timestamp()
os.utime(new_path, (mtime, mtime))
# Flip the new file into the cached location. This does not
# modify the mtime.
os.rename(new_path, cached_copy_path)
return cached_copy_path
def _handle_authentication_response(self):
has_error = flask.request.args.get('error', False, lambda x: bool(int(x)))
if has_error:
if 'error' in flask.session:
return self._show_error_response(flask.session.pop('error'))
return 'Something went wrong.'
if flask.session.pop('fragment_encoded_response', False):
return importlib_resources.read_binary('flask_pyoidc', 'parse_fragment.html').decode('utf-8')
is_processing_fragment_encoded_response = flask.request.method == 'POST'
if is_processing_fragment_encoded_response:
auth_resp = flask.request.form
else:
auth_resp = flask.request.args
logger.info(flask.session)
client = self.clients[UserSession(flask.session).current_provider]
authn_resp = client.parse_authentication_response(auth_resp)
logger.debug('received authentication response: %s', authn_resp.to_json())
try:
logger.debug(f"Keys in session: {flask.session.keys()}")
logger.debug(f"Values in session: {flask.session.values()}")
result = AuthResponseHandler(client).process_auth_response(authn_resp,
flask.session.pop('state'),
flask.session.pop('nonce'))
except AuthResponseErrorResponseError as e:
return self._handle_error_response(e.error_response, is_processing_fragment_encoded_response)
except AuthResponseProcessError as e:
return self._handle_error_response({'error': 'unexpected_error', 'error_description': str(e)},
is_processing_fragment_encoded_response)
if current_app.config.get('OIDC_SESSION_PERMANENT', True):
flask.session.permanent = True
UserSession(flask.session).update(result.access_token,
result.id_token_claims,
result.id_token_jwt,
result.userinfo_claims)
destination = flask.session.pop('destination')
if is_processing_fragment_encoded_response:
# if the POST request was from the JS page handling fragment encoded responses we need to return
# the destination URL as the response body
return destination
return redirect(destination)
async def test_get_installation_access_token(self):
gh = MockGitHubAPI()
installation_id = 6789
app_id = 12345
private_key = importlib_resources.read_binary(rsa_key_samples,
"test_rsa_key")
await apps.get_installation_access_token(
gh,
installation_id=installation_id,
app_id=app_id,
private_key=private_key)
assert gh.url == "https://api.github.com/app/installations/6789/access_tokens"
assert gh.body == b""
async def _http_static(self, req: HTTPRequest, path: str) -> HTTPResponse:
if path == "__init__.py":
return HTTPResponse.build(404)
try:
# Note we don't support nested resources, this is fine for the moment
# and it prevent us from malicious path containing `..`
data = importlib_resources.read_binary(http_static_module, path)
except (FileNotFoundError, ValueError):
return HTTPResponse.build(404)
headers = {}
content_type, _ = mimetypes.guess_type(path)
if content_type:
headers[b"content-Type"] = content_type.encode("ascii")
return HTTPResponse.build(200, headers=headers, data=data)
def test_get_jwt(self, time_mock):
app_id = 12345
time_mock.return_value = 1587069751.5588422
# test file copied from https://github.com/jpadilla/pyjwt/blob/master/tests/keys/testkey_rsa
private_key = importlib_resources.read_binary(rsa_key_samples,
"test_rsa_key")
result = apps.get_jwt(app_id=app_id, private_key=private_key)
expected_payload = {
"iat": 1587069751,
"exp": 1587069751 + (10 * 60),
"iss": app_id,
}
assert result == jwt.encode(expected_payload,
private_key,
algorithm="RS256").decode("utf-8")
def create_and_populate_analysis_directory(directory: Path, reads1: Path,
reads2: Path, library_type: str):
try:
directory.mkdir()
except OSError as e:
logger.error(e)
sys.exit(1)
# Write the configuration file
configuration = read_binary("blr", CONFIGURATION_FILE_NAME)
with (directory / CONFIGURATION_FILE_NAME).open("wb") as f:
f.write(configuration)
# Update with library type into
change_config(directory / CONFIGURATION_FILE_NAME,
[("library_type", library_type)])
create_symlink(reads1, directory, "reads.1.fastq.gz")
create_symlink(reads2, directory, "reads.2.fastq.gz")
def resource_context(*path, **kwargs):
"""Provide a context manager that yields a pathlib.Path object to a resource file or directory.
If the resource does not already exist on its own on the file system,
a temporary directory/file will be created. If the directory/file was created, it
will be deleted upon exiting the context manager (no exception is
raised if the directory was deleted prior to the context manager
exiting).
"""
if len(path) == 0:
raise TypeError("must provide a path")
final_name = path[-1]
package = ".".join([RESOURCE_MODULE] + list(path[:-1]))
ignore = kwargs.pop("ignore", (".DS_Store", "__init__.py"))
if importlib_resources.is_resource(package, final_name):
# the resource is a file
with importlib_resources.path(package, final_name) as path:
yield path.absolute()
else:
# the resource is a directory
package = package + "." + final_name
# TODO if the package folder exists on the file system it would be ideal to just return that
# but importlib_resources doesn't provide a public API for that
resources = [
c for c in importlib_resources.contents(package)
if importlib_resources.is_resource(package, c) and c not in ignore
]
folder_path = pathlib.Path(tempfile.mkdtemp())
try:
for resource in resources:
with (folder_path / resource).open("wb") as handle:
handle.write(
importlib_resources.read_binary(package, resource))
yield folder_path
finally:
if folder_path.exists():
shutil.rmtree(str(folder_path))
def create_and_populate_analysis_directory(directory: Path, reads: Path,
abc_file: Path):
try:
directory.mkdir()
except OSError as e:
logger.error(e)
sys.exit(1)
# Write the configuration file
configuration = read_binary("dbspro", CONFIGURATION_FILE_NAME)
with (directory / CONFIGURATION_FILE_NAME).open("wb") as f:
f.write(configuration)
# Write ABC fasta file with ^ prior to sequence (used in cutadapt command)
filename_as_string = str(directory) + "/" + ABC_FILE_NAME
with dnaio.open(filename_as_string, mode='w', fileformat="fasta") as open_out, \
dnaio.open(abc_file, mode='r', fileformat="fasta") as open_in:
for abc in open_in:
if not abc.sequence.startswith("^"):
abc.sequence = "^" + abc.sequence
open_out.write(abc)
create_symlink(reads, directory, "reads.fastq.gz")
def test_read_binary_does_not_keep_open(self):
c = resources.read_binary('ziptestdata', 'binary.file')
self.zip_path.unlink()
del c
## ccAF: __init__.py ##
## ______ ______ __ __ ##
## /\ __ \ /\ ___\ /\ \/\ \ ##
## \ \ __ \ \ \___ \ \ \ \_\ \ ##
## \ \_\ \_\ \/\_____\ \ \_____\ ##
## \/_/\/_/ \/_____/ \/_____/ ##
## @Developed by: Plaisier Lab ##
## (https://plaisierlab.engineering.asu.edu/) ##
## Arizona State University ##
## 242 ISTB1, 550 E Orange St ##
## Tempe, AZ 85281 ##
## @Author: Chris Plaisier, Samantha O'Connor ##
## @License: GNU GPLv3 ##
## ##
## If this program is used in your analysis please ##
## mention who built it. Thanks. :-) ##
##########################################################
from importlib_resources import read_binary
import pickle
#import ccAF.actinn as actinn
# Version
__version__ = '1.0.0'
# Load up ccAF classifier
tmp = read_binary("ccAF", 'ccAF_1536_smaller.pkl')
ccAF = pickle.loads(tmp)
def load_dvm_parameters():
blob = read_binary('hit_acs', 'DVM-Parameter_v2.10.0-HIT.csv')
parlist = load_csv(blob.splitlines(), 'utf-8')
return dicti({p['name']: p for p in parlist})
def load_resource(package, resource):
"""Load yaml document from package resource."""
return safe_load(read_binary(package, resource))
def load_resource(package, resource):
"""Load yaml document from package resource."""
return safe_load(read_binary(package, resource))
def test_read_submodule_resource(self):
submodule = import_module('ziptestdata.subdirectory')
result = resources.read_binary(submodule, 'binary.file')
self.assertEqual(result, b'\0\1\2\3')
def _handle_authentication_response(self):
has_error = flask.request.args.get("error", False,
lambda x: bool(int(x)))
if has_error:
if "error" in flask.session:
return self._show_error_response(flask.session.pop("error"))
return "Something went wrong."
if flask.session.pop("fragment_encoded_response", False):
return importlib_resources.read_binary(
"flask_pyoidc", "parse_fragment.html").decode("utf-8")
is_processing_fragment_encoded_response = flask.request.method == "POST"
if is_processing_fragment_encoded_response:
auth_resp = flask.request.form
else:
auth_resp = flask.request.args
client = self.clients[UserSession(flask.session).current_provider]
authn_resp = client.parse_authentication_response(auth_resp,
skew=self.skew)
logger.debug("received authentication response: %s",
authn_resp.to_json())
try:
auth_handler = AuthResponseHandler(client)
result = auth_handler.process_auth_response(
authn_resp,
flask.session.pop("state"),
flask.session.pop("nonce"),
skew=self.skew)
except AuthResponseErrorResponseError as e:
return self._handle_error_response(
e.error_response, is_processing_fragment_encoded_response)
except AuthResponseProcessError as e:
return self._handle_error_response(
{
"error": "unexpected_error",
"error_description": str(e)
},
is_processing_fragment_encoded_response,
)
if current_app.config.get("OIDC_SESSION_PERMANENT", True):
flask.session.permanent = True
UserSession(flask.session).update(
result.access_token,
result.id_token_claims,
result.id_token_jwt,
result.userinfo_claims,
)
destination = flask.session.pop("destination")
if is_processing_fragment_encoded_response:
# if the POST request was from the JS page handling fragment encoded responses we need to return
# the destination URL as the response body
return destination
return redirect(destination)
def ndjson_bytes(self):
return BytesIO(
importlib_resources.read_binary("tests.unit.h_api.fixtures",
"bulk_api.ndjson"))
def test_read_submodule_resource_by_name(self):
result = resources.read_binary('ziptestdata.subdirectory',
'binary.file')
self.assertEqual(result, b'\0\1\2\3')
def test_read_binary(self):
result = resources.read_binary(self.data, 'binary.file')
self.assertEqual(result, b'\0\1\2\3')
def execute(self, package, path):
resources.read_binary(package, path)
