def __init__(
self,
hostname: str,
auth: Gen3Auth,
download_list: List[Downloadable],
):
"""
Initialize the DownloadManager so that is ready to start downloading.
Note the downloadable objects are required so that all tokens are available
to support the download.
Args:
hostname (str): Gen3 commons home commons
auth (Gen3Auth) : Gen3 authentication
download_list (List[Downloadable]): list of objects to download
"""
self.hostname = hostname
self.access_token = auth.get_access_token()
self.metadata = Gen3Metadata(auth)
self.wts_endpoints = wts_external_oidc(hostname)
self.resolved_compact_drs = {}
# add COMMONS host as a DRSEndpoint as it does not use the WTS
self.known_hosts = {
self.hostname:
KnownDRSEndpoint(
hostname=self.hostname,
access_token=self.access_token,
use_wts=False,
expire=datetime.fromtimestamp(
decode_token(self.access_token)["exp"]),
)
}
self.download_list = download_list
self.resolve_objects(self.download_list)
def test_auth_init_outside_workspace():
"""
Test that a Gen3Auth instance can be initialized when the
required parameters are included.
"""
# missing parameters
with pytest.raises(ValueError):
Gen3Auth()
# working initialization
endpoint = "localhost"
refresh_token = "my-refresh-token"
auth = Gen3Auth(endpoint=endpoint, refresh_token=refresh_token)
assert auth._endpoint == endpoint
assert auth._refresh_token == refresh_token
assert auth._use_wts == False
def mock_gen3_auth():
mock_auth = MockAuth()
# patch as __init__ has method call
with patch("gen3.auth.endpoint_from_token") as mock_endpoint_from_token:
mock_endpoint_from_token().return_value = mock_auth.endpoint
return Gen3Auth(endpoint=mock_auth.endpoint,
refresh_token=mock_auth.refresh_token)
def submission_client(endpoint=DEFAULT_ENDPOINT, refresh_file=DEFAULT_CREDENTIALS_PATH):
"""Create authorized client."""
auth = Gen3Auth(endpoint, refresh_file=refresh_file)
assert auth, 'should return an auth client'
submission_client = Gen3Submission(endpoint, auth)
assert submission_client, 'should return a submission client'
assert 'delete_program' in dir(submission_client), 'should have a delete_program method'
assert 'create_program' in dir(submission_client), 'should have a create_program method'
return submission_client
def test_auth_init_in_workspace(monkeypatch):
"""
Test that a Gen3Auth instance can be initialized with no parameters
when working inside a workspace ("NAMESPACE" environment variable),
if the workspace-token-service is available.
"""
monkeypatch.setenv("NAMESPACE", "sdk-tests")
with patch("gen3.auth.requests") as mock_request:
# unable to communicate with the WTS
mock_request.get().status_code = 403
with pytest.raises(ValueError):
Gen3Auth()
# can communicate with the WTS
mock_request.get().status_code = 200
auth = Gen3Auth()
assert auth._use_wts == True
def main():
with open("/creds.json") as creds_file:
job_name = "ingest-metadata-manifest"
creds = json.load(creds_file).get(job_name, {})
if not creds:
logging.warning(
f"No configuration found for '{job_name}' job. "
"Attempting to continue anyway..."
)
# Only use provided authz requirement if resources are not empty
access_authz_requirement = JOB_REQUIRES
if creds.get("job_requires", {}).get("job_access_req"):
access_authz_requirement = creds.get("job_requires")
# check if user has sower and ingestion policies
is_allowed, message = check_user_permission(ACCESS_TOKEN, access_authz_requirement)
if not is_allowed:
print("[out]: {}".format(message["message"]))
sys.exit()
input_data_json = json.loads(INPUT_DATA)
download_file(input_data_json["URL"], MANIFEST)
commons_host_url = "http://revproxy-service/"
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
# use available access token instead of attempting refresh
# NOTE: This means this will only continue so long as the access
# token isn't expired, which defaults to 20 minutes.
auth = Gen3Auth(commons_host_url, refresh_token=ACCESS_TOKEN)
auth._access_token = ACCESS_TOKEN
loop.run_until_complete(
metadata.async_ingest_metadata_manifest(
commons_host_url,
manifest_file=MANIFEST,
metadata_source=input_data_json["metadata_source"],
auth=auth,
)
)
log_file_presigned_url = upload_file_to_s3_and_generate_presigned_url(
creds.get("bucket"), "manifest_ingestion.log"
)
print("[out] {}".format(log_file_presigned_url))
def index_object_manifest_cli(
commons_url,
manifest_file,
thread_num,
api_key,
auth,
replace_urls,
manifest_file_delimiter,
out_manifest_file,
):
"""
Commandline interface for indexing a given manifest to indexd
Args:
commons_url (str): root domain for common
manifest_file (str): the full path to the manifest
thread_num (int): number of threads being requested
api_key (str): the path to api key
auth(str): the basic auth
replace_urls(bool): Replace urls or not
NOTE: if both api_key and auth are specified, it will ignore the later and
take the former as a default
manifest_file_delimiter(str): manifest's delimiter
out_manifest_file(str): path to the output manifest
"""
if api_key:
auth = Gen3Auth(commons_url, refresh_file=api_key)
else:
auth = tuple(auth.split(",")) if auth else None
files, headers = index_object_manifest(
commons_url + "/index",
manifest_file,
int(thread_num),
auth,
replace_urls,
manifest_file_delimiter,
)
if out_manifest_file:
_write_csv(os.path.join(CURRENT_DIR, out_manifest_file), files,
headers)
def run(ctx):
ctx.ensure_object(dict)
try:
auth = Gen3Auth(
ctx.obj["data_commons"], refresh_file=ctx.obj["credentials_file"]
)
auth_url = ctx.obj["data_commons"] + "/user/credentials/cdis/access_token"
# get access token
auth_request = requests.post(auth_url, json=auth._refresh_token)
access_token = auth_request.json()["access_token"]
# set up sower requests
headers = {"Authorization": "Bearer " + access_token}
data = {
"Action": "import",
"input": {
"guid": ctx.obj["guid"],
"db": ctx.obj["database_name"],
},
}
sower_url = ctx.obj["data_commons"] + "/job/dispatch"
sower_request = requests.post(sower_url, json=data, headers=headers)
if sower_request.status_code != 200:
print("sower job start failed, below is the error report")
print(sower_request.text)
raise
else:
print(json.dumps(sower_request.json()))
print("Job hass been started")
job_id = sower_request.json()["uid"]
job_status_url = ctx.obj["data_commons"] + "/job/status" + job_id
except Exception:
click.secho("Failed!", fg="red", bold=True, err=True)
raise
else:
click.secho("Done!", fg="green", err=True, bold=True)
import warnings
import os
import pandas as pd
import requests
import uuid
import json
import fnmatch
import hashlib
warnings.filterwarnings('ignore')
res2Drug={"Ethambutol":'conferring resistance to ethambutol',"Isoniazid":'conferring resistance to isoniazid',"Pyrazinamide":'conferring resistance to pyrazinamide',"Rifampicin":'conferring resistance to rifampicin'}
#res2Drug={"ethambutol":'ethambutol',"isoniazid":'isoniazid',"pyrazinamide":'pyrazinamide',"rifampicin":'rifampicin'}
# Set up Gen3 SDK
endpoint = "https://tb.niaiddata.org/"
auth = Gen3Auth(endpoint, refresh_file = "/home/jovyan/pd/credentials.json")
sub = Gen3Submission(endpoint, auth)
file = Gen3File(endpoint, auth)
ariba_output_dir = '/home/jovyan/pd/nb_output/tb/ariba/output'
if not os.path.exists(ariba_output_dir):
os.makedirs(ariba_output_dir)
def query_file(project,chunk,offset,drug_resistant):
# SDK submission class call peregrine to retrieve data, and convert json to flat json
params = ""
for key, value in drug_resistant.items():
params += ",{}:\"{}\"".format(key, value)
query = """
{
subject(project_id:"%s", first:%d, offset:%d, order_by_asc:"submitter_id" %s){
# api = 'https://dcf-interop.kidsfirstdrc.org/' #Kids First # Import the Gen3 Python SDK from my local copy of it downloaded from GitHub # use this command to get the sdk: `git clone [email protected]:cgmeyer/gen3sdk-python.git` import pandas as pd import sys git_dir='/Users/christopher/Documents/GitHub' sdk_dir='/cgmeyer/gen3sdk-python' sys.path.insert(1, '{}{}'.format(git_dir,sdk_dir)) from expansion.expansion import Gen3Expansion from migration.migration import Gen3Migration from gen3.submission import Gen3Submission from gen3.auth import Gen3Auth auth = Gen3Auth(api, refresh_file=creds) sub = Gen3Submission(api, auth) # Initialize an instance this class, using your creds in 'auth' # run my local working copy %run /Users/christopher/Documents/GitHub/cgmeyer/gen3sdk-python/expansion/expansion.py # Some additional functions in Gen3Expansion class exp = Gen3Expansion(api, auth) # Initialize an instance, using its functions like exp.get_project_tsvs() # download the sdk files directly from GitHub and run in ipython: #!wget https://raw.githubusercontent.com/uc-cdis/gen3sdk-python/master/gen3/submission.py #%run ./submission.py #!wget https://raw.githubusercontent.com/cgmeyer/gen3sdk-python/master/expansion/expansion.py #%run ./expansion.py #!wget https://raw.githubusercontent.com/cgmeyer/gen3sdk-python/master/migration/migration.py #%run ./migration.py # Download and configure MacOsX gen3-client
import json
from gen3.auth import Gen3Auth
from gen3.submission import Gen3Submission
# settings
SUBMIT_ENDPOINT = "xxx"
SUBMIT_PROGRAM = 'xxx'
SUBMIT_PROJECT = 'xxx'
CREDS_FILE = 'xxx'
BREAK_IF_ERROR = True
DATA_FOLDER = 'S039-1' # name of the folder containing the JSON files to submit
STEP = 50 # how many items to submit at a time
# setup Gen3 SDK
SDK_AUTH = Gen3Auth(SUBMIT_ENDPOINT, refresh_file=CREDS_FILE)
SDK_SUB = Gen3Submission(SUBMIT_ENDPOINT, SDK_AUTH)
# submit in the dictionary order
order = ['study', 'subject', 'sample']
def submit_json(data_json):
response = SDK_SUB.submit_node(SUBMIT_PROGRAM, SUBMIT_PROJECT, data_json)
return json.loads(response)
def delete_node(uuid):
api_url = "{}/api/v0/submission/{}/{}/entities/{}".format(
SUBMIT_ENDPOINT, SUBMIT_PROGRAM, SUBMIT_PROJECT, uuid)
response = requests.delete(api_url, auth=SDK_AUTH).text
def build_auth(cred=None):
auth = Gen3Auth()
if cred is not None:
assert os.path.exists(cred), f"{cred} not found"
auth = Gen3Auth(GEN3_URL, refresh_file=cred)
return auth
titlefont=dict(family="Arial, sans-serif", size=15, color="black"),
showticklabels=True,
),
)
fig.update_layout(legend_orientation="h", showlegend=True)
fig.update_traces(mode="lines+markers")
fig.update_layout(legend=dict(y=-0.23, traceorder="reversed", font_size=16))
fig.update_layout(hovermode="x")
fig.update_layout(legend=dict(xanchor="center", x=0.5, y=-0.22))
fig.write_html("5_days_moving_averages_JHU.html")
# Generating plot using Illinois department of public health data
api = "https://chicagoland.pandemicresponsecommons.org/"
creds = "credentials.json"
auth = Gen3Auth(api, creds)
sub = Gen3Submission(api, auth)
program = "open"
project = "IDPH"
summary_clinical = sub.export_node(program, project, "summary_clinical", "tsv",
"summary_clinical_idph.tsv")
idph = pd.read_csv("./summary_clinical_idph.tsv", sep="\t")
idph1 = idph[["date", "confirmed", "deaths", "testing"]]
data_day = idph1.groupby(["date"]).sum()
# Start plotting
fig = go.Figure()
for i in data_day.columns:
def dry(ctx):
ctx.ensure_object(dict)
try:
print("Checking Credentials\n")
auth = Gen3Auth(
ctx.obj["data_commons"], refresh_file=ctx.obj["credentials_file"]
)
auth_url = ctx.obj["data_commons"] + "/user/credentials/cdis/access_token"
# get access token
auth_request = requests.post(auth_url, json=auth._refresh_token)
token_flag = True
if auth_request.status_code != 200:
click.secho(
"There was an error with getting an access token. Are your api credentials valid and are you using the correct commons?",
fg="red",
bold=True,
err=True,
)
token_flag = False
else:
click.secho("Access Token OK!\n", fg="green", err=True, bold=True)
access_token = auth_request.json()["access_token"]
file = Gen3File(ctx.obj["data_commons"], auth)
print("Checking GUID:\n")
guid_flag = True
signed_url = file.get_presigned_url(ctx.obj["guid"], protocol="s3")
if not isinstance(signed_url, dict):
click.secho(
"Could not get a presigned url check your GUID",
fg="red",
bold=True,
err=True,
)
guid_flag = False
if guid_flag:
click.secho("GUID OK!\n", fg="green", err=True, bold=True)
# set up sower requests
headers = {"Authorization": "Bearer " + access_token}
print("Checking sower permissions:\n")
sower_job_list = ctx.obj["data_commons"] + "job/list"
sower_test = requests.get(sower_job_list, headers=headers)
sower_flag = True
if sower_test.status_code != 200:
click.secho(
"There was an error with sower access. Check sower permissions!",
fg="red",
bold=True,
err=True,
)
sower_flag = False
else:
click.secho("Sower permissions OK!\n", fg="green", err=True, bold=True)
if not guid_flag or not sower_flag:
click.secho(
"Something is wrong. Check GUID or Sower permissions",
fg="red",
bold=True,
err=True,
)
except Exception:
click.secho("Failed!", fg="red", bold=True, err=True)
raise
else:
click.secho("Done!", fg="green", err=True, bold=True)