def node_record_counts(self, project_id):
query_txt = """{node (first:-1, project_id:"%s"){type}}""" % (
project_id)
res = Gen3Submission.query(query_txt)
df = json_normalize(res['data']['node'])
counts = Counter(df['type'])
df = pd.DataFrame.from_dict(counts, orient='index').reset_index()
df = df.rename(columns={'index': 'node', 0: 'count'})
return df
def submission_client(endpoint=DEFAULT_ENDPOINT, refresh_file=DEFAULT_CREDENTIALS_PATH):
"""Create authorized client."""
auth = Gen3Auth(endpoint, refresh_file=refresh_file)
assert auth, 'should return an auth client'
submission_client = Gen3Submission(endpoint, auth)
assert submission_client, 'should return a submission client'
assert 'delete_program' in dir(submission_client), 'should have a delete_program method'
assert 'create_program' in dir(submission_client), 'should have a create_program method'
return submission_client
def submit_metadata(self):
print("Submitting data...")
# Gen3 submission via python SDK
sub = Gen3Submission(self.base_url, TokenAuth(self.access_token))
project_id = self.program_name + "-" + self.project_code
sub.submit_file(
project_id,
os.path.join(TEMP_DIR, "summary_location_submission.tsv"),
chunk_size=100,
)
sub.submit_file(
project_id,
os.path.join(TEMP_DIR, "summary_socio_demographic_submission.tsv"),
chunk_size=100,
)
# clean up
shutil.rmtree(TEMP_DIR)
import os
import pandas as pd
import requests
import uuid
import json
import fnmatch
import hashlib
warnings.filterwarnings('ignore')
res2Drug={"Ethambutol":'conferring resistance to ethambutol',"Isoniazid":'conferring resistance to isoniazid',"Pyrazinamide":'conferring resistance to pyrazinamide',"Rifampicin":'conferring resistance to rifampicin'}
#res2Drug={"ethambutol":'ethambutol',"isoniazid":'isoniazid',"pyrazinamide":'pyrazinamide',"rifampicin":'rifampicin'}
# Set up Gen3 SDK
endpoint = "https://tb.niaiddata.org/"
auth = Gen3Auth(endpoint, refresh_file = "/home/jovyan/pd/credentials.json")
sub = Gen3Submission(endpoint, auth)
file = Gen3File(endpoint, auth)
ariba_output_dir = '/home/jovyan/pd/nb_output/tb/ariba/output'
if not os.path.exists(ariba_output_dir):
os.makedirs(ariba_output_dir)
def query_file(project,chunk,offset,drug_resistant):
# SDK submission class call peregrine to retrieve data, and convert json to flat json
params = ""
for key, value in drug_resistant.items():
params += ",{}:\"{}\"".format(key, value)
query = """
{
subject(project_id:"%s", first:%d, offset:%d, order_by_asc:"submitter_id" %s){
submitter_id
# Import the Gen3 Python SDK from my local copy of it downloaded from GitHub # use this command to get the sdk: `git clone [email protected]:cgmeyer/gen3sdk-python.git` import pandas as pd import sys git_dir='/Users/christopher/Documents/GitHub' sdk_dir='/cgmeyer/gen3sdk-python' sys.path.insert(1, '{}{}'.format(git_dir,sdk_dir)) from expansion.expansion import Gen3Expansion from migration.migration import Gen3Migration from gen3.submission import Gen3Submission from gen3.auth import Gen3Auth auth = Gen3Auth(api, refresh_file=creds) sub = Gen3Submission(api, auth) # Initialize an instance this class, using your creds in 'auth' # run my local working copy %run /Users/christopher/Documents/GitHub/cgmeyer/gen3sdk-python/expansion/expansion.py # Some additional functions in Gen3Expansion class exp = Gen3Expansion(api, auth) # Initialize an instance, using its functions like exp.get_project_tsvs() # download the sdk files directly from GitHub and run in ipython: #!wget https://raw.githubusercontent.com/uc-cdis/gen3sdk-python/master/gen3/submission.py #%run ./submission.py #!wget https://raw.githubusercontent.com/cgmeyer/gen3sdk-python/master/expansion/expansion.py #%run ./expansion.py #!wget https://raw.githubusercontent.com/cgmeyer/gen3sdk-python/master/migration/migration.py #%run ./migration.py # Download and configure MacOsX gen3-client mkdir -p ~/.gen3
from gen3.auth import Gen3Auth
from gen3.submission import Gen3Submission
# settings
SUBMIT_ENDPOINT = "xxx"
SUBMIT_PROGRAM = 'xxx'
SUBMIT_PROJECT = 'xxx'
CREDS_FILE = 'xxx'
BREAK_IF_ERROR = True
DATA_FOLDER = 'S039-1' # name of the folder containing the JSON files to submit
STEP = 50 # how many items to submit at a time
# setup Gen3 SDK
SDK_AUTH = Gen3Auth(SUBMIT_ENDPOINT, refresh_file=CREDS_FILE)
SDK_SUB = Gen3Submission(SUBMIT_ENDPOINT, SDK_AUTH)
# submit in the dictionary order
order = ['study', 'subject', 'sample']
def submit_json(data_json):
response = SDK_SUB.submit_node(SUBMIT_PROGRAM, SUBMIT_PROJECT, data_json)
return json.loads(response)
def delete_node(uuid):
api_url = "{}/api/v0/submission/{}/{}/entities/{}".format(
SUBMIT_ENDPOINT, SUBMIT_PROGRAM, SUBMIT_PROJECT, uuid)
response = requests.delete(api_url, auth=SDK_AUTH).text
return response
def sub():
return Gen3Submission("http://localhost/api", None)
def sub():
"""
Mock Gen3Submission with MockAuth
"""
return Gen3Submission(MockAuth())
showticklabels=True,
),
)
fig.update_layout(legend_orientation="h", showlegend=True)
fig.update_traces(mode="lines+markers")
fig.update_layout(legend=dict(y=-0.23, traceorder="reversed", font_size=16))
fig.update_layout(hovermode="x")
fig.update_layout(legend=dict(xanchor="center", x=0.5, y=-0.22))
fig.write_html("5_days_moving_averages_JHU.html")
# Generating plot using Illinois department of public health data
api = "https://chicagoland.pandemicresponsecommons.org/"
creds = "credentials.json"
auth = Gen3Auth(api, creds)
sub = Gen3Submission(api, auth)
program = "open"
project = "IDPH"
summary_clinical = sub.export_node(program, project, "summary_clinical", "tsv",
"summary_clinical_idph.tsv")
idph = pd.read_csv("./summary_clinical_idph.tsv", sep="\t")
idph1 = idph[["date", "confirmed", "deaths", "testing"]]
data_day = idph1.groupby(["date"]).sum()
# Start plotting
fig = go.Figure()
for i in data_day.columns:
fig.add_trace(
def submission_client(terra_auth_url, user_email, gen3_endpoint):
"""Query terra_auth_url for access token and ensure inserted onto all gen3 requests."""
auth = Gen3TerraAuth(terra_auth_url=terra_auth_url, user_email=user_email)
return Gen3Submission(gen3_endpoint, auth)
def sub():
return Gen3Submission(MockAuth())