def init_api(dev_token, platform):
"""Initialize api for the platform used"""
if platform == 'igor':
api = sbg.Api(url="https://api.sbgenomics.com/v2",
token=dev_token,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
elif platform == 'cgc':
api = sbg.Api(url="https://cgc-api.sbgenomics.com/v2",
token=dev_token,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
elif platform == 'cavatica':
api = sbg.Api(url="https://cavatica-api.sbgenomics.com/v2",
token=dev_token,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
else:
raise ValueError('Platform not defined')
return api
def init_api(profile='default', platform=None, dev_token=None, endpoint=None):
"""
Initialize SBG API using credentials located inside
$HOME/.sevenbridges/credentials or by provided dev_token and platform.
:param profile: profile listed inside $HOME/.sevenbridges/credentials.
Example: cgc, default, cavatica
:param platform: Predefined platform.
Available: igor, cgc, f4c, eu, cn, cavatica
:param dev_token: developer token from platform
:param endpoint: api endpoint
:return: sbg.Api
"""
if platform and not endpoint:
endpoint = get_endpoint(platform)
if dev_token and not endpoint:
endpoint = get_endpoint('igor')
if dev_token and endpoint:
api = sbg.Api(url=endpoint,
token=dev_token,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
else:
c = sbg.Config(profile=profile)
api = sbg.Api(config=c,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
return api
def get_profile(profile):
if profile == ".":
api = sbg.Api()
else:
api = sbg.Api(config=sbg.Config(profile))
# Least disruptive way to add in our user agent
api.headers["User-Agent"] = "sbpack/{} via {}".format(
__version__, api.headers["User-Agent"])
return api
def getTaskOutputs(task_id, ddir):
config = sbg.Config(profile='cgc')
api = sbg.Api(config=config)
try:
task = api.tasks.get(id=task_id)
if task.status != 'COMPLETED':
print('Task status:{}'.format(task.status))
sys.exit()
for okey, oitem in task.outputs.items():
print(okey)
if oitem.__class__ == sbg.File:
print(oitem.id)
print(oitem.name)
drsClient = sbcgcDRSClient('~/.keys/sevenbridges_keys.json')
drsResponse = drsClient.getObject(oitem.id)
print(drsResponse)
drsURL = drsClient.getAccessURL(oitem.id, 's3')
dPath = os.path.expanduser(ddir + oitem.name)
if ddir != None:
download(drsURL, dPath)
else:
print(oitem)
except sbg.NotFound as e:
print('Task not found')
except sbg.SbgError as e:
print(e.__class__)
print(e.message)
def main(yaml_fp, download_links_fp):
logger, config = load_config(yaml_fp)
api = sb.Api(url=config['api-url'], token=config['token'])
download_links(api=api,
logger=logger,
config=config,
download_links_fp=download_links_fp)
def __init__(self):
"""
To set up, add API_URL and AUTH_TOKEN to your environment
API_URL is "https://cgc-api.sbgenomics.com/v2" for all users
AUTH_TOKEN can be found in the Developer Settings on SBG or CGC
"""
self.api = sbg.Api(config=sbg.Config(url=environ['API_URL'],
token=environ['AUTH_TOKEN']))
def setUpClass(cls):
"""
Get input files from the test project and start test task
Returns:
"""
# Log file prefix formatting
prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace(
'-', '_')
logging.basicConfig(
filename=f'{prefix_date}_{cls.log_filename}.log',
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
cls.APP = 'smgogarten/genesis-relatedness/ld-pruning-pipeline'
cls.inputs = {}
cls.TASK_NAME = 'UnitTest_LD_pruning'
cls.metadata_status = 'fail'
cls.naming_status = 'fail'
if not cls.project:
c = sbg.Config(profile=cls.profile_name)
print(c)
cls.session = sbg.Api(
config=c,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
cls.project_name = 'smgogarten/genesis-relatedness'
# SET INPUTS'
cls.project = cls.session.projects.get(id=cls.project_name)
cls.inputs['gds_file'] = cls.session.files.query(names=[
'1KG_phase3_subset_chr21.gds', '1KG_phase3_subset_chr22.gds',
'1KG_phase3_subset_chrX.gds'
],
project=cls.project)
cls.inputs['sample_include_file_pruning'] = cls.session.files.query(
names=['sample_include.RData'], project=cls.project)[0]
cls.inputs['sample_include_file_gds'] = cls.session.files.query(
names=['sample_include.RData'], project=cls.project)[0]
cls.inputs['out_prefix'] = 'unittest'
cls.inputs['ld_r_threshold'] = 0.32
cls.inputs['ld_win_size'] = 10
cls.inputs['maf_threshold'] = 0.01
cls.inputs['missing_threshold'] = 0.01
cls.inputs['exclude_pca_corr'] = False
cls.inputs['genome_build'] = 'hg19'
cls.inputs['autosome_only'] = False
cls.log = logging.getLogger("#unit_test")
cls.log.info(f" Starting {cls.APP} test")
# RUN TASKS
try:
cls.task = cls.session.tasks.create(name=cls.TASK_NAME,
project=cls.project,
app=cls.APP,
inputs=cls.inputs,
run=True)
cls.log.info(f" Running {cls.APP} task")
cls.log.info(f"#task_id {cls.task.id}")
except:
cls.log.info(f" I was unable to run {cls.APP} task")
def __init__(self, instance, project):
#self.outdir = outdir
self.project_id = project
config = sbg.Config(profile=instance)
self.api = sbg.Api(config=config)
try:
project = self.api.projects.get(id=self.project_id)
#print ("Found " + project.name)
except sbg.SbgError as e:
print(e.message)
def _connect(self):
if self.api_endpoint and self.api_endpoint_version and self.access_token:
self.base_url = '%s/%s' % (self.api_endpoint,
self.api_endpoint_version)
self.api = sbg.Api(url=self.base_url,
token=self.access_token,
advance_access=True,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
def main(project_name, app_name, port_name):
config_file = sbg.Config(profile='cgc')
api = sbg.Api(config=config_file,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
my_project = get_project(api, project_name)
my_app = get_app(api, my_project, app_name)
my_files = list(api.files.query(project=my_project, limit=100).all())
all_tasks = begin_tasks(api, my_project, my_app, my_files)
return
def main():
arguments = docopt(__doc__, version='2018.11')
if arguments['--quiet']:
logger.setLevel(level=logging.ERROR)
else:
logger.setLevel(level=logging.DEBUG)
config = sbg.Config(arguments["--api-profile"])
api = sbg.Api(config=config,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
cwl_doc = recursive_workflow_load(arguments["WORKFLOW"])
if "id" not in cwl_doc:
cwl_doc["id"] = pathlib.PurePath(arguments["WORKFLOW"]).stem
uname = api.users.me().username
full_project_id = get_project(api, uname, arguments["--project"])
logger.debug("Using project: {}".format(full_project_id))
app = upload_app(api, full_project_id, cwl_doc)
logger.debug("Using app: {}".format(app.id))
job = load_job(arguments["JOB"])
job = fill_out_job_defaults(job, cwl_doc)
task_name = "{}: {}".format(
app.name,
hashlib.sha1(str(job).encode()).hexdigest()[:10])
job = resolve_job(api, full_project_id, job,
pathlib.PurePath(arguments["JOB"]))
task = api.tasks.create(name=task_name,
project=full_project_id,
app=app,
inputs=job,
interruptible=False,
run=True)
while task.status not in ["ABORTED", "COMPLETED", "FAILED"]:
task.reload()
time.sleep(float(arguments["--poll-interval"]) * 60)
# Output required in CWL Output Object Document format
# which expects property name enclosed in double quotes
print(
json.dumps(resolve_output_dict(task.outputs, arguments["--outdir"]),
indent=4,
separators=(',', ': ')))
def setUpClass(cls):
"""
Get input files from the test project and start test task
Returns:
"""
# Log file prefix formatting
prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace(
'-', '_')
logging.basicConfig(
filename=f'{prefix_date}_{cls.log_filename}.log',
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
cls.APP = 'smgogarten/genesis-relatedness/king-pipeline'
cls.inputs = {}
cls.TASK_NAME = 'UnitTest_KING_IBDseg'
cls.metadata_status = 'fail'
cls.naming_status = 'fail'
if not cls.project:
c = sbg.Config(profile=cls.profile_name)
print(c)
cls.session = sbg.Api(
config=c,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
cls.project_name = 'smgogarten/genesis-relatedness'
# SET INPUTS'
cls.project = cls.session.projects.get(id=cls.project_name)
cls.inputs['gds_file'] = cls.session.files.query(
names=['ALL.pruned.gds'], project=cls.project)[0]
cls.inputs['sample_include_file'] = cls.session.files.query(
names=['sample_include.RData'], project=cls.project)[0]
cls.inputs['phenotype_file'] = cls.session.files.query(
names=['1KG_phase3_subset_annot.RData'], project=cls.project)[0]
cls.inputs['out_prefix'] = 'unittest'
cls.inputs['kinship_plot_threshold'] = 0.044
cls.inputs['group'] = "Population"
cls.inputs['sparse_threshold'] = 0.022
cls.log = logging.getLogger("#unit_test")
cls.log.info(f" Starting {cls.APP} test")
# RUN TASKS
try:
cls.task = cls.session.tasks.create(name=cls.TASK_NAME,
project=cls.project,
app=cls.APP,
inputs=cls.inputs,
run=True)
cls.log.info(f" Running {cls.APP} task")
cls.log.info(f"#task_id {cls.task.id}")
except:
cls.log.info(f" I was unable to run {cls.APP} task")
def main(yaml_fp, create_draft_tasks, run_draft_tasks, check_status,
lower_bound_group_size, upper_bound_group_size):
logger, config = load_config(yaml_fp)
api = sb.Api(url=config['api-url'], token=config['token'])
if create_draft_tasks:
create_tasks(api, logger, config, lower_bound_group_size,
upper_bound_group_size)
elif run_draft_tasks:
run_tasks(api, logger, config)
elif check_status:
show_status(api)
else:
raise ValueError('Please select one of --create-draft-tasks, '
'--run-draft-tasks or --check-status')
def setUpClass(cls):
"""
Get input files from the test project and start test task
Returns:
"""
# Log file prefix formatting
prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace(
'-', '_')
logging.basicConfig(
filename=f'{prefix_date}_{cls.log_filename}.log',
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
cls.APP = 'smgogarten/qc-for-gwas-development/heterozygosity-by-sample'
cls.inputs = {}
cls.TASK_NAME = 'UnitTest_HetBySample'
cls.metadata_status = 'fail'
cls.naming_status = 'fail'
if not cls.project:
c = sbg.Config(profile=cls.profile_name)
print(c)
cls.session = sbg.Api(
config=c,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
cls.project_name = 'smgogarten/qc-for-gwas-development'
# SET INPUTS'
cls.project = cls.session.projects.get(id=cls.project_name)
cls.inputs['gds_file'] = cls.session.files.query(
names=[input_gds_file], project=cls.project)[0]
cls.inputs['out_prefix'] = input_out_prefix
cls.inputs['maf_min'] = input_maf_min
cls.inputs['maf_max'] = input_maf_max
cls.inputs['pass_only'] = input_pass_only
cls.inputs['cpu'] = input_cpu
cls.log = logging.getLogger("#unit_test")
cls.log.info(f" Starting {cls.APP} test")
# RUN TASKS
try:
cls.task = cls.session.tasks.create(name=cls.TASK_NAME,
project=cls.project,
app=cls.APP,
inputs=cls.inputs,
run=True)
cls.log.info(f" Running {cls.APP} task")
cls.log.info(f"#task_id {cls.task.id}")
except:
cls.log.info(f" I was unable to run {cls.APP} task")
def init_api(dev_token):
"""Initialize api for the platform used"""
platforms = {'igor': "https://api.sbgenomics.com/v2",
'cgc': "https://cgc-api.sbgenomics.com/v2",
'cavatica': "https://cavatica-api.sbgenomics.com/v2"}
for p in platforms:
try:
api = sbg.Api(url=platforms[p], token=dev_token,
error_handlers=[rate_limit_sleeper, maintenance_sleeper, general_error_sleeper])
api.users.me()
break
except:
continue
else:
raise ValueError('Invalid token')
return api
def main(project_name, cwl_file, sample_file):
api = sbg.Api(os.environ["CGC_API_URL"], os.environ["CGC_AUTH_TOKEN"])
project = [
p for p in api.projects.query(limit=None).all() if p.id == project_name
][0]
version = 0
app_id = "%s/%s/%s" % (
project_name, os.path.splitext(os.path.basename(cwl_file))[0].replace(
"-standalone", ""), version)
with open(cwl_file) as in_handle:
cwl = json.load(in_handle)
# _debug()
with advance_access(api):
app = api.apps.install_app(id=app_id, raw=cwl)
with open(sample_file) as in_handle:
inputs = json.load(in_handle)
current_time = datetime.datetime.now().strftime("%m-%d-%Y_%H-%M-%S")
task_name = "%s_%s" % (os.path.basename(app_id), current_time)
task = api.tasks.create(name=task_name,
project=project.id,
app=app.id,
inputs=inputs,
run=False)
def _get_api_client(config):
import sevenbridges as sbg
c = sbg.Config(profile=config.get("profile", "default"))
api = sbg.Api(config=c, advance_access=True)
return api
from datetime import datetime
import time
import os
import argparse
parser = argparse.ArgumentParser()
parser.add_argument("-p", "--project", help = "Cavatica project name. (Example: kfdrc-harmonization/sd-bhjxbdqk-03)", type = str, required = True)
parser.add_argument("-wt","--waittime", help = "Waitting time(seconds) of self-recheck for each time. (Default: 3600 seconds)", type = int, required = False)
parser.add_argument("-o","--outputlog", help = "Output log file. (Default: sbg_task_monitor.log)", type = str, required = False)
args = parser.parse_args()
# c = sbg.Config(profile='cavatica')
# api = sbg.Api(config=c)
base_url = 'https://cavatica-api.sbgenomics.com/v2/'
api = sbg.Api(url = base_url, token = os.environ['SB_AUTH_TOKEN'])
# api = sbg.Api(url = os.environ['SB_API_ENDPOINT'], token = os.environ['SB_AUTH_TOKEN'])
project = args.project
waittime = args.waittime or 3600
logfile = args.outputlog or 'sbg_task_monitor.log'
run = list()
draft = list()
log = open(logfile, 'w+')
## first check
tasks = api.tasks.query(project=project).all()
for task in tasks:
if "RUNNING" in task.status:
run.append(task)
ARGS = args
if len(sys.argv) == 1:
parser.print_help()
sys.exit()
#MAIN BLOCK OF CODE:
headers = {
'content-type': 'application/json',
'Accept': 'application/json',
'X-SBG-Advance-Access': 'advance',
'X-SBG-Auth-Token': ARGS['token']
} #use for folders API which doesn't have a python interface yet.
api = sbg.Api(url=ARGS['api_endpoint'], token=ARGS['token'])
API = api
PP = pprint.PrettyPrinter(depth=99)
if ARGS['auto_task_id']:
cwd = os.getcwd()
cwd = os.path.abspath(cwd)
cwd = os.path.normpath(cwd)
eprint(("what is the cwd? --> %s") % (cwd))
ARGS['task_id'] = cwd.split("/")[5]
#print(ARGS['task_id'])
if ARGS['task_id']:
eprint(("what is the task_id? --> %s") % (ARGS['task_id']))
ARGS['task_details'] = API.tasks.get(id=ARGS['task_id'])
import argparse
import csv
import os
import time
import boto3
import sevenbridges as sbg
breakpoint()
# print datetime.datetime.now()
base_url = "https://cavatica-api.sbgenomics.com/v2/"
token = os.environ["CAVATICA_ZYK_TOKEN"]
api = sbg.Api(url=base_url, token=token)
parser = argparse.ArgumentParser()
parser.add_argument("-list", required=True)
parser.add_argument("-vol", required=True)
args = parser.parse_args()
input_file = csv.DictReader(open(args.list))
task_list = []
for i in input_file:
task_list.append(i)
drc = boto3.session.Session(profile_name="chop")
client = drc.client("s3")
v = api.volumes.get(args.vol)
exports = []
edict = {}
from sevenbridges.http.error_handlers import rate_limit_sleeper, maintenance_sleeper
import sevenbridges as sbg
import os
token = os.environ['CAVATIC_TURBO_TOKEN']
url = 'https://cavatica-api.sbgenomics.com/v2/'
api = sbg.Api(url,
token,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
pjt = 'kfdrc-harmonization/pbta-lancet-vardict-analysis'
tasks = list(api.tasks.query(pjt).all())
for t in tasks:
if t.status == 'COMPLETED':
cost = t.price.amount
time = (t.end_time - t.start_time).total_seconds() / 3600
print "{}\t{}\t{}".format(t.name, cost, time)
def main(yaml_fp, data_dp):
logger, config = load_config(yaml_fp)
api = sb.Api(url=config['api-url'], token=config['token'])
bam_inputs = retrieve_files(api=api, config=config)
sort_files(bam_inputs=bam_inputs, data_dp=data_dp)
#!/usr/bin/env python
import sevenbridges as sbg
import sys
config = sbg.Config(profile='cavatica')
api = sbg.Api(config=config)
project = sys.argv[1]
with open(sys.argv[2]) as f:
flist = f.read().splitlines()
sys.stderr.write('Getting files for project ' + project + '\n')
# files = api.files.query(project=project, names=flist)
for fn in flist:
sys.stderr.write('Uploading file/dir ' + fn + '\n')
up = api.files.upload(fn, project, fn)
import sevenbridges as sbg
from sevenbridges.errors import SbgError
from sevenbridges.http.error_handlers import *
import datetime
import binpacking
import re
import os
import math
print("SBG library imported.")
prof = 'cgc'
config_file = sbg.Config(profile=prof)
api = sbg.Api(config=config_file,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
print "Api Configured!!"
print "Api Username : "******"Found project %s with project details %s" %
(my_project.name, my_project.description))
input_port_reference = "reference"
input_port_bam_file = "BAMfile"
input_port_targets = "targets"
reference_list = [
default="default",
)
args = parser.parse_args()
print(f"Args: {args.__dict__}")
# Create the download location if it doesn't exist
args.download_location = Path(args.download_location)
args.download_location.mkdir(parents=True, exist_ok=True)
# Connect to the API
config_file = sbg.Config(profile=args.sbg_profile)
api = sbg.Api(
config=config_file,
error_handlers=[
sbg.http.error_handlers.rate_limit_sleeper,
sbg.http.error_handlers.maintenance_sleeper,
],
)
# Generate the project object
project = fetch_project(api, args.project_name, args.project_id)
all_files = list_files_recursively(api, api.files.query(project=project),
project)
if args.project_path:
path_name = Path(args.project_path)
all_files = [
f for f in all_files if not os.path.relpath(
f.metadata["parent_file_name"], path_name).startswith("..")
]
token_file = value
elif op == "-f":
member_file = value
elif op == "-o":
output_file = value
import sevenbridges as sbg
f = open(token_file)
info = f.readline().strip('\n')
f.close()
dict = {}
for line in open(member_file):
if line == '':
break
index = line.find('\n')
key = line[:index]
dict[key] = 1
api = sbg.Api(url='https://cavatica-api.sbgenomics.com/v2', token=info)
filewriter = open(output_file, 'w')
for project in api.projects.query().all():
for member in project.get_members().all():
#filewriter.write(project.id,project.name,member.id)
print('\t'.join([project.id, project.name, member.id]),
file=filewriter)
#if dict.has_key(member.id): #python2
if member.id in dict.keys():
project.remove_member(user=member.id)
print("already imported " + f.name)
#setCGCMetadata(samp, file)
else:
# Import file to the project
imp = api.imports.submit_import(volume=myVolume,
project=myProject,
location=CRAMPATH + filename)
#%% CODE (broken into blocks below, this wilfl eventually become an iPython notebook
if __name__ == "__main__":
config = sbg.Config(profile='cgc')
#api = sbg.Api()
api = sbg.Api(config=config,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
# list all projects you are part of
try:
project_id = TARGET_PROJECT
project = api.projects.get(id=project_id)
print("Found " + project.name)
except SbgError as e:
print(e.message)
# List all volumes available
myVolume = api.volumes.get(VOLUME)
tree = ET.parse('../sbcgcapi/SRA_Submission_phs001524/run.xml')
root = tree.getroot()
for samp in root:
data=data, headers=headers)
response_dict = response.json() if response.json() else {}
if response.status_code / 100 != 2:
print(response_dict)
# print('Error Code: %i.' % (response_dict['code']))
#print(response_dict['more_info'])
raise Exception('Server responded with status code %s.' \
% response.status_code)
return response_dict
auth_token = 'b5764171da2e446f9fffdf2047eb0035'
os.environ['SB_API_ENDPOINT'] = 'https://cgc-api.sbgenomics.com/v2/'
os.environ['SB_AUTH_TOKEN'] = auth_token
api = sbg.Api()
tumors = list(set(list(manifest_data['DISEASE.ABBV'])))
tumors = [str(x) for x in tumors]
tumors = [str(x) for x in tumors if x != 'nan']
for tumor in tumors:
#if tumor in ['TARGET']:
# continue
if tumor not in 'TARGET':
continue
diseases = list(
set(
list(manifest_data.loc[manifest_data['DISEASE.ABBV'] == tumor,
'DISEASE.TYPE'])))
from __future__ import division
import json
from requests import request
from termcolor import colored
from math import ceil
import sevenbridges as sbg
import os
api = sbg.Api(url='https://api.sbgenomics.com/v2',
token='f060ce1cf6fd4c28b8d82e99114d7553')
print api.limit
print api.remaining
print api.reset_time
def api_call(path, method='GET', query=None, data=None, token=None):
base_url = 'https://cgc-datasets-api.sbgenomics.com/datasets/tcga/v0/'
data = json.dumps(data) if isinstance(data, dict) \
or isinstance(data, list) else None
headers = {
'X-SBG-Auth-Token': token,
'Accept': 'application/json',
'Content-type': 'application/json',
}
response = request(method,
base_url + path,
inputs=task.inputs,
description=task.name)
except SbgError as e:
logger.error("Draft task was not created!", exc_info=e)
raise SbgError("Draft task was not created!")
if __name__ == '__main__':
print('got to main')
parser = argparse.ArgumentParser()
parser.add_argument("option",
nargs="?",
help="create|run|status|rerun_failed")
args = parser.parse_args()
sb_config = sb.Config(url=config['api-url'], token=config['token'])
api = sb.Api(config=sb_config)
if args.option == 'create':
create_draft_tasks(api)
if args.option == 'run':
run_tasks(api)
if args.option == 'status':
status(api)
if args.option == 'rerun_failed':
rerun_failed(api)