def init_api(profile='default', platform=None, dev_token=None, endpoint=None):
"""
Initialize SBG API using credentials located inside
$HOME/.sevenbridges/credentials or by provided dev_token and platform.
:param profile: profile listed inside $HOME/.sevenbridges/credentials.
Example: cgc, default, cavatica
:param platform: Predefined platform.
Available: igor, cgc, f4c, eu, cn, cavatica
:param dev_token: developer token from platform
:param endpoint: api endpoint
:return: sbg.Api
"""
if platform and not endpoint:
endpoint = get_endpoint(platform)
if dev_token and not endpoint:
endpoint = get_endpoint('igor')
if dev_token and endpoint:
api = sbg.Api(url=endpoint,
token=dev_token,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
else:
c = sbg.Config(profile=profile)
api = sbg.Api(config=c,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
return api
def getTaskOutputs(task_id, ddir):
config = sbg.Config(profile='cgc')
api = sbg.Api(config=config)
try:
task = api.tasks.get(id=task_id)
if task.status != 'COMPLETED':
print('Task status:{}'.format(task.status))
sys.exit()
for okey, oitem in task.outputs.items():
print(okey)
if oitem.__class__ == sbg.File:
print(oitem.id)
print(oitem.name)
drsClient = sbcgcDRSClient('~/.keys/sevenbridges_keys.json')
drsResponse = drsClient.getObject(oitem.id)
print(drsResponse)
drsURL = drsClient.getAccessURL(oitem.id, 's3')
dPath = os.path.expanduser(ddir + oitem.name)
if ddir != None:
download(drsURL, dPath)
else:
print(oitem)
except sbg.NotFound as e:
print('Task not found')
except sbg.SbgError as e:
print(e.__class__)
print(e.message)
def __init__(self):
"""
To set up, add API_URL and AUTH_TOKEN to your environment
API_URL is "https://cgc-api.sbgenomics.com/v2" for all users
AUTH_TOKEN can be found in the Developer Settings on SBG or CGC
"""
self.api = sbg.Api(config=sbg.Config(url=environ['API_URL'],
token=environ['AUTH_TOKEN']))
def setUpClass(cls):
"""
Get input files from the test project and start test task
Returns:
"""
# Log file prefix formatting
prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace(
'-', '_')
logging.basicConfig(
filename=f'{prefix_date}_{cls.log_filename}.log',
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
cls.APP = 'smgogarten/genesis-relatedness/ld-pruning-pipeline'
cls.inputs = {}
cls.TASK_NAME = 'UnitTest_LD_pruning'
cls.metadata_status = 'fail'
cls.naming_status = 'fail'
if not cls.project:
c = sbg.Config(profile=cls.profile_name)
print(c)
cls.session = sbg.Api(
config=c,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
cls.project_name = 'smgogarten/genesis-relatedness'
# SET INPUTS'
cls.project = cls.session.projects.get(id=cls.project_name)
cls.inputs['gds_file'] = cls.session.files.query(names=[
'1KG_phase3_subset_chr21.gds', '1KG_phase3_subset_chr22.gds',
'1KG_phase3_subset_chrX.gds'
],
project=cls.project)
cls.inputs['sample_include_file_pruning'] = cls.session.files.query(
names=['sample_include.RData'], project=cls.project)[0]
cls.inputs['sample_include_file_gds'] = cls.session.files.query(
names=['sample_include.RData'], project=cls.project)[0]
cls.inputs['out_prefix'] = 'unittest'
cls.inputs['ld_r_threshold'] = 0.32
cls.inputs['ld_win_size'] = 10
cls.inputs['maf_threshold'] = 0.01
cls.inputs['missing_threshold'] = 0.01
cls.inputs['exclude_pca_corr'] = False
cls.inputs['genome_build'] = 'hg19'
cls.inputs['autosome_only'] = False
cls.log = logging.getLogger("#unit_test")
cls.log.info(f" Starting {cls.APP} test")
# RUN TASKS
try:
cls.task = cls.session.tasks.create(name=cls.TASK_NAME,
project=cls.project,
app=cls.APP,
inputs=cls.inputs,
run=True)
cls.log.info(f" Running {cls.APP} task")
cls.log.info(f"#task_id {cls.task.id}")
except:
cls.log.info(f" I was unable to run {cls.APP} task")
def get_profile(profile):
if profile == ".":
api = sbg.Api()
else:
api = sbg.Api(config=sbg.Config(profile))
# Least disruptive way to add in our user agent
api.headers["User-Agent"] = "sbpack/{} via {}".format(
__version__, api.headers["User-Agent"])
return api
def __init__(self, instance, project):
#self.outdir = outdir
self.project_id = project
config = sbg.Config(profile=instance)
self.api = sbg.Api(config=config)
try:
project = self.api.projects.get(id=self.project_id)
#print ("Found " + project.name)
except sbg.SbgError as e:
print(e.message)
def main(project_name, app_name, port_name):
config_file = sbg.Config(profile='cgc')
api = sbg.Api(config=config_file,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
my_project = get_project(api, project_name)
my_app = get_app(api, my_project, app_name)
my_files = list(api.files.query(project=my_project, limit=100).all())
all_tasks = begin_tasks(api, my_project, my_app, my_files)
return
def main():
arguments = docopt(__doc__, version='2018.11')
if arguments['--quiet']:
logger.setLevel(level=logging.ERROR)
else:
logger.setLevel(level=logging.DEBUG)
config = sbg.Config(arguments["--api-profile"])
api = sbg.Api(config=config,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
cwl_doc = recursive_workflow_load(arguments["WORKFLOW"])
if "id" not in cwl_doc:
cwl_doc["id"] = pathlib.PurePath(arguments["WORKFLOW"]).stem
uname = api.users.me().username
full_project_id = get_project(api, uname, arguments["--project"])
logger.debug("Using project: {}".format(full_project_id))
app = upload_app(api, full_project_id, cwl_doc)
logger.debug("Using app: {}".format(app.id))
job = load_job(arguments["JOB"])
job = fill_out_job_defaults(job, cwl_doc)
task_name = "{}: {}".format(
app.name,
hashlib.sha1(str(job).encode()).hexdigest()[:10])
job = resolve_job(api, full_project_id, job,
pathlib.PurePath(arguments["JOB"]))
task = api.tasks.create(name=task_name,
project=full_project_id,
app=app,
inputs=job,
interruptible=False,
run=True)
while task.status not in ["ABORTED", "COMPLETED", "FAILED"]:
task.reload()
time.sleep(float(arguments["--poll-interval"]) * 60)
# Output required in CWL Output Object Document format
# which expects property name enclosed in double quotes
print(
json.dumps(resolve_output_dict(task.outputs, arguments["--outdir"]),
indent=4,
separators=(',', ': ')))
def setUpClass(cls):
"""
Get input files from the test project and start test task
Returns:
"""
# Log file prefix formatting
prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace(
'-', '_')
logging.basicConfig(
filename=f'{prefix_date}_{cls.log_filename}.log',
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
cls.APP = 'smgogarten/genesis-relatedness/king-pipeline'
cls.inputs = {}
cls.TASK_NAME = 'UnitTest_KING_IBDseg'
cls.metadata_status = 'fail'
cls.naming_status = 'fail'
if not cls.project:
c = sbg.Config(profile=cls.profile_name)
print(c)
cls.session = sbg.Api(
config=c,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
cls.project_name = 'smgogarten/genesis-relatedness'
# SET INPUTS'
cls.project = cls.session.projects.get(id=cls.project_name)
cls.inputs['gds_file'] = cls.session.files.query(
names=['ALL.pruned.gds'], project=cls.project)[0]
cls.inputs['sample_include_file'] = cls.session.files.query(
names=['sample_include.RData'], project=cls.project)[0]
cls.inputs['phenotype_file'] = cls.session.files.query(
names=['1KG_phase3_subset_annot.RData'], project=cls.project)[0]
cls.inputs['out_prefix'] = 'unittest'
cls.inputs['kinship_plot_threshold'] = 0.044
cls.inputs['group'] = "Population"
cls.inputs['sparse_threshold'] = 0.022
cls.log = logging.getLogger("#unit_test")
cls.log.info(f" Starting {cls.APP} test")
# RUN TASKS
try:
cls.task = cls.session.tasks.create(name=cls.TASK_NAME,
project=cls.project,
app=cls.APP,
inputs=cls.inputs,
run=True)
cls.log.info(f" Running {cls.APP} task")
cls.log.info(f"#task_id {cls.task.id}")
except:
cls.log.info(f" I was unable to run {cls.APP} task")
def setUpClass(cls):
"""
Get input files from the test project and start test task
Returns:
"""
# Log file prefix formatting
prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace(
'-', '_')
logging.basicConfig(
filename=f'{prefix_date}_{cls.log_filename}.log',
filemode='a',
format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
cls.APP = 'smgogarten/qc-for-gwas-development/heterozygosity-by-sample'
cls.inputs = {}
cls.TASK_NAME = 'UnitTest_HetBySample'
cls.metadata_status = 'fail'
cls.naming_status = 'fail'
if not cls.project:
c = sbg.Config(profile=cls.profile_name)
print(c)
cls.session = sbg.Api(
config=c,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
cls.project_name = 'smgogarten/qc-for-gwas-development'
# SET INPUTS'
cls.project = cls.session.projects.get(id=cls.project_name)
cls.inputs['gds_file'] = cls.session.files.query(
names=[input_gds_file], project=cls.project)[0]
cls.inputs['out_prefix'] = input_out_prefix
cls.inputs['maf_min'] = input_maf_min
cls.inputs['maf_max'] = input_maf_max
cls.inputs['pass_only'] = input_pass_only
cls.inputs['cpu'] = input_cpu
cls.log = logging.getLogger("#unit_test")
cls.log.info(f" Starting {cls.APP} test")
# RUN TASKS
try:
cls.task = cls.session.tasks.create(name=cls.TASK_NAME,
project=cls.project,
app=cls.APP,
inputs=cls.inputs,
run=True)
cls.log.info(f" Running {cls.APP} task")
cls.log.info(f"#task_id {cls.task.id}")
except:
cls.log.info(f" I was unable to run {cls.APP} task")
def main(yaml_fp, create_draft_tasks, run_draft_tasks, check_status,
lower_bound_group_size, upper_bound_group_size):
logger, config = load_config(yaml_fp)
sb_config = sb.Config(url=config['api-url'], token=config['token'])
api = sb.Api(config=sb_config)
if create_draft_tasks:
create_tasks(api, logger, config, lower_bound_group_size,
upper_bound_group_size)
elif run_draft_tasks:
run_tasks(api, logger, config)
elif check_status:
show_status(api)
else:
raise ValueError('Please select one of --create-draft-tasks, '
'--run-draft-tasks or --check-status')
inputs=task.inputs,
description=task.name)
except SbgError as e:
logger.error("Draft task was not created!", exc_info=e)
raise SbgError("Draft task was not created!")
if __name__ == '__main__':
print('got to main')
parser = argparse.ArgumentParser()
parser.add_argument("option",
nargs="?",
help="create|run|status|rerun_failed")
args = parser.parse_args()
sb_config = sb.Config(url=config['api-url'], token=config['token'])
api = sb.Api(config=sb_config)
if args.option == 'create':
create_draft_tasks(api)
if args.option == 'run':
run_tasks(api)
if args.option == 'status':
status(api)
if args.option == 'rerun_failed':
rerun_failed(api)
print(api.remaining)
if len(files) > 0:
for f in files:
print("already imported " + f.name)
#setCGCMetadata(samp, file)
else:
# Import file to the project
imp = api.imports.submit_import(volume=myVolume,
project=myProject,
location=CRAMPATH + filename)
#%% CODE (broken into blocks below, this wilfl eventually become an iPython notebook
if __name__ == "__main__":
config = sbg.Config(profile='cgc')
#api = sbg.Api()
api = sbg.Api(config=config,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
# list all projects you are part of
try:
project_id = TARGET_PROJECT
project = api.projects.get(id=project_id)
print("Found " + project.name)
except SbgError as e:
print(e.message)
# List all volumes available
myVolume = api.volumes.get(VOLUME)
#!/usr/bin/env python
import sevenbridges as sbg
import sys
config = sbg.Config(profile='cavatica')
api = sbg.Api(config=config)
project = sys.argv[1]
with open(sys.argv[2]) as f:
flist = f.read().splitlines()
sys.stderr.write('Getting files for project ' + project + '\n')
# files = api.files.query(project=project, names=flist)
for fn in flist:
sys.stderr.write('Uploading file/dir ' + fn + '\n')
up = api.files.upload(fn, project, fn)
Info about setting up the configuration can be found at
https://docs.sevenbridges.com/docs/store-credentials-to-access-seven-bridges-client-applications-and-libraries
Default is 'default'
""",
default="default",
)
args = parser.parse_args()
print(f"Args: {args.__dict__}")
# Create the download location if it doesn't exist
args.download_location = Path(args.download_location)
args.download_location.mkdir(parents=True, exist_ok=True)
# Connect to the API
config_file = sbg.Config(profile=args.sbg_profile)
api = sbg.Api(
config=config_file,
error_handlers=[
sbg.http.error_handlers.rate_limit_sleeper,
sbg.http.error_handlers.maintenance_sleeper,
],
)
# Generate the project object
project = fetch_project(api, args.project_name, args.project_id)
all_files = list_files_recursively(api, api.files.query(project=project),
project)
if args.project_path:
path_name = Path(args.project_path)
import sevenbridges as sbg
from sevenbridges.errors import SbgError
from sevenbridges.http.error_handlers import *
import datetime
import binpacking
import re
import os
import math
print("SBG library imported.")
prof = 'cgc'
config_file = sbg.Config(profile=prof)
api = sbg.Api(config=config_file,
error_handlers=[
rate_limit_sleeper, maintenance_sleeper,
general_error_sleeper
])
print "Api Configured!!"
print "Api Username : "******"Found project %s with project details %s" %
(my_project.name, my_project.description))
input_port_reference = "reference"
input_port_bam_file = "BAMfile"
input_port_targets = "targets"
reference_list = [
def _get_api_client(config):
import sevenbridges as sbg
c = sbg.Config(profile=config.get("profile", "default"))
api = sbg.Api(config=c, advance_access=True)
return api
authToken = args.authToken
dataFormatFilter = args.dataFormatFilter
# diseaseTypeList = args.diseaseTypeList
dataFormatFilterLabel = {'data_format': dataFormatFilter}
print("\n")
print("CGC project name :%r" % cgcProjectName)
print("CGC user name :%r" % cgcUserName)
# Set up API configuration
os.environ['API_URL'] = 'https://cgc-api.sbgenomics.com/v2'
os.environ['AUTH_TOKEN'] = authToken
api_config = sbg.Config()
api = sbg.Api(config=api_config)
print("\n")
print("Grabbing list of files, their names, and IDs...")
# Grab list of files, their names and IDs
file_list = api.files.query(
project=cgcUserName+'/'+cgcProjectName,
metadata = dataFormatFilterLabel)
print("Extracting file names...")
f_names = [f.name for f in file_list.all()] # Grab filenames
print("Extracting file IDs...")
f_id = [f.id for f in file_list.all()] # Grab file IDs
# Grab and store filenames; convert from Unicode to Python strings
filenames = []
"""
Download missing files using a cavativa manifest and a dir of existing file
"""
import sevenbridges as sbg
from sevenbridges.errors import SbgError
from sevenbridges.http.error_handlers import rate_limit_sleeper, maintenance_sleeper
import sys
import re
import concurrent.futures
import os
import pdb
config = sbg.Config(profile='turbo')
api = sbg.Api(config=config, error_handlers=[rate_limit_sleeper, maintenance_sleeper])
def check_dl_file(entry):
try:
info = entry.rstrip('\n').split(',')
path = dir + info[1]
if not os.path.isfile(path):
getme = api.files.get(info[0])
getme.download(path)
except Exception as e:
sys.stderr.write(str(e) + "\n")
sys.stderr.write("Got an error downloading " + info[1] + "\n")
dir = sys.argv[1] + "/"
manifest = open(sys.argv[2])
head = next(manifest)
i = 1
n = 100
#!/usr/bin/python
import sys, argparse, pprint, os, errno, requests, json
import sevenbridges as sbg
api_config = sbg.Config(profile='my-profile')
api = sbg.Api(config=api_config)
user = api.users.me()
#print "What is the user? --> ", user
exit
API_BASEPATH = 'https://api.sbgenomics.com/v2'
def mkdir_p(path):
try:
os.makedirs(path)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST and os.path.isdir(path):
pass
else:
raise
def make_full_path(dir_name, project_name, headers):
folder_array = dir_name.split("/")
url = API_BASEPATH + '/' + 'projects/' + project_name
r = requests.get(url, headers=headers)
r = json.loads(r.text) #hash me.
dest='output',
help='Output log file name')
parser.add_argument(
'-x',
'--prefix',
action='store',
dest='prefix',
help='Task prefix ro run. Set to ALL if any draft task should run')
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
config = sbg.Config(profile=args.profile)
api = sbg.Api(config=config,
error_handlers=[rate_limit_sleeper, maintenance_sleeper])
limit = int(args.num_jobs)
draft_tasks = list(api.tasks.query(project=args.project, status='DRAFT').all())
running_tasks = list(
api.tasks.query(project=args.project, status='RUNNING').all())
queued_tasks = list(
api.tasks.query(project=args.project, status='QUEUED').all())
cur_run = len(running_tasks) + len(queued_tasks)
cur_draft = len(draft_tasks)
out_fh = open(args.output, 'a')
out_fh.write(date_time() + 'Checking draft/running jobs for project ' +
args.project + '\n')
if cur_draft == 0:
