def init_api(profile='default', platform=None, dev_token=None, endpoint=None): """ Initialize SBG API using credentials located inside $HOME/.sevenbridges/credentials or by provided dev_token and platform. :param profile: profile listed inside $HOME/.sevenbridges/credentials. Example: cgc, default, cavatica :param platform: Predefined platform. Available: igor, cgc, f4c, eu, cn, cavatica :param dev_token: developer token from platform :param endpoint: api endpoint :return: sbg.Api """ if platform and not endpoint: endpoint = get_endpoint(platform) if dev_token and not endpoint: endpoint = get_endpoint('igor') if dev_token and endpoint: api = sbg.Api(url=endpoint, token=dev_token, error_handlers=[ rate_limit_sleeper, maintenance_sleeper, general_error_sleeper ]) else: c = sbg.Config(profile=profile) api = sbg.Api(config=c, error_handlers=[ rate_limit_sleeper, maintenance_sleeper, general_error_sleeper ]) return api
def getTaskOutputs(task_id, ddir): config = sbg.Config(profile='cgc') api = sbg.Api(config=config) try: task = api.tasks.get(id=task_id) if task.status != 'COMPLETED': print('Task status:{}'.format(task.status)) sys.exit() for okey, oitem in task.outputs.items(): print(okey) if oitem.__class__ == sbg.File: print(oitem.id) print(oitem.name) drsClient = sbcgcDRSClient('~/.keys/sevenbridges_keys.json') drsResponse = drsClient.getObject(oitem.id) print(drsResponse) drsURL = drsClient.getAccessURL(oitem.id, 's3') dPath = os.path.expanduser(ddir + oitem.name) if ddir != None: download(drsURL, dPath) else: print(oitem) except sbg.NotFound as e: print('Task not found') except sbg.SbgError as e: print(e.__class__) print(e.message)
def __init__(self): """ To set up, add API_URL and AUTH_TOKEN to your environment API_URL is "https://cgc-api.sbgenomics.com/v2" for all users AUTH_TOKEN can be found in the Developer Settings on SBG or CGC """ self.api = sbg.Api(config=sbg.Config(url=environ['API_URL'], token=environ['AUTH_TOKEN']))
def setUpClass(cls): """ Get input files from the test project and start test task Returns: """ # Log file prefix formatting prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace( '-', '_') logging.basicConfig( filename=f'{prefix_date}_{cls.log_filename}.log', filemode='a', format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) cls.APP = 'smgogarten/genesis-relatedness/ld-pruning-pipeline' cls.inputs = {} cls.TASK_NAME = 'UnitTest_LD_pruning' cls.metadata_status = 'fail' cls.naming_status = 'fail' if not cls.project: c = sbg.Config(profile=cls.profile_name) print(c) cls.session = sbg.Api( config=c, error_handlers=[rate_limit_sleeper, maintenance_sleeper]) cls.project_name = 'smgogarten/genesis-relatedness' # SET INPUTS' cls.project = cls.session.projects.get(id=cls.project_name) cls.inputs['gds_file'] = cls.session.files.query(names=[ '1KG_phase3_subset_chr21.gds', '1KG_phase3_subset_chr22.gds', '1KG_phase3_subset_chrX.gds' ], project=cls.project) cls.inputs['sample_include_file_pruning'] = cls.session.files.query( names=['sample_include.RData'], project=cls.project)[0] cls.inputs['sample_include_file_gds'] = cls.session.files.query( names=['sample_include.RData'], project=cls.project)[0] cls.inputs['out_prefix'] = 'unittest' cls.inputs['ld_r_threshold'] = 0.32 cls.inputs['ld_win_size'] = 10 cls.inputs['maf_threshold'] = 0.01 cls.inputs['missing_threshold'] = 0.01 cls.inputs['exclude_pca_corr'] = False cls.inputs['genome_build'] = 'hg19' cls.inputs['autosome_only'] = False cls.log = logging.getLogger("#unit_test") cls.log.info(f" Starting {cls.APP} test") # RUN TASKS try: cls.task = cls.session.tasks.create(name=cls.TASK_NAME, project=cls.project, app=cls.APP, inputs=cls.inputs, run=True) cls.log.info(f" Running {cls.APP} task") cls.log.info(f"#task_id {cls.task.id}") except: cls.log.info(f" I was unable to run {cls.APP} task")
def get_profile(profile): if profile == ".": api = sbg.Api() else: api = sbg.Api(config=sbg.Config(profile)) # Least disruptive way to add in our user agent api.headers["User-Agent"] = "sbpack/{} via {}".format( __version__, api.headers["User-Agent"]) return api
def __init__(self, instance, project): #self.outdir = outdir self.project_id = project config = sbg.Config(profile=instance) self.api = sbg.Api(config=config) try: project = self.api.projects.get(id=self.project_id) #print ("Found " + project.name) except sbg.SbgError as e: print(e.message)
def main(project_name, app_name, port_name): config_file = sbg.Config(profile='cgc') api = sbg.Api(config=config_file, error_handlers=[ rate_limit_sleeper, maintenance_sleeper, general_error_sleeper ]) my_project = get_project(api, project_name) my_app = get_app(api, my_project, app_name) my_files = list(api.files.query(project=my_project, limit=100).all()) all_tasks = begin_tasks(api, my_project, my_app, my_files) return
def main(): arguments = docopt(__doc__, version='2018.11') if arguments['--quiet']: logger.setLevel(level=logging.ERROR) else: logger.setLevel(level=logging.DEBUG) config = sbg.Config(arguments["--api-profile"]) api = sbg.Api(config=config, error_handlers=[ rate_limit_sleeper, maintenance_sleeper, general_error_sleeper ]) cwl_doc = recursive_workflow_load(arguments["WORKFLOW"]) if "id" not in cwl_doc: cwl_doc["id"] = pathlib.PurePath(arguments["WORKFLOW"]).stem uname = api.users.me().username full_project_id = get_project(api, uname, arguments["--project"]) logger.debug("Using project: {}".format(full_project_id)) app = upload_app(api, full_project_id, cwl_doc) logger.debug("Using app: {}".format(app.id)) job = load_job(arguments["JOB"]) job = fill_out_job_defaults(job, cwl_doc) task_name = "{}: {}".format( app.name, hashlib.sha1(str(job).encode()).hexdigest()[:10]) job = resolve_job(api, full_project_id, job, pathlib.PurePath(arguments["JOB"])) task = api.tasks.create(name=task_name, project=full_project_id, app=app, inputs=job, interruptible=False, run=True) while task.status not in ["ABORTED", "COMPLETED", "FAILED"]: task.reload() time.sleep(float(arguments["--poll-interval"]) * 60) # Output required in CWL Output Object Document format # which expects property name enclosed in double quotes print( json.dumps(resolve_output_dict(task.outputs, arguments["--outdir"]), indent=4, separators=(',', ': ')))
def setUpClass(cls): """ Get input files from the test project and start test task Returns: """ # Log file prefix formatting prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace( '-', '_') logging.basicConfig( filename=f'{prefix_date}_{cls.log_filename}.log', filemode='a', format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) cls.APP = 'smgogarten/genesis-relatedness/king-pipeline' cls.inputs = {} cls.TASK_NAME = 'UnitTest_KING_IBDseg' cls.metadata_status = 'fail' cls.naming_status = 'fail' if not cls.project: c = sbg.Config(profile=cls.profile_name) print(c) cls.session = sbg.Api( config=c, error_handlers=[rate_limit_sleeper, maintenance_sleeper]) cls.project_name = 'smgogarten/genesis-relatedness' # SET INPUTS' cls.project = cls.session.projects.get(id=cls.project_name) cls.inputs['gds_file'] = cls.session.files.query( names=['ALL.pruned.gds'], project=cls.project)[0] cls.inputs['sample_include_file'] = cls.session.files.query( names=['sample_include.RData'], project=cls.project)[0] cls.inputs['phenotype_file'] = cls.session.files.query( names=['1KG_phase3_subset_annot.RData'], project=cls.project)[0] cls.inputs['out_prefix'] = 'unittest' cls.inputs['kinship_plot_threshold'] = 0.044 cls.inputs['group'] = "Population" cls.inputs['sparse_threshold'] = 0.022 cls.log = logging.getLogger("#unit_test") cls.log.info(f" Starting {cls.APP} test") # RUN TASKS try: cls.task = cls.session.tasks.create(name=cls.TASK_NAME, project=cls.project, app=cls.APP, inputs=cls.inputs, run=True) cls.log.info(f" Running {cls.APP} task") cls.log.info(f"#task_id {cls.task.id}") except: cls.log.info(f" I was unable to run {cls.APP} task")
def setUpClass(cls): """ Get input files from the test project and start test task Returns: """ # Log file prefix formatting prefix_date = str(datetime.datetime.now()).replace(' ', '_').replace( '-', '_') logging.basicConfig( filename=f'{prefix_date}_{cls.log_filename}.log', filemode='a', format='%(asctime)s,%(msecs)d %(name)s %(levelname)s %(message)s', datefmt='%H:%M:%S', level=logging.INFO) cls.APP = 'smgogarten/qc-for-gwas-development/heterozygosity-by-sample' cls.inputs = {} cls.TASK_NAME = 'UnitTest_HetBySample' cls.metadata_status = 'fail' cls.naming_status = 'fail' if not cls.project: c = sbg.Config(profile=cls.profile_name) print(c) cls.session = sbg.Api( config=c, error_handlers=[rate_limit_sleeper, maintenance_sleeper]) cls.project_name = 'smgogarten/qc-for-gwas-development' # SET INPUTS' cls.project = cls.session.projects.get(id=cls.project_name) cls.inputs['gds_file'] = cls.session.files.query( names=[input_gds_file], project=cls.project)[0] cls.inputs['out_prefix'] = input_out_prefix cls.inputs['maf_min'] = input_maf_min cls.inputs['maf_max'] = input_maf_max cls.inputs['pass_only'] = input_pass_only cls.inputs['cpu'] = input_cpu cls.log = logging.getLogger("#unit_test") cls.log.info(f" Starting {cls.APP} test") # RUN TASKS try: cls.task = cls.session.tasks.create(name=cls.TASK_NAME, project=cls.project, app=cls.APP, inputs=cls.inputs, run=True) cls.log.info(f" Running {cls.APP} task") cls.log.info(f"#task_id {cls.task.id}") except: cls.log.info(f" I was unable to run {cls.APP} task")
def main(yaml_fp, create_draft_tasks, run_draft_tasks, check_status, lower_bound_group_size, upper_bound_group_size): logger, config = load_config(yaml_fp) sb_config = sb.Config(url=config['api-url'], token=config['token']) api = sb.Api(config=sb_config) if create_draft_tasks: create_tasks(api, logger, config, lower_bound_group_size, upper_bound_group_size) elif run_draft_tasks: run_tasks(api, logger, config) elif check_status: show_status(api) else: raise ValueError('Please select one of --create-draft-tasks, ' '--run-draft-tasks or --check-status')
inputs=task.inputs, description=task.name) except SbgError as e: logger.error("Draft task was not created!", exc_info=e) raise SbgError("Draft task was not created!") if __name__ == '__main__': print('got to main') parser = argparse.ArgumentParser() parser.add_argument("option", nargs="?", help="create|run|status|rerun_failed") args = parser.parse_args() sb_config = sb.Config(url=config['api-url'], token=config['token']) api = sb.Api(config=sb_config) if args.option == 'create': create_draft_tasks(api) if args.option == 'run': run_tasks(api) if args.option == 'status': status(api) if args.option == 'rerun_failed': rerun_failed(api)
print(api.remaining) if len(files) > 0: for f in files: print("already imported " + f.name) #setCGCMetadata(samp, file) else: # Import file to the project imp = api.imports.submit_import(volume=myVolume, project=myProject, location=CRAMPATH + filename) #%% CODE (broken into blocks below, this wilfl eventually become an iPython notebook if __name__ == "__main__": config = sbg.Config(profile='cgc') #api = sbg.Api() api = sbg.Api(config=config, error_handlers=[rate_limit_sleeper, maintenance_sleeper]) # list all projects you are part of try: project_id = TARGET_PROJECT project = api.projects.get(id=project_id) print("Found " + project.name) except SbgError as e: print(e.message) # List all volumes available myVolume = api.volumes.get(VOLUME)
#!/usr/bin/env python import sevenbridges as sbg import sys config = sbg.Config(profile='cavatica') api = sbg.Api(config=config) project = sys.argv[1] with open(sys.argv[2]) as f: flist = f.read().splitlines() sys.stderr.write('Getting files for project ' + project + '\n') # files = api.files.query(project=project, names=flist) for fn in flist: sys.stderr.write('Uploading file/dir ' + fn + '\n') up = api.files.upload(fn, project, fn)
Info about setting up the configuration can be found at https://docs.sevenbridges.com/docs/store-credentials-to-access-seven-bridges-client-applications-and-libraries Default is 'default' """, default="default", ) args = parser.parse_args() print(f"Args: {args.__dict__}") # Create the download location if it doesn't exist args.download_location = Path(args.download_location) args.download_location.mkdir(parents=True, exist_ok=True) # Connect to the API config_file = sbg.Config(profile=args.sbg_profile) api = sbg.Api( config=config_file, error_handlers=[ sbg.http.error_handlers.rate_limit_sleeper, sbg.http.error_handlers.maintenance_sleeper, ], ) # Generate the project object project = fetch_project(api, args.project_name, args.project_id) all_files = list_files_recursively(api, api.files.query(project=project), project) if args.project_path: path_name = Path(args.project_path)
import sevenbridges as sbg from sevenbridges.errors import SbgError from sevenbridges.http.error_handlers import * import datetime import binpacking import re import os import math print("SBG library imported.") prof = 'cgc' config_file = sbg.Config(profile=prof) api = sbg.Api(config=config_file, error_handlers=[ rate_limit_sleeper, maintenance_sleeper, general_error_sleeper ]) print "Api Configured!!" print "Api Username : "******"Found project %s with project details %s" % (my_project.name, my_project.description)) input_port_reference = "reference" input_port_bam_file = "BAMfile" input_port_targets = "targets" reference_list = [
def _get_api_client(config): import sevenbridges as sbg c = sbg.Config(profile=config.get("profile", "default")) api = sbg.Api(config=c, advance_access=True) return api
authToken = args.authToken dataFormatFilter = args.dataFormatFilter # diseaseTypeList = args.diseaseTypeList dataFormatFilterLabel = {'data_format': dataFormatFilter} print("\n") print("CGC project name :%r" % cgcProjectName) print("CGC user name :%r" % cgcUserName) # Set up API configuration os.environ['API_URL'] = 'https://cgc-api.sbgenomics.com/v2' os.environ['AUTH_TOKEN'] = authToken api_config = sbg.Config() api = sbg.Api(config=api_config) print("\n") print("Grabbing list of files, their names, and IDs...") # Grab list of files, their names and IDs file_list = api.files.query( project=cgcUserName+'/'+cgcProjectName, metadata = dataFormatFilterLabel) print("Extracting file names...") f_names = [f.name for f in file_list.all()] # Grab filenames print("Extracting file IDs...") f_id = [f.id for f in file_list.all()] # Grab file IDs # Grab and store filenames; convert from Unicode to Python strings filenames = []
""" Download missing files using a cavativa manifest and a dir of existing file """ import sevenbridges as sbg from sevenbridges.errors import SbgError from sevenbridges.http.error_handlers import rate_limit_sleeper, maintenance_sleeper import sys import re import concurrent.futures import os import pdb config = sbg.Config(profile='turbo') api = sbg.Api(config=config, error_handlers=[rate_limit_sleeper, maintenance_sleeper]) def check_dl_file(entry): try: info = entry.rstrip('\n').split(',') path = dir + info[1] if not os.path.isfile(path): getme = api.files.get(info[0]) getme.download(path) except Exception as e: sys.stderr.write(str(e) + "\n") sys.stderr.write("Got an error downloading " + info[1] + "\n") dir = sys.argv[1] + "/" manifest = open(sys.argv[2]) head = next(manifest) i = 1 n = 100
#!/usr/bin/python import sys, argparse, pprint, os, errno, requests, json import sevenbridges as sbg api_config = sbg.Config(profile='my-profile') api = sbg.Api(config=api_config) user = api.users.me() #print "What is the user? --> ", user exit API_BASEPATH = 'https://api.sbgenomics.com/v2' def mkdir_p(path): try: os.makedirs(path) except OSError as exc: # Python >2.5 if exc.errno == errno.EEXIST and os.path.isdir(path): pass else: raise def make_full_path(dir_name, project_name, headers): folder_array = dir_name.split("/") url = API_BASEPATH + '/' + 'projects/' + project_name r = requests.get(url, headers=headers) r = json.loads(r.text) #hash me.
dest='output', help='Output log file name') parser.add_argument( '-x', '--prefix', action='store', dest='prefix', help='Task prefix ro run. Set to ALL if any draft task should run') if len(sys.argv) == 1: parser.print_help() sys.exit(1) args = parser.parse_args() config = sbg.Config(profile=args.profile) api = sbg.Api(config=config, error_handlers=[rate_limit_sleeper, maintenance_sleeper]) limit = int(args.num_jobs) draft_tasks = list(api.tasks.query(project=args.project, status='DRAFT').all()) running_tasks = list( api.tasks.query(project=args.project, status='RUNNING').all()) queued_tasks = list( api.tasks.query(project=args.project, status='QUEUED').all()) cur_run = len(running_tasks) + len(queued_tasks) cur_draft = len(draft_tasks) out_fh = open(args.output, 'a') out_fh.write(date_time() + 'Checking draft/running jobs for project ' + args.project + '\n') if cur_draft == 0: