def synapse_login(): try: syn = synapseclient.login() except Exception as e: print("Please provide your synapse username/email and password (You will only be prompted once)") Username = raw_input("Username: ") Password = getpass.getpass() syn = synapseclient.login(email=Username, password=Password,rememberMe=True) return syn
def perform_main(args): if args.synapse_user is not None: syn = synapseclient.login(args.synapse_user,args.password,rememberMe=True) else: syn = synapseclient.login() if 'func' in args: try: args.func(args,syn) except Exception as ex: print(ex)
def main(): import argparse import sys parser = argparse.ArgumentParser() parser.add_argument('--dryrun', action="store_true", default=False, help="Run without making changes [default: %(default)s]") parser.add_argument('--storetable', action="store_true", default=False) parser.add_argument('--projectId', type=str) args = parser.parse_args() syn = synapseclient.login(silent=True) annots = PCBCAnnotations.PCBCAnnotations(syn) a = PCBCAnnotations.PCBCAnnotationTableUpdate(syn, annots) my_synapse_id = None if not args.dryrun: my_synapse_id = synapseHelpers.thisCodeInSynapse(parentId='syn2758110', syn=syn) a.update_annots_synapse(executed=my_synapse_id, dryrun=args.dryrun) if args.storetable: tbl = a.update_annots_table_synapse(projectId=args.projectId, dryrun=args.dryrun)
def login(): """Log into Synapse. If cached info not found, prompt user. Returns: syn: Synapse object """ try: syn = synapseclient.login(silent=True) except Exception: print("Cached credentials not found; please provide", "your Synapse username and password.") username = input("Synapse username: "******"Synapse password: "******"utf-8") syn = synapseclient.login(username=username, password=password, rememberMe=True, silent=True) return syn
def main(): if not os.path.exists(TEST_SUBMISSION_PATH): os.makedirs(TEST_SUBMISSION_PATH) syn = sc.login() args = readargs() args.filetype = "csv" if not args.filetype else args.filetype indexCols = None if not args.indexCols else args.indexCols.split(",") writeSubmissions(args.sampleSubmission, indexCols, args.filetype) storeSubmissions(syn, args.evaluationQueue, args.synProject, args.filetype) logger.info("Finished")
def main(): import argparse import sys parser = argparse.ArgumentParser() parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout) parser.add_argument('--storetable', action="store_true", default=False) parser.add_argument('--projectId', type=str) args = parser.parse_args() syn = synapseclient.login(silent=True) ## mRNA metadata from bam, bed, and fpkm files pcbcmeta = BootstrapExperimentalMetadata.MRNABamMetadata(syn) pcbc_mrna_bam = pcbcmeta() pcbcmeta = BootstrapExperimentalMetadata.MRNABedMetadata(syn) pcbc_mrna_bed = pcbcmeta() pcbcmeta = BootstrapExperimentalMetadata.MRNAFpkmMetadata(syn) pcbc_mrna_fpkm = pcbcmeta() ## miRNA metadata from Fastq and Expr files pcbcmeta = BootstrapExperimentalMetadata.MIRNAFastqMetadata(syn) pcbc_mirna_fastq = pcbcmeta() pcbcmeta = BootstrapExperimentalMetadata.MIRNAExprMetadata(syn) pcbc_mirna_expr = pcbcmeta() pcbcmeta = BootstrapExperimentalMetadata.MethylMetadata(syn) pcbc_methyl = pcbcmeta() pcbc_sample_metadata = pd.concat([pcbc_mrna_bam, pcbc_mrna_bed, pcbc_mrna_fpkm, pcbc_mirna_fastq, pcbc_mirna_expr, pcbc_methyl], ignore_index=True) _keepcols = ["name", "id", "UID", "C4_Cell_Line_ID", "Diffname_short", "dataType", "fileType"] pcbc_sample_metadata = pcbc_sample_metadata[_keepcols] pcbc_sample_metadata.to_csv(args.outfile, sep="\t", index=False) if args.storetable: tbl = BootstrapExperimentalMetadata.to_table(syn, pcbc_sample_metadata, args.projectId)
def main(): args = read_args() syn = synapseclient.login() df = download_from_synapse(syn=syn, source_table=args.synapse_source, limit=args.num) session = boto.Session(profile_name=args.aws_profile) copy_to_s3(df=df, session=session, bucket=args.s3_bucket, basekey=args.s3_prefix)
def main(): log.info('Parsing arguments...') args = parse_arguments() if args.team_name_tsv is not None: team_name_dict = parse_team_name_tsv(args.team_name_tsv) else: team_name_dict = None print(team_name_dict) syn = synapseclient.login() t0 = time.perf_counter() while True: try: if not args.update_wiki_only: evaluation = syn.getEvaluation(args.eval_queue_id) # init multiprocessing pool = multiprocessing.Pool(args.nth) # distribute jobs ret_vals = [] for submission, status in syn.getSubmissionBundles( evaluation, status='RECEIVED'): ret_vals.append( pool.apply_async(validate_submission, (submission, status, args, syn))) # gather for r in ret_vals: r.get(BIG_INT) pool.close() pool.join() update_wiki_for_round2(syn, team_name_dict, args) except Exception as ex1: st = StringIO() traceback.print_exc(file=st) message = st.getvalue() subject = 'Server error:' if args.send_msg_to_admin: send_message(syn, args.admin_id, subject, message) log.error(message) log.info('Waiting for new submissions...') while time.perf_counter() - t0 < args.period: time.sleep(60) t0 = time.perf_counter() log.info('All done')
def validateFiles(synId, center): """ This function walks through each center's input directory and validates every file """ syn = synapseclient.login() walked = synu.walk(syn, synId) clinicalpair = [] invalid = [] validFiles = [] for dirpath, dirname, filenames in walked: for name, synid in filenames: file = [] if name.endswith(".vcf") and VALIDATE_FILENAME['vcf'] % center in name: fileType = "vcf" elif name.endswith(".bed") and VALIDATE_FILENAME['bed'] % center in name: fileType = "bed" elif VALIDATE_FILENAME['maf'] % center == name: fileType = "maf" elif VALIDATE_FILENAME['cnv'] % center == name: fileType = "cnv" elif VALIDATE_FILENAME['fusions'] % center == name: fileType = "fusions" elif VALIDATE_FILENAME['seg'] % center == name: fileType = "seg" elif VALIDATE_FILENAME['clinical'][0] % center == name: fileType = "clinical" elif name in VALIDATE_FILENAME['clinical'][1:]: clinicalpair.append(synid) if len(clinicalpair) == 2: fileType = "clinical" else: fileType = None else: fileType = None if fileType is not None: if len(clinicalpair) == 2: #Need to pass in both filepath and synid for processing files file = [(syn.get(i).path,i) for i in clinicalpair] clinicalpair = [] else: file = [(syn.get(synid).path,synid)] #Validation only takes in a list of filepaths paths = [i[0] for i in file] message, valid = validate_genie.main(fileType, paths, center) else: print("%s: Cannot be processed" % name) valid = False invalid.append(name) if not valid: invalid.append(name) else: validFiles.extend(file) print(", ".join(invalid) + " can't be processed!") return(validFiles)
def main(): credentials = get_credentials() syn = sc.login(email = credentials['synapseUsername'], password = credentials['synapsePassword']) new_users = get_new_users(syn) if isinstance(new_users, tuple): # returned error message table_row = create_table_row(new_users[0], new_users[1], new_users[2], new_users[3]) syn.store(sc.Table(OUTPUT_TABLE, [table_row])) return duplicated_numbers = new_users.phone_number.duplicated(keep = False) if any(duplicated_numbers): duplicates = new_users.loc[duplicated_numbers] if len(duplicates.guid) == len(duplicates.guid.unique()): table_row = create_table_row("Error: It looks like you accidentally " "entered an incorrect guid and tried to " "submit a corrected one immediately " "afterwards. Please delete the duplicate " "phone number from the Users table " "(syn16784393) and this table (syn16786935) " "and resubmit.", duplicates.phone_number.iloc[0], "", duplicates.visit_date.iloc[0]) syn.store(sc.Table(OUTPUT_TABLE, [table_row])) return to_append_to_table = [] for i, user in new_users.iterrows(): phone_number = int(user.phone_number) guid = user.guid visit_date = int(user.visit_date) print("phone_number: ", phone_number) print("guid: ", guid) print("visit_date: ", visit_date) try: if not (len(str(phone_number)) == 10 and str(phone_number).isdigit()): table_row = create_table_row("Error: The phone number is improperly " "formatted. Please enter a valid, 10-digit " "number", phone_number, guid, visit_date) else: bridge = get_bridge_client(credentials['bridgeUsername'], credentials['bridgePassword']) participant_info = get_participant_info(bridge, phone_number) status = process_request(bridge, participant_info, phone_number, guid) table_row = create_table_row(status, phone_number, guid, visit_date) except Exception as e: table_row = create_table_row("Error: One of the fields is improperly " "formatted. Console output: {0}".format(e), -1, guid, visit_date) to_append_to_table.append(table_row) if len(to_append_to_table): syn.store(sc.Table(OUTPUT_TABLE, to_append_to_table))
def validate_submission(evaluation, submission): """ Find the right validation function and validate the submission. :returns: (True, message) if validated, (False, message) if validation fails or throws exception """ config = config_evaluations_map[int(evaluation.id)] validation_func = config['validation_function'] syn = synapseclient.login() results = validation_func(evaluation, submission.filePath, syn) return results
def validate_submission(evaluation, submission, token): """ Find the right validation function and validate the submission. :returns: (True, message) if validated, (False, message) if validation fails or throws exception """ config = config_evaluations_map[int(evaluation.id)] validation_func = config['validation_function'] syn = synapseclient.login() results = validation_func(evaluation,submission,syn, token) return results
def synapseLogin(): """ First tries to login to synapse by finding the local auth key cached on user's computing platform, if not found, prompts the user to provide their synapse user name and password, then caches the auth key on their computing platform. :return: """ try: syn = synapseclient.login() except Exception as e: print( 'Please provide your synapse username/email and password (You will only be prompted once)' ) username = input("Username: "******"Password for " + username + ": ").encode('utf-8')) syn = synapseclient.login(email=username, password=password, rememberMe=True) return syn
def main(): args = get_env_var_args() syn = sc.login(args["synapseUsername"], args["synapsePassword"]) table_mapping = get_table_mapping(syn, args["tableMapping"]) relevant_healthcodes = get_relevant_healthcodes( syn, participants_table=args["participantsTable"], substudy=args["substudy"]) synapsebridgehelpers.export_tables(syn=syn, table_mapping=table_mapping, identifier_col="healthCode", identifier=relevant_healthcodes, copy_file_handles=True)
def connect_to_synapse(): """ Connect to synapse.org. You need to set the right environment variables: - SYN_EMAIL: Your synapse.org email. - SYN_PWD: Your synapse.org password. :return: (Synapse) synapse connection instance """ syn_email = os.getenv('SYN_EMAIL') syn_pwd = os.getenv('SYN_PWD') assert syn_email, 'The SYN_EMAIL environment variable is not set.' assert syn_pwd, 'The SYN_PWD environment variable is not set.' return synapseclient.login(syn_email, syn_pwd)
def main(): syn = synapseclient.login(email='*****@*****.**', password='******', rememberMe=True) print "Querying the Walking training table..." #Query 'walking training table' for walk data recordIDs and healthCodes. INPUT_WALKING_ACTIVITY_TABLE_SYNID = "syn10733842" actv_walking_syntable = syn.tableQuery( ('SELECT * FROM {0}').format(INPUT_WALKING_ACTIVITY_TABLE_SYNID)) actv_walking = actv_walking_syntable.asDataFrame() actv_walking[["recordId", "phoneInfo"]].to_csv('meta-data-testing.csv', index=False, header=True) print "Done!"
def synapse_login(username=None, password=None): """ This function logs into synapse for you if credentials are saved. If not saved, then user is prompted username and password. :returns: Synapseclient object """ try: syn = synapseclient.login(silent=True) except Exception: if username is None and password is None: raise ValueError( "Please specify --syn_user, --syn_pass to specify your Synapse " "login. Please view https://docs.synapse.org/articles/client_configuration.html" "to learn about logging into Synapse via the Python client.") else: syn = synapseclient.login( email=username, password=password, silent=True) return(syn) return result
def synapse_login(): """ This function logs into synapse for you if credentials are saved. If not saved, then user is prompted username and password. :returns: Synapseclient object """ try: syn = synapseclient.login(silent=True) except Exception as e: logger.info( "Please provide your synapse username/email and password (You will only be prompted once), or write your username and password to your bash profile under GENIE_USER and GENIE_PASS" ) Username = os.getenv("GENIE_USER") Password = os.getenv("GENIE_PASS") if Username is None or Password is None: Username = raw_input("Username: ") Password = getpass.getpass() syn = synapseclient.login(email=Username, password=Password, rememberMe=True, silent=True) return (syn)
def login(self): logging.info('Logging into Synapse...') self._username = self._username or os.getenv('SYNAPSE_USER') self._password = self._password or os.getenv('SYNAPSE_PASSWORD') if self._username == None: self._username = input('Synapse username: '******'Synapse password: ') self._synapse_client = synapseclient.login(self._username, self._password, silent=True)
def stop_hanging_docker_submissions(): """Stops hanging docker submissions""" syn = synapseclient.login() client = docker.from_env() running_containers = client.containers.list() for container in running_containers: try: status = syn.getSubmissionStatus(container.name) if status.status == "INVALID": print("stopping: " + container.name) container.stop() container.remove() except Exception: print("Not a synapse submission / unable to remove container")
def run(self): self._synapse_client = synapseclient.login(self._parent._username, self._parent._password, silent=True) while not self.exit_thread: self._lock.acquire() if not self._queue.empty(): folder_obj = self._queue.get() self._lock.release() self.upload_file_to_synapse(folder_obj["file_info"], folder_obj["folder_path"]) else: self._lock.release() time.sleep(.10)
def make_manifests_main(slistcsv, target_dir=".", prefix='chess-'): ''' Make manifests given a list of samples in a CSV file Arguments: slistcsv: path to the input CSV file target_dir: where the manifest files will be created prefix: common prefix to all three manifests Value: a tuple of the three manifests, each one a pandas DataFrame Details: The side effect of the function is to write the manifests in the target directory with the given prefix. ''' syn = synapseclient.login() def do_one_sample(subject, tissue): m = make_manifests(subject=subject, tissue=tissue, syn=syn, target_dir=target_dir, matching_sample_ids=True) return (m) samples = pd.read_csv(slistcsv) lomanifests = [ do_one_sample(subject=samples.iloc[i][0], tissue=samples.iloc[i][1]) for i in samples.index ] kinds = ['nichd_btb02', 'genomics_subject02', 'genomics_sample03'] def do_one_manifest(kind): m = [row[kind] for row in lomanifests] m = pd.concat(m) csvpath = target_dir + os.path.sep + prefix + kinds[kind] + '.csv' templ_synid = manifest_template_synids[kinds[kind]] templ_df, templ_syn = get_manifest(templ_synid, syn, download_dir=target_dir) templ_p = target_dir + os.path.sep + templ_syn.properties.name write_manifest(m, templ_p, csvpath) return (m) manifests = tuple(do_one_manifest(k) for k in range(len(kinds))) return (manifests)
def validate_writeup(submission, goldstandard_path, public=True, admin=None): ''' Validates challenge writeup Args: submission: Submission object goldstandard_path: Unused syn: Synapse object public: If the writeup needs to be public. Defaults to True admin: Specify Synapse userid that writeup needs to be shared with Returns: (True, message) if validated, (False, message) if validation fails or throws exception ''' from synapseclient.exceptions import SynapseHTTPError from synapseclient import AUTHENTICATED_USERS import synapseclient syn = synapseclient.login() # Add in users to share this with share_with = [] try: if public: message = "Please make your private project ({}) public".format( submission['entityId']) share_with.append(message) ent = \ syn.getPermissions(submission['entityId'], AUTHENTICATED_USERS) assert "READ" in ent and "DOWNLOAD" in ent, message ent = syn.getPermissions(submission['entityId']) assert "READ" in ent, message if admin is not None: message = ( "Please share your private directory ({}) with the Synapse" " user `{}` with `Can Download` permissions.".format( submission['entityId'], admin)) share_with.append(message) ent = syn.getPermissions(submission['entityId'], admin) assert "READ" in ent and "DOWNLOAD" in ent, message except SynapseHTTPError as e: if e.response.status_code == 403: raise AssertionError("\n".join(share_with)) else: raise (e) return True, "Validated!"
def submit(self, name): folderid = 'syn10932057' import synapseclient from synapseclient import File, Evaluation syn = synapseclient.login() # upload the file to the synapse project folder submissionfile = File(os.path.join(self.summary_path, name + ".csv"), parent = folderid) submissionfile = syn.store(submissionfile) team_entity = syn.getTeam("Spreedictors") submission = syn.submit(evaluation = 9606375, entity = submissionfile, name = "Spreedictor_{}".format(name), team = team_entity) syn.logout()
def push(filePath, parentId, annotationFile, provenanceFile, method, branch, token, commitMessage): syn = synapseclient.login() with open(annotationFile, 'r') as f: entries = f.read().strip().split('\n') annotations = {s[0] : s[1] for s in [pair.split(',') for pair in entries]} annotations['method'] = method with open(provenanceFile, 'r') as csvfile: reader = csv.DictReader(csvfile, delimiter=',') used = [r['provenance'] for r in reader if r['executed'] == 'FALSE'] executed= [r['provenance'] for r in reader if r['executed'] == 'TRUE'] g = github.Github(token) #repo = g.get_repo("Sage-Bionetworks/metanetworkSynapse") #config = repo.get_contents("config.sh", ref=branch) config = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="config.sh") #thisScript = repo.get_contents("pushToSynapse.py", ref=branch) thisScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="pushToSynapse.py",raw=False) networkScriptName = get_ns_name(method) #networkScript = repo.get_contents("networkScripts/%s.sh" % networkScriptName, #ref=branch) networkScript = getPermalink(gh=g,repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="networkScripts/%s.sh" % networkScriptName, raw=False) if method == "rankConsensus" or method == "bic": q = syn.chunkedQuery("select id, name from entity where entity.parentId=='%s'" % parentId) [used.append(i['entity.id']) for i in q] #submissionScript = repo.get_contents("submissionConsensus.sh", ref=branch) submissionScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="submissionConsensus.sh",raw=False) #pushScript = repo.get_contents("pushConsensus.sh", ref=branch) pushScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="pushConsensus.sh",raw=False) #buildScript = repo.get_contents("buildConsensus.R", ref=branch) buildScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="buildConsensus.R",raw=False) else: #submissionScript = repo.get_contents("submission.sh", ref=branch) submissionScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="submission.sh",raw=False) #pushScript = repo.get_contents("pushNetworks.sh", ref=branch) pushScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="pushNetworks.sh",raw=False) if networkScriptName in ['c3net', 'wgcnaTOM', 'mrnet']: buildScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="buildOtherNet.R",raw=False) else: buildScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="buildMpiNet.R",raw=False) executed += [config, networkScript, buildScript, submissionScript, pushScript, thisScript] activity = synapseclient.Activity(name='Network Inference', description=method, used=used, executed=executed) synFile = synapseclient.File(filePath, parent=parentId) synFile.properties['versionComment'] = commitMessage synEntity = syn.store(obj=synFile, activity=activity) syn.setAnnotations(synEntity, annotations)
def main(): """Main function.""" syn = synapseclient.login(silent=True) with open("projects_and_cores.tsv", "w") as out: out.write("\t".join([ "Center Name", "Project/Core", "Number", "Title", "Description Markdown" ]) + "\n") project_view = syn.tableQuery( "select id from syn10142562 where grantType in ('U01', 'U54')" ).asDataFrame() for syn_id in project_view.id: project = syn.get(syn_id, downloadFile=False) project_wiki = syn.getWikiHeaders(project) print(syn_id) # diagnostic. # U01 grants only has one project and the description is # listed on the main Wiki page. if project.grantType[0] == "U01": project_page_id = project_wiki[0].id project_page = syn.getWiki(project, project_page_id).markdown project_desc = parse_u01_project(project_page) out.write("\t".join([ project.centerName[0], "Project", "1", "", repr(project_desc) ]) + "\n") # Whereas U54 grants can have multiple projects (and cores), # which are listed on another page. else: for page in project_wiki: if re.search(r"Projects.*Core", page.title): project_page_id = page.id break project_page = syn.getWiki(project, project_page_id).markdown projects, cores = parse_u54_center(project.centerName[0], project_page) out.write(projects) if cores: out.write(cores) syn.logout()
def main(): args = get_env_var_args() syn = sc.login(args["synapseUsername"], args["synapsePassword"]) table_mapping = get_table_mapping(syn, args["tableMapping"]) relevant_healthcodes = get_relevant_healthcodes( syn, participants_table=args["participantsTable"], substudy=args["substudy"]) if args["additionalHealthcodeJson"] is not None: additional_healthcodes = get_additional_healthcodes( syn, synapse_id=args["additionalHealthcodeJson"]) relevant_healthcodes = list( {*relevant_healthcodes, *additional_healthcodes}) synapsebridgehelpers.export_tables(syn=syn, table_mapping=table_mapping, identifier_col="healthCode", identifier=relevant_healthcodes, copy_file_handles=True)
def main(): args = parser.parse_args() syn = sc.login() h1 = pickle.load(open(syn.get(args.refid).path, 'rb')) #h2 = pickle.load(open(syn.get(args.evalid).path, 'rb')) newd = {args.refName: h1} for key, val in args.evalFiles.items(): newd[key] = pickle.load(open(syn.get(val).path, 'rb')) res = hyStats.compute_all_distances(newd) fname = 'hyp2hypDistances_' + args.refName + '_to_data.csv' res.to_csv(fname) if args.synProj is not None: tab = sc.table.build_table( "Original data to " + args.refName + ' Distances', args.synProj, res) syn.store(tab)
def main(): import argparse import sys parser = argparse.ArgumentParser() parser.add_argument('--id', type=str) parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), default=sys.stdout) args = parser.parse_args() syn = synapseclient.login(silent=True) qry = syn.queryTable("select * from %s" % args.id) res = qry.asDataFrame() res.to_csv(args.outfile, sep="\t", index=False)
def main(): """Main""" parser = argparse.ArgumentParser( description='Run GENIE sponsored projects' ) parser.add_argument('sp', type=str, help='Specify sponsored project to run', choices=BPC_MAPPING.keys()) parser.add_argument('release', type=str, help='Specify bpc release') parser.add_argument("--staging", action='store_true', help="If true, files aren't uploaded onto synapse") args = parser.parse_args() syn = synapseclient.login() BPC_MAPPING[args.sp](syn, '../cbioportal', release=args.release, staging=args.staging).run()
def test_getWithEntityBundle__no_DOWNLOAD_permission_warning(): if not (other_user.get('username') and other_user.get('password')): raise SkipTest( "Test was skipped because an additional user is required for this test. Please add a username, password, and pricipalId under [test-authentication] in the .synapseConfig file if oyu wish to execute this test" ) other_syn = synapseclient.login(other_user['username'], other_user['password']) #make a temp data file path = utils.make_bogus_data_file() schedule_for_cleanup(path) #upload to synapse and set permissions to READ only entity = syn.store(File(path, parent=project)) syn.setPermissions(entity, other_user['username'], accessType=['READ']) #try to download and check that nothing wad downloaded and a warning message was printed with patch.object(other_syn.logger, "warning") as mocked_warn: entity_no_download = other_syn.get(entity['id']) mocked_warn.assert_called_once() assert_is_none(entity_no_download.path)
def test_annotation_key_lengths_json_vs_table(): """ For each annotation module, compares the length of annotation keys of json files on github with the current melted annotation synapse table keys generated by json2synapse.py """ syn = synapseclient.login() tableSynId = "syn10242922" names = schema.moduleJsonPath() currentTable = syn.tableQuery("SELECT * FROM %s" % tableSynId) currentTable = currentTable.asDataFrame() for module, path in iteritems(names): table_key_set = set( currentTable[currentTable['module'] == module].key.unique()) json_record = pandas.read_json(path) json_key_set = set(json_record['name']) assert len(json_key_set) == len(table_key_set)
def embed_url_in_synapse_wiki(url, wikiID): import synapseclient syn = synapseclient.login() wiki = syn.getWiki(wikiID) markdown = wiki['markdown'] #complete hack if len(url) > 1: url = [url[x] for x in url if x.endswith('index.html')] url = url[0] else: url = url.values()[0] #percent encoded URL import urllib url = urllib.quote(url, safe='') link_markdown = '${iframe?site=' + url + '&height=1000}' wiki['markdown'] = link_markdown wiki = syn.store(wiki) syn.onweb(wikiID)
def main(): parser = argparse.ArgumentParser(description='Download and process data.') parser.add_argument('-d', dest='data_folder', required=True) parser.add_argument('-l', dest='id_list', required=True) parser.add_argument('record_ids', nargs='+') args = parser.parse_args() syn = synapseclient.login('*****@*****.**', 'Txrxuxnxg123!') pool = Pool(8) print 'Saving to ' + args.data_folder try: ids = pd.read_csv(args.id_list, index_col=0) except Exception as e: print 'There was an exception in reading list of ids...' print ellipsis raise # read in the healthCodes of interest from demographics training table for row in args.record_ids: record_id = ids.iloc[int(row), 0] get_data(row, record_id, args.data_folder, syn)
def read_clinical(ancestry=True): # CMC_Human_clinical_metadata.csv if not os.path.exists(cmc_clinical_path): import synapseclient syn = synapseclient.login() wdir = bsmutils.get_bsmdir() + '/resources/' clinical_syn = syn.get('syn2279441', downloadLocation=wdir, ifcollision='overwrite.local') fpath = clinical_syn.path else: fpath = cmc_clinical_path clinical = pd.read_csv(fpath, index_col='Individual ID') if ancestry: ancestry = pd.read_csv(cmc_ancestry_path, sep='\t', index_col='Individual_ID') ancestry = ancestry.drop(columns=['Genotyping_Sample_ID', 'Cluster']) clinical = pd.concat([clinical, ancestry], axis=1) #clinical.columns = pd.MultiIndex.from_product([['Clinical'], calls.columns], names=['Source', 'Annotation']) return (clinical)
def main(): import csv import argparse parser = argparse.ArgumentParser("Upload raw data files.") parser.add_argument("-d", "--directory", type=str, help="Path to where raw data files are.") parser.add_argument("-k", "--key", type=str, default="name", help="Column in datafile with file name.") parser.add_argument("-p", "--parentId", type=str, help="Folder parentId to upload to.") parser.add_argument("datafile", type=str, help="csv file containing file name and annotations") args = parser.parse_args() syn = synapseclient.login(silent=True) with file(args.datafile) as f: reader = csv.DictReader(f) map(lambda x: process(x, syn, key=args.key, path=args.directory, parentId=args.parentId), reader)
def main(): args = read_args() syn = sc.login() left_col = "left_motion.json" right_col = "right_motion.json" left = download_sensor_data(syn=syn, synapse_id=TREMOR_TABLE, col=left_col, limit=args.limit) right = download_sensor_data(syn=syn, synapse_id=TREMOR_TABLE, col=right_col, limit=args.limit) store_to_s3(bucket_name=args.bucket_name, base_key=os.path.join(args.base_key, left_col), file_handle_df=left, profile_name=args.aws_profile) store_to_s3(bucket_name=args.bucket_name, base_key=os.path.join(args.base_key, right_col), file_handle_df=right, profile_name=args.aws_profile)
import pandas as pd import scipy.stats as stats import hierarchical_clustering as clust import MicroArray from toppGenePost import ToppGeneEnrichement from IPython.display import HTML EXPR_ID = 'syn2247799' #private'syn1968267' EXPR_META_ID = 'syn2278178' METH_ID = 'syn2233188' METH_META_ID = 'syn2677043' MIRSEQ_ID = 'syn2247832' #Private 'syn2233189' MIRSEQ_META_ID = 'syn2278179' BIOMART_ANNOT_ID = 'syn2706233' pd.set_option('max_columns', 200) syn = synapseclient.login() # <markdowncell> # #Download and annotate data # 1. Downloads expression/miRNA-Seq/methylation data from Synapse # 2. Gets annotations from metadata in Synapse # 3. Make sure the metadata matches up with samples # <codecell> mirseq = pd.read_csv(syn.get(MIRSEQ_ID).path, sep='\t', index_col=0) mirseq_meta = pd.read_csv(syn.get(MIRSEQ_META_ID).path, sep='\t',index_col=0) mirseq_meta = mirseq_meta.ix[mirseq.columns,:] mirseq_meta.drop(['Related C4 ID comment'],1, inplace=True) print 'Mirseq Data Size:', mirseq.shape, mirseq_meta.shape
import json import synapseclient import straw import sys if (len(sys.argv) != 2): print("Usage: python {} synapse_id".format(sys.argv[0])) sys.exit(1) user = "******" password = "******" syn = synapseclient.login(user, password) myfile=syn.get(sys.argv[1],downloadFile=False) batch_file_request = { "requestedFiles": [ { "fileHandleId": myfile._file_handle['id'], "associateObjectId": sys.argv[1], "associateObjectType": "FileEntity" } ], "includePreSignedURLs": True, "includeFileHandles": False } result = syn.restPOST(uri='/fileHandle/batch', body=json.dumps(batch_file_request), endpoint=syn.fileHandleEndpoint) url=result["requestedFiles"][0]["preSignedURL"] result2 = straw.straw('NONE', url, '1', '1', 'BP', 1000000, is_synapse=True)
def main(): import argparse parser = argparse.ArgumentParser(description='Convert JSON to Synapse Table Schema') parser.add_argument('path', type=str, help='Path (or URL) to JSON file') parser.add_argument('--projectId', type=str, help='Synapse Project ID to store schema') parser.add_argument('-n', '--dry_run', action="store_true", default=False, help='Dry run') parser.add_argument('--synapseJSONSchema', action="store_true", default=False, help="JSON is already in Synapse Table Schema format") args = parser.parse_args() syn = synapseclient.login(silent=True) project = syn.get(args.projectId) f = urllib.urlopen(path2url(args.path)) data = json.load(f) url_path = urllib.splittype(args.path)[1] filename = os.path.split(url_path)[1] schema_name = os.path.splitext(filename)[0] if args.synapseJSONSchema: schema = synapseclient.Schema(name=schema_name, parent=project) schema.columns_to_store = data else: cols = [] for k, v in data.iteritems(): # Handle null values, assume that they will be strings if not v: column_type = "STRING" elif bool in map(type, v): column_type = "BOOLEAN" elif int in map(type, v): column_type = "INTEGER" elif float in map(type, v): column_type = "DOUBLE" else: column_type = "STRING" cols.append(synapseclient.Column(name=k, columnType=column_type, enumValues=v, maximumSize=250)) schema = synapseclient.Schema(name=schema_name, columns=cols, parent=project) if args.dry_run: schema_as_list = map(dict, schema.columns_to_store) new_schema_as_list = [] _key_order = ['name', 'description', 'columnType', 'maximumSize', 'enumValues'] for col in schema_as_list: col['description'] = "" col['source'] = "" new_enum_values = [] for v in col['enumValues']: new_value_ordered_dict = collections.OrderedDict() new_value_ordered_dict['value'] = v new_value_ordered_dict['description'] = "" new_value_ordered_dict['source'] = "" new_enum_values.append(new_value_ordered_dict) col['enumValues'] = new_enum_values new_ordered_dict = collections.OrderedDict() for k in _key_order: new_ordered_dict[k] = col[k] new_schema_as_list.append(new_ordered_dict) print json.dumps(new_schema_as_list, indent=2) else: schema = syn.store(schema)
import sys import synapseclient syn = synapseclient.login(silent=True) files = syn.restGET('/entity/md5/%s' % sys.argv[1]) for item in files['results']: print sys.argv[1], item['name'], item['id'], item['versionNumber']
argparse.HelpFormatter(prog, max_help_position=40)) parser.add_argument("project_id", help='Synapse Project ID, such as "syn32071528"') parser.add_argument("-u", "--user", help="Synapse User Name", default=None) parser.add_argument("-p", "--password", help="Synapse Password", default=None) parser.add_argument("--file-db", help="SQLite database of tracking last modified dates of files to upload", default="synapse_file_upload_directory_tree.sqlite") parser.add_argument("--top", help='Topmost directory', default='.', type=str, metavar='STR') args = parser.parse_args() project_id = args.project_id user = args.user password = args.password start_path = args.top syn=synapseclient.login(user, password) #(silent=True) conn = sqlite3.connect(args.file_db) try: #----------------------------------------------------------------------------- # get cached file and directory info from SQLite: #----------------------------------------------------------------------------- c = conn.cursor() c.execute('CREATE TABLE IF NOT EXISTS files (path TEXT, mtime REAL);') c.execute('CREATE TABLE IF NOT EXISTS folders (path TEXT, synapse_id TEXT);') print('Reading file table...') previous_uploads = {} result = c.execute('SELECT path, mtime from files;') for row in result:
configuration = '/home/boiseb01/.synapse-key.txt' if not os.path.isfile(configuration): print("Error: " + configuration + " is not a file") sys.exit(1) #print("DEBUG configuration " + configuration) reader = open(configuration) key1= reader.readline().strip() key2 = reader.readline().strip() #print("DEBUG attempting login with " + key1) synapse = synapseclient.login(key1, key2) #synapse = synapseclient.login() print("DEBUG Got connection to synapse") handles = [] handles.append("syn2343226") handles.append("syn2343245") handles.append("syn2344779") handles.append("syn2342038") #~/.synapse-key.txt mustDownload = True location = "." for handle in handles: