def synapse_login():
try:
syn = synapseclient.login()
except Exception as e:
print("Please provide your synapse username/email and password (You will only be prompted once)")
Username = raw_input("Username: ")
Password = getpass.getpass()
syn = synapseclient.login(email=Username, password=Password,rememberMe=True)
return syn
def perform_main(args): if args.synapse_user is not None: syn = synapseclient.login(args.synapse_user,args.password,rememberMe=True) else: syn = synapseclient.login() if 'func' in args: try: args.func(args,syn) except Exception as ex: print(ex)
def main():
import argparse
import sys
parser = argparse.ArgumentParser()
parser.add_argument('--dryrun', action="store_true",
default=False,
help="Run without making changes [default: %(default)s]")
parser.add_argument('--storetable', action="store_true", default=False)
parser.add_argument('--projectId', type=str)
args = parser.parse_args()
syn = synapseclient.login(silent=True)
annots = PCBCAnnotations.PCBCAnnotations(syn)
a = PCBCAnnotations.PCBCAnnotationTableUpdate(syn, annots)
my_synapse_id = None
if not args.dryrun:
my_synapse_id = synapseHelpers.thisCodeInSynapse(parentId='syn2758110', syn=syn)
a.update_annots_synapse(executed=my_synapse_id, dryrun=args.dryrun)
if args.storetable:
tbl = a.update_annots_table_synapse(projectId=args.projectId, dryrun=args.dryrun)
def login():
"""Log into Synapse. If cached info not found, prompt user.
Returns:
syn: Synapse object
"""
try:
syn = synapseclient.login(silent=True)
except Exception:
print("Cached credentials not found; please provide",
"your Synapse username and password.")
username = input("Synapse username: "******"Synapse password: "******"utf-8")
syn = synapseclient.login(username=username,
password=password,
rememberMe=True,
silent=True)
return syn
def main():
if not os.path.exists(TEST_SUBMISSION_PATH):
os.makedirs(TEST_SUBMISSION_PATH)
syn = sc.login()
args = readargs()
args.filetype = "csv" if not args.filetype else args.filetype
indexCols = None if not args.indexCols else args.indexCols.split(",")
writeSubmissions(args.sampleSubmission, indexCols, args.filetype)
storeSubmissions(syn, args.evaluationQueue, args.synProject, args.filetype)
logger.info("Finished")
def main():
import argparse
import sys
parser = argparse.ArgumentParser()
parser.add_argument('outfile', nargs='?',
type=argparse.FileType('w'),
default=sys.stdout)
parser.add_argument('--storetable',
action="store_true",
default=False)
parser.add_argument('--projectId', type=str)
args = parser.parse_args()
syn = synapseclient.login(silent=True)
## mRNA metadata from bam, bed, and fpkm files
pcbcmeta = BootstrapExperimentalMetadata.MRNABamMetadata(syn)
pcbc_mrna_bam = pcbcmeta()
pcbcmeta = BootstrapExperimentalMetadata.MRNABedMetadata(syn)
pcbc_mrna_bed = pcbcmeta()
pcbcmeta = BootstrapExperimentalMetadata.MRNAFpkmMetadata(syn)
pcbc_mrna_fpkm = pcbcmeta()
## miRNA metadata from Fastq and Expr files
pcbcmeta = BootstrapExperimentalMetadata.MIRNAFastqMetadata(syn)
pcbc_mirna_fastq = pcbcmeta()
pcbcmeta = BootstrapExperimentalMetadata.MIRNAExprMetadata(syn)
pcbc_mirna_expr = pcbcmeta()
pcbcmeta = BootstrapExperimentalMetadata.MethylMetadata(syn)
pcbc_methyl = pcbcmeta()
pcbc_sample_metadata = pd.concat([pcbc_mrna_bam, pcbc_mrna_bed, pcbc_mrna_fpkm,
pcbc_mirna_fastq, pcbc_mirna_expr,
pcbc_methyl],
ignore_index=True)
_keepcols = ["name", "id", "UID", "C4_Cell_Line_ID",
"Diffname_short", "dataType", "fileType"]
pcbc_sample_metadata = pcbc_sample_metadata[_keepcols]
pcbc_sample_metadata.to_csv(args.outfile, sep="\t", index=False)
if args.storetable:
tbl = BootstrapExperimentalMetadata.to_table(syn, pcbc_sample_metadata,
args.projectId)
def main():
args = read_args()
syn = synapseclient.login()
df = download_from_synapse(syn=syn,
source_table=args.synapse_source,
limit=args.num)
session = boto.Session(profile_name=args.aws_profile)
copy_to_s3(df=df,
session=session,
bucket=args.s3_bucket,
basekey=args.s3_prefix)
def main():
log.info('Parsing arguments...')
args = parse_arguments()
if args.team_name_tsv is not None:
team_name_dict = parse_team_name_tsv(args.team_name_tsv)
else:
team_name_dict = None
print(team_name_dict)
syn = synapseclient.login()
t0 = time.perf_counter()
while True:
try:
if not args.update_wiki_only:
evaluation = syn.getEvaluation(args.eval_queue_id)
# init multiprocessing
pool = multiprocessing.Pool(args.nth)
# distribute jobs
ret_vals = []
for submission, status in syn.getSubmissionBundles(
evaluation, status='RECEIVED'):
ret_vals.append(
pool.apply_async(validate_submission,
(submission, status, args, syn)))
# gather
for r in ret_vals:
r.get(BIG_INT)
pool.close()
pool.join()
update_wiki_for_round2(syn, team_name_dict, args)
except Exception as ex1:
st = StringIO()
traceback.print_exc(file=st)
message = st.getvalue()
subject = 'Server error:'
if args.send_msg_to_admin:
send_message(syn, args.admin_id, subject, message)
log.error(message)
log.info('Waiting for new submissions...')
while time.perf_counter() - t0 < args.period:
time.sleep(60)
t0 = time.perf_counter()
log.info('All done')
def validateFiles(synId, center):
"""
This function walks through each center's input directory and validates every file
"""
syn = synapseclient.login()
walked = synu.walk(syn, synId)
clinicalpair = []
invalid = []
validFiles = []
for dirpath, dirname, filenames in walked:
for name, synid in filenames:
file = []
if name.endswith(".vcf") and VALIDATE_FILENAME['vcf'] % center in name:
fileType = "vcf"
elif name.endswith(".bed") and VALIDATE_FILENAME['bed'] % center in name:
fileType = "bed"
elif VALIDATE_FILENAME['maf'] % center == name:
fileType = "maf"
elif VALIDATE_FILENAME['cnv'] % center == name:
fileType = "cnv"
elif VALIDATE_FILENAME['fusions'] % center == name:
fileType = "fusions"
elif VALIDATE_FILENAME['seg'] % center == name:
fileType = "seg"
elif VALIDATE_FILENAME['clinical'][0] % center == name:
fileType = "clinical"
elif name in VALIDATE_FILENAME['clinical'][1:]:
clinicalpair.append(synid)
if len(clinicalpair) == 2:
fileType = "clinical"
else:
fileType = None
else:
fileType = None
if fileType is not None:
if len(clinicalpair) == 2:
#Need to pass in both filepath and synid for processing files
file = [(syn.get(i).path,i) for i in clinicalpair]
clinicalpair = []
else:
file = [(syn.get(synid).path,synid)]
#Validation only takes in a list of filepaths
paths = [i[0] for i in file]
message, valid = validate_genie.main(fileType, paths, center)
else:
print("%s: Cannot be processed" % name)
valid = False
invalid.append(name)
if not valid:
invalid.append(name)
else:
validFiles.extend(file)
print(", ".join(invalid) + " can't be processed!")
return(validFiles)
def main():
credentials = get_credentials()
syn = sc.login(email = credentials['synapseUsername'],
password = credentials['synapsePassword'])
new_users = get_new_users(syn)
if isinstance(new_users, tuple): # returned error message
table_row = create_table_row(new_users[0], new_users[1],
new_users[2], new_users[3])
syn.store(sc.Table(OUTPUT_TABLE, [table_row]))
return
duplicated_numbers = new_users.phone_number.duplicated(keep = False)
if any(duplicated_numbers):
duplicates = new_users.loc[duplicated_numbers]
if len(duplicates.guid) == len(duplicates.guid.unique()):
table_row = create_table_row("Error: It looks like you accidentally "
"entered an incorrect guid and tried to "
"submit a corrected one immediately "
"afterwards. Please delete the duplicate "
"phone number from the Users table "
"(syn16784393) and this table (syn16786935) "
"and resubmit.", duplicates.phone_number.iloc[0],
"", duplicates.visit_date.iloc[0])
syn.store(sc.Table(OUTPUT_TABLE, [table_row]))
return
to_append_to_table = []
for i, user in new_users.iterrows():
phone_number = int(user.phone_number)
guid = user.guid
visit_date = int(user.visit_date)
print("phone_number: ", phone_number)
print("guid: ", guid)
print("visit_date: ", visit_date)
try:
if not (len(str(phone_number)) == 10 and str(phone_number).isdigit()):
table_row = create_table_row("Error: The phone number is improperly "
"formatted. Please enter a valid, 10-digit "
"number",
phone_number, guid, visit_date)
else:
bridge = get_bridge_client(credentials['bridgeUsername'],
credentials['bridgePassword'])
participant_info = get_participant_info(bridge, phone_number)
status = process_request(bridge, participant_info,
phone_number, guid)
table_row = create_table_row(status, phone_number,
guid, visit_date)
except Exception as e:
table_row = create_table_row("Error: One of the fields is improperly "
"formatted. Console output: {0}".format(e),
-1, guid, visit_date)
to_append_to_table.append(table_row)
if len(to_append_to_table):
syn.store(sc.Table(OUTPUT_TABLE, to_append_to_table))
def validate_submission(evaluation, submission):
"""
Find the right validation function and validate the submission.
:returns: (True, message) if validated, (False, message) if
validation fails or throws exception
"""
config = config_evaluations_map[int(evaluation.id)]
validation_func = config['validation_function']
syn = synapseclient.login()
results = validation_func(evaluation, submission.filePath, syn)
return results
def validate_submission(evaluation, submission, token):
"""
Find the right validation function and validate the submission.
:returns: (True, message) if validated, (False, message) if
validation fails or throws exception
"""
config = config_evaluations_map[int(evaluation.id)]
validation_func = config['validation_function']
syn = synapseclient.login()
results = validation_func(evaluation,submission,syn, token)
return results
def synapseLogin():
"""
First tries to login to synapse by finding the local auth key cached on user's computing platform, if not found,
prompts the user to provide their synapse user name and password, then caches the auth key on their computing
platform.
:return:
"""
try:
syn = synapseclient.login()
except Exception as e:
print(
'Please provide your synapse username/email and password (You will only be prompted once)'
)
username = input("Username: "******"Password for " + username + ": ").encode('utf-8'))
syn = synapseclient.login(email=username,
password=password,
rememberMe=True)
return syn
def main():
args = get_env_var_args()
syn = sc.login(args["synapseUsername"], args["synapsePassword"])
table_mapping = get_table_mapping(syn, args["tableMapping"])
relevant_healthcodes = get_relevant_healthcodes(
syn,
participants_table=args["participantsTable"],
substudy=args["substudy"])
synapsebridgehelpers.export_tables(syn=syn,
table_mapping=table_mapping,
identifier_col="healthCode",
identifier=relevant_healthcodes,
copy_file_handles=True)
def connect_to_synapse():
"""
Connect to synapse.org. You need to set the right environment variables:
- SYN_EMAIL: Your synapse.org email.
- SYN_PWD: Your synapse.org password.
:return: (Synapse) synapse connection instance
"""
syn_email = os.getenv('SYN_EMAIL')
syn_pwd = os.getenv('SYN_PWD')
assert syn_email, 'The SYN_EMAIL environment variable is not set.'
assert syn_pwd, 'The SYN_PWD environment variable is not set.'
return synapseclient.login(syn_email, syn_pwd)
def connect_to_synapse():
"""
Connect to synapse.org. You need to set the right environment variables:
- SYN_EMAIL: Your synapse.org email.
- SYN_PWD: Your synapse.org password.
:return: (Synapse) synapse connection instance
"""
syn_email = os.getenv('SYN_EMAIL')
syn_pwd = os.getenv('SYN_PWD')
assert syn_email, 'The SYN_EMAIL environment variable is not set.'
assert syn_pwd, 'The SYN_PWD environment variable is not set.'
return synapseclient.login(syn_email, syn_pwd)
def main():
syn = synapseclient.login(email='*****@*****.**',
password='******',
rememberMe=True)
print "Querying the Walking training table..."
#Query 'walking training table' for walk data recordIDs and healthCodes.
INPUT_WALKING_ACTIVITY_TABLE_SYNID = "syn10733842"
actv_walking_syntable = syn.tableQuery(
('SELECT * FROM {0}').format(INPUT_WALKING_ACTIVITY_TABLE_SYNID))
actv_walking = actv_walking_syntable.asDataFrame()
actv_walking[["recordId", "phoneInfo"]].to_csv('meta-data-testing.csv',
index=False,
header=True)
print "Done!"
def synapse_login(username=None, password=None):
"""
This function logs into synapse for you if credentials are saved.
If not saved, then user is prompted username and password.
:returns: Synapseclient object
"""
try:
syn = synapseclient.login(silent=True)
except Exception:
if username is None and password is None:
raise ValueError(
"Please specify --syn_user, --syn_pass to specify your Synapse "
"login. Please view https://docs.synapse.org/articles/client_configuration.html"
"to learn about logging into Synapse via the Python client.")
else:
syn = synapseclient.login(
email=username,
password=password,
silent=True)
return(syn)
return result
def synapse_login():
"""
This function logs into synapse for you if credentials are saved.
If not saved, then user is prompted username and password.
:returns: Synapseclient object
"""
try:
syn = synapseclient.login(silent=True)
except Exception as e:
logger.info(
"Please provide your synapse username/email and password (You will only be prompted once), or write your username and password to your bash profile under GENIE_USER and GENIE_PASS"
)
Username = os.getenv("GENIE_USER")
Password = os.getenv("GENIE_PASS")
if Username is None or Password is None:
Username = raw_input("Username: ")
Password = getpass.getpass()
syn = synapseclient.login(email=Username,
password=Password,
rememberMe=True,
silent=True)
return (syn)
def login(self):
logging.info('Logging into Synapse...')
self._username = self._username or os.getenv('SYNAPSE_USER')
self._password = self._password or os.getenv('SYNAPSE_PASSWORD')
if self._username == None:
self._username = input('Synapse username: '******'Synapse password: ')
self._synapse_client = synapseclient.login(self._username,
self._password,
silent=True)
def stop_hanging_docker_submissions():
"""Stops hanging docker submissions"""
syn = synapseclient.login()
client = docker.from_env()
running_containers = client.containers.list()
for container in running_containers:
try:
status = syn.getSubmissionStatus(container.name)
if status.status == "INVALID":
print("stopping: " + container.name)
container.stop()
container.remove()
except Exception:
print("Not a synapse submission / unable to remove container")
def run(self):
self._synapse_client = synapseclient.login(self._parent._username,
self._parent._password,
silent=True)
while not self.exit_thread:
self._lock.acquire()
if not self._queue.empty():
folder_obj = self._queue.get()
self._lock.release()
self.upload_file_to_synapse(folder_obj["file_info"],
folder_obj["folder_path"])
else:
self._lock.release()
time.sleep(.10)
def make_manifests_main(slistcsv, target_dir=".", prefix='chess-'):
'''
Make manifests given a list of samples in a CSV file
Arguments:
slistcsv: path to the input CSV file
target_dir: where the manifest files will be created
prefix: common prefix to all three manifests
Value:
a tuple of the three manifests, each one a pandas DataFrame
Details:
The side effect of the function is to write the manifests in the target
directory with the given prefix.
'''
syn = synapseclient.login()
def do_one_sample(subject, tissue):
m = make_manifests(subject=subject,
tissue=tissue,
syn=syn,
target_dir=target_dir,
matching_sample_ids=True)
return (m)
samples = pd.read_csv(slistcsv)
lomanifests = [
do_one_sample(subject=samples.iloc[i][0], tissue=samples.iloc[i][1])
for i in samples.index
]
kinds = ['nichd_btb02', 'genomics_subject02', 'genomics_sample03']
def do_one_manifest(kind):
m = [row[kind] for row in lomanifests]
m = pd.concat(m)
csvpath = target_dir + os.path.sep + prefix + kinds[kind] + '.csv'
templ_synid = manifest_template_synids[kinds[kind]]
templ_df, templ_syn = get_manifest(templ_synid,
syn,
download_dir=target_dir)
templ_p = target_dir + os.path.sep + templ_syn.properties.name
write_manifest(m, templ_p, csvpath)
return (m)
manifests = tuple(do_one_manifest(k) for k in range(len(kinds)))
return (manifests)
def validate_writeup(submission, goldstandard_path, public=True, admin=None):
'''
Validates challenge writeup
Args:
submission: Submission object
goldstandard_path: Unused
syn: Synapse object
public: If the writeup needs to be public. Defaults to True
admin: Specify Synapse userid that writeup needs to be
shared with
Returns:
(True, message) if validated, (False, message) if
validation fails or throws exception
'''
from synapseclient.exceptions import SynapseHTTPError
from synapseclient import AUTHENTICATED_USERS
import synapseclient
syn = synapseclient.login()
# Add in users to share this with
share_with = []
try:
if public:
message = "Please make your private project ({}) public".format(
submission['entityId'])
share_with.append(message)
ent = \
syn.getPermissions(submission['entityId'], AUTHENTICATED_USERS)
assert "READ" in ent and "DOWNLOAD" in ent, message
ent = syn.getPermissions(submission['entityId'])
assert "READ" in ent, message
if admin is not None:
message = (
"Please share your private directory ({}) with the Synapse"
" user `{}` with `Can Download` permissions.".format(
submission['entityId'], admin))
share_with.append(message)
ent = syn.getPermissions(submission['entityId'], admin)
assert "READ" in ent and "DOWNLOAD" in ent, message
except SynapseHTTPError as e:
if e.response.status_code == 403:
raise AssertionError("\n".join(share_with))
else:
raise (e)
return True, "Validated!"
def submit(self, name):
folderid = 'syn10932057'
import synapseclient
from synapseclient import File, Evaluation
syn = synapseclient.login()
# upload the file to the synapse project folder
submissionfile = File(os.path.join(self.summary_path, name + ".csv"),
parent = folderid)
submissionfile = syn.store(submissionfile)
team_entity = syn.getTeam("Spreedictors")
submission = syn.submit(evaluation = 9606375,
entity = submissionfile, name = "Spreedictor_{}".format(name),
team = team_entity)
syn.logout()
def push(filePath, parentId, annotationFile, provenanceFile, method, branch, token, commitMessage):
syn = synapseclient.login()
with open(annotationFile, 'r') as f:
entries = f.read().strip().split('\n')
annotations = {s[0] : s[1] for s in [pair.split(',') for pair in entries]}
annotations['method'] = method
with open(provenanceFile, 'r') as csvfile:
reader = csv.DictReader(csvfile, delimiter=',')
used = [r['provenance'] for r in reader if r['executed'] == 'FALSE']
executed= [r['provenance'] for r in reader if r['executed'] == 'TRUE']
g = github.Github(token)
#repo = g.get_repo("Sage-Bionetworks/metanetworkSynapse")
#config = repo.get_contents("config.sh", ref=branch)
config = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="config.sh")
#thisScript = repo.get_contents("pushToSynapse.py", ref=branch)
thisScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="pushToSynapse.py",raw=False)
networkScriptName = get_ns_name(method)
#networkScript = repo.get_contents("networkScripts/%s.sh" % networkScriptName,
#ref=branch)
networkScript = getPermalink(gh=g,repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="networkScripts/%s.sh" % networkScriptName, raw=False)
if method == "rankConsensus" or method == "bic":
q = syn.chunkedQuery("select id, name from entity where entity.parentId=='%s'" % parentId)
[used.append(i['entity.id']) for i in q]
#submissionScript = repo.get_contents("submissionConsensus.sh", ref=branch)
submissionScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="submissionConsensus.sh",raw=False)
#pushScript = repo.get_contents("pushConsensus.sh", ref=branch)
pushScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="pushConsensus.sh",raw=False)
#buildScript = repo.get_contents("buildConsensus.R", ref=branch)
buildScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="buildConsensus.R",raw=False)
else:
#submissionScript = repo.get_contents("submission.sh", ref=branch)
submissionScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="submission.sh",raw=False)
#pushScript = repo.get_contents("pushNetworks.sh", ref=branch)
pushScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="pushNetworks.sh",raw=False)
if networkScriptName in ['c3net', 'wgcnaTOM', 'mrnet']:
buildScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="buildOtherNet.R",raw=False)
else:
buildScript = getPermalink(gh=g, repo="Sage-Bionetworks/metanetworkSynapse", ref="master", path="buildMpiNet.R",raw=False)
executed += [config, networkScript, buildScript, submissionScript, pushScript, thisScript]
activity = synapseclient.Activity(name='Network Inference', description=method, used=used, executed=executed)
synFile = synapseclient.File(filePath, parent=parentId)
synFile.properties['versionComment'] = commitMessage
synEntity = syn.store(obj=synFile, activity=activity)
syn.setAnnotations(synEntity, annotations)
def main():
"""Main function."""
syn = synapseclient.login(silent=True)
with open("projects_and_cores.tsv", "w") as out:
out.write("\t".join([
"Center Name", "Project/Core", "Number", "Title",
"Description Markdown"
]) + "\n")
project_view = syn.tableQuery(
"select id from syn10142562 where grantType in ('U01', 'U54')"
).asDataFrame()
for syn_id in project_view.id:
project = syn.get(syn_id, downloadFile=False)
project_wiki = syn.getWikiHeaders(project)
print(syn_id) # diagnostic.
# U01 grants only has one project and the description is
# listed on the main Wiki page.
if project.grantType[0] == "U01":
project_page_id = project_wiki[0].id
project_page = syn.getWiki(project, project_page_id).markdown
project_desc = parse_u01_project(project_page)
out.write("\t".join([
project.centerName[0], "Project", "1", "",
repr(project_desc)
]) + "\n")
# Whereas U54 grants can have multiple projects (and cores),
# which are listed on another page.
else:
for page in project_wiki:
if re.search(r"Projects.*Core", page.title):
project_page_id = page.id
break
project_page = syn.getWiki(project, project_page_id).markdown
projects, cores = parse_u54_center(project.centerName[0],
project_page)
out.write(projects)
if cores:
out.write(cores)
syn.logout()
def main():
args = get_env_var_args()
syn = sc.login(args["synapseUsername"], args["synapsePassword"])
table_mapping = get_table_mapping(syn, args["tableMapping"])
relevant_healthcodes = get_relevant_healthcodes(
syn,
participants_table=args["participantsTable"],
substudy=args["substudy"])
if args["additionalHealthcodeJson"] is not None:
additional_healthcodes = get_additional_healthcodes(
syn, synapse_id=args["additionalHealthcodeJson"])
relevant_healthcodes = list(
{*relevant_healthcodes, *additional_healthcodes})
synapsebridgehelpers.export_tables(syn=syn,
table_mapping=table_mapping,
identifier_col="healthCode",
identifier=relevant_healthcodes,
copy_file_handles=True)
def main():
args = parser.parse_args()
syn = sc.login()
h1 = pickle.load(open(syn.get(args.refid).path, 'rb'))
#h2 = pickle.load(open(syn.get(args.evalid).path, 'rb'))
newd = {args.refName: h1}
for key, val in args.evalFiles.items():
newd[key] = pickle.load(open(syn.get(val).path, 'rb'))
res = hyStats.compute_all_distances(newd)
fname = 'hyp2hypDistances_' + args.refName + '_to_data.csv'
res.to_csv(fname)
if args.synProj is not None:
tab = sc.table.build_table(
"Original data to " + args.refName + ' Distances', args.synProj,
res)
syn.store(tab)
def main():
import argparse
import sys
parser = argparse.ArgumentParser()
parser.add_argument('--id', type=str)
parser.add_argument('outfile', nargs='?',
type=argparse.FileType('w'),
default=sys.stdout)
args = parser.parse_args()
syn = synapseclient.login(silent=True)
qry = syn.queryTable("select * from %s" % args.id)
res = qry.asDataFrame()
res.to_csv(args.outfile, sep="\t", index=False)
def main():
"""Main"""
parser = argparse.ArgumentParser(
description='Run GENIE sponsored projects'
)
parser.add_argument('sp', type=str,
help='Specify sponsored project to run',
choices=BPC_MAPPING.keys())
parser.add_argument('release', type=str,
help='Specify bpc release')
parser.add_argument("--staging", action='store_true',
help="If true, files aren't uploaded onto synapse")
args = parser.parse_args()
syn = synapseclient.login()
BPC_MAPPING[args.sp](syn, '../cbioportal', release=args.release,
staging=args.staging).run()
def main():
import argparse
import sys
parser = argparse.ArgumentParser()
parser.add_argument('--id', type=str)
parser.add_argument('outfile',
nargs='?',
type=argparse.FileType('w'),
default=sys.stdout)
args = parser.parse_args()
syn = synapseclient.login(silent=True)
qry = syn.queryTable("select * from %s" % args.id)
res = qry.asDataFrame()
res.to_csv(args.outfile, sep="\t", index=False)
def test_getWithEntityBundle__no_DOWNLOAD_permission_warning():
if not (other_user.get('username') and other_user.get('password')):
raise SkipTest(
"Test was skipped because an additional user is required for this test. Please add a username, password, and pricipalId under [test-authentication] in the .synapseConfig file if oyu wish to execute this test"
)
other_syn = synapseclient.login(other_user['username'],
other_user['password'])
#make a temp data file
path = utils.make_bogus_data_file()
schedule_for_cleanup(path)
#upload to synapse and set permissions to READ only
entity = syn.store(File(path, parent=project))
syn.setPermissions(entity, other_user['username'], accessType=['READ'])
#try to download and check that nothing wad downloaded and a warning message was printed
with patch.object(other_syn.logger, "warning") as mocked_warn:
entity_no_download = other_syn.get(entity['id'])
mocked_warn.assert_called_once()
assert_is_none(entity_no_download.path)
def test_annotation_key_lengths_json_vs_table():
"""
For each annotation module, compares the length of annotation keys of json files on github with the current melted
annotation synapse table keys generated by json2synapse.py
"""
syn = synapseclient.login()
tableSynId = "syn10242922"
names = schema.moduleJsonPath()
currentTable = syn.tableQuery("SELECT * FROM %s" % tableSynId)
currentTable = currentTable.asDataFrame()
for module, path in iteritems(names):
table_key_set = set(
currentTable[currentTable['module'] == module].key.unique())
json_record = pandas.read_json(path)
json_key_set = set(json_record['name'])
assert len(json_key_set) == len(table_key_set)
def embed_url_in_synapse_wiki(url, wikiID):
import synapseclient
syn = synapseclient.login()
wiki = syn.getWiki(wikiID)
markdown = wiki['markdown']
#complete hack
if len(url) > 1:
url = [url[x] for x in url if x.endswith('index.html')]
url = url[0]
else:
url = url.values()[0]
#percent encoded URL
import urllib
url = urllib.quote(url, safe='')
link_markdown = '${iframe?site=' + url + '&height=1000}'
wiki['markdown'] = link_markdown
wiki = syn.store(wiki)
syn.onweb(wikiID)
def main():
parser = argparse.ArgumentParser(description='Download and process data.')
parser.add_argument('-d', dest='data_folder', required=True)
parser.add_argument('-l', dest='id_list', required=True)
parser.add_argument('record_ids', nargs='+')
args = parser.parse_args()
syn = synapseclient.login('*****@*****.**', 'Txrxuxnxg123!')
pool = Pool(8)
print 'Saving to ' + args.data_folder
try:
ids = pd.read_csv(args.id_list, index_col=0)
except Exception as e:
print 'There was an exception in reading list of ids...'
print ellipsis
raise
# read in the healthCodes of interest from demographics training table
for row in args.record_ids:
record_id = ids.iloc[int(row), 0]
get_data(row, record_id, args.data_folder, syn)
def read_clinical(ancestry=True):
# CMC_Human_clinical_metadata.csv
if not os.path.exists(cmc_clinical_path):
import synapseclient
syn = synapseclient.login()
wdir = bsmutils.get_bsmdir() + '/resources/'
clinical_syn = syn.get('syn2279441',
downloadLocation=wdir,
ifcollision='overwrite.local')
fpath = clinical_syn.path
else:
fpath = cmc_clinical_path
clinical = pd.read_csv(fpath, index_col='Individual ID')
if ancestry:
ancestry = pd.read_csv(cmc_ancestry_path,
sep='\t',
index_col='Individual_ID')
ancestry = ancestry.drop(columns=['Genotyping_Sample_ID', 'Cluster'])
clinical = pd.concat([clinical, ancestry], axis=1)
#clinical.columns = pd.MultiIndex.from_product([['Clinical'], calls.columns], names=['Source', 'Annotation'])
return (clinical)
def main():
import csv
import argparse
parser = argparse.ArgumentParser("Upload raw data files.")
parser.add_argument("-d", "--directory", type=str, help="Path to where raw data files are.")
parser.add_argument("-k", "--key", type=str, default="name", help="Column in datafile with file name.")
parser.add_argument("-p", "--parentId", type=str, help="Folder parentId to upload to.")
parser.add_argument("datafile", type=str, help="csv file containing file name and annotations")
args = parser.parse_args()
syn = synapseclient.login(silent=True)
with file(args.datafile) as f:
reader = csv.DictReader(f)
map(lambda x: process(x, syn, key=args.key, path=args.directory,
parentId=args.parentId),
reader)
def main():
args = read_args()
syn = sc.login()
left_col = "left_motion.json"
right_col = "right_motion.json"
left = download_sensor_data(syn=syn,
synapse_id=TREMOR_TABLE,
col=left_col,
limit=args.limit)
right = download_sensor_data(syn=syn,
synapse_id=TREMOR_TABLE,
col=right_col,
limit=args.limit)
store_to_s3(bucket_name=args.bucket_name,
base_key=os.path.join(args.base_key, left_col),
file_handle_df=left,
profile_name=args.aws_profile)
store_to_s3(bucket_name=args.bucket_name,
base_key=os.path.join(args.base_key, right_col),
file_handle_df=right,
profile_name=args.aws_profile)
import pandas as pd
import scipy.stats as stats
import hierarchical_clustering as clust
import MicroArray
from toppGenePost import ToppGeneEnrichement
from IPython.display import HTML
EXPR_ID = 'syn2247799' #private'syn1968267'
EXPR_META_ID = 'syn2278178'
METH_ID = 'syn2233188'
METH_META_ID = 'syn2677043'
MIRSEQ_ID = 'syn2247832' #Private 'syn2233189'
MIRSEQ_META_ID = 'syn2278179'
BIOMART_ANNOT_ID = 'syn2706233'
pd.set_option('max_columns', 200)
syn = synapseclient.login()
# <markdowncell>
# #Download and annotate data
# 1. Downloads expression/miRNA-Seq/methylation data from Synapse
# 2. Gets annotations from metadata in Synapse
# 3. Make sure the metadata matches up with samples
# <codecell>
mirseq = pd.read_csv(syn.get(MIRSEQ_ID).path, sep='\t', index_col=0)
mirseq_meta = pd.read_csv(syn.get(MIRSEQ_META_ID).path, sep='\t',index_col=0)
mirseq_meta = mirseq_meta.ix[mirseq.columns,:]
mirseq_meta.drop(['Related C4 ID comment'],1, inplace=True)
print 'Mirseq Data Size:', mirseq.shape, mirseq_meta.shape
import json
import synapseclient
import straw
import sys
if (len(sys.argv) != 2):
print("Usage: python {} synapse_id".format(sys.argv[0]))
sys.exit(1)
user = "******"
password = "******"
syn = synapseclient.login(user, password)
myfile=syn.get(sys.argv[1],downloadFile=False)
batch_file_request = {
"requestedFiles": [
{
"fileHandleId": myfile._file_handle['id'],
"associateObjectId": sys.argv[1],
"associateObjectType": "FileEntity"
}
],
"includePreSignedURLs": True,
"includeFileHandles": False
}
result = syn.restPOST(uri='/fileHandle/batch', body=json.dumps(batch_file_request), endpoint=syn.fileHandleEndpoint)
url=result["requestedFiles"][0]["preSignedURL"]
result2 = straw.straw('NONE', url, '1', '1', 'BP', 1000000, is_synapse=True)
def main():
import argparse
parser = argparse.ArgumentParser(description='Convert JSON to Synapse Table Schema')
parser.add_argument('path', type=str, help='Path (or URL) to JSON file')
parser.add_argument('--projectId', type=str, help='Synapse Project ID to store schema')
parser.add_argument('-n', '--dry_run', action="store_true", default=False,
help='Dry run')
parser.add_argument('--synapseJSONSchema', action="store_true",
default=False,
help="JSON is already in Synapse Table Schema format")
args = parser.parse_args()
syn = synapseclient.login(silent=True)
project = syn.get(args.projectId)
f = urllib.urlopen(path2url(args.path))
data = json.load(f)
url_path = urllib.splittype(args.path)[1]
filename = os.path.split(url_path)[1]
schema_name = os.path.splitext(filename)[0]
if args.synapseJSONSchema:
schema = synapseclient.Schema(name=schema_name, parent=project)
schema.columns_to_store = data
else:
cols = []
for k, v in data.iteritems():
# Handle null values, assume that they will be strings
if not v:
column_type = "STRING"
elif bool in map(type, v):
column_type = "BOOLEAN"
elif int in map(type, v):
column_type = "INTEGER"
elif float in map(type, v):
column_type = "DOUBLE"
else:
column_type = "STRING"
cols.append(synapseclient.Column(name=k, columnType=column_type,
enumValues=v, maximumSize=250))
schema = synapseclient.Schema(name=schema_name, columns=cols, parent=project)
if args.dry_run:
schema_as_list = map(dict, schema.columns_to_store)
new_schema_as_list = []
_key_order = ['name', 'description', 'columnType', 'maximumSize', 'enumValues']
for col in schema_as_list:
col['description'] = ""
col['source'] = ""
new_enum_values = []
for v in col['enumValues']:
new_value_ordered_dict = collections.OrderedDict()
new_value_ordered_dict['value'] = v
new_value_ordered_dict['description'] = ""
new_value_ordered_dict['source'] = ""
new_enum_values.append(new_value_ordered_dict)
col['enumValues'] = new_enum_values
new_ordered_dict = collections.OrderedDict()
for k in _key_order:
new_ordered_dict[k] = col[k]
new_schema_as_list.append(new_ordered_dict)
print json.dumps(new_schema_as_list, indent=2)
else:
schema = syn.store(schema)
import sys
import synapseclient
syn = synapseclient.login(silent=True)
files = syn.restGET('/entity/md5/%s' % sys.argv[1])
for item in files['results']:
print sys.argv[1], item['name'], item['id'], item['versionNumber']
argparse.HelpFormatter(prog, max_help_position=40))
parser.add_argument("project_id",
help='Synapse Project ID, such as "syn32071528"')
parser.add_argument("-u", "--user", help="Synapse User Name", default=None)
parser.add_argument("-p", "--password", help="Synapse Password", default=None)
parser.add_argument("--file-db", help="SQLite database of tracking last modified dates of files to upload", default="synapse_file_upload_directory_tree.sqlite")
parser.add_argument("--top",
help='Topmost directory',
default='.', type=str, metavar='STR')
args = parser.parse_args()
project_id = args.project_id
user = args.user
password = args.password
start_path = args.top
syn=synapseclient.login(user, password) #(silent=True)
conn = sqlite3.connect(args.file_db)
try:
#-----------------------------------------------------------------------------
# get cached file and directory info from SQLite:
#-----------------------------------------------------------------------------
c = conn.cursor()
c.execute('CREATE TABLE IF NOT EXISTS files (path TEXT, mtime REAL);')
c.execute('CREATE TABLE IF NOT EXISTS folders (path TEXT, synapse_id TEXT);')
print('Reading file table...')
previous_uploads = {}
result = c.execute('SELECT path, mtime from files;')
for row in result:
configuration = '/home/boiseb01/.synapse-key.txt'
if not os.path.isfile(configuration):
print("Error: " + configuration + " is not a file")
sys.exit(1)
#print("DEBUG configuration " + configuration)
reader = open(configuration)
key1= reader.readline().strip()
key2 = reader.readline().strip()
#print("DEBUG attempting login with " + key1)
synapse = synapseclient.login(key1, key2)
#synapse = synapseclient.login()
print("DEBUG Got connection to synapse")
handles = []
handles.append("syn2343226")
handles.append("syn2343245")
handles.append("syn2344779")
handles.append("syn2342038")
#~/.synapse-key.txt
mustDownload = True
location = "."
for handle in handles:
