def testCustomConfigFile(syn, schedule_for_cleanup):
if os.path.isfile(client.CONFIG_FILE):
configPath = './CONFIGFILE'
shutil.copyfile(client.CONFIG_FILE, configPath)
schedule_for_cleanup(configPath)
syn2 = Synapse(configPath=configPath)
syn2.login()
else:
raise ValueError(
"Please supply a username and password in the configuration file.")
def submit_to_challenge(filename, challenge, label, retry=True):
try:
client = Synapse()
client.login()
evaluation = client.getEvaluation(CODES[challenge])
filename = filename + '.gct' if challenge == 'sc1' else filename + '.zip'
myfile = File(RESULTS_FOLDER + filename, parent=PROJECT_ID)
myfile = client.store(myfile)
client.submit(evaluation, myfile, name=label, teamName=TEAM)
except:
if retry:
submit_to_challenge(filename, challenge, label, retry=False)
else:
print 'failed to submit', label, 'to', challenge
def syn():
"""
Create a logged in Synapse instance that can be shared by all tests in the session.
"""
print("Python version:", sys.version)
syn = Synapse(debug=False, skip_checks=True)
print("Testing against endpoints:")
print(" " + syn.repoEndpoint)
print(" " + syn.authEndpoint)
print(" " + syn.fileHandleEndpoint)
print(" " + syn.portalEndpoint + "\n")
syn.logger = logging.getLogger(SILENT_LOGGER_NAME)
syn.login()
return syn
def main():
# Parse command-line arguments
args = parse_arguments()
# Set up Synapse
syn = Synapse()
syn.login(args.username, args.password, rememberMe=args.remember)
# Retrieve Synapse entity (e.g., project, folder)
entity = syn.get(args.synid, downloadFile=False)
log("Entity", entity)
# Retrieve team
team = syn.getTeam(args.team) # TODO: Handle users with try-catch
log("Team", team)
# Assign specified permissions for given entity and team
permissions = syn.setPermissions(entity,
team.id,
accessType=args.permissions)
log("Permissions", permissions)
# Celebrate
print("Success!")
# ------------------------
import pandas as pd
from msda import kmeans
from synapseclient import Synapse
from msda import pca
import matplotlib.pyplot as plt
import numpy as np
from msda import enrichr_api as ai
from msda.clustering import plot_clustermap as pc
from msda import mapping
from msda import preprocessing as pr
# Load files from synapse
# -----------------------
syn = Synapse()
syn.login()
df = pd.read_csv('total_MS.csv')
# Wrangle data
# -----------
cols = df.columns.tolist()
new_cols = [c.replace('0p5', '0.25') for c in cols]
col_map = {o: n for o, n in zip(cols, new_cols)}
df = df.rename(columns=col_map)
df.insert(2, 'HGNC_Gene_Name',
[mapping.get_name_from_symbol(sy) for sy in df.Gene_Symbol.tolist()])
dfc = df.replace([0], np.nan).dropna().copy()
samples = dfc.columns.tolist()[6:]
days = [s.split('_')[-1] for s in samples]
class KrakenDownload(object):
"""Utility to download Kraken DB and place them in a local directory
::
from sequana import KrakenDownload
kd = KrakenDownload()
kd.download('toydb')
kd.download('minikraken')
A large database (8Gb) is available on synapse and has the following DOI::
doi:10.7303/syn6171000
It can be downloaded manually or if you have a Synapse login
(https://www.synapse.org), you can use::
from sequana import KrakenDownload
kd = KrakenDownload()
kd.downloaded("sequana_db1")
"""
dv = DevTools()
def download(self, name, verbose=True):
if name == "minikraken":
self._download_minikraken(verbose=verbose)
elif name == "toydb":
self._download_kraken_toydb(verbose=verbose)
elif name == "sequana_db1":
self._download_sequana_db1(verbose=verbose)
else:
raise ValueError(
"name must be toydb or minikraken, or sequana_db1")
def _download_kraken_toydb(self, verbose=True):
"""Download the kraken DB toy example from sequana_data into
.config/sequana directory
Checks the md5 checksums. About 32Mb of data
"""
dv = DevTools()
base = sequana_config_path + os.sep + "kraken_toydb"
taxondir = base + os.sep + "taxonomy"
dv.mkdir(base)
dv.mkdir(taxondir)
baseurl = "https://github.com/sequana/data/raw/master/"
# download only if required
logger.info("Downloading the database into %s" % base)
md5sums = [
"28661f8baf0514105b0c6957bec0fc6e",
"97a39d44ed86cadea470352d6f69748d",
"d91a0fcbbc0f4bbac918755b6400dea6",
"c8bae69565af2170ece194925b5fdeb9"
]
filenames = [
"database.idx", "database.kdb", "taxonomy/names.dmp",
"taxonomy/nodes.dmp"
]
for filename, md5sum in zip(filenames, md5sums):
url = baseurl + "kraken_toydb/%s" % filename
filename = base + os.sep + filename
if os.path.exists(filename) and md5(filename) == md5sum:
logger.warning("%s already present" % filename)
else:
logger.info("Downloading %s" % url)
wget(url, filename)
def _download_minikraken(self, verbose=True):
dv = DevTools()
base = sequana_config_path + os.sep + ""
taxondir = base + os.sep + "taxonomy"
dv.mkdir(base)
dv.mkdir(taxondir)
logger.info("Downloading minikraken (4Gb)")
filename = base + os.sep + "minikraken.tgz"
if os.path.exists(filename) and md5(
filename) == "30eab12118158d0b31718106785195e2":
logger.warning("%s already present" % filename)
else:
wget("https://ccb.jhu.edu/software/kraken/dl/minikraken.tgz",
filename)
# unzipping. requires tar and gzip
def _download_from_synapse(self, synid, target_dir):
try:
from synapseclient import Synapse
except ImportError:
raise ImportError(
"Please install synapseclient using 'pip install synapseclient'"
)
try:
self._synapse.get(synid, downloadLocation=target_dir)
except:
self._synapse = Synapse()
self._synapse.login()
self._synapse.get(synid, downloadLocation=target_dir)
def _download_sequana_db1(self, verbose=True):
dbname = "sequana_db1"
from easydev import md5
dir1 = sequana_config_path + os.sep + dbname
dir2 = dir1 + os.sep + "taxonomy"
self.dv.mkdir(dir1)
self.dv.mkdir(dir2)
logger.info(
"Downloading about 8Gb of data (if not already downloaded) from"
" Synapse into %s" % dir1)
from os.path import exists
filename = dir1 + "ena_list.txt"
if exists(filename) and md5(
filename) == "a9cc6268f3338d1632c4712a412593f2":
pass
else:
self._download_from_synapse('syn6171700', dir1)
# database.idx
filename = dir1 + "database.idx"
if exists(filename) and md5(
filename) == "2fa4a99a4f52f2f04c5a965adb1534ac":
pass
else:
self._download_from_synapse('syn6171017', dir1)
# database.kdb ; this one is large (8Gb)
filename = dir1 + "database.kdb"
if exists(filename) and md5(
filename) == "ff698696bfc88fe83bc201937cd9cbdf":
pass
else:
self._download_from_synapse('syn6171107', dir1)
# Then, the taxonomy directory
filename = dir1 + "names.dmp"
if exists(filename) and md5(
filename) == "10bc7a63c579de02112d125a51fd65d0":
pass
else:
self._download_from_synapse('syn6171286', dir2)
filename = dir1 + "nodes.dmp"
if exists(filename) and md5(
filename) == "a68af5a60434e2067c4a0a16df873980":
pass
else:
self._download_from_synapse('syn6171289', dir2)
filename = dir1 + "taxons.txt"
if exists(filename) and md5(
filename) == "e78fbb43b3b41cbf4511d6af16c0287f":
pass
else:
self._download_from_synapse('syn6171290', dir2)
logger.info('done. You should have a kraken DB in %s' % dir1)
# The annotations
wget(
"https://github.com/sequana/data/raw/master/sequana_db1/annotations.csv",
dir1 + os.sep + "annotations.csv")
def _syn_login(login: str) -> Synapse:
"""Login to synapse. Password must be stored in environment variable SYNAPSE_PASS"""
assert 'SYNAPSE_PASS' in os.environ, 'SYNAPSE_PASS must be set as an environment variable'
syn = Synapse()
syn.login(login, os.environ['SYNAPSE_PASS'])
return syn
class KrakenDownload(object):
"""Utility to download Kraken DB and place them in a local directory
::
from sequana import KrakenDownload
kd = KrakenDownload()
kd.download('toydb')
kd.download('minikraken')
A large database (8Gb) is available on synapse and has the following DOI::
doi:10.7303/syn6171000
It can be downloaded manually or if you have a Synapse login
(https://www.synapse.org), you can use::
from sequana import KrakenDownload
kd = KrakenDownload()
kd.downloaded("sequana_db1")
"""
dv = DevTools()
def download(self, name, verbose=True):
if name == "minikraken":
self._download_minikraken(verbose=verbose)
elif name == "toydb":
self._download_kraken_toydb(verbose=verbose)
elif name == "sequana_db1":
self._download_sequana_db1(verbose=verbose)
else:
raise ValueError("name must be toydb or minikraken, or sequana_db1")
def _download_kraken_toydb(self, verbose=True):
"""Download the kraken DB toy example from sequana_data into
.config/sequana directory
Checks the md5 checksums. About 32Mb of data
"""
dv = DevTools()
base = sequana_config_path + os.sep + "kraken_toydb"
taxondir = base + os.sep + "taxonomy"
dv.mkdir(base)
dv.mkdir(taxondir)
baseurl = "https://github.com/sequana/data/raw/master/"
# download only if required
logger.info("Downloading the database into %s" % base)
md5sums = [
"28661f8baf0514105b0c6957bec0fc6e",
"97a39d44ed86cadea470352d6f69748d",
"d91a0fcbbc0f4bbac918755b6400dea6",
"c8bae69565af2170ece194925b5fdeb9"]
filenames = [
"database.idx",
"database.kdb",
"taxonomy/names.dmp",
"taxonomy/nodes.dmp"]
for filename, md5sum in zip(filenames, md5sums):
url = baseurl + "kraken_toydb/%s" % filename
filename = base + os.sep + filename
if os.path.exists(filename) and md5(filename) == md5sum:
logger.warning("%s already present" % filename)
else:
logger.info("Downloading %s" % url)
wget(url, filename)
def _download_minikraken(self, verbose=True):
dv = DevTools()
base = sequana_config_path + os.sep + ""
taxondir = base + os.sep + "taxonomy"
dv.mkdir(base)
dv.mkdir(taxondir)
logger.info("Downloading minikraken (4Gb)")
filename = base + os.sep + "minikraken.tgz"
if os.path.exists(filename) and md5(filename) == "30eab12118158d0b31718106785195e2":
logger.warning("%s already present" % filename)
else:
wget("https://ccb.jhu.edu/software/kraken/dl/minikraken.tgz", filename)
# unzipping. requires tar and gzip
def _download_from_synapse(self, synid, target_dir):
try:
from synapseclient import Synapse
except ImportError:
raise ImportError("Please install synapseclient using 'pip install synapseclient'")
try:
self._synapse.get(synid, downloadLocation=target_dir)
except:
self._synapse = Synapse()
self._synapse.login()
self._synapse.get(synid, downloadLocation=target_dir)
def _download_sequana_db1(self, verbose=True):
dbname = "sequana_db1"
from easydev import md5
dir1 = sequana_config_path + os.sep + dbname
dir2 = dir1 + os.sep + "taxonomy"
self.dv.mkdir(dir1)
self.dv.mkdir(dir2)
logger.info("Downloading about 8Gb of data (if not already downloaded) from"
" Synapse into %s" % dir1)
from os.path import exists
filename = dir1 + "ena_list.txt"
if exists(filename) and md5(filename) == "a9cc6268f3338d1632c4712a412593f2":
pass
else:
self._download_from_synapse('syn6171700', dir1)
# database.idx
filename = dir1 + "database.idx"
if exists(filename) and md5(filename) == "2fa4a99a4f52f2f04c5a965adb1534ac":
pass
else:
self._download_from_synapse('syn6171017', dir1)
# database.kdb ; this one is large (8Gb)
filename = dir1 + "database.kdb"
if exists(filename) and md5(filename) == "ff698696bfc88fe83bc201937cd9cbdf":
pass
else:
self._download_from_synapse('syn6171107', dir1)
# Then, the taxonomy directory
filename = dir1 + "names.dmp"
if exists(filename) and md5(filename) == "10bc7a63c579de02112d125a51fd65d0":
pass
else:
self._download_from_synapse('syn6171286', dir2)
filename = dir1 + "nodes.dmp"
if exists(filename) and md5(filename) == "a68af5a60434e2067c4a0a16df873980":
pass
else:
self._download_from_synapse('syn6171289', dir2)
filename = dir1 + "taxons.txt"
if exists(filename) and md5(filename) == "e78fbb43b3b41cbf4511d6af16c0287f":
pass
else:
self._download_from_synapse('syn6171290', dir2)
logger.info('done. You should have a kraken DB in %s' % dir1)
# The annotations
wget("https://github.com/sequana/data/raw/master/sequana_db1/annotations.csv",
dir1 + os.sep + "annotations.csv")
# Download Synapse reference and example data using inputs.json
# into "./data"
from getpass import getpass
from synapseclient import Synapse
import json
print '''
This script helps download the reference and example data
from synapse.org. You need a Synapse user account.
https://www.synapse.org/
Data will be downloaded to ./data
'''
username = raw_input("Enter your Synapse account email: ")
password = getpass()
syn = Synapse()
print 'Logging in to synapse.org...'
syn.login(username, password, rememberMe=True)
print 'Loading synapse_inputs.json'
inputs = json.load(open('synapse_inputs.json'))
for k, v in inputs.items():
print 'Downloading "{}"'.format(k)
syn.get(v, downloadLocation='data')
if ((aws_access_key_id is None) or (aws_secret_access_key is None)):
break
else:
s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key))
iamConnections.append(IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key))
i=i+1
if (len(s3Connections)==0):
raise("No AWS crdentials provided")
MAXIMUM_USER_NAME_LENGTH = 63
## connect to Synapse
syn = Synapse()
syn.login(synapseUserId, synapseUserPw)
ownUserProfile = syn.getUserProfile()
ownPrincipalId = ownUserProfile['ownerId']
## get all Participants for Evaluation
participants = syn.restGET("/evaluation/"+evaluationId+'/participant?limit=99999')['results']
print "total number of results: "+str(len(participants))
## For each participant
participantList = []
anyNewUsers = False
for i,part in enumerate(participants):
## add to a list the user's first name, last name, email address, user name and principal ID
## "user name" is defined as <firstName>.<lastName>.<principalId>.wcpe.sagebase.org
partId = part['userId']
up = syn.getUserProfile(partId)
break
else:
s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key))
iamConnections.append(
IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
)
i = i + 1
if len(s3Connections) == 0:
raise ("No AWS crdentials provided")
MAXIMUM_USER_NAME_LENGTH = 63
## connect to Synapse
syn = Synapse()
syn.login(synapseUserId, synapseUserPw)
ownUserProfile = syn.getUserProfile()
ownPrincipalId = ownUserProfile["ownerId"]
## get all Participants for Evaluation
participants = syn.restGET("/evaluation/" + evaluationId + "/participant?limit=99999")["results"]
print "total number of results: " + str(len(participants))
## For each participant
participantList = []
anyNewUsers = False
for i, part in enumerate(participants):
## add to a list the user's first name, last name, email address, user name and principal ID
## "user name" is defined as <firstName>.<lastName>.<principalId>.wcpe.sagebase.org
partId = part["userId"]
up = syn.getUserProfile(partId)