def testCustomConfigFile(syn, schedule_for_cleanup): if os.path.isfile(client.CONFIG_FILE): configPath = './CONFIGFILE' shutil.copyfile(client.CONFIG_FILE, configPath) schedule_for_cleanup(configPath) syn2 = Synapse(configPath=configPath) syn2.login() else: raise ValueError( "Please supply a username and password in the configuration file.")
def submit_to_challenge(filename, challenge, label, retry=True): try: client = Synapse() client.login() evaluation = client.getEvaluation(CODES[challenge]) filename = filename + '.gct' if challenge == 'sc1' else filename + '.zip' myfile = File(RESULTS_FOLDER + filename, parent=PROJECT_ID) myfile = client.store(myfile) client.submit(evaluation, myfile, name=label, teamName=TEAM) except: if retry: submit_to_challenge(filename, challenge, label, retry=False) else: print 'failed to submit', label, 'to', challenge
def syn(): """ Create a logged in Synapse instance that can be shared by all tests in the session. """ print("Python version:", sys.version) syn = Synapse(debug=False, skip_checks=True) print("Testing against endpoints:") print(" " + syn.repoEndpoint) print(" " + syn.authEndpoint) print(" " + syn.fileHandleEndpoint) print(" " + syn.portalEndpoint + "\n") syn.logger = logging.getLogger(SILENT_LOGGER_NAME) syn.login() return syn
def main(): # Parse command-line arguments args = parse_arguments() # Set up Synapse syn = Synapse() syn.login(args.username, args.password, rememberMe=args.remember) # Retrieve Synapse entity (e.g., project, folder) entity = syn.get(args.synid, downloadFile=False) log("Entity", entity) # Retrieve team team = syn.getTeam(args.team) # TODO: Handle users with try-catch log("Team", team) # Assign specified permissions for given entity and team permissions = syn.setPermissions(entity, team.id, accessType=args.permissions) log("Permissions", permissions) # Celebrate print("Success!")
# ------------------------ import pandas as pd from msda import kmeans from synapseclient import Synapse from msda import pca import matplotlib.pyplot as plt import numpy as np from msda import enrichr_api as ai from msda.clustering import plot_clustermap as pc from msda import mapping from msda import preprocessing as pr # Load files from synapse # ----------------------- syn = Synapse() syn.login() df = pd.read_csv('total_MS.csv') # Wrangle data # ----------- cols = df.columns.tolist() new_cols = [c.replace('0p5', '0.25') for c in cols] col_map = {o: n for o, n in zip(cols, new_cols)} df = df.rename(columns=col_map) df.insert(2, 'HGNC_Gene_Name', [mapping.get_name_from_symbol(sy) for sy in df.Gene_Symbol.tolist()]) dfc = df.replace([0], np.nan).dropna().copy() samples = dfc.columns.tolist()[6:] days = [s.split('_')[-1] for s in samples]
class KrakenDownload(object): """Utility to download Kraken DB and place them in a local directory :: from sequana import KrakenDownload kd = KrakenDownload() kd.download('toydb') kd.download('minikraken') A large database (8Gb) is available on synapse and has the following DOI:: doi:10.7303/syn6171000 It can be downloaded manually or if you have a Synapse login (https://www.synapse.org), you can use:: from sequana import KrakenDownload kd = KrakenDownload() kd.downloaded("sequana_db1") """ dv = DevTools() def download(self, name, verbose=True): if name == "minikraken": self._download_minikraken(verbose=verbose) elif name == "toydb": self._download_kraken_toydb(verbose=verbose) elif name == "sequana_db1": self._download_sequana_db1(verbose=verbose) else: raise ValueError( "name must be toydb or minikraken, or sequana_db1") def _download_kraken_toydb(self, verbose=True): """Download the kraken DB toy example from sequana_data into .config/sequana directory Checks the md5 checksums. About 32Mb of data """ dv = DevTools() base = sequana_config_path + os.sep + "kraken_toydb" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) baseurl = "https://github.com/sequana/data/raw/master/" # download only if required logger.info("Downloading the database into %s" % base) md5sums = [ "28661f8baf0514105b0c6957bec0fc6e", "97a39d44ed86cadea470352d6f69748d", "d91a0fcbbc0f4bbac918755b6400dea6", "c8bae69565af2170ece194925b5fdeb9" ] filenames = [ "database.idx", "database.kdb", "taxonomy/names.dmp", "taxonomy/nodes.dmp" ] for filename, md5sum in zip(filenames, md5sums): url = baseurl + "kraken_toydb/%s" % filename filename = base + os.sep + filename if os.path.exists(filename) and md5(filename) == md5sum: logger.warning("%s already present" % filename) else: logger.info("Downloading %s" % url) wget(url, filename) def _download_minikraken(self, verbose=True): dv = DevTools() base = sequana_config_path + os.sep + "" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) logger.info("Downloading minikraken (4Gb)") filename = base + os.sep + "minikraken.tgz" if os.path.exists(filename) and md5( filename) == "30eab12118158d0b31718106785195e2": logger.warning("%s already present" % filename) else: wget("https://ccb.jhu.edu/software/kraken/dl/minikraken.tgz", filename) # unzipping. requires tar and gzip def _download_from_synapse(self, synid, target_dir): try: from synapseclient import Synapse except ImportError: raise ImportError( "Please install synapseclient using 'pip install synapseclient'" ) try: self._synapse.get(synid, downloadLocation=target_dir) except: self._synapse = Synapse() self._synapse.login() self._synapse.get(synid, downloadLocation=target_dir) def _download_sequana_db1(self, verbose=True): dbname = "sequana_db1" from easydev import md5 dir1 = sequana_config_path + os.sep + dbname dir2 = dir1 + os.sep + "taxonomy" self.dv.mkdir(dir1) self.dv.mkdir(dir2) logger.info( "Downloading about 8Gb of data (if not already downloaded) from" " Synapse into %s" % dir1) from os.path import exists filename = dir1 + "ena_list.txt" if exists(filename) and md5( filename) == "a9cc6268f3338d1632c4712a412593f2": pass else: self._download_from_synapse('syn6171700', dir1) # database.idx filename = dir1 + "database.idx" if exists(filename) and md5( filename) == "2fa4a99a4f52f2f04c5a965adb1534ac": pass else: self._download_from_synapse('syn6171017', dir1) # database.kdb ; this one is large (8Gb) filename = dir1 + "database.kdb" if exists(filename) and md5( filename) == "ff698696bfc88fe83bc201937cd9cbdf": pass else: self._download_from_synapse('syn6171107', dir1) # Then, the taxonomy directory filename = dir1 + "names.dmp" if exists(filename) and md5( filename) == "10bc7a63c579de02112d125a51fd65d0": pass else: self._download_from_synapse('syn6171286', dir2) filename = dir1 + "nodes.dmp" if exists(filename) and md5( filename) == "a68af5a60434e2067c4a0a16df873980": pass else: self._download_from_synapse('syn6171289', dir2) filename = dir1 + "taxons.txt" if exists(filename) and md5( filename) == "e78fbb43b3b41cbf4511d6af16c0287f": pass else: self._download_from_synapse('syn6171290', dir2) logger.info('done. You should have a kraken DB in %s' % dir1) # The annotations wget( "https://github.com/sequana/data/raw/master/sequana_db1/annotations.csv", dir1 + os.sep + "annotations.csv")
def _syn_login(login: str) -> Synapse: """Login to synapse. Password must be stored in environment variable SYNAPSE_PASS""" assert 'SYNAPSE_PASS' in os.environ, 'SYNAPSE_PASS must be set as an environment variable' syn = Synapse() syn.login(login, os.environ['SYNAPSE_PASS']) return syn
class KrakenDownload(object): """Utility to download Kraken DB and place them in a local directory :: from sequana import KrakenDownload kd = KrakenDownload() kd.download('toydb') kd.download('minikraken') A large database (8Gb) is available on synapse and has the following DOI:: doi:10.7303/syn6171000 It can be downloaded manually or if you have a Synapse login (https://www.synapse.org), you can use:: from sequana import KrakenDownload kd = KrakenDownload() kd.downloaded("sequana_db1") """ dv = DevTools() def download(self, name, verbose=True): if name == "minikraken": self._download_minikraken(verbose=verbose) elif name == "toydb": self._download_kraken_toydb(verbose=verbose) elif name == "sequana_db1": self._download_sequana_db1(verbose=verbose) else: raise ValueError("name must be toydb or minikraken, or sequana_db1") def _download_kraken_toydb(self, verbose=True): """Download the kraken DB toy example from sequana_data into .config/sequana directory Checks the md5 checksums. About 32Mb of data """ dv = DevTools() base = sequana_config_path + os.sep + "kraken_toydb" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) baseurl = "https://github.com/sequana/data/raw/master/" # download only if required logger.info("Downloading the database into %s" % base) md5sums = [ "28661f8baf0514105b0c6957bec0fc6e", "97a39d44ed86cadea470352d6f69748d", "d91a0fcbbc0f4bbac918755b6400dea6", "c8bae69565af2170ece194925b5fdeb9"] filenames = [ "database.idx", "database.kdb", "taxonomy/names.dmp", "taxonomy/nodes.dmp"] for filename, md5sum in zip(filenames, md5sums): url = baseurl + "kraken_toydb/%s" % filename filename = base + os.sep + filename if os.path.exists(filename) and md5(filename) == md5sum: logger.warning("%s already present" % filename) else: logger.info("Downloading %s" % url) wget(url, filename) def _download_minikraken(self, verbose=True): dv = DevTools() base = sequana_config_path + os.sep + "" taxondir = base + os.sep + "taxonomy" dv.mkdir(base) dv.mkdir(taxondir) logger.info("Downloading minikraken (4Gb)") filename = base + os.sep + "minikraken.tgz" if os.path.exists(filename) and md5(filename) == "30eab12118158d0b31718106785195e2": logger.warning("%s already present" % filename) else: wget("https://ccb.jhu.edu/software/kraken/dl/minikraken.tgz", filename) # unzipping. requires tar and gzip def _download_from_synapse(self, synid, target_dir): try: from synapseclient import Synapse except ImportError: raise ImportError("Please install synapseclient using 'pip install synapseclient'") try: self._synapse.get(synid, downloadLocation=target_dir) except: self._synapse = Synapse() self._synapse.login() self._synapse.get(synid, downloadLocation=target_dir) def _download_sequana_db1(self, verbose=True): dbname = "sequana_db1" from easydev import md5 dir1 = sequana_config_path + os.sep + dbname dir2 = dir1 + os.sep + "taxonomy" self.dv.mkdir(dir1) self.dv.mkdir(dir2) logger.info("Downloading about 8Gb of data (if not already downloaded) from" " Synapse into %s" % dir1) from os.path import exists filename = dir1 + "ena_list.txt" if exists(filename) and md5(filename) == "a9cc6268f3338d1632c4712a412593f2": pass else: self._download_from_synapse('syn6171700', dir1) # database.idx filename = dir1 + "database.idx" if exists(filename) and md5(filename) == "2fa4a99a4f52f2f04c5a965adb1534ac": pass else: self._download_from_synapse('syn6171017', dir1) # database.kdb ; this one is large (8Gb) filename = dir1 + "database.kdb" if exists(filename) and md5(filename) == "ff698696bfc88fe83bc201937cd9cbdf": pass else: self._download_from_synapse('syn6171107', dir1) # Then, the taxonomy directory filename = dir1 + "names.dmp" if exists(filename) and md5(filename) == "10bc7a63c579de02112d125a51fd65d0": pass else: self._download_from_synapse('syn6171286', dir2) filename = dir1 + "nodes.dmp" if exists(filename) and md5(filename) == "a68af5a60434e2067c4a0a16df873980": pass else: self._download_from_synapse('syn6171289', dir2) filename = dir1 + "taxons.txt" if exists(filename) and md5(filename) == "e78fbb43b3b41cbf4511d6af16c0287f": pass else: self._download_from_synapse('syn6171290', dir2) logger.info('done. You should have a kraken DB in %s' % dir1) # The annotations wget("https://github.com/sequana/data/raw/master/sequana_db1/annotations.csv", dir1 + os.sep + "annotations.csv")
# Download Synapse reference and example data using inputs.json # into "./data" from getpass import getpass from synapseclient import Synapse import json print ''' This script helps download the reference and example data from synapse.org. You need a Synapse user account. https://www.synapse.org/ Data will be downloaded to ./data ''' username = raw_input("Enter your Synapse account email: ") password = getpass() syn = Synapse() print 'Logging in to synapse.org...' syn.login(username, password, rememberMe=True) print 'Loading synapse_inputs.json' inputs = json.load(open('synapse_inputs.json')) for k, v in inputs.items(): print 'Downloading "{}"'.format(k) syn.get(v, downloadLocation='data')
if ((aws_access_key_id is None) or (aws_secret_access_key is None)): break else: s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key)) iamConnections.append(IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)) i=i+1 if (len(s3Connections)==0): raise("No AWS crdentials provided") MAXIMUM_USER_NAME_LENGTH = 63 ## connect to Synapse syn = Synapse() syn.login(synapseUserId, synapseUserPw) ownUserProfile = syn.getUserProfile() ownPrincipalId = ownUserProfile['ownerId'] ## get all Participants for Evaluation participants = syn.restGET("/evaluation/"+evaluationId+'/participant?limit=99999')['results'] print "total number of results: "+str(len(participants)) ## For each participant participantList = [] anyNewUsers = False for i,part in enumerate(participants): ## add to a list the user's first name, last name, email address, user name and principal ID ## "user name" is defined as <firstName>.<lastName>.<principalId>.wcpe.sagebase.org partId = part['userId'] up = syn.getUserProfile(partId)
break else: s3Connections.append(S3Connection(aws_access_key_id, aws_secret_access_key)) iamConnections.append( IAMConnection(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key) ) i = i + 1 if len(s3Connections) == 0: raise ("No AWS crdentials provided") MAXIMUM_USER_NAME_LENGTH = 63 ## connect to Synapse syn = Synapse() syn.login(synapseUserId, synapseUserPw) ownUserProfile = syn.getUserProfile() ownPrincipalId = ownUserProfile["ownerId"] ## get all Participants for Evaluation participants = syn.restGET("/evaluation/" + evaluationId + "/participant?limit=99999")["results"] print "total number of results: " + str(len(participants)) ## For each participant participantList = [] anyNewUsers = False for i, part in enumerate(participants): ## add to a list the user's first name, last name, email address, user name and principal ID ## "user name" is defined as <firstName>.<lastName>.<principalId>.wcpe.sagebase.org partId = part["userId"] up = syn.getUserProfile(partId)