def main(uargs):
uargs['--config'] = os.path.abspath(os.path.expanduser(uargs['--config']))
conf = ConfigObj(uargs['--config'], configspec=get_configspec())
try:
conf_args = conf[uargs['--profile']]
except KeyError:
msg = 'Profile "{}" not found in config file'
raise KeyError, msg.format(uargs['--profile'])
# selecting config args
keys = ['clientKey', 'clientSecret', 'apiServer', 'apiVersion', 'appSessionId', 'accessToken']
conf_args = [conf_args[x] for x in keys]
myAPI = BaseSpaceAPI(*conf_args)
# setting user
if uargs['--user'] is not None:
user = myAPI.getUserById(uargs['--user'])
else:
user = myAPI.getUserById('current')
# all samples for project
if uargs['project']:
samples = myAPI.getSamplesByProject(uargs['<Id>'])
for sample in samples:
file_download(myAPI, sample.Id)
if uargs['sample']:
file_download(myAPI, uargs['<Id>'])
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-p', '--profile', default="DEFAULT", help="the .basespacepy.cfg profile to load")
parser.add_argument('-d', '--dry', action='store_true', default=False, help="dry run; return size of selected items")
parser.add_argument('-f', '--force', action='store_true', default=False, help="force overwrite; otherwise cat counters on new filenames")
parser.add_argument('-j', '--project', required=True, nargs="+", help="project to download; can accept multiple values")
parser.add_argument('-t', '--type', choices=['b','f','bam','fastq'], default='f', help='type of file to download')
args = parser.parse_args()
myAPI = BaseSpaceAPI(profile=args.profile, timeout=500)
user = myAPI.getUserById('current')
qp = QueryParameters.QueryParameters({'Limit':1024})
projects = user.getProjects(myAPI, qp)
if args.type in ['b', 'bam']:
download = downloadProjectBam
elif args.type in ['f', 'fastq']:
download = downloadProjectFastq
userProjs = stringsToBSObj(projects, args.project)
for lostProj in set(args.project) - set([str(x) for x in userProjs]):
warning("cannot find " + str(lostProj))
TotalSize = 0
for project in userProjs:
TotalSize += download(project , myAPI, args.dry, force=args.force)
if len(userProjs) > 1:
print(humanFormat(TotalSize) + "\tTotal")
def Main():
args = ParseArg()
client_key = args.key
client_secret = args.secret
token = args.token
folder = args.directory
BaseSpaceUrl = 'https://api.basespace.illumina.com/'
myAPI = BaseSpaceAPI(client_key,client_secret,BaseSpaceUrl,"v1pre3","AA",token)
user = myAPI.getUserById('current')
print >> sys.stderr, "\nUser name: %s\n"%(str(user))
Projects = myAPI.getProjectByUser()
Found = False
for p in Projects:
if p.Name == args.project:
print >>sys.stderr, " Find project %s with ID: %s. "%(p.Name, p.Id)
Project = p
Found = True
break
if not Found:
print >>sys.stderr, " Could not find project %s, from user %s, please check your token." %(args.project,str(user))
sys.exit(0)
Samples=Project.getSamples(myAPI)
print >> sys.stderr, "Samples for this project: " + str(Samples)
for s in Samples:
print >>sys.stderr," Downloading files in sample " + str(s)
subfolder=folder+"/"+str(s)
if not os.path.exists(subfolder):
os.makedirs(subfolder)
for f in s.getFiles(myAPI):
print >> sys.stderr," "+str(f)
f.downloadFile(myAPI,subfolder)
def calc_quantity(self, file_num, user_id, live_purchase):
"""
Calculates quantity of product needed to purchase from analyzing the provided file
If live_purchase is True, free trials are decremented from the db
"""
db=current.db
self.file_num = file_num
self.free_trial = False
if(self.prod_name == current.product_names['AlignmentQC']):
user_row = db(db.auth_user.id==user_id).select().first()
app = db(db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, "", user_row.access_token)
input_file = bs_api.getFileById(file_num)
# BAM files less than 100 MB are free
if input_file.Size < 100*(2**20):
self.prod_quantity = 0
else:
# determine if free trial
trial_row = db((db.free_trial.user_id==user_row.id) &
(db.free_trial.product_id==self.prod_id) &
(db.free_trial.trials > 0)).select().first()
if trial_row:
# decrement trials in db
if live_purchase:
trial_row.update_record(trials=int(trial_row.trials) - 1)
self.prod_quantity = 0
self.free_trial = True
else:
self.prod_quantity = 1
else:
raise UnrecognizedProductException(self.prod_name)
def Main():
args = ParseArg()
client_key = args.key
client_secret = args.secret
token = args.token
folder = args.directory
BaseSpaceUrl = 'https://api.basespace.illumina.com/'
myAPI = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, "v1pre3",
"AA", token)
user = myAPI.getUserById('current')
print >> sys.stderr, "\nUser name: %s\n" % (str(user))
Projects = myAPI.getProjectByUser()
Found = False
for p in Projects:
if p.Name == args.project:
print >> sys.stderr, " Find project %s with ID: %s. " % (p.Name,
p.Id)
Project = p
Found = True
break
if not Found:
print >> sys.stderr, " Could not find project %s, from user %s, please check your token." % (
args.project, str(user))
sys.exit(0)
Samples = Project.getSamples(myAPI)
print >> sys.stderr, "Samples for this project: " + str(Samples)
for s in Samples:
print >> sys.stderr, " Downloading files in sample " + str(s)
subfolder = folder + "/" + str(s)
if not os.path.exists(subfolder):
os.makedirs(subfolder)
for f in s.getFiles(myAPI):
print >> sys.stderr, " " + str(f)
f.downloadFile(myAPI, subfolder)
def __init__(self,
project_id=None,
project_name=None,
get_all_projects=False):
super(BaseSpace, self).__init__()
# BaseSpace credentials
creds = self._get_credentials()
self.client_key = creds['client_id']
self.client_secret = creds['client_secret']
self.access_token = creds['access_token']
self.version = creds['version']
self.api_server = creds['api_server']
self.api = BaseSpaceAPI(self.client_key,
self.client_secret,
self.api_server,
self.version,
AccessToken=self.access_token)
self.params = qp(pars={'Limit': 1024, 'SortDir': 'Desc'})
if project_id is not None:
self.project_id = project_id
self.project_name = None
elif project_name is not None:
self.project_name = project_name
self.project_id = self._get_project_id_from_name(project_name)
else:
self.project_id = None
self.project_name = None
# self.project_id, self.project_name = self._user_selected_project_id()
self._runs = None
def get_auth_code_util(scope):
"""
Initiate OAuth2 to get auth code for the given scope, returns url to redirect to so user can confirm scope via oauth2 dialog
"""
if not scope:
scope = ""
app = current.db(current.db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id,app.client_secret,app.baseSpaceUrl,app.version, current.session.app_session_num)
url = bs_api.getWebVerificationCode(scope,app.redirect_uri)
return url
def __init__(self):
self.qp = {}
self.rest_method = 'GET'
self.postData = None
self.headerParams = None
self.list_request = False
# TODO change to unit_tests, but need to add projects/run to account?
self.myAPI = BaseSpaceAPI(profile="ps_native_hoth")
self.api = APIClient(self.myAPI.getAccessToken(), self.myAPI.apiServer)
def get_access_token_util(auth_code):
"""
Given an auth code, retrieve and return the access token from BaseSpace
"""
app = current.db(current.db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id,app.client_secret,app.baseSpaceUrl,app.version,current.session.app_session_num)
bs_api.updatePrivileges(auth_code)
access_token = bs_api.getAccessToken()
# set token in session var
current.session.token = access_token
return access_token
def get_file_url(self, file_num, app_session_id):
"""
Returns the S3 link to the provided file
"""
db = current.db
# get access token for app session's user (can't use current user since may be accessing from scheduler worker)
ssn_row = db(db.app_session.id==app_session_id).select().first()
user_row = db(db.auth_user.id==ssn_row.user_id).select().first()
app = db(db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token)
f = bs_api.getFileById(file_num)
return f.getFileUrl(bs_api)
def download_Project(project_Name, output_folder):
# initialize an authentication object using the key and secret from your app
# Fill in with your own values
'''
client_key = <my key>
client_secret = <my secret>
AppSessionId = <my appSession id>
BaseSpaceUrl = 'https://api.basespace.illumina.com/'
version = 'v1pre3'
accessToken = <my acceseToken>
'''
myAPI = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId,AccessToken=accessToken)
# Retrieve current user
user = myAPI.getUserById('current')
user=str(user)
id_name=user.split(':')
#print id_name[0]
# Retrieve all the project associated to that user
projects=myAPI.getProjectByUser(id_name[0], queryPars=QueryParameters( {'Limit': '100'}))
project_found=0
for project in projects:
project=str(project)
nameProject_id=project.split('-')
if str(project_Name) in str(nameProject_id):
project_found=1
id_project=nameProject_id[1].split('=')
id_project=id_project[1]
samples=myAPI.getSamplesByProject(id_project, queryPars=QueryParameters( {'Limit': '100'}))
print "There are "+str(len(samples))+" samples in the requested project ("+str(project_Name)+" - ID_PROJECT "+str(id_project)+")"
if not os.path.exists(output_folder):
os.makedirs(output_folder)
print time.ctime()+" START DOWNLOADING"
for file in samples:
file_out=file.getFiles(myAPI)
#print file_out
for fastq in file_out:
fastq.downloadFile(myAPI,output_folder)
print time.ctime()+" FILE "+str(fastq)+" DOWNLOADED"
path_file=join(output_folder,str(fastq))
path_S3=join(str(project_Name),str(fastq))
s3_upload(path_file,"bmi-ngs",path_S3)
print time.ctime()+" DOWNLOAD COMPLETED"
if project_found==0:
print "Project Not Found"
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--profile',
default="DEFAULT",
help="the .basespacepy.cfg profile to load")
parser.add_argument('-d',
'--dry',
action='store_true',
default=False,
help="dry run; return size of selected items")
parser.add_argument(
'-f',
'--force',
action='store_true',
default=False,
help="force overwrite; otherwise cat counters on new filenames")
parser.add_argument(
'-r',
'--run',
default=[],
nargs="+",
help="run name to download; can accept multiple values")
parser.add_argument(
'--file',
default=[],
nargs="+",
help=
"specific file(s) to pull from each run; can accept multiple values")
args = parser.parse_args()
myAPI = BaseSpaceAPI(profile=args.profile, timeout=500)
user = myAPI.getUserById('current')
qp = QueryParameters.QueryParameters({'Limit': 1024})
runs = user.getRuns(myAPI, qp)
userRuns = stringsToBSObj(runs, args.run)
if not args.run:
userRuns = runs
for lostRun in set(args.run) - set([str(x) for x in userRuns]):
warning("cannot find " + str(lostRun))
TotalSize = 0
userFiles = args.file
for run in userRuns:
# must create a copy of userFiles or the downloadRun function will strip entries from this instance of the list
TotalSize += downloadRun(run,
myAPI,
args.dry,
files=[x for x in userFiles],
force=args.force)
def create_new_project_and_upload_fastq(project_name, sample_data_list,
experiment_name):
'''
A function for uploading afstq files to basespace after creating a new project
:param project_name: A project name
:param sample_data_list: Sample data list containing following information
sample_name (str)
read_count (str)
read_length (str)
fastq_path (list)
:param experiment_name: Experiment name
:retruns: None
'''
try:
myAPI = BaseSpaceAPI()
project = myAPI.createProject(project_name) # create new project
appResults = project.\
createAppResult(\
myAPI,
"FastqUpload",
"uploading project data",
appSessionId='') # create new app results for project
myAppSession = appResults.AppSession # get app session
__create_sample_and_upload_data(\
api=myAPI,
appSessionId=myAppSession.Id,
appSession=myAppSession,
project_id=project.Id,
sample_data_list=sample_data_list,
exp_name=experiment_name
) # upload fastq
myAppSession.setStatus(
myAPI, 'complete',
"finished file upload") # mark app session a complete
except:
if myAppSession and \
len(project.getAppResults(myAPI,statuses=['Running']))>0:
myAppSession.setStatus(
myAPI, 'complete',
"failed file upload") # comment for failed jobs
raise
def __init__(self):
self.qp = {}
self.rest_method = 'GET'
self.postData = None
self.headerParams=None
self.list_request = False
# TODO change to unit_tests, but need to add projects/run to account?
self.myAPI = BaseSpaceAPI(profile="ps_native_hoth")
self.api = APIClient(self.myAPI.getAccessToken(), self.myAPI.apiServer)
def download_file(self, file_num, local_dir, app_session_id):
"""
Download a file from BaseSpace into the provided directory (created if doesn't exist)
"""
db = current.db
# get access token for app session's user (can't use current user since may be accessing from scheduler worker)
ssn_row = db(db.app_session.id==app_session_id).select().first()
user_row = db(db.auth_user.id==ssn_row.user_id).select().first()
# get file info from BaseSpace
app = db(db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token)
f = bs_api.getFileById(file_num)
# create local_path dir if it doesn't exist
if not os.path.exists(local_dir):
os.makedirs(local_dir)
# write downloaded data to new file
f.downloadFile(bs_api,local_dir)
self.local_path = os.path.join(local_dir, f.Name)
def _writeback_app_result_files(self):
"""
Writeback all files in output_files list, and create corresponding entries in the local db
"""
db = current.db
# get BaseSpace API
ssn_row = db(db.app_session.id==self.app_session_id).select().first()
user_row = db(db.auth_user.id==ssn_row.user_id).select().first()
app = db(db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token)
app_result = bs_api.getAppResultById(self.app_result_num)
# upload files to BaseSpace
for f in self.output_files:
# currently not using a dir name to write to
bs_file = app_result.uploadFile(bs_api, f.local_path, f.file_name, '', 'text/plain')
# add file to local db
db.output_file.insert(app_result_id=f.app_result_id,
file_num=bs_file.Id,
file_name=f.file_name,
local_path=f.local_path,
file_type=f.file_type)
db.commit()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--profile',
default="DEFAULT",
help="the .basespacepy.cfg profile to load")
parser.add_argument('-j',
'--project',
required=True,
nargs="+",
help="project to download; can accept multiple values")
args = parser.parse_args()
myAPI = BaseSpaceAPI(profile=args.profile, timeout=500)
user = myAPI.getUserById('current')
qp = QueryParameters.QueryParameters({'Limit': 1024})
projects = user.getProjects(myAPI, qp)
userProjs = stringsToBSObj(projects, args.project)
for lostProj in set(args.project) - set([str(x) for x in userProjs]):
warning("cannot find " + str(lostProj))
fullSampleMetadata = pd.DataFrame()
fullFileMetadata = pd.DataFrame()
for project in userProjs:
smout, fmout = downloadProjectMetadata(project, myAPI)
fullSampleMetadata = fullSampleMetadata.append(smout)
fullFileMetadata = fullFileMetadata.append(fmout)
thisInstant = str(datetime.datetime.today()).replace(' ', ';')
fullSampleMetadata.to_csv('fullSampleMetadata.' + thisInstant + '.txt',
sep='\t',
header=True,
index=False)
fullFileMetadata.to_csv('fullFileMetadata.' + thisInstant + '.txt',
sep='\t',
header=True,
index=False)
def update_status(self, local_status, message, bs_ssn_status=None):
"""
Update db with provided status and detailed message, and update BaseSpace App Session status if provided
"""
db = current.db
# update status in local db
self.status=local_status
self.message=message
ssn_row = db(db.app_session.id==self.app_session_id).select().first()
ssn_row.update_record(status=self.status, message=self.message)
db.commit()
# optionally update status of AppSession in BaseSpace -- limited to 128 chars
if bs_ssn_status:
# get BaseSpace API
ssn_row = db(db.app_session.id==self.app_session_id).select().first()
user_row = db(db.auth_user.id==ssn_row.user_id).select().first()
app = db(db.app_data.id > 0).select().first()
# handle exceptions in caller
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token)
app_result = bs_api.getAppResultById(self.app_result_num)
app_ssn = app_result.AppSession
app_ssn.setStatus(bs_api, bs_ssn_status, message[:128])
from __future__ import print_function
import sys, os, glob, logging
from argparse import ArgumentParser
from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI
from BaseSpacePy.model.QueryParameters import QueryParameters as qp
list_options = qp({'Limit': 1024})
logging.basicConfig(
level=logging.INFO,
format='[%(asctime)s] %(message)s',
datefmt='%d/%m/%Y %H:%M:%S',
)
bs = BaseSpaceAPI()
user = bs.getUserById('current')
logging.info("User Name: %s", user)
projects = bs.getProjectByUser(list_options)
project_list = [project.Name for project in projects]
cli = ArgumentParser()
cli.add_argument(
'project',
nargs='?',
help=
'Which project to download files from. When not specified, list projects instead.'
)
cli.add_argument(
'--dry-run',
accessToken =
appSessionId =
apiServer = https://api.cloud-hoth.illumina.com/
apiVersion = v1pre3
"""
# Alternately, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = "https://api.basespace.illumina.com/" # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = "v1pre3"
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId)
else:
myAPI = BaseSpaceAPI(profile="DEFAULT")
# First, let's grab the genome with id=4
myGenome = myAPI.getGenomeById("4")
print "\nThe Genome is " + str(myGenome)
print "We can get more information from the genome object"
print "Id: " + myGenome.Id
print "Href: " + myGenome.Href
print "DisplayName: " + myGenome.DisplayName
# Get a list of all genomes
allGenomes = myAPI.getAvailableGenomes()
print "\nGenomes \n" + str(allGenomes)
# initialize an authentication object using the key and secret from your app
# FILL IN WITH YOUR APP VALUES HERE!
client_key = ""
client_secret = ""
AppSessionId = ""
# test if client variables have been set
helper.checkClientVars({'client_key':client_key,'client_secret':client_secret,'AppSessionId':AppSessionId})
BaseSpaceUrl = 'https://api.cloud-endor.illumina.com/'
version = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
BSapi = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId)
# Using the basespaceApi we can request the appSession object corresponding to the AppSession id supplied
myAppSession = BSapi.getAppSession()
print myAppSession
# An app session contains a referal to one or more appLaunchObjects which reference the data module
# the user launched the app on. This can be a list of projects, samples, or a mixture of objects
print "\nType of data the app was triggered on can be seen in 'references'"
print myAppSession.References
# We can also get a handle to the user who started the AppSession
print "\nWe can get a handle for the user who triggered the app\n" + str(myAppSession.UserCreatedBy)
# Let's have a closer look at the appSessionLaunchObject
myReference = myAppSession.References[0]
def download_basespace_files(config_file_path=None,
client_key=None,
client_secret=None,
access_token=None,
project_id_list=None,
project_name_list=None,
sample_id_list=None,
sample_name_list=None,
dry_run=False,
output_directory=None,
recreate_basespace_dir_tree=True):
# Check input parameters / load from config file / defaults
if not project_id_list: project_id_list = []
if not project_name_list: project_name_list = []
if not sample_id_list: sample_id_list = []
if not sample_name_list: sample_name_list = []
if not output_directory:
output_directory = os.getcwd()
print_stderr(
"Output directory not specified; using current directory ({})".
format(output_directory))
else:
output_directory = os.path.abspath(output_directory)
if not dry_run:
safe_makedir(output_directory)
config_dict = {}
if config_file_path:
config_parser = ConfigParser()
config_parser.read(config_file_path)
config_dict = config_parser._defaults
if not client_key: client_key = config_dict.get('clientkey')
if not client_secret: client_secret = config_dict.get('clientsecret')
if not access_token: access_token = config_dict.get('accesstoken')
if not (client_key and client_secret and access_token):
missing_params = []
if not client_key: missing_params.append("client_key")
if not client_secret: missing_params.append("client_secret")
if not access_token: missing_params.append("access_token")
print_stderr(
'Error: Required parameters not supplied either in config '
'file ({}) or via arguments.'.format(config_file_path,
', '.join(missing_params)))
sys.exit(1)
app_session_id = config_dict.get("appsessionid") or ""
api_server = config_dict.get(
"apiserver") or "https://api.basespace.illumina.com"
api_version = config_dict.get("apiversion") or "v1pre3"
# Get the API connection object
myAPI = BaseSpaceAPI(clientKey=client_key,
clientSecret=client_secret,
apiServer=api_server,
version=api_version,
appSessionId=app_session_id,
AccessToken=access_token)
basespace_projects = myAPI.getProjectByUser(qp({'Limit': 1024}))
user = myAPI.getUserById('current')
# If user specified projects, get them by name or id
project_objects = []
if project_name_list:
project_objects.extend(
_select_from_object(filter_list=project_name_list,
search_list=basespace_projects,
key_attr="Name",
obj_type="project",
user=user))
if project_id_list:
digit_pattern = re.compile(r'^\d+$')
project_filtered_id_list = []
for project_id in project_id_list:
if not digit_pattern.match(project_id):
print_stderr(
'Error: Invalid format for user-specified project id '
'"{}": project ids are strictly numeric. Did you mean '
'to pass this as a project name?'.format(project_id))
else:
project_filtered_id_list.append(project_id)
project_objects.extend(
_select_from_object(filter_list=project_filtered_id_list,
search_list=basespace_projects,
key_attr="Id",
obj_type="project",
user=user))
if not (project_name_list or project_id_list):
# Get all projects if none are specified by user
project_objects = basespace_projects
basespace_samples = []
for project_obj in project_objects:
basespace_samples.extend(project_obj.getSamples(myAPI))
sample_objects = []
if sample_name_list:
sample_objects.extend(
_select_from_object(filter_list=sample_name_list,
search_list=basespace_samples,
key_attr="Name",
obj_type="sample",
user=user))
if sample_id_list:
digit_pattern = re.compile(r'^\d+$')
sample_filtered_id_list = []
for sample_id in sample_id_list:
if not digit_pattern.match(sample_id):
print_stderr(
'Error: Invalid format for user-specified sample id '
'"{}": sample ids are strictly numeric. Did you mean '
'to pass this as a sample name?'.format(sample_id))
else:
sample_filtered_id_list.append(sample_id)
sample_objects.extend(
_select_from_object(filter_list=sample_filtered_id_list,
search_list=basespace_samples,
key_attr="Id",
obj_type="sample",
user=user))
if not (sample_name_list or sample_id_list):
# Get all samples if none are specified by user
sample_objects = basespace_samples
files_to_download = []
for sample_obj in sample_objects:
files_to_download.extend(sample_obj.getFiles(myAPI))
if files_to_download:
print_stderr("Found {} files to download: ".format(
len(files_to_download)))
for file_obj in files_to_download:
print_stderr("\t- {}".format(file_obj))
print_stderr('Downloading files to output directory {}'.format(
output_directory))
if recreate_basespace_dir_tree:
print_stderr(
"Recreating BaseSpace project directory tree for file.")
if dry_run:
print_stderr("-> Dry run: not downloading any data.")
for i, file_obj in enumerate(files_to_download):
print_stderr('[{}/{}] Downloading file "{}"'.format(
i + 1, len(files_to_download), file_obj))
if not dry_run:
file_obj.downloadFile(api=myAPI,
localDir=output_directory,
createBsDir=recreate_basespace_dir_tree)
print_stderr('Download completed; files are located in "{}"'.format(
output_directory))
else:
print_stderr("Error: no files found to download.")
def download(clientKey=None, clientSecret=None, accessToken=None, sampleId=None, projectId=None, sampleName=None, projectName=None, outputDirectory='\.', createBsDir=True):
'''
Downloads sample-level files.
Project Id and project name should
not be specified together; similarly sample Id and sample name should not be
specified together.
1. If only a project Id or only a project name is given, all files for all
samples will be downloaded within that project. If additionally a sample Id or
sample name is given, then only the first matching sample within the project
will be downloaded.
2. If only a sample Id is given, then all files for that sample will be downloaded.
3. If only a sample name is given, then all files within the first project
containing a sample with matching name will be downloaded.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param sampleId the BaseSpace sample identifier
:param projectId the BaseSpace project identifier
:param sampleName the BaseSpace sample name
:param projectName the BaseSpace project name
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
projectLimit = 1024
sampleLimit = 1024
sampleFileLimit = 1024
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
sampleToFiles = {}
if None != projectId:
sampleToFiles = Samples.__get_files_to_download(myAPI, projectId, sampleId, sampleName, sampleLimit, sampleFileLimit)
else:
offset = 0
while True:
myProjects = myAPI.getProjectByUser(qp({'Limit' : projectLimit, 'Offset' : offset}))
if len(myProjects) == 0:
break
for project in myProjects:
projectId = project.Id
sys.stderr.write("project.Name: " + str(project.Name) + " projectName: " + str(projectName) + '\n')
if None != projectName and project.Name != projectName:
continue
sampleToFiles = Samples.__get_files_to_download(myAPI, projectId, sampleId, sampleName, sampleLimit, sampleFileLimit)
if 0 < len(sampleToFiles):
break
if 0 < len(sampleToFiles):
break
offset += projectLimit
numFiles = sum([len(sampleToFiles[sampleId]) for sampleId in sampleToFiles])
print "Will download files from %d ." % numFiles
i = 0
for sampleId in sampleToFiles:
for sampleFile in sampleToFiles[sampleId]:
print 'Downloading (%d/%d): %s' % ((i+1), numFiles, str(sampleFile))
print "BaseSpace File Path: %s" % sampleFile.Path
print "Sample Id: %s" % sampleId
if not options.dryRun:
if createBsDir:
sampleOutputDirectory = os.path.join(outputDirectory, sampleId)
else:
sampleOutputDirectory = outputDirectory
sampleFile.downloadFile(myAPI, sampleOutputDirectory, createBsDir=createBsDir)
i = i + 1
print "Download complete."
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# Using the basespaceApi we can request the appSession object corresponding to the AppSession id supplied
myAppSession = myAPI.getAppSession()
print myAppSession
# An app session contains a referal to one or more appLaunchObjects which reference the data module
# the user launched the app on. This can be a list of projects, samples, or a mixture of objects
print "\nType of data the app was triggered on can be seen in 'references'"
print myAppSession.References
# We can also get a handle to the user who started the AppSession
print "\nWe can get a handle for the user who triggered the app\n" + str(myAppSession.UserCreatedBy)
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = "https://api.basespace.illumina.com/" # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = "v1pre3"
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId)
else:
myAPI = BaseSpaceAPI(profile="DEFAULT")
# Now we'll do some work of our own. First get a project to work on
# we'll need write permission, for the project we are working on
# meaning we will need get a new token and instantiate a new BaseSpaceAPI
p = myAPI.getProjectById("89")
# Assuming we have write access to the project
# we will list the current App Results for the project
appRes = p.getAppResults(myAPI, statuses=["Running"])
print "\nThe current running AppResults are \n" + str(appRes)
# now let's do some work!
def download(clientKey=None, clientSecret=None, accessToken=None, runId=None, runName=None, outputDirectory='\.', createBsDir=True):
'''
Downloads run-level files.
Run Id and run name should not be specified together.
All files for a given run will be downloaded based on either the unique run ID, or
the first run found with matching experiment name.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param runId the BaseSpace run identifier
:param runName the BaseSpace run experiment name
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 1024
runLimit = 100
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
expName = None
if runId:
run = myAPI.getRunById(Id=runId)
runFiles = Runs.__get_files_to_download(myAPI, run.Id, fileLimit)
expName = run.ExperimentName
else:
runs = myAPI.getAccessibleRunsByUser(qp({'Limit' : runLimit}))
for run in runs:
runId = run.Id
if runName and runName == run.ExperimentName:
expName = run.ExperimentName
runFiles = Samples.__get_files_to_download(myAPI, runId)
if 0 < len(runFiles):
break
if not expName:
if runName:
print 'Could not find a run with name: %s' % runName
else:
print 'Could not find a run for user'
sys.exit(1)
numFiles = len(runFiles)
print "Will download files from %d ." % numFiles
i = 0
for runFile in runFiles:
outDir = os.path.join(outputDirectory, expName)
print 'Downloading (%d/%d): %s' % ((i+1), numFiles, str())
print "BaseSpace File Path: %s" % runFile.Path
print "Destination File Path: %s" % os.path.join(outDir, runFile.Name)
if not options.dryRun:
runFile.downloadFile(myAPI, outDir, createBsDir=createBsDir)
i = i + 1
print "Download complete."
This script demonstrates how to access Samples and AppResults from a projects and how to work with the available
file data for such instances.
"""
# FILL IN WITH YOUR APP VALUES HERE!
client_key = ""
client_secret = ""
AppSessionId = ""
accessToken = ""
helper.checkClientVars({'client_key':client_key,'client_secret':client_secret,'AppSessionId':AppSessionId})
BaseSpaceUrl = 'https://api.basespace.illumina.com/'
version = 'v1pre3'
# First, create a client for making calls for this user session
myAPI = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId,AccessToken=accessToken)
user = myAPI.getUserById('current')
myProjects = myAPI.getProjectByUser('current')
# Let's list all the AppResults and samples for these projects
for singleProject in myProjects:
print "# " + str(singleProject)
appResults = singleProject.getAppResults(myAPI)
print " The App results for project " + str(singleProject) + " are \n\t" + str(appResults)
samples = singleProject.getSamples(myAPI)
print " The samples for project " + str(singleProject) + " are \n\t" + str(samples)
#
## we'll take a further look at the files belonging to the sample and
##analyses from the last project in the loop above
for a in appResults:
def download(clientKey=None,
clientSecret=None,
accessToken=None,
appResultId=None,
fileNameRegexesInclude=list(),
fileNameRegexesOmit=list(),
outputDirectory='\.',
createBsDir=True,
force=False,
numRetries=3):
'''
Downloads App Result files.
Provide an App Result identifier, and optionally regexes to include or omit files
based on their names (path not included). Omission takes precedence over inclusion.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param appResultId the BaseSpace App Result identifier
:param fileNameRegexesInclude a list of regexes on which to include files based on name
:param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include)
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
:param force use the force: overwrite existing files if true, false otherwise
:param numRetries the number of retries for a single download API call
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 10000
sleepTime = 1.0
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer,
apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
appResult = myAPI.getAppResultById(Id=appResultId)
print "Retrieving files from the App Result: " + str(appResult)
# Get all the files from the AppResult
filesToDownload = appResult.getFiles(myAPI,
queryPars=qp({'Limit':
fileLimit}))
# Filter file names based on the include or omit regexes
includePatterns = [
re.compile(pattern) for pattern in fileNameRegexesInclude
]
omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit]
def includePatternMatch(f):
if not includePatterns:
return True
for pattern in includePatterns:
if pattern.match(f):
return True
return False
def omitPatternMatch(f):
if not omitPatterns:
return False
for pattern in omitPatterns:
if pattern.match(f):
return True
return False
def keepFile(f):
return includePatternMatch(f) and not omitPatternMatch(f)
filesToDownload = [f for f in filesToDownload if keepFile(str(f))]
print "Will download %d files." % len(filesToDownload)
for i in range(len(filesToDownload)):
appResultFile = filesToDownload[i]
print 'Downloading (%d/%d): %s' % (
(i + 1), len(filesToDownload), str(appResultFile))
print "File Path: %s" % appResultFile.Path
if not options.dryRun:
outputPath = str(appResultFile.Path)
if not createBsDir:
outputPath = os.path.basename(outputPath)
if os.path.exists(outputPath):
if force:
print "Overwritting: %s" % outputPath
else:
print "Skipping existing file: %s" % outputPath
continue
else:
print "Downloading to: %s" % outputPath
retryIdx = 0
retryException = None
while retryIdx < numRetries:
try:
appResultFile.downloadFile(myAPI,
outputDirectory,
createBsDir=createBsDir)
except BaseSpaceException.ServerResponseException as e:
retryIdx += 1
time.sleep(sleepTime)
retryException = e
else:
break
if retryIdx == numRetries:
raise retryException
print "Download complete."
class TestSDK(object):
'''
Compares objects from BaseSpace REST API to SDK objects, including pickled objects
'''
def __init__(self):
self.qp = {}
self.rest_method = 'GET'
self.postData = None
self.headerParams = None
self.list_request = False
# TODO change to unit_tests, but need to add projects/run to account?
self.myAPI = BaseSpaceAPI(profile="ps_native_hoth")
self.api = APIClient(self.myAPI.getAccessToken(), self.myAPI.apiServer)
def compare_dict_to_obj(self, rest_dict, p_obj):
"""
Compare a dictionary from a REST API response and a SDK object for identity.
"""
for r_key, r_val in six.iteritems(rest_dict):
# confirm that the key from REST api exists in stored object
try:
p_val = getattr(p_obj, r_key)
except AttributeError:
print("REST API attribute '" + r_key +
"' doesn't exist in object")
else:
self.classify_rest_item(r_val, p_val, r_key)
def compare_list_to_obj(self, rest_list, p_obj, r_key):
"""
Compare a list from a REST API response and an SDK object for identity.
"""
if type(p_obj) != list:
print("Attribute '" + r_key +
"' is a list in the REST API but not in the object")
elif len(p_obj) != len(rest_list):
print("Attribute '" + r_key +
"' has different list length between REST API and object")
else:
for r_val, p_val in map(None, rest_list, p_obj):
self.classify_rest_item(r_val, p_val, r_key)
def compare_builtin_to_obj(self, rest_val, p_obj, r_key):
"""
Compare a built-in type from a REST API response and an SDK object for identity.
"""
# convert unicode to ascii for comparisons
if isinstance(rest_val, six.text_type):
rest_val = rest_val.encode('ascii', 'ignore')
# don't compare values for datetimes
if r_key in [
'DateCreated', 'DateModified', 'DateUploadCompleted',
'DateUploadStarted'
]:
pass
elif rest_val != p_obj:
print("REST API attribute '" + r_key + "' has value '" +
str(rest_val) + "' doesn't match object value '" +
str(p_obj) + "'")
def classify_rest_item(self, r_val, p_val, r_key):
"""
Determine the input REST item's type and call method to compare to input object
"""
if type(r_val) in [int, str, bool, float, six.text_type]:
self.compare_builtin_to_obj(r_val, p_val, r_key)
elif type(r_val) == dict:
self.compare_dict_to_obj(r_val, p_val)
elif type(r_val) == list:
self.compare_list_to_obj(r_val, p_val, r_key)
else:
print("REST API attribute'" + r_key +
"' has an unrecognized attribute type")
def test_rest_vs_sdk(self):
"""
Compares REST API response and python SDK object for identify, for an API method
"""
sdk_obj = self.call_sdk()
rest_obj = self.call_rest_api()
# TODO passing Response here, SDK doesn't currently capture other items at this level (e.g. Notifications)
if self.list_request:
self.compare_list_to_obj(rest_obj['Response']['Items'], sdk_obj,
"BASE")
else:
self.compare_dict_to_obj(rest_obj['Response'], sdk_obj)
def call_rest_api(self):
"""
Call the REST API for this object
"""
return self.api.callAPI(self.rest_path,
self.rest_method,
queryParams=self.qp,
postData=self.postData,
headerParams=self.headerParams)
def create_pickle_from_sdk(self, pickle_path):
"""
Stores a pickled object in the provided path (include file name) for the object returned for this SDK method
"""
sdk_obj = self.call_sdk()
with open(pickle_path, 'w') as f:
Pickle.dump(sdk_obj, f)
def get_pickle(self, pickle_path):
"""
Retrieves a pickled object from the provided path (include file name), for this API test
"""
with open(pickle_path, 'r') as f:
sdk_obj = Pickle.load(f)
return sdk_obj
def test_rest_vs_pickle(self, pickle_path):
"""
Compares REST API response and a stored object for identify, for an API method
"""
p_obj = self.get_pickle(pickle_path)
rest_obj = self.call_rest_api()
self.compare_dict_to_obj(rest_obj['Response'], p_obj)
"""
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
user = myAPI.getUserById('current')
myProjects = myAPI.getProjectByUser('current')
# Let's list all the AppResults and samples for these projects
for singleProject in myProjects:
print "# " + str(singleProject)
appResults = singleProject.getAppResults(myAPI)
print " The App results for project " + str(singleProject) + " are \n\t" + str(appResults)
samples = singleProject.getSamples(myAPI)
print " The samples for project " + str(singleProject) + " are \n\t" + str(samples)
accessToken =
appSessionId =
apiServer = https://api.cloud-hoth.illumina.com/
apiVersion = v1pre3
"""
# Alternately, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# First, let's grab the genome with id=4
myGenome = myAPI.getGenomeById('4')
print("\nThe Genome is " + str(myGenome))
print("We can get more information from the genome object")
print('Id: ' + myGenome.Id)
print('Href: ' + myGenome.Href)
print('DisplayName: ' + myGenome.DisplayName)
# Get a list of all genomes
allGenomes = myAPI.getAvailableGenomes()
print("\nGenomes \n" + str(allGenomes))
NOTE: You will need to fill client values for your app below
"""
# FILL IN WITH YOUR APP VALUES HERE!
client_key = ""
client_secret = ""
AppSessionId = ""
# test if client variables have been set
helper.checkClientVars({'client_key':client_key,'client_secret':client_secret,'AppSessionId':AppSessionId})
BaseSpaceUrl = 'https://api.basespace.illumina.com/'
version = 'v1pre3'
BSapi = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId)
# First, get the verification code and uri for scope 'browse global'
deviceInfo = BSapi.getVerificationCode('browse global')
print "\n URL for user to visit and grant access: "
print deviceInfo['verification_with_code_uri']
## PAUSE HERE
# Have the user visit the verification uri to grant us access
print "\nPlease visit the uri within 15 seconds and grant access"
print deviceInfo['verification_with_code_uri']
webbrowser.open_new(deviceInfo['verification_with_code_uri'])
time.sleep(15)
## PAUSE HERE
# Once the user has granted us access to objects we requested, we can
class TestSDK(object):
'''
Compares objects from BaseSpace REST API to SDK objects, including pickled objects
'''
def __init__(self):
self.qp = {}
self.rest_method = 'GET'
self.postData = None
self.headerParams=None
self.list_request = False
# TODO change to unit_tests, but need to add projects/run to account?
self.myAPI = BaseSpaceAPI(profile="ps_native_hoth")
self.api = APIClient(self.myAPI.getAccessToken(), self.myAPI.apiServer)
def compare_dict_to_obj(self, rest_dict, p_obj):
"""
Compare a dictionary from a REST API response and a SDK object for identity.
"""
for r_key, r_val in rest_dict.iteritems():
# confirm that the key from REST api exists in stored object
try:
p_val = getattr(p_obj, r_key)
except AttributeError:
print "REST API attribute '" + r_key + "' doesn't exist in object"
else:
self.classify_rest_item(r_val, p_val, r_key)
def compare_list_to_obj(self, rest_list, p_obj, r_key):
"""
Compare a list from a REST API response and an SDK object for identity.
"""
if type(p_obj) != list:
print "Attribute '" + r_key + "' is a list in the REST API but not in the object"
elif len(p_obj) != len(rest_list):
print "Attribute '" + r_key + "' has different list length between REST API and object"
else:
for r_val, p_val in map(None, rest_list, p_obj):
self.classify_rest_item(r_val, p_val, r_key)
def compare_builtin_to_obj(self, rest_val, p_obj, r_key):
"""
Compare a built-in type from a REST API response and an SDK object for identity.
"""
# convert unicode to ascii for comparisons
if isinstance(rest_val, unicode):
rest_val = rest_val.encode('ascii','ignore')
# don't compare values for datetimes
if r_key in ['DateCreated', 'DateModified', 'DateUploadCompleted', 'DateUploadStarted']:
pass
elif rest_val != p_obj:
print "REST API attribute '" + r_key + "' has value '" + str(rest_val) + "' doesn't match object value '" + str(p_obj) + "'"
def classify_rest_item(self, r_val, p_val, r_key):
"""
Determine the input REST item's type and call method to compare to input object
"""
if type(r_val) in [ int, str, bool, float, unicode]:
self.compare_builtin_to_obj(r_val, p_val, r_key)
elif type(r_val) == dict:
self.compare_dict_to_obj(r_val, p_val)
elif type(r_val) == list:
self.compare_list_to_obj(r_val, p_val, r_key)
else:
print "REST API attribute'" + r_key + "' has an unrecognized attribute type"
def test_rest_vs_sdk(self):
"""
Compares REST API response and python SDK object for identify, for an API method
"""
sdk_obj = self.call_sdk()
rest_obj = self.call_rest_api()
# TODO passing Response here, SDK doesn't currently capture other items at this level (e.g. Notifications)
if self.list_request:
self.compare_list_to_obj(rest_obj['Response']['Items'], sdk_obj, "BASE")
else:
self.compare_dict_to_obj(rest_obj['Response'], sdk_obj)
def call_rest_api(self):
"""
Call the REST API for this object
"""
return self.api.callAPI(self.rest_path, self.rest_method, queryParams=self.qp, postData=self.postData, headerParams=self.headerParams)
def create_pickle_from_sdk(self, pickle_path):
"""
Stores a pickled object in the provided path (include file name) for the object returned for this SDK method
"""
sdk_obj = self.call_sdk()
with open(pickle_path, 'w') as f:
Pickle.dump(sdk_obj, f)
def get_pickle(self, pickle_path):
"""
Retrieves a pickled object from the provided path (include file name), for this API test
"""
with open(pickle_path, 'r') as f:
sdk_obj = Pickle.load(f)
return sdk_obj
def test_rest_vs_pickle(self, pickle_path):
"""
Compares REST API response and a stored object for identify, for an API method
"""
p_obj = self.get_pickle(pickle_path)
rest_obj = self.call_rest_api()
self.compare_dict_to_obj(rest_obj['Response'], p_obj)
import sys, os, glob, logging
from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI
from BaseSpacePy.model.QueryParameters import QueryParameters as qp
listOptions = qp({'Limit': 1024})
logging.basicConfig(level=logging.INFO,
format='[%(asctime)s] %(message)s',
datefmt='%d/%m/%Y %H:%M:%S')
myAPI = BaseSpaceAPI()
p = sys.argv[1]
user = myAPI.getUserById('current')
logging.info("User Name: %s" % str(user))
projects = myAPI.getProjectByUser(listOptions)
project_list = [project.Name for project in projects]
try:
idx = project_list.index(sys.argv[1])
project = projects[idx]
except ValueError:
message = '"%s" is not in your projects. Available projects are:\n%s' % \
(sys.argv[1],
'\n'.join(project_list))
logging.error(message)
sys.exit(1)
downloaded = glob.glob('*fastq.gz') # get already downloaded fastq
logging.info("Retrieving samples from project %s" % sys.argv[1])
samples = project.getSamples(myAPI, listOptions)
def upload(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), inputDirectory='\.', dryRun=False, numRetries=3):
'''
Creates an App Result and uploads files.
TODO
Provide an App Result identifier, and optionally regexes to include or omit files
based on their names (path not included). Omission takes precedence over inclusion.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param appResultId the BaseSpace App Result identifier
:param fileNameRegexesInclude a list of regexes on which to include files based on name
:param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include)
:param inputDirectory the root input directory
:param numRetries the number of retries for a single download API call
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 10000
sleepTime = 1.0
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT', clientKey=clientKey, clientSecret=clientSecret, AccessToken=accessToken)
# get the current user
user = myAPI.getUserById('current')
# get the app result
appResult = myAPI.getAppResultById(Id=appResultId)
appSession = appResult.AppSession
print "Uploading files to the App Result: " + str(appResult)
# Filter file names based on the include or omit regexes
includePatterns = [re.compile(pattern) for pattern in fileNameRegexesInclude]
omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit]
def includePatternMatch(f):
if not includePatterns:
return True
for pattern in includePatterns:
if pattern.match(f):
return True
return False
def omitPatternMatch(f):
if not omitPatterns:
return False
for pattern in omitPatterns:
if pattern.match(f):
return True
return False
def keepFile(f):
return includePatternMatch(f) and not omitPatternMatch(f)
# walk the current directory structure
for root, dirs, files in os.walk(inputDirectory):
for fileName in files:
localPath = os.path.join(root, fileName)
directory = root.replace(inputDirectory, "")
if AppResults.isBinaryContent(fileName):
contentType = 'application/octet-stream'
else:
contentType = 'text/plain'
if keepFile(fileName):
print "Uploading file: %s" % localPath
if not options.dryRun:
retryIdx = 0
retryException = None
while retryIdx < numRetries:
try:
appResult.uploadFile(api=myAPI, localPath=localPath, fileName=fileName, directory=directory, contentType=contentType)
except BaseSpaceException.ServerResponseException as e:
retryIdx += 1
time.sleep(sleepTime)
retryException = e
else:
break
if retryIdx == numRetries:
raise retryException
print "Upload complete"
for global browsing has been obtained.
"""
# FILL IN WITH YOUR APP VALUES HERE!
client_key = ""
client_secret = ""
AppSessionId = ""
accessToken = ""
# test if client variables have been set
helper.checkClientVars({'client_key':client_key,'client_secret':client_secret,'AppSessionId':AppSessionId})
BaseSpaceUrl = 'https://api.cloud-endor.illumina.com/'
version = 'v1pre3'
# First, create a client for making calls for this user session
myAPI = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId,AccessToken=accessToken)
# First, let's grab the genome with id=4
myGenome = myAPI.getGenomeById('4')
print "\nThe Genome is " + str(myGenome)
print "We can get more information from the genome object"
print 'Id: ' + myGenome.Id
print 'Href: ' + myGenome.Href
print 'DisplayName: ' + myGenome.DisplayName
# Get a list of all genomes
allGenomes = myAPI.getAvailableGenomes()
print "\nGenomes \n" + str(allGenomes)
# Let's have a look at the current user
user = myAPI.getUserById('current')
import json
import os
import unittest
from BaseSpacePy.api.AppLaunchHelpers import AppSessionMetaDataRaw, AppSessionMetaDataSDK, LaunchSpecification, \
LaunchPayload
from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI
from BaseSpacePy.model.AppSessionResponse import AppSessionResponse
api = BaseSpaceAPI()
mydir = os.path.dirname(os.path.abspath(__file__))
app_session_path = os.path.join(mydir, "appsession.json")
app_name = "BWA Whole Genome Sequencing v1.0"
app_id = "279279"
app_properties = [
{'Type': 'string', 'Name': 'Input.AnnotationSource'},
{'Type': 'string[]', 'Name': 'Input.FlagPCRDuplicates-id'},
{'Type': 'string', 'Name': 'Input.genome-id'},
{'Type': 'string', 'Name': 'Input.GQX-id'},
{'Type': 'project', 'Name': 'Input.project-id'},
{'Type': 'sample', 'Name': 'Input.sample-id'},
{'Type': 'string', 'Name': 'Input.StrandBias-id'}
]
app_property_names = [
'AnnotationSource',
'FlagPCRDuplicates-id',
'genome-id',
'GQX-id',
'project-id',
'sample-id',
and upload result files to it and download files from it.
"""
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion,
appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# Now we'll do some work of our own. First get a project to work on
# we'll need write permission, for the project we are working on
# meaning we will need get a new token and instantiate a new BaseSpaceAPI
p = myAPI.getProjectById('89')
# Assuming we have write access to the project
# we will list the current App Results for the project
appRes = p.getAppResults(myAPI, statuses=['Running'])
print("\nThe current running AppResults are \n" + str(appRes))
# now let's do some work!
# to create an appResults for a project, simply give the name and description
def download(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), outputDirectory='\.', createBsDir=True, force=False, numRetries=3):
'''
Downloads App Result files.
Provide an App Result identifier, and optionally regexes to include or omit files
based on their names (path not included). Omission takes precedence over inclusion.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param appResultId the BaseSpace App Result identifier
:param fileNameRegexesInclude a list of regexes on which to include files based on name
:param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include)
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
:param force use the force: overwrite existing files if true, false otherwise
:param numRetries the number of retries for a single download API call
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 10000
sleepTime = 1.0
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
appResult = myAPI.getAppResultById(Id=appResultId)
print "Retrieving files from the App Result: " + str(appResult)
# Get all the files from the AppResult
filesToDownload = appResult.getFiles(myAPI, queryPars=qp({'Limit' : fileLimit}))
# Filter file names based on the include or omit regexes
includePatterns = [re.compile(pattern) for pattern in fileNameRegexesInclude]
omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit]
def includePatternMatch(f):
if not includePatterns:
return True
for pattern in includePatterns:
if pattern.match(f):
return True
return False
def omitPatternMatch(f):
if not omitPatterns:
return False
for pattern in omitPatterns:
if pattern.match(f):
return True
return False
def keepFile(f):
return includePatternMatch(f) and not omitPatternMatch(f)
filesToDownload = [f for f in filesToDownload if keepFile(str(f))]
print "Will download %d files." % len(filesToDownload)
for i in range(len(filesToDownload)):
appResultFile = filesToDownload[i]
print 'Downloading (%d/%d): %s' % ((i+1), len(filesToDownload), str(appResultFile))
print "File Path: %s" % appResultFile.Path
if not options.dryRun:
outputPath = str(appResultFile.Path)
if not createBsDir:
outputPath = os.path.basename(outputPath)
if os.path.exists(outputPath):
if force:
print "Overwritting: %s" % outputPath
else:
print "Skipping existing file: %s" % outputPath
continue
else:
print "Downloading to: %s" % outputPath
retryIdx = 0
retryException = None
while retryIdx < numRetries:
try:
appResultFile.downloadFile(myAPI, outputDirectory, createBsDir=createBsDir)
except BaseSpaceException.ServerResponseException as e:
retryIdx += 1
time.sleep(sleepTime)
retryException = e
else:
break
if retryIdx == numRetries:
raise retryException
print "Download complete."
python's pickle module.
"""
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion,
appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# First, get the verification code and uri for scope 'browse global'
deviceInfo = myAPI.getVerificationCode('browse global')
print "\n URL for user to visit and grant access: "
print deviceInfo['verification_with_code_uri']
## PAUSE HERE
# Have the user visit the verification uri to grant us access
print "\nPlease visit the uri within 15 seconds and grant access"
print deviceInfo['verification_with_code_uri']
webbrowser.open_new(deviceInfo['verification_with_code_uri'])
time.sleep(15)
## PAUSE HERE
class BaseSpace(object):
def __init__(self,
project_id=None,
project_name=None,
get_all_projects=False):
super(BaseSpace, self).__init__()
# BaseSpace credentials
creds = self._get_credentials()
self.client_key = creds['client_id']
self.client_secret = creds['client_secret']
self.access_token = creds['access_token']
self.version = creds['version']
self.api_server = creds['api_server']
self.api = BaseSpaceAPI(self.client_key,
self.client_secret,
self.api_server,
self.version,
AccessToken=self.access_token)
self.params = qp(pars={'Limit': 1024, 'SortDir': 'Desc'})
if project_id is not None:
self.project_id = project_id
self.project_name = None
elif project_name is not None:
self.project_name = project_name
self.project_id = self._get_project_id_from_name(project_name)
else:
self.project_id = None
self.project_name = None
# self.project_id, self.project_name = self._user_selected_project_id()
self._runs = None
@property
def runs(self):
if self._runs is None:
self._runs = self.api.getAccessibleRunsByUser(
queryPars=self.params)
return self._runs
def _get_credentials(self):
# BaseSpace credentials file should be in JSON format
cred_file = os.path.expanduser('~/.abstar/basespace_credentials')
cred_handle = open(cred_file, 'r')
return json.load(cred_handle)
def _get_project_id_from_name(self):
projects = self.api.getProjectByUser(queryPars=self.params)
for project in projects:
name = project.Name.encode('ascii', 'ignore')
if name == self.project_name:
return project.Id
print('No projects matched the given project name ({})'.format(name))
sys.exit(1)
def _user_selected_project_id(self):
projects = self.api.getProjectByUser(queryPars=self.params)
self.print_basespace_project()
offset = 0
while True:
for i, project in enumerate(projects[offset * 25:(offset * 25) +
25]):
project_name = project.Name.encode('ascii', 'ignore')
print('[ {} ] {}'.format(i + (offset * 25), project_name))
print('')
project_index = raw_input(
"Select the project number (or 'next' to see more projects): ")
try:
project_index = int(project_index)
return projects[project_index].Id, projects[
project_index].Name.encode('ascii', 'ignore')
except:
offset += 1
return projects[project_index].Id, projects[project_index].Name.encode(
'ascii', 'ignore')
def _get_projects(self, start=0):
projects = self.api.getProjectByUser(queryPars=self.params)
self.print_basespace_project()
for i, project in enumerate(projects[:25]):
project_name = project.Name.encode('ascii', 'ignore')
print('[ {} ] {}'.format(i, project_name))
print('')
return projects
def _get_samples(self, project_id):
samples = []
offset = 0
while True:
query_params = qp(pars={
'Limit': 1024,
'SortDir': 'Asc',
'Offset': offset * 1024
})
s = self.api.getSamplesByProject(self.project_id,
queryPars=query_params)
if not s:
break
samples.extend(s)
offset += 1
return samples
def _get_files(self):
files = []
samples = self._get_samples(self.project_id)
for sample in samples:
files.extend(
self.api.getFilesBySample(sample.Id, queryPars=self.params))
return files
def download(self, direc):
if all([self.project_id is None, self.project_name is None]):
self.project_id, self.project_name = self._user_selected_project_id(
)
files = self._get_files()
self.print_download_info(files)
start = time.time()
for i, f in enumerate(files):
# self.log.write('[ {} ] {}\n'.format(i, str(f)))
logger.info('[ {} ] {}'.format(i, str(f)))
f.downloadFile(self.api, direc)
end = time.time()
self.print_completed_download_info(start, end)
return len(files)
def print_basespace_project(self):
print('')
print('')
print('========================================')
print('BaseSpace Project Selection')
print('========================================')
print('')
def print_download_info(self, files):
logger.info('')
logger.info('')
logger.info('========================================')
logger.info('Downloading files from BaseSpace')
logger.info('========================================')
logger.info('')
logger.info('Identified {0} files for download.'.format(len(files)))
logger.info('')
def print_completed_download_info(self, start, end):
logger.info('')
logger.info('Download completed in {0} seconds'.format(end - start))
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('-k', '--key', dest='key', required=True,
type=str,
help='specify client key')
parser.add_argument('-s', '--secret', dest='secret', required=True,
type=str,
help='specify client secret')
parser.add_argument('-t', '--token', dest='token', required=True,
type=str,
help='specify access token')
parser.add_argument('-r', '--run', dest='run', required=False,
type=str,
help='specify the run name to download')
parser.add_argument('-d', '--directory', dest='directory', required=False,
type=str,
default='./',
help='specify download directory. \
The default is the current directory')
parser.add_argument('--offset', dest='offset', required=False,
type=int,
default=0,
help='specify the starting offset to read. \
The default is 0')
parser.add_argument('-n', '--num_items', dest='num_items', required=False,
type=int,
default=10,
help='specify the maximum number of items to return \
(max 1024). The default is 10')
parser.add_argument('-e', '--excluded_path', dest='excluded_path', required=False,
type=str,
default=None,
help='specify files to skip (comma separated). \
If file paths contain \
this(ese) strings, it will be skipped.)\
The default is None')
args = parser.parse_args()
client_key = args.key
client_secret = args.secret
client_token = args.token
run_name = args.run
download_directory = args.directory
if args.excluded_path:
excluded_file_path_strings = args.excluded_path.split(',')
else:
excluded_file_path_strings = []
num_items = args.num_items
offset = args.offset
base_space_url = 'https://api.basespace.illumina.com/'
my_bs_api = BaseSpaceAPI(client_key,
client_secret,
base_space_url,
'v1pre3',
'',
client_token)
user = my_bs_api.getUserById('current')
print('User: {}'.format(str(user)),
sep='', file=sys.stderr)
runs = user.getRuns(my_bs_api, queryPars=qp({'Limit': num_items}))
print('Run(s): {}'.format(runs), sep='', file=sys.stderr)
if run_name:
run = runs[[index for index, value
in enumerate(runs) if value.Name == run_name][0]]
print('Total size ({}): {} GB'.format(run.Name,
run.TotalSize / 1000000000),
sep='', file=sys.stderr)
print('Offset: {}'.format(offset),
sep='', file=sys.stderr)
print('Number of items to return: {}'.format(num_items),
sep='', file=sys.stderr)
for f in run.getFiles(my_bs_api,
queryPars=qp({'Limit': num_items,
'Offset': offset})):
file_path = f.Path
if any([i in file_path for i in excluded_file_path_strings]):
print('Skipping file: {}'.format(file_path),
file=sys.stderr)
else:
print('Downloading file: {}'.format(file_path),
'...', sep=' ', end='', file=sys.stderr)
try:
f.downloadFile(my_bs_api, download_directory, createBsDir=True)
etag = f.getFileS3metadata(my_bs_api)['etag']
file_path = download_directory + '/' + file_path
if len(etag) == 32:
f_md5 = md5_hash(file_path)
if f_md5 == etag:
print(' done (md5 correct)!',
file=sys.stderr)
else:
print(' error (md5 incorrect)!',
f.Id,
etag,
f_md5,
file=sys.stderr)
else:
if f.Size == os.path.getsize(file_path):
print(' done (file size correct)!',
file=sys.stderr)
else:
print(' error (file size incorrect)!',
f.Id,
etag,
file=sys.stderr)
except Exception as e:
print(' error ({})!!'.format(e),
file=sys.stderr)
exception message: 'Forbidden: App credentials do not match AppSession application'.
"""
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion,
appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# Using the basespaceApi we can request the appSession object corresponding to the AppSession id supplied
myAppSession = myAPI.getAppSession()
print myAppSession
# An app session contains a referal to one or more appLaunchObjects which reference the data module
# the user launched the app on. This can be a list of projects, samples, or a mixture of objects
print "\nType of data the app was triggered on can be seen in 'references'"
print myAppSession.References
# We can also get a handle to the user who started the AppSession
print "\nWe can get a handle for the user who triggered the app\n" + str(
myAppSession.UserCreatedBy)
def main(uargs):
uargs['--config'] = os.path.abspath(os.path.expanduser(uargs['--config']))
conf = ConfigObj(uargs['--config'], configspec=get_configspec())
try:
conf_args = conf[uargs['--profile']]
except KeyError:
msg = 'Profile "{}" not found in config file'
raise KeyError, msg.format(uargs['--profile'])
# selecting config args
keys = ['clientKey', 'clientSecret', 'apiServer', 'apiVersion', 'appSessionId', 'accessToken']
conf_args = [conf_args[x] for x in keys]
myAPI = BaseSpaceAPI(*conf_args)
# setting user
if uargs['--user'] is not None:
user = myAPI.getUserById(uargs['--user'])
else:
user = myAPI.getUserById('current')
# user projects
projects = myAPI.getProjectByUser()
print "## The projects for this user are:"
project_headers = ['ProjectID','Name','UserOwnedBy','DateCreated']
sample_headers = ['SampleID','NumReadsRaw','NumReadsPF','IsPairedEnd','Status']
print '\t'.join(project_headers + sample_headers)
for project in projects:
project_data = [project.Id,
project.Name,
project.UserOwnedBy,
project.DateCreated]
samples = myAPI.getSamplesByProject(project.Id)
for sample in samples:
sample_data = [sample.Id,
sample.NumReadsRaw,
sample.NumReadsPF,
sample.IsPairedEnd,
sample.Status]
print '\t'.join([str(x) for x in project_data + sample_data])
#files = myAPI.getSampleFilesById(sample.Id)
#for f in files:
#fo = myAPI.getFileById(f.Id)
# myAPI.fileDownload(f.Id, '.')
# user runs
print "## The runs for this user are:"
print '\t'.join(['RunID','ExperimentName','UserOwnedBy',
'DateCreated', 'Status'])
runs = user.getRuns(myAPI)
for run in runs:
data = [run.Id,
run.ExperimentName,
run.UserOwnedBy,
run.DateCreated,
run.Status]
print '\t'.join([str(x) for x in data])
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(0, os.path.abspath(os.path.sep.join([SCRIPT_DIR, "..", "..", "basespace-python-sdk", "src"])))
from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI
from BaseSpacePy.model.QueryParameters import QueryParameters
from collections import defaultdict
from memoize import memoized
from operator import attrgetter
import Repository
NUMREADS_ATTR = "NumReadsPF"
READ1_ATTR = "Read1"
READ2_ATTR = "Read2"
PAIRED_END_ATTR = "IsPairedEnd"
baseSpaceAPI = BaseSpaceAPI()
noLimitQP = QueryParameters({ "Limit" : 1000 })
def OrganiseSamples(allSampleList):
"""
GetSamplesInProject() returns a flat list of samples.
This rearranges it a little to make it easier to work with
@param allSampleList: (list of BaseSpace sample objects)
@return (dict): sample_name -> list of BaseSpace sample objects
"""
sampleListsByName = defaultdict(list)
for sample in allSampleList:
# don't forget! a BaseSpace "Sample" is a bundle of fastq files
# there could be more than one bundle with the same sample name
help="Target project name (default: %(default)s)")
parser.add_argument("--dir",
"-d",
type=str,
default="upload",
help="Target upload directory (default: %(default)s)")
parser.add_argument("--sdir",
"-s",
type=str,
default="",
help="Target subdirectory (default: %(default)s)")
parser.add_argument("--name", "-n", type=str, help="Target file name")
args = parser.parse_args()
print "Connecting to BaseSpace API server..."
myAPI = BaseSpaceAPI()
user = myAPI.getUserById('current')
print "Connected to server, username: %s" % user
p = myAPI.createProject(args.project)
appResults = p.createAppResult(myAPI, args.dir, "", appSessionId="")
myAppSession = appResults.AppSession
myAppSession.setStatus(myAPI, "running", "uploading data")
if not args.name or len(args.file):
args.name = os.path.basename(args.file)
print "Uploading file %(file)s to %(project)s:%(dir)s/%(sdir)s/%(name)s" % args.__dict__
data', and click the Import button for the Project named 'BaseSpaceDemo'.
"""
"""
NOTE: You will need to provide the credentials for your app (available in the developer portal).
You can do this with a master config file (preferred), or by filling in values below.
"""
# If you're not using a config file, fill in you app's credentials here:
clientKey = ""
clientSecret = ""
appSessionId = ""
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
if clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion,
appSessionId)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
user = myAPI.getUserById('current')
myProjects = myAPI.getProjectByUser('current')
# Let's list all the AppResults and samples for these projects
for singleProject in myProjects:
print "# " + str(singleProject)
appResults = singleProject.getAppResults(myAPI)
print " The App results for project " + str(
singleProject) + " are \n\t" + str(appResults)
samples = singleProject.getSamples(myAPI)
print " The samples for project " + str(
singleProject) + " are \n\t" + str(samples)
def download(clientKey=None, clientSecret=None, accessToken=None, sampleId=None, projectId=None, sampleName=None, projectName=None, outputDirectory='\.', createBsDir=True):
'''
Downloads sample-level files.
Project Id and project name should
not be specified together; similarly sample Id and sample name should not be
specified together.
1. If only a project Id or only a project name is given, all files for all
samples will be downloaded within that project. If additionally a sample Id or
sample name is given, then only the first matching sample within the project
will be downloaded.
2. If only a sample Id is given, then all files for that sample will be downloaded.
3. If only a sample name is given, then all files within the first project
containing a sample with matching name will be downloaded.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param sampleId the BaseSpace sample identifier
:param projectId the BaseSpace project identifier
:param sampleName the BaseSpace sample name
:param projectName the BaseSpace project name
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
projectLimit = 100
sampleLimit = 1024
sampleFileLimit = 1024
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
sampleToFiles = {}
if None != projectId:
sampleToFiles = Samples.__get_files_to_download(myAPI, projectId, sampleId, sampleName, sampleLimit, sampleFileLimit)
else:
myProjects = myAPI.getProjectByUser(qp({'Limit' : projectLimit}))
for project in myProjects:
projectId = project.Id
if None != projectName and project.Name != projectName:
continue
sampleToFiles = Samples.__get_files_to_download(myAPI, projectId, sampleId, sampleName, sampleLimit, sampleFileLimit)
if 0 < len(sampleToFiles):
break
numFiles = sum([len(sampleToFiles[sampleId]) for sampleId in sampleToFiles])
print("will download files from %d ." % numFiles)
i = 0
for sampleId in sampleToFiles:
for sampleFile in sampleToFiles[sampleId]:
print('Downloading (%d/%d): %s' % ((i+1), numFiles, str(sampleFile)))
print("BaseSpace File Path: %s" % sampleFile.Path)
print("Sample Id: %s" % sampleId)
if not options.dryRun:
if createBsDir:
sampleOutputDirectory = os.path.join(outputDirectory, sampleId)
else:
sampleOutputDirectory = outputDirectory
sampleFile.downloadFile(myAPI, sampleOutputDirectory, createBsDir=createBsDir)
i = i + 1
print("FASTQ file downloading complete.")
# initialize an authentication object using the key and secret from your app
# FILL IN WITH YOUR APP VALUES HERE!
client_key = ""
client_secret = ""
AppSessionId = ""
# test if client variables have been set
helper.checkClientVars({'client_key':client_key,'client_secret':client_secret,'AppSessionId':AppSessionId})
BaseSpaceUrl = 'https://api.basespace.illumina.com/'
version = 'v1pre3'
# First we will initialize a BaseSpace API object using our app information and the appSessionId
BSapi = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId)
# Using the basespaceApi we can request the appSession object corresponding to the AppSession id supplied
myAppSession = BSapi.getAppSession()
print myAppSession
# An app session contains a referal to one or more appLaunchObjects which reference the data module
# the user launched the app on. This can be a list of projects, samples, or a mixture of objects
print "\nType of data the app was triggered on can be seen in 'references'"
print myAppSession.References
# We can also get a handle to the user who started the AppSession
print "\nWe can get a handle for the user who triggered the app\n" + str(myAppSession.UserCreatedBy)
# Let's have a closer look at the appSessionLaunchObject
myReference = myAppSession.References[0]
This script demonstrates how to create a new AppResults object, change its state
and upload result files to it and download files from it.
"""
# FILL IN WITH YOUR APP VALUES HERE!
client_key = ""
client_secret = ""
AppSessionId = ""
accessToken = ""
# test if client variables have been set
helper.checkClientVars({'client_key':client_key,'client_secret':client_secret,'AppSessionId':AppSessionId})
BaseSpaceUrl = 'https://api.basespace.illumina.com'
version = 'v1pre3'
# First, create a client for making calls for this user session
myBaseSpaceAPI = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId,AccessToken=accessToken)
# Now we'll do some work of our own. First get a project to work on
# we'll need write permission, for the project we are working on
# meaning we will need get a new token and instantiate a new BaseSpaceAPI
p = myBaseSpaceAPI.getProjectById('89')
# Assuming we have write access to the project
# we will list the current App Results for the project
appRes = p.getAppResults(myBaseSpaceAPI,statuses=['Running'])
print "\nThe current running AppResults are \n" + str(appRes)
# now let's do some work!
# to create an appResults for a project, simply give the name and description
appResults = p.createAppResult(myBaseSpaceAPI,"testing","this is my results",appSessionId='')
def download_basespace_files(config_file_path=None, client_key=None, client_secret=None, access_token=None,
project_id_list=None, project_name_list=None, sample_id_list=None, sample_name_list=None,
dry_run=False, output_directory=None, recreate_basespace_dir_tree=True):
# Check input parameters / load from config file / defaults
if not project_id_list: project_id_list = []
if not project_name_list: project_name_list = []
if not sample_id_list: sample_id_list = []
if not sample_name_list: sample_name_list = []
if not output_directory:
output_directory = os.getcwd()
print_stderr("Output directory not specified; using current directory ({})".format(output_directory))
else:
output_directory = os.path.abspath(output_directory)
if not dry_run:
safe_makedir(output_directory)
config_dict = {}
if config_file_path:
config_parser = ConfigParser()
config_parser.read(config_file_path)
config_dict = config_parser._defaults
if not client_key: client_key = config_dict.get('clientkey')
if not client_secret: client_secret = config_dict.get('clientsecret')
if not access_token: access_token = config_dict.get('accesstoken')
if not (client_key and client_secret and access_token):
missing_params = []
if not client_key: missing_params.append("client_key")
if not client_secret: missing_params.append("client_secret")
if not access_token: missing_params.append("access_token")
print_stderr('Error: Required parameters not supplied either in config '
'file ({}) or via arguments.'.format(config_file_path,
', '.join(missing_params)))
sys.exit(1)
app_session_id = config_dict.get("appsessionid") or ""
api_server = config_dict.get("apiserver") or "https://api.basespace.illumina.com"
api_version = config_dict.get("apiversion") or "v1pre3"
# Get the API connection object
myAPI = BaseSpaceAPI(clientKey=client_key, clientSecret=client_secret,
apiServer=api_server, version=api_version,
appSessionId=app_session_id, AccessToken=access_token)
basespace_projects = myAPI.getProjectByUser(qp({'Limit' : 1024}))
user = myAPI.getUserById('current')
# If user specified projects, get them by name or id
project_objects = []
if project_name_list:
project_objects.extend(_select_from_object(filter_list=project_name_list,
search_list=basespace_projects,
key_attr="Name",
obj_type="project",
user=user))
if project_id_list:
digit_pattern = re.compile(r'^\d+$')
project_filtered_id_list = []
for project_id in project_id_list:
if not digit_pattern.match(project_id):
print_stderr('Error: Invalid format for user-specified project id '
'"{}": project ids are strictly numeric. Did you mean '
'to pass this as a project name?'.format(project_id))
else:
project_filtered_id_list.append(project_id)
project_objects.extend(_select_from_object(filter_list=project_filtered_id_list,
search_list=basespace_projects,
key_attr="Id",
obj_type="project",
user=user))
if not (project_name_list or project_id_list):
# Get all projects if none are specified by user
project_objects = basespace_projects
basespace_samples = []
for project_obj in project_objects:
basespace_samples.extend(project_obj.getSamples(myAPI))
sample_objects = []
if sample_name_list:
sample_objects.extend(_select_from_object(filter_list=sample_name_list,
search_list=basespace_samples,
key_attr="Name",
obj_type="sample",
user=user))
if sample_id_list:
digit_pattern = re.compile(r'^\d+$')
sample_filtered_id_list = []
for sample_id in sample_id_list:
if not digit_pattern.match(sample_id):
print_stderr('Error: Invalid format for user-specified sample id '
'"{}": sample ids are strictly numeric. Did you mean '
'to pass this as a sample name?'.format(sample_id))
else:
sample_filtered_id_list.append(sample_id)
sample_objects.extend(_select_from_object(filter_list=sample_filtered_id_list,
search_list=basespace_samples,
key_attr="Id",
obj_type="sample",
user=user))
if not (sample_name_list or sample_id_list):
# Get all samples if none are specified by user
sample_objects = basespace_samples
files_to_download = []
for sample_obj in sample_objects:
files_to_download.extend(sample_obj.getFiles(myAPI))
if files_to_download:
print_stderr("Found {} files to download: ".format(len(files_to_download)))
for file_obj in files_to_download:
print_stderr("\t- {}".format(file_obj))
print_stderr('Downloading files to output directory {}'.format(output_directory))
if recreate_basespace_dir_tree:
print_stderr("Recreating BaseSpace project directory tree for file.")
if dry_run:
print_stderr("-> Dry run: not downloading any data.")
for i, file_obj in enumerate(files_to_download):
print_stderr('[{}/{}] Downloading file "{}"'.format(i+1, len(files_to_download),
file_obj))
if not dry_run:
file_obj.downloadFile(api=myAPI, localDir=output_directory,
createBsDir=recreate_basespace_dir_tree)
print_stderr('Download completed; files are located in "{}"'.format(output_directory))
else:
print_stderr("Error: no files found to download.")
"""
import os
import sys
# Add relative path libraries
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
sys.path.append(os.path.abspath(os.path.sep.join([SCRIPT_DIR, "..", "lib"])))
sys.path.append(os.path.abspath(os.path.sep.join([SCRIPT_DIR, "..", "..", "basespace-python-sdk", "src"])))
from BaseSpacePy.api.BaseSpaceAPI import BaseSpaceAPI
import Repository
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description='Create a project')
parser.add_argument('-n', '--name', type=str, required=True, dest="name", help='name of project')
parser.add_argument('-p', '--path', type=str, required=True, dest="path", help='path where project should write')
args = parser.parse_args()
baseSpaceAPI = BaseSpaceAPI()
if not os.path.exists(args.path):
print "must specify an output directory that already exists!"
sys.exit(1)
print "attempting to create/retrieve BaseSpace project"
project = baseSpaceAPI.createProject(args.name)
print "got Id: %s" % project.Id
Repository.AddProject(args.name, args.path, project.Id)
