def main(uargs): uargs['--config'] = os.path.abspath(os.path.expanduser(uargs['--config'])) conf = ConfigObj(uargs['--config'], configspec=get_configspec()) try: conf_args = conf[uargs['--profile']] except KeyError: msg = 'Profile "{}" not found in config file' raise KeyError, msg.format(uargs['--profile']) # selecting config args keys = ['clientKey', 'clientSecret', 'apiServer', 'apiVersion', 'appSessionId', 'accessToken'] conf_args = [conf_args[x] for x in keys] myAPI = BaseSpaceAPI(*conf_args) # setting user if uargs['--user'] is not None: user = myAPI.getUserById(uargs['--user']) else: user = myAPI.getUserById('current') # all samples for project if uargs['project']: samples = myAPI.getSamplesByProject(uargs['<Id>']) for sample in samples: file_download(myAPI, sample.Id) if uargs['sample']: file_download(myAPI, uargs['<Id>'])
def download_Project(project_Name, output_folder): # initialize an authentication object using the key and secret from your app # Fill in with your own values ''' client_key = <my key> client_secret = <my secret> AppSessionId = <my appSession id> BaseSpaceUrl = 'https://api.basespace.illumina.com/' version = 'v1pre3' accessToken = <my acceseToken> ''' myAPI = BaseSpaceAPI(client_key, client_secret, BaseSpaceUrl, version, AppSessionId,AccessToken=accessToken) # Retrieve current user user = myAPI.getUserById('current') user=str(user) id_name=user.split(':') #print id_name[0] # Retrieve all the project associated to that user projects=myAPI.getProjectByUser(id_name[0], queryPars=QueryParameters( {'Limit': '100'})) project_found=0 for project in projects: project=str(project) nameProject_id=project.split('-') if str(project_Name) in str(nameProject_id): project_found=1 id_project=nameProject_id[1].split('=') id_project=id_project[1] samples=myAPI.getSamplesByProject(id_project, queryPars=QueryParameters( {'Limit': '100'})) print "There are "+str(len(samples))+" samples in the requested project ("+str(project_Name)+" - ID_PROJECT "+str(id_project)+")" if not os.path.exists(output_folder): os.makedirs(output_folder) print time.ctime()+" START DOWNLOADING" for file in samples: file_out=file.getFiles(myAPI) #print file_out for fastq in file_out: fastq.downloadFile(myAPI,output_folder) print time.ctime()+" FILE "+str(fastq)+" DOWNLOADED" path_file=join(output_folder,str(fastq)) path_S3=join(str(project_Name),str(fastq)) s3_upload(path_file,"bmi-ngs",path_S3) print time.ctime()+" DOWNLOAD COMPLETED" if project_found==0: print "Project Not Found"
def main(uargs): uargs['--config'] = os.path.abspath(os.path.expanduser(uargs['--config'])) conf = ConfigObj(uargs['--config'], configspec=get_configspec()) try: conf_args = conf[uargs['--profile']] except KeyError: msg = 'Profile "{}" not found in config file' raise KeyError, msg.format(uargs['--profile']) # selecting config args keys = ['clientKey', 'clientSecret', 'apiServer', 'apiVersion', 'appSessionId', 'accessToken'] conf_args = [conf_args[x] for x in keys] myAPI = BaseSpaceAPI(*conf_args) # setting user if uargs['--user'] is not None: user = myAPI.getUserById(uargs['--user']) else: user = myAPI.getUserById('current') # user projects projects = myAPI.getProjectByUser() print "## The projects for this user are:" project_headers = ['ProjectID','Name','UserOwnedBy','DateCreated'] sample_headers = ['SampleID','NumReadsRaw','NumReadsPF','IsPairedEnd','Status'] print '\t'.join(project_headers + sample_headers) for project in projects: project_data = [project.Id, project.Name, project.UserOwnedBy, project.DateCreated] samples = myAPI.getSamplesByProject(project.Id) for sample in samples: sample_data = [sample.Id, sample.NumReadsRaw, sample.NumReadsPF, sample.IsPairedEnd, sample.Status] print '\t'.join([str(x) for x in project_data + sample_data]) #files = myAPI.getSampleFilesById(sample.Id) #for f in files: #fo = myAPI.getFileById(f.Id) # myAPI.fileDownload(f.Id, '.') # user runs print "## The runs for this user are:" print '\t'.join(['RunID','ExperimentName','UserOwnedBy', 'DateCreated', 'Status']) runs = user.getRuns(myAPI) for run in runs: data = [run.Id, run.ExperimentName, run.UserOwnedBy, run.DateCreated, run.Status] print '\t'.join([str(x) for x in data])
class BaseSpace(object): def __init__(self, project_id=None, project_name=None, get_all_projects=False): super(BaseSpace, self).__init__() # BaseSpace credentials creds = self._get_credentials() self.client_key = creds['client_id'] self.client_secret = creds['client_secret'] self.access_token = creds['access_token'] self.version = creds['version'] self.api_server = creds['api_server'] self.api = BaseSpaceAPI(self.client_key, self.client_secret, self.api_server, self.version, AccessToken=self.access_token) self.params = qp(pars={'Limit': 1024, 'SortDir': 'Desc'}) if project_id is not None: self.project_id = project_id self.project_name = None elif project_name is not None: self.project_name = project_name self.project_id = self._get_project_id_from_name(project_name) else: self.project_id = None self.project_name = None # self.project_id, self.project_name = self._user_selected_project_id() self._runs = None @property def runs(self): if self._runs is None: self._runs = self.api.getAccessibleRunsByUser( queryPars=self.params) return self._runs def _get_credentials(self): # BaseSpace credentials file should be in JSON format cred_file = os.path.expanduser('~/.abstar/basespace_credentials') cred_handle = open(cred_file, 'r') return json.load(cred_handle) def _get_project_id_from_name(self): projects = self.api.getProjectByUser(queryPars=self.params) for project in projects: name = project.Name.encode('ascii', 'ignore') if name == self.project_name: return project.Id print('No projects matched the given project name ({})'.format(name)) sys.exit(1) def _user_selected_project_id(self): projects = self.api.getProjectByUser(queryPars=self.params) self.print_basespace_project() offset = 0 while True: for i, project in enumerate(projects[offset * 25:(offset * 25) + 25]): project_name = project.Name.encode('ascii', 'ignore') print('[ {} ] {}'.format(i + (offset * 25), project_name)) print('') project_index = raw_input( "Select the project number (or 'next' to see more projects): ") try: project_index = int(project_index) return projects[project_index].Id, projects[ project_index].Name.encode('ascii', 'ignore') except: offset += 1 return projects[project_index].Id, projects[project_index].Name.encode( 'ascii', 'ignore') def _get_projects(self, start=0): projects = self.api.getProjectByUser(queryPars=self.params) self.print_basespace_project() for i, project in enumerate(projects[:25]): project_name = project.Name.encode('ascii', 'ignore') print('[ {} ] {}'.format(i, project_name)) print('') return projects def _get_samples(self, project_id): samples = [] offset = 0 while True: query_params = qp(pars={ 'Limit': 1024, 'SortDir': 'Asc', 'Offset': offset * 1024 }) s = self.api.getSamplesByProject(self.project_id, queryPars=query_params) if not s: break samples.extend(s) offset += 1 return samples def _get_files(self): files = [] samples = self._get_samples(self.project_id) for sample in samples: files.extend( self.api.getFilesBySample(sample.Id, queryPars=self.params)) return files def download(self, direc): if all([self.project_id is None, self.project_name is None]): self.project_id, self.project_name = self._user_selected_project_id( ) files = self._get_files() self.print_download_info(files) start = time.time() for i, f in enumerate(files): # self.log.write('[ {} ] {}\n'.format(i, str(f))) logger.info('[ {} ] {}'.format(i, str(f))) f.downloadFile(self.api, direc) end = time.time() self.print_completed_download_info(start, end) return len(files) def print_basespace_project(self): print('') print('') print('========================================') print('BaseSpace Project Selection') print('========================================') print('') def print_download_info(self, files): logger.info('') logger.info('') logger.info('========================================') logger.info('Downloading files from BaseSpace') logger.info('========================================') logger.info('') logger.info('Identified {0} files for download.'.format(len(files))) logger.info('') def print_completed_download_info(self, start, end): logger.info('') logger.info('Download completed in {0} seconds'.format(end - start))