def _writeback_app_result_files(self): """ Writeback all files in output_files list, and create corresponding entries in the local db """ db = current.db # get BaseSpace API ssn_row = db(db.app_session.id==self.app_session_id).select().first() user_row = db(db.auth_user.id==ssn_row.user_id).select().first() app = db(db.app_data.id > 0).select().first() bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token) app_result = bs_api.getAppResultById(self.app_result_num) # upload files to BaseSpace for f in self.output_files: # currently not using a dir name to write to bs_file = app_result.uploadFile(bs_api, f.local_path, f.file_name, '', 'text/plain') # add file to local db db.output_file.insert(app_result_id=f.app_result_id, file_num=bs_file.Id, file_name=f.file_name, local_path=f.local_path, file_type=f.file_type) db.commit()
def update_status(self, local_status, message, bs_ssn_status=None): """ Update db with provided status and detailed message, and update BaseSpace App Session status if provided """ db = current.db # update status in local db self.status=local_status self.message=message ssn_row = db(db.app_session.id==self.app_session_id).select().first() ssn_row.update_record(status=self.status, message=self.message) db.commit() # optionally update status of AppSession in BaseSpace -- limited to 128 chars if bs_ssn_status: # get BaseSpace API ssn_row = db(db.app_session.id==self.app_session_id).select().first() user_row = db(db.auth_user.id==ssn_row.user_id).select().first() app = db(db.app_data.id > 0).select().first() # handle exceptions in caller bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token) app_result = bs_api.getAppResultById(self.app_result_num) app_ssn = app_result.AppSession app_ssn.setStatus(bs_api, bs_ssn_status, message[:128])
def upload(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), inputDirectory='\.', dryRun=False, numRetries=3): ''' Creates an App Result and uploads files. TODO Provide an App Result identifier, and optionally regexes to include or omit files based on their names (path not included). Omission takes precedence over inclusion. :param clientKey the Illumina developer app client key :param clientSecret the Illumina developer app client secret :param accessToken the Illumina developer app access token :param appResultId the BaseSpace App Result identifier :param fileNameRegexesInclude a list of regexes on which to include files based on name :param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include) :param inputDirectory the root input directory :param numRetries the number of retries for a single download API call ''' appSessionId = '' apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/' apiVersion = 'v1pre3' fileLimit = 10000 sleepTime = 1.0 # init the API if None != clientKey: myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken) else: myAPI = BaseSpaceAPI(profile='DEFAULT', clientKey=clientKey, clientSecret=clientSecret, AccessToken=accessToken) # get the current user user = myAPI.getUserById('current') # get the app result appResult = myAPI.getAppResultById(Id=appResultId) appSession = appResult.AppSession print "Uploading files to the App Result: " + str(appResult) # Filter file names based on the include or omit regexes includePatterns = [re.compile(pattern) for pattern in fileNameRegexesInclude] omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit] def includePatternMatch(f): if not includePatterns: return True for pattern in includePatterns: if pattern.match(f): return True return False def omitPatternMatch(f): if not omitPatterns: return False for pattern in omitPatterns: if pattern.match(f): return True return False def keepFile(f): return includePatternMatch(f) and not omitPatternMatch(f) # walk the current directory structure for root, dirs, files in os.walk(inputDirectory): for fileName in files: localPath = os.path.join(root, fileName) directory = root.replace(inputDirectory, "") if AppResults.isBinaryContent(fileName): contentType = 'application/octet-stream' else: contentType = 'text/plain' if keepFile(fileName): print "Uploading file: %s" % localPath if not options.dryRun: retryIdx = 0 retryException = None while retryIdx < numRetries: try: appResult.uploadFile(api=myAPI, localPath=localPath, fileName=fileName, directory=directory, contentType=contentType) except BaseSpaceException.ServerResponseException as e: retryIdx += 1 time.sleep(sleepTime) retryException = e else: break if retryIdx == numRetries: raise retryException print "Upload complete"
print appResults.Id print "\nThe app results also comes with a reference to our AppSession" myAppSession = appResults.AppSession print myAppSession # we can change the status of our AppSession and add a status-summary as follows myAppSession.setStatus(myBaseSpaceAPI,'needsattention',"We worked hard, but encountered some trouble.") print "\nAfter a change of status of the app sessions we get\n" + str(myAppSession) # we'll set our appSession back to running so we can do some more work myAppSession.setStatus(myBaseSpaceAPI,'running',"Back on track") ### Let's list all AppResults again and see if our new object shows up appRes = p.getAppResults(myBaseSpaceAPI,statuses=['Running']) print "\nThe updated app results are \n" + str(appRes) appResult2 = myBaseSpaceAPI.getAppResultById(appResults.Id) print appResult2 ## Now we will make another AppResult ## and try to upload a file to it appResults2 = p.createAppResult(myBaseSpaceAPI,"My second AppResult","This one I will upload to") appResults2.uploadFile(myBaseSpaceAPI, '/home/mkallberg/Desktop/testFile2.txt', 'BaseSpaceTestFile.txt', '/mydir/', 'text/plain') print "\nMy AppResult number 2 \n" + str(appResults2) ## let's see if our new file made it appResultFiles = appResults2.getFiles(myBaseSpaceAPI) print "\nThese are the files in the appResult" print appResultFiles f = appResultFiles[-1] # we can even download our newly uploaded file
print appResults.Id print "\nThe app results also comes with a reference to our AppSession" myAppSession = appResults.AppSession print myAppSession # we can change the status of our AppSession and add a status-summary as follows myAppSession.setStatus(myAPI, "needsattention", "We worked hard, but encountered some trouble.") print "\nAfter a change of status of the app sessions we get\n" + str(myAppSession) # we'll set our appSession back to running so we can do some more work myAppSession.setStatus(myAPI, "running", "Back on track") ### Let's list all AppResults again and see if our new object shows up appRes = p.getAppResults(myAPI, statuses=["Running"]) print "\nThe updated app results are \n" + str(appRes) appResult2 = myAPI.getAppResultById(appResults.Id) print appResult2 ## Now we will make another AppResult ## and try to upload a file to it appResults2 = p.createAppResult(myAPI, "My second AppResult", "This one I will upload to") appResults2.uploadFile(myAPI, "/home/mkallberg/Desktop/testFile2.txt", "BaseSpaceTestFile.txt", "/mydir/", "text/plain") print "\nMy AppResult number 2 \n" + str(appResults2) ## let's see if our new file made it appResultFiles = appResults2.getFiles(myAPI) print "\nThese are the files in the appResult" print appResultFiles f = appResultFiles[-1] # we can even download our newly uploaded file
def download(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), outputDirectory='\.', createBsDir=True, force=False, numRetries=3): ''' Downloads App Result files. Provide an App Result identifier, and optionally regexes to include or omit files based on their names (path not included). Omission takes precedence over inclusion. :param clientKey the Illumina developer app client key :param clientSecret the Illumina developer app client secret :param accessToken the Illumina developer app access token :param appResultId the BaseSpace App Result identifier :param fileNameRegexesInclude a list of regexes on which to include files based on name :param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include) :param outputDirectory the root output directory :param createBsDir true to recreate the path structure within BaseSpace, false otherwise :param force use the force: overwrite existing files if true, false otherwise :param numRetries the number of retries for a single download API call ''' appSessionId = '' apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/' apiVersion = 'v1pre3' fileLimit = 10000 sleepTime = 1.0 # init the API if None != clientKey: myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken) else: myAPI = BaseSpaceAPI(profile='DEFAULT') # get the current user user = myAPI.getUserById('current') appResult = myAPI.getAppResultById(Id=appResultId) print "Retrieving files from the App Result: " + str(appResult) # Get all the files from the AppResult filesToDownload = appResult.getFiles(myAPI, queryPars=qp({'Limit': fileLimit})) # Filter file names based on the include or omit regexes includePatterns = [ re.compile(pattern) for pattern in fileNameRegexesInclude ] omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit] def includePatternMatch(f): if not includePatterns: return True for pattern in includePatterns: if pattern.match(f): return True return False def omitPatternMatch(f): if not omitPatterns: return False for pattern in omitPatterns: if pattern.match(f): return True return False def keepFile(f): return includePatternMatch(f) and not omitPatternMatch(f) filesToDownload = [f for f in filesToDownload if keepFile(str(f))] print "Will download %d files." % len(filesToDownload) for i in range(len(filesToDownload)): appResultFile = filesToDownload[i] print 'Downloading (%d/%d): %s' % ( (i + 1), len(filesToDownload), str(appResultFile)) print "File Path: %s" % appResultFile.Path if not options.dryRun: outputPath = str(appResultFile.Path) if not createBsDir: outputPath = os.path.basename(outputPath) if os.path.exists(outputPath): if force: print "Overwritting: %s" % outputPath else: print "Skipping existing file: %s" % outputPath continue else: print "Downloading to: %s" % outputPath retryIdx = 0 retryException = None while retryIdx < numRetries: try: appResultFile.downloadFile(myAPI, outputDirectory, createBsDir=createBsDir) except BaseSpaceException.ServerResponseException as e: retryIdx += 1 time.sleep(sleepTime) retryException = e else: break if retryIdx == numRetries: raise retryException print "Download complete."
def download(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), outputDirectory='\.', createBsDir=True, force=False, numRetries=3): ''' Downloads App Result files. Provide an App Result identifier, and optionally regexes to include or omit files based on their names (path not included). Omission takes precedence over inclusion. :param clientKey the Illumina developer app client key :param clientSecret the Illumina developer app client secret :param accessToken the Illumina developer app access token :param appResultId the BaseSpace App Result identifier :param fileNameRegexesInclude a list of regexes on which to include files based on name :param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include) :param outputDirectory the root output directory :param createBsDir true to recreate the path structure within BaseSpace, false otherwise :param force use the force: overwrite existing files if true, false otherwise :param numRetries the number of retries for a single download API call ''' appSessionId = '' apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/' apiVersion = 'v1pre3' fileLimit = 10000 sleepTime = 1.0 # init the API if None != clientKey: myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken) else: myAPI = BaseSpaceAPI(profile='DEFAULT') # get the current user user = myAPI.getUserById('current') appResult = myAPI.getAppResultById(Id=appResultId) print "Retrieving files from the App Result: " + str(appResult) # Get all the files from the AppResult filesToDownload = appResult.getFiles(myAPI, queryPars=qp({'Limit' : fileLimit})) # Filter file names based on the include or omit regexes includePatterns = [re.compile(pattern) for pattern in fileNameRegexesInclude] omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit] def includePatternMatch(f): if not includePatterns: return True for pattern in includePatterns: if pattern.match(f): return True return False def omitPatternMatch(f): if not omitPatterns: return False for pattern in omitPatterns: if pattern.match(f): return True return False def keepFile(f): return includePatternMatch(f) and not omitPatternMatch(f) filesToDownload = [f for f in filesToDownload if keepFile(str(f))] print "Will download %d files." % len(filesToDownload) for i in range(len(filesToDownload)): appResultFile = filesToDownload[i] print 'Downloading (%d/%d): %s' % ((i+1), len(filesToDownload), str(appResultFile)) print "File Path: %s" % appResultFile.Path if not options.dryRun: outputPath = str(appResultFile.Path) if not createBsDir: outputPath = os.path.basename(outputPath) if os.path.exists(outputPath): if force: print "Overwritting: %s" % outputPath else: print "Skipping existing file: %s" % outputPath continue else: print "Downloading to: %s" % outputPath retryIdx = 0 retryException = None while retryIdx < numRetries: try: appResultFile.downloadFile(myAPI, outputDirectory, createBsDir=createBsDir) except BaseSpaceException.ServerResponseException as e: retryIdx += 1 time.sleep(sleepTime) retryException = e else: break if retryIdx == numRetries: raise retryException print "Download complete."
print("\nThe app results also comes with a reference to our AppSession") myAppSession = appResults.AppSession print(myAppSession) # we can change the status of our AppSession and add a status-summary as follows myAppSession.setStatus(myAPI, 'needsattention', "We worked hard, but encountered some trouble.") print("\nAfter a change of status of the app sessions we get\n" + str(myAppSession)) # we'll set our appSession back to running so we can do some more work myAppSession.setStatus(myAPI, 'running', "Back on track") ### Let's list all AppResults again and see if our new object shows up appRes = p.getAppResults(myAPI, statuses=['Running']) print("\nThe updated app results are \n" + str(appRes)) appResult2 = myAPI.getAppResultById(appResults.Id) print(appResult2) ## Now we will make another AppResult ## and try to upload a file to it appResults2 = p.createAppResult(myAPI, "My second AppResult", "This one I will upload to") appResults2.uploadFile(myAPI, '/home/mkallberg/Desktop/testFile2.txt', 'BaseSpaceTestFile.txt', '/mydir/', 'text/plain') print("\nMy AppResult number 2 \n" + str(appResults2)) ## let's see if our new file made it appResultFiles = appResults2.getFiles(myAPI) print("\nThese are the files in the appResult") print(appResultFiles) f = appResultFiles[-1]