def _writeback_app_result_files(self):
"""
Writeback all files in output_files list, and create corresponding entries in the local db
"""
db = current.db
# get BaseSpace API
ssn_row = db(db.app_session.id==self.app_session_id).select().first()
user_row = db(db.auth_user.id==ssn_row.user_id).select().first()
app = db(db.app_data.id > 0).select().first()
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token)
app_result = bs_api.getAppResultById(self.app_result_num)
# upload files to BaseSpace
for f in self.output_files:
# currently not using a dir name to write to
bs_file = app_result.uploadFile(bs_api, f.local_path, f.file_name, '', 'text/plain')
# add file to local db
db.output_file.insert(app_result_id=f.app_result_id,
file_num=bs_file.Id,
file_name=f.file_name,
local_path=f.local_path,
file_type=f.file_type)
db.commit()
def update_status(self, local_status, message, bs_ssn_status=None):
"""
Update db with provided status and detailed message, and update BaseSpace App Session status if provided
"""
db = current.db
# update status in local db
self.status=local_status
self.message=message
ssn_row = db(db.app_session.id==self.app_session_id).select().first()
ssn_row.update_record(status=self.status, message=self.message)
db.commit()
# optionally update status of AppSession in BaseSpace -- limited to 128 chars
if bs_ssn_status:
# get BaseSpace API
ssn_row = db(db.app_session.id==self.app_session_id).select().first()
user_row = db(db.auth_user.id==ssn_row.user_id).select().first()
app = db(db.app_data.id > 0).select().first()
# handle exceptions in caller
bs_api = BaseSpaceAPI(app.client_id, app.client_secret, app.baseSpaceUrl, app.version, ssn_row.app_session_num, user_row.access_token)
app_result = bs_api.getAppResultById(self.app_result_num)
app_ssn = app_result.AppSession
app_ssn.setStatus(bs_api, bs_ssn_status, message[:128])
def upload(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), inputDirectory='\.', dryRun=False, numRetries=3):
'''
Creates an App Result and uploads files.
TODO
Provide an App Result identifier, and optionally regexes to include or omit files
based on their names (path not included). Omission takes precedence over inclusion.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param appResultId the BaseSpace App Result identifier
:param fileNameRegexesInclude a list of regexes on which to include files based on name
:param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include)
:param inputDirectory the root input directory
:param numRetries the number of retries for a single download API call
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 10000
sleepTime = 1.0
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT', clientKey=clientKey, clientSecret=clientSecret, AccessToken=accessToken)
# get the current user
user = myAPI.getUserById('current')
# get the app result
appResult = myAPI.getAppResultById(Id=appResultId)
appSession = appResult.AppSession
print "Uploading files to the App Result: " + str(appResult)
# Filter file names based on the include or omit regexes
includePatterns = [re.compile(pattern) for pattern in fileNameRegexesInclude]
omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit]
def includePatternMatch(f):
if not includePatterns:
return True
for pattern in includePatterns:
if pattern.match(f):
return True
return False
def omitPatternMatch(f):
if not omitPatterns:
return False
for pattern in omitPatterns:
if pattern.match(f):
return True
return False
def keepFile(f):
return includePatternMatch(f) and not omitPatternMatch(f)
# walk the current directory structure
for root, dirs, files in os.walk(inputDirectory):
for fileName in files:
localPath = os.path.join(root, fileName)
directory = root.replace(inputDirectory, "")
if AppResults.isBinaryContent(fileName):
contentType = 'application/octet-stream'
else:
contentType = 'text/plain'
if keepFile(fileName):
print "Uploading file: %s" % localPath
if not options.dryRun:
retryIdx = 0
retryException = None
while retryIdx < numRetries:
try:
appResult.uploadFile(api=myAPI, localPath=localPath, fileName=fileName, directory=directory, contentType=contentType)
except BaseSpaceException.ServerResponseException as e:
retryIdx += 1
time.sleep(sleepTime)
retryException = e
else:
break
if retryIdx == numRetries:
raise retryException
print "Upload complete"
print appResults.Id print "\nThe app results also comes with a reference to our AppSession" myAppSession = appResults.AppSession print myAppSession # we can change the status of our AppSession and add a status-summary as follows myAppSession.setStatus(myBaseSpaceAPI,'needsattention',"We worked hard, but encountered some trouble.") print "\nAfter a change of status of the app sessions we get\n" + str(myAppSession) # we'll set our appSession back to running so we can do some more work myAppSession.setStatus(myBaseSpaceAPI,'running',"Back on track") ### Let's list all AppResults again and see if our new object shows up appRes = p.getAppResults(myBaseSpaceAPI,statuses=['Running']) print "\nThe updated app results are \n" + str(appRes) appResult2 = myBaseSpaceAPI.getAppResultById(appResults.Id) print appResult2 ## Now we will make another AppResult ## and try to upload a file to it appResults2 = p.createAppResult(myBaseSpaceAPI,"My second AppResult","This one I will upload to") appResults2.uploadFile(myBaseSpaceAPI, '/home/mkallberg/Desktop/testFile2.txt', 'BaseSpaceTestFile.txt', '/mydir/', 'text/plain') print "\nMy AppResult number 2 \n" + str(appResults2) ## let's see if our new file made it appResultFiles = appResults2.getFiles(myBaseSpaceAPI) print "\nThese are the files in the appResult" print appResultFiles f = appResultFiles[-1] # we can even download our newly uploaded file
print appResults.Id print "\nThe app results also comes with a reference to our AppSession" myAppSession = appResults.AppSession print myAppSession # we can change the status of our AppSession and add a status-summary as follows myAppSession.setStatus(myAPI, "needsattention", "We worked hard, but encountered some trouble.") print "\nAfter a change of status of the app sessions we get\n" + str(myAppSession) # we'll set our appSession back to running so we can do some more work myAppSession.setStatus(myAPI, "running", "Back on track") ### Let's list all AppResults again and see if our new object shows up appRes = p.getAppResults(myAPI, statuses=["Running"]) print "\nThe updated app results are \n" + str(appRes) appResult2 = myAPI.getAppResultById(appResults.Id) print appResult2 ## Now we will make another AppResult ## and try to upload a file to it appResults2 = p.createAppResult(myAPI, "My second AppResult", "This one I will upload to") appResults2.uploadFile(myAPI, "/home/mkallberg/Desktop/testFile2.txt", "BaseSpaceTestFile.txt", "/mydir/", "text/plain") print "\nMy AppResult number 2 \n" + str(appResults2) ## let's see if our new file made it appResultFiles = appResults2.getFiles(myAPI) print "\nThese are the files in the appResult" print appResultFiles f = appResultFiles[-1] # we can even download our newly uploaded file
def download(clientKey=None,
clientSecret=None,
accessToken=None,
appResultId=None,
fileNameRegexesInclude=list(),
fileNameRegexesOmit=list(),
outputDirectory='\.',
createBsDir=True,
force=False,
numRetries=3):
'''
Downloads App Result files.
Provide an App Result identifier, and optionally regexes to include or omit files
based on their names (path not included). Omission takes precedence over inclusion.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param appResultId the BaseSpace App Result identifier
:param fileNameRegexesInclude a list of regexes on which to include files based on name
:param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include)
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
:param force use the force: overwrite existing files if true, false otherwise
:param numRetries the number of retries for a single download API call
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 10000
sleepTime = 1.0
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer,
apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
appResult = myAPI.getAppResultById(Id=appResultId)
print "Retrieving files from the App Result: " + str(appResult)
# Get all the files from the AppResult
filesToDownload = appResult.getFiles(myAPI,
queryPars=qp({'Limit':
fileLimit}))
# Filter file names based on the include or omit regexes
includePatterns = [
re.compile(pattern) for pattern in fileNameRegexesInclude
]
omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit]
def includePatternMatch(f):
if not includePatterns:
return True
for pattern in includePatterns:
if pattern.match(f):
return True
return False
def omitPatternMatch(f):
if not omitPatterns:
return False
for pattern in omitPatterns:
if pattern.match(f):
return True
return False
def keepFile(f):
return includePatternMatch(f) and not omitPatternMatch(f)
filesToDownload = [f for f in filesToDownload if keepFile(str(f))]
print "Will download %d files." % len(filesToDownload)
for i in range(len(filesToDownload)):
appResultFile = filesToDownload[i]
print 'Downloading (%d/%d): %s' % (
(i + 1), len(filesToDownload), str(appResultFile))
print "File Path: %s" % appResultFile.Path
if not options.dryRun:
outputPath = str(appResultFile.Path)
if not createBsDir:
outputPath = os.path.basename(outputPath)
if os.path.exists(outputPath):
if force:
print "Overwritting: %s" % outputPath
else:
print "Skipping existing file: %s" % outputPath
continue
else:
print "Downloading to: %s" % outputPath
retryIdx = 0
retryException = None
while retryIdx < numRetries:
try:
appResultFile.downloadFile(myAPI,
outputDirectory,
createBsDir=createBsDir)
except BaseSpaceException.ServerResponseException as e:
retryIdx += 1
time.sleep(sleepTime)
retryException = e
else:
break
if retryIdx == numRetries:
raise retryException
print "Download complete."
def download(clientKey=None, clientSecret=None, accessToken=None, appResultId=None, fileNameRegexesInclude=list(), fileNameRegexesOmit=list(), outputDirectory='\.', createBsDir=True, force=False, numRetries=3):
'''
Downloads App Result files.
Provide an App Result identifier, and optionally regexes to include or omit files
based on their names (path not included). Omission takes precedence over inclusion.
:param clientKey the Illumina developer app client key
:param clientSecret the Illumina developer app client secret
:param accessToken the Illumina developer app access token
:param appResultId the BaseSpace App Result identifier
:param fileNameRegexesInclude a list of regexes on which to include files based on name
:param fileNameRegexesOmit a list of regexes on which to omit files based on name (takes precedence over include)
:param outputDirectory the root output directory
:param createBsDir true to recreate the path structure within BaseSpace, false otherwise
:param force use the force: overwrite existing files if true, false otherwise
:param numRetries the number of retries for a single download API call
'''
appSessionId = ''
apiServer = 'https://api.basespace.illumina.com/' # or 'https://api.cloud-hoth.illumina.com/'
apiVersion = 'v1pre3'
fileLimit = 10000
sleepTime = 1.0
# init the API
if None != clientKey:
myAPI = BaseSpaceAPI(clientKey, clientSecret, apiServer, apiVersion, appSessionId, accessToken)
else:
myAPI = BaseSpaceAPI(profile='DEFAULT')
# get the current user
user = myAPI.getUserById('current')
appResult = myAPI.getAppResultById(Id=appResultId)
print "Retrieving files from the App Result: " + str(appResult)
# Get all the files from the AppResult
filesToDownload = appResult.getFiles(myAPI, queryPars=qp({'Limit' : fileLimit}))
# Filter file names based on the include or omit regexes
includePatterns = [re.compile(pattern) for pattern in fileNameRegexesInclude]
omitPatterns = [re.compile(pattern) for pattern in fileNameRegexesOmit]
def includePatternMatch(f):
if not includePatterns:
return True
for pattern in includePatterns:
if pattern.match(f):
return True
return False
def omitPatternMatch(f):
if not omitPatterns:
return False
for pattern in omitPatterns:
if pattern.match(f):
return True
return False
def keepFile(f):
return includePatternMatch(f) and not omitPatternMatch(f)
filesToDownload = [f for f in filesToDownload if keepFile(str(f))]
print "Will download %d files." % len(filesToDownload)
for i in range(len(filesToDownload)):
appResultFile = filesToDownload[i]
print 'Downloading (%d/%d): %s' % ((i+1), len(filesToDownload), str(appResultFile))
print "File Path: %s" % appResultFile.Path
if not options.dryRun:
outputPath = str(appResultFile.Path)
if not createBsDir:
outputPath = os.path.basename(outputPath)
if os.path.exists(outputPath):
if force:
print "Overwritting: %s" % outputPath
else:
print "Skipping existing file: %s" % outputPath
continue
else:
print "Downloading to: %s" % outputPath
retryIdx = 0
retryException = None
while retryIdx < numRetries:
try:
appResultFile.downloadFile(myAPI, outputDirectory, createBsDir=createBsDir)
except BaseSpaceException.ServerResponseException as e:
retryIdx += 1
time.sleep(sleepTime)
retryException = e
else:
break
if retryIdx == numRetries:
raise retryException
print "Download complete."
print("\nThe app results also comes with a reference to our AppSession")
myAppSession = appResults.AppSession
print(myAppSession)
# we can change the status of our AppSession and add a status-summary as follows
myAppSession.setStatus(myAPI, 'needsattention',
"We worked hard, but encountered some trouble.")
print("\nAfter a change of status of the app sessions we get\n" +
str(myAppSession))
# we'll set our appSession back to running so we can do some more work
myAppSession.setStatus(myAPI, 'running', "Back on track")
### Let's list all AppResults again and see if our new object shows up
appRes = p.getAppResults(myAPI, statuses=['Running'])
print("\nThe updated app results are \n" + str(appRes))
appResult2 = myAPI.getAppResultById(appResults.Id)
print(appResult2)
## Now we will make another AppResult
## and try to upload a file to it
appResults2 = p.createAppResult(myAPI, "My second AppResult",
"This one I will upload to")
appResults2.uploadFile(myAPI, '/home/mkallberg/Desktop/testFile2.txt',
'BaseSpaceTestFile.txt', '/mydir/', 'text/plain')
print("\nMy AppResult number 2 \n" + str(appResults2))
## let's see if our new file made it
appResultFiles = appResults2.getFiles(myAPI)
print("\nThese are the files in the appResult")
print(appResultFiles)
f = appResultFiles[-1]
