def getService(serviceName, apiVersion):
# For some reason, the drive API suggests 'v3' for the api version, but gmail wants 'v1'. Ok.
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
homeTempDirectory = getSaddlebagsDirectory()
#jsonFileLocation = join(homeTempDirectory, 'credentials.json')
jsonFileLocation = join(homeTempDirectory, 'client_secrets.json')
tokenPickleLocation = join(homeTempDirectory, 'token.pickle')
if os.path.exists(tokenPickleLocation):
with open(tokenPickleLocation, 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
jsonFileLocation, SCOPES)
creds = flow.run_local_server()
# Save the credentials for the next run
with open(tokenPickleLocation, 'wb') as token:
pickle.dump(creds, token)
service = build(serviceName, apiVersion, credentials=creds)
return service
def initializeLog():
logFileLocation = join(getSaddlebagsDirectory(), 'Saddlebags.Log.txt')
# If there is no "globals" yet, then I haven't loaded a config yet, lets default to 'DEBUG' level.
if (not ("globalVariables" in globals())
or getConfigurationValue('logging') is None):
logLevelText = 'DEBUG'
else:
logLevelText = getConfigurationValue('logging')
logLevel = getattr(logging, logLevelText.upper())
logFormatter = logging.Formatter(
"%(asctime)s:%(name)s:%(levelname)s:%(message)s")
rootLogger = logging.getLogger()
# Remove handlers, It's easiest for me to add my own.
rootLogger.handlers = []
rootLogger.setLevel(logLevel)
# Add the Stream Handler to print to console.
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
# Add the File Handler to log to a file.
fileHandler = logging.FileHandler(logFileLocation)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
def getService(serviceName, apiVersion):
# For some reason, the drive API suggests 'v3' for the api version, but gmail wants 'v1'. Ok.
creds = None
# The file token.pickle stores the user's access and refresh tokens, and is
# created automatically when the authorization flow completes for the first
# time.
homeTempDirectory = getSaddlebagsDirectory()
#jsonFileLocation = join(homeTempDirectory, 'credentials.json')
jsonFileLocation = join(homeTempDirectory, 'client_secrets.json')
tokenPickleLocation = join(homeTempDirectory, 'token.pickle')
if os.path.exists(tokenPickleLocation):
with open(tokenPickleLocation, 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
jsonFileLocation, SCOPES)
creds = flow.run_local_server()
# Save the credentials for the next run
with open(tokenPickleLocation, 'wb') as token:
pickle.dump(creds, token)
service = build(serviceName, apiVersion, credentials=creds)
return service
def performFullSubmission(submissionText):
logging.info('Uploading Submission to ENA')
# Determine a working directory. Folder underneath executable called temp.
try:
workingDirectory = getSaddlebagsDirectory()
logging.info('I can work in this directory:' + workingDirectory)
if not isdir(workingDirectory):
logging.info('Making Directory:' + workingDirectory)
makedirs(workingDirectory)
except Exception:
logging.error ('Cannot Initialize Working Directory')
logging.error (exc_info())
messagebox.showinfo('Working Directory Error',
'Sorry, I failed to create this working directory:\n'
+ str(workingDirectory)
+ '\n and I cannot continue.\nMaybe this is a '
+ 'permissions issue, are these folders read only?\n'
+ str(exc_info()[1]))
return
enaUsername = getConfigurationValue('ena_username')
enaPassword = getConfigurationValue('ena_password')
if(enaUsername is None
or len(enaUsername) < 1
or enaPassword is None
or len(enaPassword) < 1):
messagebox.showinfo('Missing Login Credentials',
'You must provide ENA username and password.\n'
'Please use the "Submission Options" button.')
logging.error('Missing ENA Username or Password.' + '\n')
return
else:
logging.info('ENA Username and Password exist.' + '\n')
useTestServers = (int(getConfigurationValue('test_submission')) == 1)
# Are you sure?
if useTestServers:
logging.info('Using Test ENA Server.' + '\n')
result = messagebox.askquestion("Submit to TEST / DEMO environment", "You are about to submit a sequence to the\n\nTEST / DEMO ENA environment.\n\nAre You Sure?", icon='warning')
else:
logging.info('Using Production ENA Server.' + '\n')
result = messagebox.askquestion("Submit to LIVE / PROD environment", "You are about to submit a sequence to the\n\nLIVE / PROD ENA environment.\n\nAre You Sure?", icon='warning')
if result == 'yes':
pass
else:
return
# TODO: Existing project? Maybe I should check if the study/project exists, before I get started
# Give my submission a filename. SOmething with a datetime stamp
try:
# This includes a "seconds" measure, should be pretty unique.
dateTimeNow = '{:%Y_%m_%d_%H_%M_%S}'.format(datetime.now())
submissionShortFileName = 'HLA_Submission_' + dateTimeNow + '.txt'
submissionFileName = join(workingDirectory, submissionShortFileName)
zippedShortFileName = submissionShortFileName + '.gz'
zippedFileName = join(workingDirectory, zippedShortFileName)
md5FileName = zippedFileName + '.md5'
#submissionText = self.submOutputGuiObject.get('1.0', 'end')
outputFileObject = open(submissionFileName, 'w')
outputFileObject.write(submissionText)
logging.info('Submission Text:\n' + submissionText)
outputFileObject.close()
except Exception:
logging.error ('Cannot Write Submission Flatfile')
logging.error (exc_info())
messagebox.showinfo('Cannot Write Submission Flatfile',
'Sorry, I failed to create the submission file:\n'
+ str(submissionText)
+ '\n and I cannot continue.\nMaybe this is a '
+ 'permissions issue, are these folders read only?\n'
+ str(exc_info()[1]))
logging.error('Failure to create submission file:' + str(exc_info()[1]) + '\n')
return
logging.info('Submission file was created:\n' + str(submissionFileName) + '\n')
# gzip the submission file. Make a gz file.
try:
#zippedFileName = submissionFileName + '.gz'
with open(submissionFileName, 'rb') as fileIn, gzipOpen(zippedFileName, 'wb') as fileOut:
copyfileobj(fileIn, fileOut)
except Exception:
logging.error ('Cannot Compress Submission File')
logging.error (exc_info())
messagebox.showinfo('Cannot Compress Submission File',
'Sorry, I failed to compress the submission file:\n'
+ str(zippedFileName)
+ '\n and I cannot continue.\n'
+ str(exc_info()[1]))
logging.error('Failure to create zip file:' + str(exc_info()[1]) + '\n')
return
logging.info('Zip file was created:\n' + str(zippedFileName) + '\n')
# Calculate an MD5SUM
try:
#md5FileName = zippedFileName + '.md5'
md5HashValue = writeMd5(zippedFileName,md5FileName)
except Exception:
logging.error ('Cannot Calculate MD5')
logging.error (exc_info())
messagebox.showinfo('Cannot Calculate an Md5 checksum',
'Sorry, I failed to calculate an md5 checksum\nand I cannot continue.\n'
+ str(exc_info()[1]))
logging.error('Failure to create zip file:' + str(exc_info()[1]) + '\n')
return
logging.info('md5 file was created:\n' + str(md5FileName) + '\n')
# Use FTP to send the file to ENA
try:
if useTestServers:
ftpServerAddress = getConfigurationValue('ena_ftp_upload_site_test')
else:
ftpServerAddress = getConfigurationValue('ena_ftp_upload_site_prod')
#logging.info ('attempting to open ftp connection')
ftp = FTP(ftpServerAddress)
ftp.login(getConfigurationValue('ena_username'), getConfigurationValue('ena_password'))
ftp.storbinary('STOR ' + '/' + split(zippedFileName)[1], open(zippedFileName, 'rb'), 1024)
ftp.storbinary('STOR ' + '/' + split(md5FileName)[1], open(md5FileName, 'rb'), 1024)
ftp.close()
# is that it? Easy.
except Exception:
logging.error ('Cannot Upload to FTP site')
logging.error (exc_info())
messagebox.showinfo('Cannot Upload to FTP site',
'Sorry, I failed to upload your submission files to the ENA FTP site\nand I cannot continue.\n'
+ str(exc_info()[1]))
logging.error('Failure to upload to FTP site:' + str(exc_info()[1]) + '\n')
return
logging.info('Submission and MD5 successfully uploaded.\n')
# Handle the new project
# effectively, study = project
# existing study = 1, new study = 2
newProject = (getConfigurationValue('choose_project') == '2')
if newProject:
# Generate Project and Project Submission XML Files
try:
projectFileName = join(workingDirectory, 'project.xml')
projectText = createProjectXML(projectFileName)
#logging.info ('My project text looks like this:\n' + projectText)
projectSubmissionFileName = join(workingDirectory, 'project_submission.xml')
projectSubmissionText = createProjectSubmissionXML(projectSubmissionFileName
,'proj_sub_' + dateTimeNow
,'project.xml')
#logging.info('I made this project text:\n' + projectText)
#logging.info('I made this project submission text:\n' + projectSubmissionText)
except Exception:
logging.error ('Cannot Create Project Submission XML')
logging.error (exc_info())
messagebox.showinfo('Cannot Create Project Submission XML',
'Sorry, I failed to create a project XML file\nand I cannot continue.\n'
+ str(exc_info()[1]))
logging.error('Failure to create project submission file:' + str(exc_info()[1]) + '\n')
return
logging.info('Project Submission XML files were created.\n')
logging.info('Project Text:\n' + projectText)
logging.info('Project Submission Text:\n' + projectText)
# Use REST to submit this project
try:
# Return value should be a tuple:
# (Success, ProjectAccession, Messages[])
(projectSubmissionSuccess, projectAccessionNumber, projectErrorMessages) = performProjectSubmission(projectSubmissionFileName,projectFileName)
if(projectSubmissionSuccess):
# Great. The project was created successfully.
# Lets use this new study accession moving forward.
assignConfigurationValue('study_accession', projectAccessionNumber)
assignConfigurationValue('choose_project','1')
pass
else:
messageText = ('There was a problem in the Project Submission.\n'
+ 'I cannot continue.\n'
+ 'These messages were reported by ENA:\n')
for errorMessage in projectErrorMessages:
messageText += ('\n' + errorMessage + '\n')
messagebox.showinfo('Cannot Submit Project XML via REST', messageText)
logging.error('Failure to submit project submission file:' + str(exc_info()[1]) + '\n' + messageText + '\n')
return
except Exception:
logging.error ('Cannot Submit Project XML')
logging.error (exc_info())
messagebox.showinfo('Cannot Submit Project XML',
'Sorry, I failed to submit the project XML file\nand I cannot continue.\n'
+ str(exc_info()[1]))
logging.error('Failure to upload project submission file:' + str(exc_info()[1]) + '\n')
return
logging.info('New study has been uploaded, accession:' + str(getConfigurationValue('study_accession')) + '\n')
# existing project, we will use the supplied accession #
else:
logging.info('Using existing study accession:' + str(getConfigurationValue('study_accession')) + '\n')
# projectAccessionNumber = getConfigurationValue('study_accession')
pass
# Generate Analysis and Analysis Submission xmls
try:
analysisFileName = join(workingDirectory, 'analysis.xml')
analysisText = createAnalysisXML(analysisFileName, md5HashValue, zippedShortFileName)
analysisSubmissionFileName = join(workingDirectory, 'analysis_submission.xml')
analysisSubmissionText = createAnalysisSubmissionXML(analysisSubmissionFileName
,'analysis_sub_' + dateTimeNow
,'analysis.xml')
except Exception:
logging.error('Cannot Create Analysis Submission XML')
logging.error (exc_info())
messagebox.showinfo('Cannot Create Analysis Submission XML',
'Sorry, I failed to create a Analysis XML file\nand I cannot continue.\n'
+ str(exc_info()[1]))
logging.error('Failure to create analysis submission file:' + str(exc_info()[1]) + '\n')
return
logging.info('Analysis Submission XML files were created.\n')
# Use REST to submit this analysis
try:
# Return value should be a tuple:
# (Success, analysisAccessionNumber, Messages[])
(analysisSubmissionSuccess, analysisAccessionNumber, analysisErrorMessages) = performAnalysisSubmission(analysisSubmissionFileName,analysisFileName)
if(analysisSubmissionSuccess):
# Great. The analysis was created successfully.
pass
else:
messageText = ('There was a problem in the Analysis Submission.\n'
+ 'I cannot continue.\n'
+ 'These messages were reported by ENA:\n')
for errorMessage in analysisErrorMessages:
messageText += ('\n' + errorMessage + '\n')
messagebox.showinfo('Cannot Submit Analysis XML via REST', messageText)
logging.error('Failure to submit analysis submission file:' + str(exc_info()[1]) + '\n')
return
except Exception:
logging.error ('Cannot Submit Analysis XML')
logging.error (exc_info())
messagebox.showinfo('Cannot Submit Analysis XML via REST',
'Sorry, I failed to submit the analysis XML file\nand I cannot continue.\n'
+ str(exc_info()[1]))
return
logging.info('New analysis has been Uploaded, accession:' + str(analysisAccessionNumber) + '\n')
# Popup message with Results
messagebox.showinfo('Success uploading submission to ENA.',
'The sequence and analysis was uploaded to EMBL ENA Successfully.\n\n'
+ 'For your reference:\n\n'
+ 'You can use this Project/Study accession\nnumber on future submissions:\n'
+ 'Study Accession:' + str(getConfigurationValue('study_accession') + '\n\n')
+ 'Use the Analysis Accession number if you\ncontact EMBL regarding this\nsequence submission:\n'
+ 'Analysis Accession:' + str(analysisAccessionNumber) + '\n\n'
+ 'Find your submission files here:\n'
+ workingDirectory + '\n\n'
+ 'If EMBL successfully validates your sequence, you will\n'
+ 'receive an email with an ENA Sequence accession number.\n'
+ 'This accession number is necessary for IPD-IMGT/HLA submission.\n'
+ 'Contact EMBL Support with your\nAnalysis Accession # if it has been\nmore than 48 hours since submission.\n'
)
def performFullEnaSubmission(submission, submissionBatch):
logging.info('Performing an EMBL/ENA Submission.')
submissionText = submission.enaSubmissionText
# This includes a "seconds" measure, should be pretty unique.
# TODO: I could add milliseconds to make this more unique.
# TODO: I'm worried about batch submissions, is it a problem if the submission files have the same name? I think it's not a problem.
dateTimeNow = '{:%Y_%m_%d_%H_%M_%S_%f}'.format(datetime.now())
# TODO: If the submissionText is None, generate a submission now.
if (submissionText is None or len(submissionText) < 5):
logging.error(
'Oh no, there is no submission text. I probably forgot to generate a submission. Fix this bug please!'
)
#generate a submission text now......
if (checkENAPrerequisites):
useTestServers = (int(getConfigurationValue('test_submission')) == 1)
sequenceName = submission.localAlleleName
# Are you sure? Test or Live?
if useTestServers:
logging.info('Using Test ENA Server.' + '\n')
result = showYesNoBox(
"Submit to TEST / DEMO environment",
"You are about to submit " + str(sequenceName) +
" to the\n\nTEST / DEMO ENA environment.\n\nAre You Sure?")
else:
logging.info('Using Production ENA Server.' + '\n')
result = showYesNoBox(
"Submit to LIVE / PROD environment",
"You are about to submit " + str(sequenceName) +
" to the\n\nLIVE / PROD ENA environment.\n\nAre You Sure?")
if result:
pass
else:
logging.error(
'Submission aborted by the user, because we do not want to submit to the Test/Live server'
)
return
# set some parameters real quick.
jarFileLocation = findJarFile()
logging.info('I will use this webin cli jar file:' +
str(jarFileLocation))
saddlebagsDirectory = getSaddlebagsDirectory()
workingDirectory = join(saddlebagsDirectory, 'submission_temp')
logging.info('I\'m working in this directory:' + str(workingDirectory))
enaUserName = submissionBatch.enaUserName
enaPassword = submissionBatch.enaPassword
# Check the credentials, do they look okay?
if (enaUserName is None or len(enaUserName) < 1):
logging.warning('Missing ENA Username.' + '\n')
enaUserName = getInfoBox(
"ENA Username Please",
"You must provide ENA Username for submission.")
submissionBatch.enaUserName = enaUserName
else:
logging.info('ENA Username ok.' + '\n')
if (enaPassword is None or len(enaPassword) < 1):
logging.warning('Missing ENA Password.' + '\n')
enaPassword = getInfoBox(
"ENA Password Please",
"You must provide ENA Password for user " + enaUserName +
" for submission.\nSaddlebags will not store your password anywhere."
)
submissionBatch.enaPassword = enaPassword
else:
logging.info('ENA Password look ok.' + '\n')
# Submission is divided into 3 stages (https://ena-docs.readthedocs.io/en/latest/general-guide/webin-cli.html)
# Stage 1 - Register Study (Registering a Sample is not necessary for submitting a sequence)
registerStudy(submissionBatch, workingDirectory, dateTimeNow)
# Stage 2 - Prepare Files
prepareSubmissionFiles(submission, submissionBatch, workingDirectory,
dateTimeNow)
# Stage 3 - Validate and Submit Files
validateAndSubmit(submission, submissionBatch, workingDirectory,
dateTimeNow)
# Report Results
# Send a summary of what was submitted.
# Delete the files.
# reportAndCleanup()
# popup message with results.
# Maybe: delete working directory, but the files first. Maybe delete output files?
else:
# TODO: Handle this better, tell the user somehow.
logging.error(
'ENA Submission Requirements are not met. Details: (TODO)')
def uploadZipToIpdHla(zipFileName):
"""Shows basic usage of the Drive v3 API.
Prints the names and ids of the first 10 files the user has access to.
"""
print('Uploading a zip file to IMGT/HLA:' + str(zipFileName))
homeTempDirectory = getSaddlebagsDirectory()
zipFileFullPath = join(homeTempDirectory, zipFileName)
#jsonFileLocation = join(homeTempDirectory, 'credentials.json')
#tokenPickleLocation = join(homeTempDirectory, 'token.pickle')
service = getGoogleDriveService()
# Call the Drive v3 API
# Get a list of 10 files.
results = service.files().list(
pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
# list the first 10 files.
if not items:
print('No files found.')
else:
print('Files:')
for item in items:
print(u'{0} ({1})'.format(item['name'], item['id']))
# TODO: Check if the file with this name already exists.
# Conversely, name the zip file with a timestamp. This is risky because
# we might accidentally send dupe zip files.
# TODO: Detect the "Saddlebags" folder name from the list of files I have access to.
# Conversely - Ask IPD to setup the upload folder.
# They will send the upload folder directory.
# Dominic suggests, I set up the initial configuration for the IPD upload.
# IPD will set up my account for bulk submissions.
# IPD sets up a shared google drive folder, which they share with a google email that I provide.
# TODO: Send an Email to ipd submissions that a file has been uploaded.
# Use google interface to do this again.
# put (Test) in the email subject.
# This folder is shared with mumc.tissuetyping.
#folder_id = '1haV3w8DYawFKo0V4-pwwXoai4JUFaqeT'
# This folder is in the personal drive of mumc.transplantationimmunology
folder_id = '1Yw6Skb_ZsQn2kCbYUjo_zTnjSf-jstxX'
print('zip file full path:' + zipFileFullPath)
# Lets try to upload the file
file_metadata = {'name': zipFileName, 'parents': [folder_id]}
media = MediaFileUpload(zipFileFullPath, mimetype='application/zip')
file = service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
print('File ID: %s' % file.get('id'))
# TODO: Somehow export the gui stuff to the GUI. Popups shouldn't happen in here.
# Popup a query. Do you want to send an email to ipd submissions?
# If yes, send an email.
# Subject = Saddlebags HLA Allele Submission
# Subject = (TEST) Saddlebags HLA Allele Submission
# Body = "IPD User (username) has uploade the file "HLASubmission.zip"
# to the shared folder "Saddlebags" (with the folder id (X))
# Body += "Since this is a test submission, this file is safe to remove."
emailSubject = '(TEST) Saddlebags HLA Allele Submission'
emailBody = ('IPD user (username) has just uploaded the file ' +
zipFileName + '\n' + 'It was placed into the folder ' +
'Saddlebags' + ' with a google folder resource id of ' +
folder_id)
if (True):
emailBody += '\nSince this is a test submission, you are free to remove the file.'
# or should the sender be "me"...maybe it's silly to pass this information in.
# I reckon I could hard-code the target email address. not sure what it is right now, [email protected]? I made that up.
sendEmail(emailSubject, emailBody, '*****@*****.**', 'me')
def assignConfigName():
# Join together the working directory, a subfolder called "saddlebags", and the config name.
assignConfigurationValue(
'config_file_location',
join(getSaddlebagsDirectory(), 'Saddlebags.Config.xml'))
def uploadZipToIpdHla(zipFileName):
"""Shows basic usage of the Drive v3 API.
Prints the names and ids of the first 10 files the user has access to.
"""
print ('Uploading a zip file to IMGT/HLA:' + str(zipFileName))
homeTempDirectory = getSaddlebagsDirectory()
zipFileFullPath = join(homeTempDirectory, zipFileName)
#jsonFileLocation = join(homeTempDirectory, 'credentials.json')
#tokenPickleLocation = join(homeTempDirectory, 'token.pickle')
service = getGoogleDriveService()
# Call the Drive v3 API
# Get a list of 10 files.
results = service.files().list(
pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get('files', [])
# list the first 10 files.
if not items:
print('No files found.')
else:
print('Files:')
for item in items:
print(u'{0} ({1})'.format(item['name'], item['id']))
# TODO: Check if the file with this name already exists.
# Conversely, name the zip file with a timestamp. This is risky because
# we might accidentally send dupe zip files.
# TODO: Detect the "Saddlebags" folder name from the list of files I have access to.
# Conversely - Ask IPD to setup the upload folder.
# They will send the upload folder directory.
# Dominic suggests, I set up the initial configuration for the IPD upload.
# IPD will set up my account for bulk submissions.
# IPD sets up a shared google drive folder, which they share with a google email that I provide.
# TODO: Send an Email to ipd submissions that a file has been uploaded.
# Use google interface to do this again.
# put (Test) in the email subject.
# This folder is shared with mumc.tissuetyping.
#folder_id = '1haV3w8DYawFKo0V4-pwwXoai4JUFaqeT'
# This folder is in the personal drive of mumc.transplantationimmunology
folder_id = '1Yw6Skb_ZsQn2kCbYUjo_zTnjSf-jstxX'
print('zip file full path:' + zipFileFullPath)
# Lets try to upload the file
file_metadata = {'name': zipFileName,
'parents': [folder_id]}
media = MediaFileUpload(zipFileFullPath,
mimetype='application/zip')
file = service.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
print('File ID: %s' % file.get('id'))
# TODO: Somehow export the gui stuff to the GUI. Popups shouldn't happen in here.
# Popup a query. Do you want to send an email to ipd submissions?
# If yes, send an email.
# Subject = Saddlebags HLA Allele Submission
# Subject = (TEST) Saddlebags HLA Allele Submission
# Body = "IPD User (username) has uploade the file "HLASubmission.zip"
# to the shared folder "Saddlebags" (with the folder id (X))
# Body += "Since this is a test submission, this file is safe to remove."
emailSubject = '(TEST) Saddlebags HLA Allele Submission'
emailBody = ('IPD user (username) has just uploaded the file ' + zipFileName + '\n' +
'It was placed into the folder ' + 'Saddlebags' + ' with a google folder resource id of ' + folder_id)
if(True):
emailBody += '\nSince this is a test submission, you are free to remove the file.'
# or should the sender be "me"...maybe it's silly to pass this information in.
# I reckon I could hard-code the target email address. not sure what it is right now, [email protected]? I made that up.
sendEmail(emailSubject, emailBody, '*****@*****.**', 'me')