def HandleFullSizeResumableUpload(self, existingDatafile):
"""
If the existing unverified DataFile upload is the correct size
in staging, then we can request its verification, but no upload
is needed.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.verificationModel\
.SetMessage("Found unverified full-size datafile "
"on staging server.")
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
self.verificationsModel.MessageUpdated(self.verificationModel)
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if existingDatafile and not self.testRun:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=self.foldersController
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def GetOrCreateExperimentForFolder(folderModel, testRun=False):
"""
See also GetExperimentForFolder, CreateExperimentForFolder
"""
try:
existingExperiment = \
ExperimentModel.GetExperimentForFolder(folderModel)
if testRun:
message = "ADDING TO EXISTING EXPERIMENT FOR FOLDER: %s\n" \
" URL: %s/%s\n" \
" Title: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
folderModel.settingsModel.GetMyTardisUrl(),
existingExperiment.GetViewUri(),
existingExperiment.GetTitle(),
folderModel.GetOwner().GetUsername())
logger.testrun(message)
return existingExperiment
except DoesNotExist, err:
if err.GetModelClass() == ExperimentModel:
return ExperimentModel.CreateExperimentForFolder(folderModel,
testRun)
else:
raise
def GetOrCreateExperimentForFolder(folderModel, testRun=False):
"""
See also GetExperimentForFolder, CreateExperimentForFolder
"""
try:
existingExperiment = \
ExperimentModel.GetExperimentForFolder(folderModel)
if testRun:
message = "ADDING TO EXISTING EXPERIMENT FOR FOLDER: %s\n" \
" URL: %s/%s\n" \
" Title: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
folderModel.settingsModel.GetMyTardisUrl(),
existingExperiment.GetViewUri(),
existingExperiment.GetTitle(),
folderModel.GetOwner().GetUsername())
logger.testrun(message)
return existingExperiment
except DoesNotExist, err:
if err.GetModelClass() == ExperimentModel:
return ExperimentModel.CreateExperimentForFolder(
folderModel, testRun)
else:
raise
def HandleUnresumableUpload(self, existingDatafile):
"""
We found an unverified datafile on the server for which
there is no point in checking for a resumable partial
upload.
This is usually because we are uploading using the POST upload method.
Or we could be using the STAGING method but failed to find any
DataFileObjects on the server for the datafile.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
logger.debug("Found unverified datafile record for \"%s\" "
"on MyTardis." % dataFilePath)
self.verificationModel.SetMessage("Found unverified datafile record.")
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE
else:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_NO_DFOS)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_NO_DFOS
self.verificationsModel.MessageUpdated(self.verificationModel)
if existingDatafile and not self.testRun:
if existingDatafile.GetMd5Sum() == \
self.settingsModel.GetFakeMd5Sum():
logger.warning("MD5(%s): %s" %
(dataFilePath, existingDatafile.GetMd5Sum()))
else:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=eventId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def StartDataUploadsForFolderWorker():
"""
Start the data uploads in a dedicated thread.
"""
logger.debug("Starting run() method for thread %s" %
threading.current_thread().name)
logger.debug("StartDataUploadsForFolderWorker")
wx.CallAfter(BeginBusyCursorIfRequired)
message = "Checking for data files on MyTardis and uploading " \
"if necessary for folder: %s" % event.folderModel.GetFolder()
logger.info(message)
app = wx.GetApp()
if app.TestRunRunning():
logger.testrun(message)
app.DisableTestAndUploadToolbarButtons()
wx.GetApp().SetPerformingLookupsAndUploads(True)
app.foldersController.StartUploadsForFolder(event.folderModel)
wx.CallAfter(EndBusyCursorIfRequired, event)
def UploadDatafile(self, event):
"""
Called in response to didntFindDatafileOnServerEvent or
unverifiedDatafileOnServerEvent.
This method runs in the main thread, so it shouldn't do anything
time-consuming or blocking, unless it launches another thread.
Because this method adds upload tasks to a queue, it is important
to note that if the queue has a maxsize set, then an attempt to
add something to the queue could block the GUI thread, making the
application appear unresponsive.
"""
folderModel = event.folderModel
dfi = event.dataFileIndex
existingUnverifiedDatafile = \
getattr(event, "existingUnverifiedDatafile", False)
if self.testRun:
if existingUnverifiedDatafile:
message = "NEEDS RE-UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
else:
message = "NEEDS UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
self.uploadsAcknowledged += 1
logger.testrun(message)
self.CountCompletedUploadsAndVerifications(event=None)
return
if folderModel not in self.uploadDatafileRunnable:
self.uploadDatafileRunnable[folderModel] = {}
bytesUploadedPreviously = getattr(event, "bytesUploadedPreviously",
None)
verificationModel = getattr(event, "verificationModel", None)
self.uploadDatafileRunnable[folderModel][dfi] = \
UploadDatafileRunnable(self, self.foldersModel, folderModel,
dfi, self.uploadsModel,
self.settingsModel,
existingUnverifiedDatafile,
verificationModel,
bytesUploadedPreviously)
self.uploadsQueue.put(self.uploadDatafileRunnable[folderModel][dfi])
self.CountCompletedUploadsAndVerifications(event=None)
def UploadDatafile(self, event):
"""
Called in response to didntFindDatafileOnServerEvent or
unverifiedDatafileOnServerEvent.
This method runs in the main thread, so it shouldn't do anything
time-consuming or blocking, unless it launches another thread.
Because this method adds upload tasks to a queue, it is important
to note that if the queue has a maxsize set, then an attempt to
add something to the queue could block the GUI thread, making the
application appear unresponsive.
"""
folderModel = event.folderModel
dfi = event.dataFileIndex
existingUnverifiedDatafile = \
getattr(event, "existingUnverifiedDatafile", False)
if self.testRun:
if existingUnverifiedDatafile:
message = "NEEDS RE-UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
else:
message = "NEEDS UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
self.uploadsAcknowledged += 1
logger.testrun(message)
self.CountCompletedUploadsAndVerifications(event=None)
return
if folderModel not in self.uploadDatafileRunnable:
self.uploadDatafileRunnable[folderModel] = {}
bytesUploadedPreviously = getattr(event, "bytesUploadedPreviously", None)
verificationModel = getattr(event, "verificationModel", None)
self.uploadDatafileRunnable[folderModel][dfi] = \
UploadDatafileRunnable(self, self.foldersModel, folderModel,
dfi, self.uploadsModel,
self.settingsModel,
existingUnverifiedDatafile,
verificationModel,
bytesUploadedPreviously)
self.uploadsQueue.put(self.uploadDatafileRunnable[folderModel][dfi])
self.CountCompletedUploadsAndVerifications(event=None)
def HandleExistingVerifiedDatafile(self):
"""
Found existing verified file on server.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.foundVerifiedDatafileEvent(
id=self.foldersController.EVT_FOUND_VERIFIED_DATAFILE,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND VERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def HandleExistingVerifiedDatafile(self):
"""
Found existing verified file on server.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.foundVerifiedDatafileEvent(
id=self.foldersController.EVT_FOUND_VERIFIED_DATAFILE,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND VERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def StartDataUploadsForFolderWorker():
"""
Start the data uploads in a dedicated thread.
"""
logger.debug("Starting run() method for thread %s"
% threading.current_thread().name)
logger.debug("StartDataUploadsForFolderWorker")
wx.CallAfter(BeginBusyCursorIfRequired)
message = "Checking for data files on MyTardis and uploading " \
"if necessary for folder: %s" % event.folderModel.GetFolder()
logger.info(message)
app = wx.GetApp()
if app.TestRunRunning():
logger.testrun(message)
app.DisableTestAndUploadToolbarButtons()
wx.GetApp().SetPerformingLookupsAndUploads(True)
app.foldersController.StartUploadsForFolder(
event.folderModel)
wx.CallAfter(EndBusyCursorIfRequired, event)
def HandleUnresumableUpload(self, existingDatafile):
"""
We found an unverified datafile on the server for which
there is no point in checking for a resumable partial
upload.
This is usually because we are uploading using the POST upload method.
Or we could be using the STAGING method but failed to find any
DataFileObjects on the server for the datafile.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
logger.debug("Found unverified datafile record for \"%s\" "
"on MyTardis." % dataFilePath)
self.verificationModel.SetMessage("Found unverified datafile record.")
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE
else:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_NO_DFOS)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_NO_DFOS
self.verificationsModel.MessageUpdated(self.verificationModel)
if existingDatafile and not self.testRun:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=eventId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def HandleFullSizeResumableUpload(self, existingDatafile):
"""
If the existing unverified DataFile upload is the correct size
in staging, then we can request its verification, but no upload
is needed.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.verificationModel\
.SetMessage("Found unverified full-size datafile "
"on staging server.")
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
self.verificationsModel.MessageUpdated(self.verificationModel)
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if existingDatafile and not self.testRun:
if existingDatafile.GetMd5Sum() == \
self.settingsModel.GetFakeMd5Sum():
logger.warning("MD5(%s): %s" %
(dataFilePath, existingDatafile.GetMd5Sum()))
else:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=self.foldersController
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def ShutDownUploadThreads(self, event=None):
# pylint: disable=too-many-branches
if self.IsShuttingDown():
return
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
if not wx.GetApp().PerformingLookupsAndUploads():
EndBusyCursorIfRequired()
return
self.SetShuttingDown(True)
message = "Shutting down upload threads..."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if hasattr(event, "failed") and event.failed:
self.SetFailed()
self.uploadsModel.CancelRemaining()
elif hasattr(event, "completed") and event.completed:
self.SetCompleted()
else:
self.SetCanceled()
self.uploadsModel.CancelRemaining()
logger.debug("Shutting down FoldersController upload worker threads.")
for _ in range(self.numUploadWorkerThreads):
self.uploadsQueue.put(None)
for thread in self.uploadWorkerThreads:
thread.join()
logger.debug("Shutting down FoldersController verification "
"worker threads.")
for _ in range(self.numVerificationWorkerThreads):
self.verificationsQueue.put(None)
for thread in self.verificationWorkerThreads:
thread.join()
self.verifyDatafileRunnable = {}
self.uploadDatafileRunnable = {}
if sys.platform == 'darwin':
sshControlMasterPool = \
OPENSSH.GetSshControlMasterPool(createIfMissing=False)
if sshControlMasterPool:
sshControlMasterPool.ShutDown()
if self.testRun:
numVerificationsCompleted = \
self.verificationsModel.GetCompletedCount()
numVerifiedUploads = \
self.verificationsModel.GetFoundVerifiedCount()
numFilesNotFoundOnServer = \
self.verificationsModel.GetNotFoundCount()
numFullSizeUnverifiedUploads = \
self.verificationsModel.GetFoundUnverifiedFullSizeCount()
numIncompleteUploads = \
self.verificationsModel.GetFoundUnverifiedNotFullSizeCount()
numFailedLookups = self.verificationsModel.GetFailedCount()
logger.testrun("")
logger.testrun("SUMMARY")
logger.testrun("")
logger.testrun("Files looked up on server: %s" %
numVerificationsCompleted)
logger.testrun("Files verified on server: %s" % numVerifiedUploads)
logger.testrun("Files not found on server: %s" %
numFilesNotFoundOnServer)
logger.testrun("Files unverified (but full size) on server: %s" %
numFullSizeUnverifiedUploads)
logger.testrun("Files unverified (and incomplete) on server: %s" %
numIncompleteUploads)
logger.testrun("Failed lookups: %s" % numFailedLookups)
logger.testrun("")
if self.Failed():
message = "Data scans and uploads failed."
elif self.Canceled():
message = "Data scans and uploads were canceled."
elif self.uploadsModel.GetFailedCount() > 0:
message = \
"Data scans and uploads completed with " \
"%d failed upload(s)." % self.uploadsModel.GetFailedCount()
elif self.Completed():
message = "Data scans and uploads completed successfully."
else:
message = "Data scans and uploads appear to have " \
"completed successfully."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if self.testRun:
logger.testrun(message)
app = wx.GetApp()
if hasattr(app, "toolbar"):
app.EnableTestAndUploadToolbarButtons()
app.SetShouldAbort(False)
if self.testRun:
app.testRunFrame.saveButton.Enable()
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
wx.GetApp().SetPerformingLookupsAndUploads(False)
self.SetShuttingDown(False)
if hasattr(app, "SetTestRunRunning"):
app.SetTestRunRunning(False)
EndBusyCursorIfRequired()
logger.debug("")
def CreateDatasetIfNecessary(folderModel, testRun=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
description = folderModel.GetFolder()
settingsModel = folderModel.settingsModel
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
experiment = folderModel.GetExperiment()
if experiment: # Could be None in test run
url = myTardisUrl + "/api/v1/dataset/?format=json" + \
"&experiments__id=" + str(experiment.GetId())
url = url + "&description=" + urllib.quote(description)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
response = requests.get(headers=headers, url=url)
existingMatchingDatasets = response.json()
numExistingMatchingDatasets = \
existingMatchingDatasets['meta']['total_count']
if numExistingMatchingDatasets == 1:
logger.debug("Found existing dataset for folder " + description)
elif numExistingMatchingDatasets > 1:
logger.debug("WARNING: Found multiple datasets for folder " +
description)
else:
numExistingMatchingDatasets = 0
if numExistingMatchingDatasets == 0:
logger.debug("Creating dataset record for folder: " + description)
description = folderModel.GetFolder()
if experiment:
experimentUri = experiment.GetResourceUri()
else:
experimentUri = None
immutable = False
datasetJson = {
"instrument": settingsModel.GetInstrument().GetResourceUri(),
"description": description,
"experiments": [experimentUri],
"immutable": immutable}
data = json.dumps(datasetJson)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
url = myTardisUrl + "/api/v1/dataset/"
if testRun:
message = "CREATING NEW DATASET FOR FOLDER: %s\n" \
" Description: %s" \
% (folderModel.GetRelPath(),
description)
if experiment: # Could be None in test run.
message += "\n In Experiment: %s/%s" \
% (folderModel.settingsModel.GetMyTardisUrl(),
experiment.GetViewUri())
logger.testrun(message)
return
response = requests.post(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
newDatasetJson = response.json()
return DatasetModel(settingsModel, newDatasetJson)
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
if response.status_code == 401:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisUsername
raise Unauthorized(message)
elif response.status_code == 500:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "An Internal Server Error occurred."
message += "\n\n"
message += "If running MyTardis in DEBUG mode, " \
"more information may be available below. " \
"Otherwise, please ask your MyTardis " \
"administrator to check in their logs " \
"for more information."
message += "\n\n"
try:
message += "ERROR: \"%s\"" \
% response.json()['error_message']
except: # pylint: disable=bare-except
message += response.text
raise InternalServerError(message)
raise Exception(response.text)
else:
existingDatasetJson = \
existingMatchingDatasets['objects'][0]
if testRun:
description = existingDatasetJson['description']
datasetId = existingDatasetJson['id']
viewUri = "dataset/%s" % datasetId
message = "ADDING TO EXISTING DATASET FOR FOLDER: %s\n" \
" URL: %s/%s\n" \
" Description: %s\n" \
" In Experiment: %s/%s" \
% (folderModel.GetRelPath(),
folderModel.settingsModel.GetMyTardisUrl(),
viewUri,
description,
folderModel.settingsModel.GetMyTardisUrl(),
folderModel.GetExperiment().GetViewUri())
logger.testrun(message)
return DatasetModel(settingsModel,
existingMatchingDatasets['objects'][0])
def CreateExperimentForFolder(folderModel, testRun=False):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
userFolderName = folderModel.GetUserFolderName()
hostname = settingsModel.GetUploaderModel().GetHostname()
location = folderModel.GetLocation()
groupFolderName = folderModel.GetGroupFolderName()
owner = folderModel.GetOwner()
ownerUsername = folderModel.GetOwner().GetUsername()
# pylint: disable=bare-except
try:
ownerUserId = folderModel.GetOwner().GetJson()['id']
except:
ownerUserId = None
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
experimentTitle = folderModel.GetExperimentTitle()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
if userFolderName:
message = "Creating experiment for uploader '%s', " \
"user folder '%s'." % (uploaderName, userFolderName)
if groupFolderName:
message += ", group folder : '%s'" % groupFolderName
elif groupFolderName:
message = "Creating experiment for uploader '%s', " \
"user group folder '%s'." % (uploaderName, groupFolderName)
else:
message = "Creating experiment for uploader '%s'" % uploaderName
logger.info(message)
if userFolderName:
description = ("Uploader: %s\n"
"User folder name: %s\n"
"Uploaded from: %s:%s"
% (uploaderName, userFolderName, hostname, location))
if groupFolderName:
description += "\nGroup folder name: %s" % groupFolderName
else:
description = ("Uploader: %s\n"
"Group folder name: %s\n"
"Uploaded from: %s:%s"
% (uploaderName, groupFolderName, hostname, location))
if testRun:
message = "CREATING NEW EXPERIMENT FOR FOLDER: %s\n" \
" Title: %s\n" \
" Description: \n" \
" Uploader: %s\n" \
" User folder name: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
experimentTitle, uploaderName, userFolderName,
ownerUsername)
logger.testrun(message)
return
experimentJson = {
"title": experimentTitle,
"description": description,
"immutable": False,
"parameter_sets": [{
"schema": "http://mytardis.org/schemas"
"/mydata/defaultexperiment",
"parameters": [{"name": "uploader",
"value": uploaderUuid},
{"name": "user_folder_name",
"value": userFolderName}]}]}
if groupFolderName:
experimentJson["parameter_sets"][0]["parameters"].append(
{"name": "group_folder_name", "value": groupFolderName})
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisDefaultUsername,
myTardisDefaultUserApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
url = myTardisUrl + "/api/v1/mydata_experiment/"
response = requests.post(headers=headers, url=url,
data=json.dumps(experimentJson))
# pylint: disable=bare-except
try:
createdExperimentJson = response.json()
createdExperiment = ExperimentModel(settingsModel,
createdExperimentJson)
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " +
str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the experiment.\n\n" \
"Please ask your MyTardis administrator to " \
"check that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message)
raise
if response.status_code == 201:
message = "Succeeded in creating experiment '%s' for uploader " \
"\"%s\" and user folder \"%s\"" \
% (experimentTitle, uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.debug(message)
facilityManagersGroup = settingsModel.GetFacility().GetManagerGroup()
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
facilityManagersGroup)
# Avoid creating a duplicate ObjectACL if the user folder's
# username matches the facility manager's username.
# Don't attempt to create an ObjectACL record for an
# invalid user (without a MyTardis user ID).
if myTardisDefaultUsername != ownerUsername and \
ownerUserId is not None:
ObjectAclModel.ShareExperimentWithUser(createdExperiment,
owner)
if folderModel.GetGroup() is not None and \
folderModel.GetGroup().GetId() != \
facilityManagersGroup.GetId():
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
folderModel.GetGroup())
else:
message = "Failed to create experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.error(message)
logger.error(headers)
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " +
str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
return createdExperiment
def CreateExperimentForFolder(folderModel, testRun=False):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
userFolderName = folderModel.GetUserFolderName()
hostname = settingsModel.GetUploaderModel().GetHostname()
location = folderModel.GetLocation()
groupFolderName = folderModel.GetGroupFolderName()
owner = folderModel.GetOwner()
ownerUsername = folderModel.GetOwner().GetUsername()
# pylint: disable=bare-except
try:
ownerUserId = folderModel.GetOwner().GetJson()['id']
except:
ownerUserId = None
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
experimentTitle = folderModel.GetExperimentTitle()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
if userFolderName:
message = "Creating experiment for uploader '%s', " \
"user folder '%s'." % (uploaderName, userFolderName)
if groupFolderName:
message += ", group folder : '%s'" % groupFolderName
elif groupFolderName:
message = "Creating experiment for uploader '%s', " \
"user group folder '%s'." % (uploaderName, groupFolderName)
else:
message = "Creating experiment for uploader '%s'" % uploaderName
logger.info(message)
if userFolderName:
description = ("Uploader: %s\n"
"User folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, userFolderName, hostname, location))
if groupFolderName:
description += "\nGroup folder name: %s" % groupFolderName
else:
description = ("Uploader: %s\n"
"Group folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, groupFolderName, hostname, location))
if testRun:
message = "CREATING NEW EXPERIMENT FOR FOLDER: %s\n" \
" Title: %s\n" \
" Description: \n" \
" Uploader: %s\n" \
" User folder name: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
experimentTitle, uploaderName, userFolderName,
ownerUsername)
logger.testrun(message)
return
experimentJson = {
"title":
experimentTitle,
"description":
description,
"immutable":
False,
"parameter_sets": [{
"schema":
"http://mytardis.org/schemas"
"/mydata/defaultexperiment",
"parameters": [{
"name": "uploader",
"value": uploaderUuid
}, {
"name": "user_folder_name",
"value": userFolderName
}]
}]
}
if groupFolderName:
experimentJson["parameter_sets"][0]["parameters"].append({
"name":
"group_folder_name",
"value":
groupFolderName
})
headers = {
"Authorization":
"ApiKey %s:%s" %
(myTardisDefaultUsername, myTardisDefaultUserApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/mydata_experiment/"
response = requests.post(headers=headers,
url=url,
data=json.dumps(experimentJson))
# pylint: disable=bare-except
try:
createdExperimentJson = response.json()
createdExperiment = ExperimentModel(settingsModel,
createdExperimentJson)
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the experiment.\n\n" \
"Please ask your MyTardis administrator to " \
"check that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message)
raise
if response.status_code == 201:
message = "Succeeded in creating experiment '%s' for uploader " \
"\"%s\" and user folder \"%s\"" \
% (experimentTitle, uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.debug(message)
facilityManagersGroup = settingsModel.GetFacility(
).GetManagerGroup()
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
facilityManagersGroup)
# Avoid creating a duplicate ObjectACL if the user folder's
# username matches the facility manager's username.
# Don't attempt to create an ObjectACL record for an
# invalid user (without a MyTardis user ID).
if myTardisDefaultUsername != ownerUsername and \
ownerUserId is not None:
ObjectAclModel.ShareExperimentWithUser(createdExperiment,
owner)
if folderModel.GetGroup() is not None and \
folderModel.GetGroup().GetId() != \
facilityManagersGroup.GetId():
ObjectAclModel.ShareExperimentWithGroup(
createdExperiment, folderModel.GetGroup())
else:
message = "Failed to create experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.error(message)
logger.error(headers)
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
return createdExperiment
def CreateDatasetIfNecessary(folderModel, testRun=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
description = folderModel.GetFolder()
settingsModel = folderModel.settingsModel
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
experiment = folderModel.GetExperiment()
if experiment: # Could be None in test run
url = myTardisUrl + "/api/v1/dataset/?format=json" + \
"&experiments__id=" + str(experiment.GetId())
url = url + "&description=" + urllib.quote(description)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey)
}
response = requests.get(headers=headers, url=url)
existingMatchingDatasets = response.json()
numExistingMatchingDatasets = \
existingMatchingDatasets['meta']['total_count']
if numExistingMatchingDatasets == 1:
logger.debug("Found existing dataset for folder " +
description)
elif numExistingMatchingDatasets > 1:
logger.debug("WARNING: Found multiple datasets for folder " +
description)
else:
numExistingMatchingDatasets = 0
if numExistingMatchingDatasets == 0:
logger.debug("Creating dataset record for folder: " + description)
description = folderModel.GetFolder()
if experiment:
experimentUri = experiment.GetResourceUri()
else:
experimentUri = None
immutable = False
datasetJson = {
"instrument": settingsModel.GetInstrument().GetResourceUri(),
"description": description,
"experiments": [experimentUri],
"immutable": immutable
}
data = json.dumps(datasetJson)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/dataset/"
if testRun:
message = "CREATING NEW DATASET FOR FOLDER: %s\n" \
" Description: %s" \
% (folderModel.GetRelPath(),
description)
if experiment: # Could be None in test run.
message += "\n In Experiment: %s/%s" \
% (folderModel.settingsModel.GetMyTardisUrl(),
experiment.GetViewUri())
logger.testrun(message)
return
response = requests.post(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
newDatasetJson = response.json()
return DatasetModel(settingsModel, newDatasetJson)
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
if response.status_code == 401:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisUsername
raise Unauthorized(message)
elif response.status_code == 500:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "An Internal Server Error occurred."
message += "\n\n"
message += "If running MyTardis in DEBUG mode, " \
"more information may be available below. " \
"Otherwise, please ask your MyTardis " \
"administrator to check in their logs " \
"for more information."
message += "\n\n"
try:
message += "ERROR: \"%s\"" \
% response.json()['error_message']
except: # pylint: disable=bare-except
message += response.text
raise InternalServerError(message)
raise Exception(response.text)
else:
existingDatasetJson = \
existingMatchingDatasets['objects'][0]
if testRun:
description = existingDatasetJson['description']
datasetId = existingDatasetJson['id']
viewUri = "dataset/%s" % datasetId
message = "ADDING TO EXISTING DATASET FOR FOLDER: %s\n" \
" URL: %s/%s\n" \
" Description: %s\n" \
" In Experiment: %s/%s" \
% (folderModel.GetRelPath(),
folderModel.settingsModel.GetMyTardisUrl(),
viewUri,
description,
folderModel.settingsModel.GetMyTardisUrl(),
folderModel.GetExperiment().GetViewUri())
logger.testrun(message)
return DatasetModel(settingsModel,
existingMatchingDatasets['objects'][0])
def ShutDownUploadThreads(self, event=None):
# pylint: disable=too-many-branches
if self.IsShuttingDown():
return
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
if not wx.GetApp().PerformingLookupsAndUploads():
EndBusyCursorIfRequired()
return
self.SetShuttingDown(True)
message = "Shutting down upload threads..."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if hasattr(event, "failed") and event.failed:
self.SetFailed()
self.uploadsModel.CancelRemaining()
elif hasattr(event, "completed") and event.completed:
self.SetCompleted()
else:
self.SetCanceled()
self.uploadsModel.CancelRemaining()
logger.debug("Shutting down FoldersController upload worker threads.")
for _ in range(self.numUploadWorkerThreads):
self.uploadsQueue.put(None)
if self.uploadMethod == UploadMethod.VIA_STAGING:
# SCP can leave orphaned SSH processes which need to be
# cleaned up.
CleanUpSshProcesses(self.settingsModel)
for thread in self.uploadWorkerThreads:
thread.join()
logger.debug("Shutting down FoldersController verification "
"worker threads.")
for _ in range(self.numVerificationWorkerThreads):
self.verificationsQueue.put(None)
for thread in self.verificationWorkerThreads:
thread.join()
self.verifyDatafileRunnable = {}
self.uploadDatafileRunnable = {}
if self.testRun:
numVerificationsCompleted = \
self.verificationsModel.GetCompletedCount()
numVerifiedUploads = \
self.verificationsModel.GetFoundVerifiedCount()
numFilesNotFoundOnServer = \
self.verificationsModel.GetNotFoundCount()
numFullSizeUnverifiedUploads = \
self.verificationsModel.GetFoundUnverifiedFullSizeCount()
numIncompleteUploads = \
self.verificationsModel.GetFoundUnverifiedNotFullSizeCount()
numFailedLookups = self.verificationsModel.GetFailedCount()
logger.testrun("")
logger.testrun("SUMMARY")
logger.testrun("")
logger.testrun("Files looked up on server: %s"
% numVerificationsCompleted)
logger.testrun("Files verified on server: %s" % numVerifiedUploads)
logger.testrun("Files not found on server: %s"
% numFilesNotFoundOnServer)
logger.testrun("Files unverified (but full size) on server: %s"
% numFullSizeUnverifiedUploads)
logger.testrun("Files unverified (and incomplete) on server: %s"
% numIncompleteUploads)
logger.testrun("Failed lookups: %s" % numFailedLookups)
logger.testrun("")
if self.Failed():
message = "Data scans and uploads failed."
elif self.Canceled():
message = "Data scans and uploads were canceled."
elif self.uploadsModel.GetFailedCount() > 0:
message = \
"Data scans and uploads completed with " \
"%d failed upload(s)." % self.uploadsModel.GetFailedCount()
elif self.Completed():
message = "Data scans and uploads completed successfully."
elapsedTime = self.uploadsModel.GetElapsedTime()
if elapsedTime and not self.testRun:
averageSpeed = "%3.1f MB/s" % \
(float(self.uploadsModel.GetCompletedSize()) / 1000000.0 \
/ elapsedTime.total_seconds())
message += " Average speed: %s" % averageSpeed
else:
message = "Data scans and uploads appear to have " \
"completed successfully."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if self.testRun:
logger.testrun(message)
app = wx.GetApp()
if hasattr(app, "toolbar"):
app.EnableTestAndUploadToolbarButtons()
app.SetShouldAbort(False)
if self.testRun:
app.testRunFrame.saveButton.Enable()
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
wx.GetApp().SetPerformingLookupsAndUploads(False)
self.SetShuttingDown(False)
if hasattr(app, "SetTestRunRunning"):
app.SetTestRunRunning(False)
EndBusyCursorIfRequired()
logger.debug("")
