def ScanDataDirs():
"""
Scan data folders, looking for datafiles to look up on MyTardis
and upload if necessary.
"""
logger.debug("Starting run() method for thread %s" % threading.current_thread().name)
wx.CallAfter(self.frame.SetStatusMessage, "Scanning data folders...")
try:
self.scanningFoldersThreadingLock.acquire()
self.SetScanningFolders(True)
logger.info("Just set ScanningFolders to True")
self.toolbar.EnableTool(self.stopTool.GetId(), True)
self.foldersModel.ScanFolders(WriteProgressUpdateToStatusBar, self.ShouldAbort)
self.SetScanningFolders(False)
self.scanningFoldersThreadingLock.release()
logger.info("Just set ScanningFolders to False")
except InvalidFolderStructure, ifs:
def ShowMessageDialog():
"""
Needs to run in the main thread.
"""
dlg = wx.MessageDialog(None, str(ifs), "MyData", wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.CallAfter(ShowMessageDialog)
self.frame.SetStatusMessage(str(ifs))
return
def ReadPublicKey(self):
"""
Read public key, including "ssh-rsa "
"""
if self.publicKeyFilePath is not None and \
os.path.exists(self.publicKeyFilePath):
with open(self.publicKeyFilePath, "r") as pubKeyFile:
return pubKeyFile.read()
elif os.path.exists(self.privateKeyFilePath):
cmdList = [OPENSSH.DoubleQuote(OPENSSH.sshKeyGen),
"-y",
"-f", OPENSSH.DoubleQuote(
GetCygwinPath(self.privateKeyFilePath))]
cmd = " ".join(cmdList)
logger.debug(cmd)
proc = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True,
startupinfo=DEFAULT_STARTUP_INFO,
creationflags=DEFAULT_CREATION_FLAGS)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise SshException(stdout)
return stdout
else:
raise SshException("Couldn't find MyData key files in ~/.ssh "
"while trying to read public key.")
def jobFunc(taskModel, tasksModel, row, col):
def taskJobFunc():
assert callable(taskModel.GetJobFunc())
title = "Starting"
message = taskModel.GetJobDesc()
Notification.notify(message, title=title)
taskModel.GetJobFunc()(*taskModel.GetJobArgs())
taskModel.SetFinishTime(datetime.now())
title = "Finished"
message = taskModel.GetJobDesc()
Notification.notify(message, title=title)
wx.CallAfter(tasksModel.RowValueChanged, row, col)
scheduleType = taskModel.GetScheduleType()
if scheduleType == "Timer":
intervalMinutes = taskModel.GetIntervalMinutes()
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(minutes=intervalMinutes)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Timer",
intervalMinutes=intervalMinutes)
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = \
"{d:%A} {d.day}/{d.month}/{d.year}".format(d=newStartTime)
wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring every %d minutes)"
% (taskModel.GetJobDesc(),
timeString, dateString, intervalMinutes))
tasksModel.AddRow(newTaskModel)
elif scheduleType == "Daily":
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(days=1)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Daily")
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = \
"{d:%A} {d.day}/{d.month}/{d.year}".format(d=newStartTime)
wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring daily)"
% (taskModel.GetJobDesc(),
timeString, dateString))
tasksModel.AddRow(newTaskModel)
thread = threading.Thread(target=taskJobFunc)
logger.debug("Starting task %s" % taskModel.GetJobDesc())
thread.start()
def CalculateMd5Sum(self, filePath, fileSize, uploadModel,
progressCallback=None):
"""
Calculate MD5 checksum.
"""
md5 = hashlib.md5()
defaultChunkSize = 128 * 1024
maxChunkSize = 16 * 1024 * 1024
chunkSize = defaultChunkSize
while (fileSize / chunkSize) > 50 and chunkSize < maxChunkSize:
chunkSize = chunkSize * 2
bytesProcessed = 0
with open(filePath, 'rb') as fileHandle:
# Note that the iter() func needs an empty byte string
# for the returned iterator to halt at EOF, since read()
# returns b'' (not just '').
for chunk in iter(lambda: fileHandle.read(chunkSize), b''):
if self.IsShuttingDown() or uploadModel.Canceled():
logger.debug("Aborting MD5 calculation for "
"%s" % filePath)
return None
md5.update(chunk)
bytesProcessed += len(chunk)
del chunk
if progressCallback:
progressCallback(bytesProcessed)
return md5.hexdigest()
def GetFacility(settingsModel, name):
"""
Get facility by name.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/facility/?format=json&name=" + urllib.quote(name)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json",
}
response = requests.get(url=url, headers=headers, stream=False)
logger.debug(response.text)
if response.status_code != 200:
message = response.text
response.close()
raise Exception(message)
facilitiesJson = response.json()
response.close()
numFacilitiesFound = facilitiesJson["meta"]["total_count"]
if numFacilitiesFound == 0:
logger.warning('Facility "%s" was not found in MyTardis' % name)
return None
else:
logger.debug("Found facility record for name '" + name + "'.")
return FacilityModel(settingsModel=settingsModel, name=name, facilityJson=facilitiesJson["objects"][0])
def ScanFolders(self, incrementProgressDialog, shouldAbort):
if self.GetCount() > 0:
self.DeleteAllRows()
if self.usersModel.GetCount() > 0:
self.usersModel.DeleteAllRows()
if self.groupsModel.GetCount() > 0:
self.groupsModel.DeleteAllRows()
dataDir = self.settingsModel.GetDataDirectory()
folderStructure = self.settingsModel.GetFolderStructure()
self.ignoreOldDatasets = self.settingsModel.IgnoreOldDatasets()
if self.ignoreOldDatasets:
seconds = {}
seconds['day'] = 24 * 60 * 60
seconds['week'] = 7 * seconds['day']
seconds['year'] = int(365.25 * seconds['day'])
seconds['month'] = seconds['year'] / 12
singularIgnoreIntervalUnit = \
self.settingsModel.GetIgnoreOldDatasetIntervalUnit().rstrip(
's')
ignoreIntervalUnitSeconds = seconds[singularIgnoreIntervalUnit]
self.ignoreIntervalNumber = \
self.settingsModel.GetIgnoreOldDatasetIntervalNumber()
self.ignoreIntervalUnit = \
self.settingsModel.GetIgnoreOldDatasetIntervalUnit()
self.ignoreIntervalSeconds = \
self.ignoreIntervalNumber * ignoreIntervalUnitSeconds
logger.debug("FoldersModel.ScanFolders(): Scanning " + dataDir + "...")
if folderStructure.startswith("Username") or \
folderStructure.startswith("Email"):
self.ScanForUserFolders(incrementProgressDialog, shouldAbort)
elif folderStructure.startswith("User Group"):
self.ScanForGroupFolders(incrementProgressDialog, shouldAbort)
else:
raise InvalidFolderStructure("Unknown folder structure.")
def OnSyncNow(self, event):
"""
Called when the "Sync Now" menu item is
selected from MyData's system tray / menu bar icon menu.
"""
logger.debug("Sync Now called from task bar menu item.")
wx.GetApp().ScanFoldersAndUpload(event)
def HandleExistingUnverifiedDatafile(self, existingDatafile):
"""
If the existing unverified DataFile was uploaded via POST, we just
need to wait for it to be verified. But if it was uploaded via
staging, we might be able to resume a partial upload.
"""
self.verificationModel.SetExistingUnverifiedDatafile(existingDatafile)
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
replicas = existingDatafile.GetReplicas()
message = "Found datafile record for %s " \
"but it has no verified replicas." % dataFilePath
logger.debug(message)
message = "Found unverified datafile record on MyTardis."
self.verificationModel.SetMessage(message)
uploadToStagingRequest = self.settingsModel.GetUploadToStagingRequest()
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING and \
uploadToStagingRequest is not None and \
uploadToStagingRequest.IsApproved() and \
len(replicas) > 0:
# Can resume partial uploads:
self.HandleResumableUpload(existingDatafile)
else:
# Can't resume partial uploads:
self.HandleUnresumableUpload(existingDatafile)
def GetUserByEmail(settingsModel, email):
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/user/?format=json&email__iexact=" + \
urllib2.quote(email)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
try:
response = requests.get(url=url, headers=headers)
except:
raise Exception(traceback.format_exc())
if response.status_code != 200:
logger.debug(url)
message = response.text
raise Exception(message)
try:
userRecordsJson = response.json()
except:
logger.error(traceback.format_exc())
raise
numUserRecordsFound = userRecordsJson['meta']['total_count']
if numUserRecordsFound == 0:
raise DoesNotExist(
message="User with email \"%s\" was not found in MyTardis"
% email,
url=url, response=response)
else:
logger.debug("Found user record for email '" + email + "'.")
return UserModel(settingsModel=settingsModel,
userRecordJson=userRecordsJson['objects'][0])
def HandleIncompleteResumableUpload(self, existingDatafile,
bytesUploadedPreviously):
"""
Resume partial upload.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.verificationModel\
.SetMessage("Found partially uploaded datafile "
"on staging server.")
self.verificationModel\
.SetStatus(VerificationStatus
.FOUND_UNVERIFIED_NOT_FULL_SIZE)
self.verificationsModel.MessageUpdated(self.verificationModel)
logger.debug("Re-uploading \"%s\" to staging, because "
"the file size is %s bytes in staging, "
"but it should be %s bytes."
% (dataFilePath,
bytesUploadedPreviously,
existingDatafile.GetSize()))
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.unverifiedDatafileOnServerEvent(
id=self.foldersController.EVT_INCOMPLETE_FILE_ON_STAGING,
foldersController=self.foldersController,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
existingUnverifiedDatafile=existingDatafile,
bytesUploadedPreviously=bytesUploadedPreviously,
verificationModel=self.verificationModel))
def GetDataFile(settingsModel, dataset, filename, directory):
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/mydata_dataset_file/?format=json" + \
"&dataset__id=" + str(dataset.GetId()) + \
"&filename=" + urllib.quote(filename) + \
"&directory=" + urllib.quote(directory)
headers = {"Authorization": "ApiKey " + myTardisUsername + ":" +
myTardisApiKey}
response = requests.get(url=url, headers=headers)
if response.status_code < 200 or response.status_code >= 300:
logger.debug("Failed to look up datafile \"%s\" "
"in dataset \"%s\"."
% (filename, dataset.GetDescription()))
logger.debug(response.text)
return None
dataFilesJson = response.json()
numDataFilesFound = dataFilesJson['meta']['total_count']
if numDataFilesFound == 0:
raise DoesNotExist(
message="Datafile \"%s\" was not found in MyTardis" % filename,
url=url, response=response)
elif numDataFilesFound > 1:
raise MultipleObjectsReturned(
message="Multiple datafiles matching %s were found in MyTardis"
% filename,
url=url, response=response)
else:
return DataFileModel(
settingsModel=settingsModel,
dataset=dataset,
dataFileJson=dataFilesJson['objects'][0])
def HandleUnverifiedNotFoundOnStaging(self, existingDatafile):
"""
File has a DataFile record, and a DataFileObject record,
marked as unverified. The file is not accessible on staging.
So we need to upload the entire file and create any subdirectories
required for it on staging.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.verificationModel\
.SetMessage("Unverified and not found on staging server.")
self.verificationModel.SetStatus(
VerificationStatus.NOT_FOUND_ON_STAGING)
self.verificationsModel.MessageUpdated(self.verificationModel)
logger.debug("Uploading \"%s\" to staging, because "
"it was not there. It does have a DataFileObject."
% dataFilePath)
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.unverifiedDatafileOnServerEvent(
id=self.foldersController.EVT_UNVERIFIED_NOT_FOUND_ON_STAGING,
foldersController=self.foldersController,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
existingUnverifiedDatafile=existingDatafile,
bytesUploadedPreviously=None,
verificationModel=self.verificationModel))
def RenameInstrumentWorker():
"""
Renames instrument in separate thread.
"""
logger.debug("Starting run() method for thread %s"
% threading.current_thread().name)
try:
wx.CallAfter(BeginBusyCursorIfRequired)
event.settingsModel.RenameInstrument(
event.facilityName,
event.oldInstrumentName,
event.newInstrumentName)
wx.CallAfter(EndBusyCursorIfRequired, event)
if event.nextEvent:
wx.PostEvent(wx.GetApp().GetMainFrame(), event.nextEvent)
except DuplicateKey:
wx.CallAfter(EndBusyCursorIfRequired, event)
def NotifyUserOfDuplicateInstrumentName():
"""
Notifies user of duplicate instrument name.
"""
message = "Instrument name \"%s\" already exists in " \
"facility \"%s\"." \
% (event.newInstrumentName,
event.facilityName)
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
event.settingsDialog.instrumentNameField.SetFocus()
event.settingsDialog.instrumentNameField.SelectAll()
wx.CallAfter(NotifyUserOfDuplicateInstrumentName)
logger.debug("Finishing run() method for thread %s"
% threading.current_thread().name)
def GetFacility(settingsModel, name):
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/facility/?format=json&name=" + \
urllib.quote(name)
headers = {'Authorization': 'ApiKey ' + myTardisUsername + ":" +
myTardisApiKey}
session = requests.Session()
response = session.get(url=url, headers=headers, stream=False)
logger.debug(response.text)
if response.status_code != 200:
message = response.text
response.close()
session.close()
raise Exception(message)
facilitiesJson = response.json()
response.close()
session.close()
numFacilitiesFound = facilitiesJson['meta']['total_count']
if numFacilitiesFound == 0:
logger.warning("Facility \"%s\" was not found in MyTardis" % name)
return None
else:
logger.debug("Found facility record for name '" + name + "'.")
return FacilityModel(
settingsModel=settingsModel, name=name,
facilityJson=facilitiesJson['objects'][0])
def ValidateSettings():
"""
Validate settings.
"""
logger.debug("Starting run() method for thread %s" % threading.current_thread().name)
# pylint: disable=bare-except
try:
wx.CallAfter(wx.BeginBusyCursor)
# pylint: disable=broad-except
try:
activeNetworkInterfaces = UploaderModel.GetActiveNetworkInterfaces()
except Exception, err:
logger.error(traceback.format_exc())
if type(err).__name__ == "WindowsError" and "The handle is invalid" in str(err):
message = (
"An error occurred, suggesting "
"that you have launched MyData.exe from a "
"Command Prompt window. Please launch it "
"from a shortcut or from a Windows Explorer "
"window instead.\n"
"\n"
"See: https://bugs.python.org/issue3905"
)
def ShowErrorDialog(message):
"""
Needs to run in the main thread.
"""
dlg = wx.MessageDialog(None, message, "MyData", wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.CallAfter(ShowErrorDialog, message)
if len(activeNetworkInterfaces) == 0:
message = (
"No active network interfaces."
"\n\n"
"Please ensure that you have an active "
"network interface (e.g. Ethernet or WiFi)."
)
def ShowDialog():
"""
Needs to run in the main thread.
"""
dlg = wx.MessageDialog(None, message, "MyData", wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.CallAfter(EndBusyCursorIfRequired)
self.frame.SetStatusMessage("")
self.frame.SetConnected(self.settingsModel.GetMyTardisUrl(), False)
wx.CallAfter(ShowDialog)
return
self.settingsValidation = self.settingsModel.Validate()
event = mde.MyDataEvent(
mde.EVT_SETTINGS_VALIDATION_FOR_REFRESH_COMPLETE, needToValidateSettings=False
)
wx.PostEvent(self.frame, event)
wx.CallAfter(EndBusyCursorIfRequired)
def CleanUp(self):
"""
Perform clean up actions on each each folder model.
"""
logger.debug("Cleaning up each FolderModel record's threads...")
for row in range(0, self.GetRowCount()):
self.foldersData[row].CleanUp()
logger.debug("Cleaned up each FolderModel record's threads...")
def OnRefreshFromToolbar(self, event):
"""
The user pressed the Refresh icon on the main windows' toolbar.
"""
logger.debug("OnRefreshFromToolbar")
self.tasksModel.DeleteAllRows()
self.settingsModel.SetScheduleType("Manually")
self.settingsModel.SetLastSettingsUpdateTrigger(LastSettingsUpdateTrigger.UI_RESPONSE)
self.scheduleController.ApplySchedule(event, runManually=True)
def TryRowDeleted(self, row):
try:
if row < self.GetCount():
self.RowDeleted(row)
else:
logger.warning("TryRowDeleted called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
except:
logger.debug(traceback.format_exc())
def ShutdownForRefresh(event):
"""
Shuts down upload threads before restarting them.
"""
if event.GetEventId() != EVT_SHUTDOWN_FOR_REFRESH:
event.Skip()
return
def ShutdownForRefreshWorker():
"""
Shuts down upload threads (in dedicated worker thread)
before restarting them.
"""
logger.debug("Starting run() method for thread %s"
% threading.current_thread().name)
logger.debug("Shutting down for refresh from %s."
% threading.current_thread().name)
# pylint: disable=bare-except
try:
wx.CallAfter(BeginBusyCursorIfRequired)
app = wx.GetApp()
app.GetScheduleController().ApplySchedule(event)
event.foldersController.ShutDownUploadThreads()
shutdownForRefreshCompleteEvent = MyDataEvent(
EVT_SHUTDOWN_FOR_REFRESH_COMPLETE,
shutdownSuccessful=True)
wx.PostEvent(app.GetMainFrame(),
shutdownForRefreshCompleteEvent)
wx.CallAfter(EndBusyCursorIfRequired, event)
except:
logger.error(traceback.format_exc())
message = "An error occurred while trying to shut down " \
"the existing data-scan-and-upload process in order " \
"to start another one.\n\n" \
"See the Log tab for details of the error."
logger.error(message)
def ShowDialog():
"""
Show error dialog in main thread.
"""
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.CallAfter(ShowDialog)
logger.debug("Finishing run() method for thread %s"
% threading.current_thread().name)
shutdownForRefreshThread = \
threading.Thread(target=ShutdownForRefreshWorker,
name="ShutdownForRefreshThread")
MYDATA_THREADS.Add(shutdownForRefreshThread)
logger.debug("Starting thread %s" % shutdownForRefreshThread.name)
shutdownForRefreshThread.start()
logger.debug("Started thread %s" % shutdownForRefreshThread.name)
def TryRowValueChanged(self, row, col):
# pylint: disable=bare-except
try:
if row < self.GetCount():
self.RowValueChanged(row, col)
else:
logger.warning("TryRowValueChanged called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
self.RowValueChanged(row, col)
except:
logger.debug(traceback.format_exc())
def RenameInstrument(event):
"""
Responds to instrument rename request from Settings dialog.
"""
if event.GetEventId() != EVT_RENAME_INSTRUMENT:
event.Skip()
return
def RenameInstrumentWorker():
"""
Renames instrument in separate thread.
"""
logger.debug("Starting run() method for thread %s"
% threading.current_thread().name)
try:
wx.CallAfter(BeginBusyCursorIfRequired)
event.settingsModel.RenameInstrument(
event.facilityName,
event.oldInstrumentName,
event.newInstrumentName)
wx.CallAfter(EndBusyCursorIfRequired, event)
if event.nextEvent:
wx.PostEvent(wx.GetApp().GetMainFrame(), event.nextEvent)
except DuplicateKey:
wx.CallAfter(EndBusyCursorIfRequired, event)
def NotifyUserOfDuplicateInstrumentName():
"""
Notifies user of duplicate instrument name.
"""
message = "Instrument name \"%s\" already exists in " \
"facility \"%s\"." \
% (event.newInstrumentName,
event.facilityName)
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
event.settingsDialog.instrumentNameField.SetFocus()
event.settingsDialog.instrumentNameField.SelectAll()
wx.CallAfter(NotifyUserOfDuplicateInstrumentName)
logger.debug("Finishing run() method for thread %s"
% threading.current_thread().name)
renameInstrumentThread = \
threading.Thread(target=RenameInstrumentWorker,
name="RenameInstrumentThread")
MYDATA_THREADS.Add(renameInstrumentThread)
logger.debug("Starting thread %s" % renameInstrumentThread.name)
renameInstrumentThread.start()
logger.debug("Started thread %s" % renameInstrumentThread.name)
def OnMyTardisSync(self, event):
"""
Called when the "MyTardis Sync" menu item is
selected from MyData's system tray / menu bar icon menu.
"""
# wx.GetApp().OnRefresh(event)
logger.debug("MyTardis Sync called from task bar menu item.")
app = wx.GetApp()
app.tasksModel.DeleteAllRows()
app.settingsModel.SetScheduleType("Manually")
app.settingsModel.SetLastSettingsUpdateTrigger(
LastSettingsUpdateTrigger.UI_RESPONSE)
app.GetScheduleController().ApplySchedule(event, runManually=True)
event.Skip()
def ShutDownDataScansAndUploads():
"""
Shut down data folder scanning, datafile lookups
(verifications) and uploads.
"""
logger.debug("Starting ShutDownDataScansAndUploads...")
# pylint: disable=bare-except
try:
wx.CallAfter(wx.BeginBusyCursor)
self.foldersController.ShutDownUploadThreads()
wx.CallAfter(EndBusyCursorIfRequired)
self.tasksModel.ShutDown()
sys.exit(0)
except:
try:
logger.debug(traceback.format_exc())
self.tasksModel.ShutDown()
# pylint: disable=protected-access
os._exit(1)
# pylint: disable=bare-except
except:
logger.debug(traceback.format_exc())
# pylint: disable=protected-access
os._exit(1)
logger.debug("Finishing run() method for thread %s" % threading.current_thread().name)
def GetDataFileFromId(settingsModel, dataFileId):
"""
Lookup datafile by ID.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = "%s/api/v1/mydata_dataset_file/%s/?format=json" \
% (myTardisUrl, dataFileId)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
response = requests.get(url=url, headers=headers)
if response.status_code == 404:
raise DoesNotExist(
message="Datafile ID \"%s\" was not found in MyTardis" % dataFileId,
url=url, response=response)
elif response.status_code < 200 or response.status_code >= 300:
logger.debug("Failed to look up datafile ID \"%s\"." % dataFileId)
logger.debug(url)
logger.debug("Status: %s" % response.status_code)
logger.debug(response.text)
return None
dataFileJson = response.json()
return DataFileModel(
settingsModel=settingsModel,
dataset=None,
dataFileJson=dataFileJson)
def OnCloseFrame(self, event):
"""
If running in background mode, don't actually close it,
just iconize it.
"""
if self.settingsModel.RunningInBackgroundMode():
self.frame.Show() # See: http://trac.wxwidgets.org/ticket/10426
self.frame.Hide()
else:
started = self.foldersController.Started()
completed = self.foldersController.Completed()
canceled = self.foldersController.Canceled()
failed = self.foldersController.Failed()
message = "Are you sure you want to close MyData?"
if started and not completed and not canceled and not failed:
message += "\n\n" \
"MyData will attempt to shut down any uploads currently " \
"in progress before exiting."
confirmationDialog = \
wx.MessageDialog(None, message, "MyData",
wx.YES | wx.NO | wx.ICON_QUESTION)
okToExit = confirmationDialog.ShowModal()
if okToExit == wx.ID_YES:
def shutDownDataScansAndUploads():
logger.debug("Starting shutDownDataScansAndUploads...")
try:
wx.CallAfter(wx.BeginBusyCursor)
self.foldersController.ShutDownUploadThreads()
def endBusyCursorIfRequired():
try:
wx.EndBusyCursor()
except wx._core.PyAssertionError, e:
if "no matching wxBeginBusyCursor()" \
not in str(e):
logger.error(str(e))
raise
wx.CallAfter(endBusyCursorIfRequired)
self.tasksModel.ShutDown()
sys.exit(0)
except:
try:
logger.debug(traceback.format_exc())
self.tasksModel.ShutDown()
os._exit(1)
except:
logger.debug(traceback.format_exc())
os._exit(1)
def validateSettings():
logger.debug("Starting run() method for thread %s"
% threading.current_thread().name)
try:
wx.CallAfter(wx.BeginBusyCursor)
activeNetworkInterfaces = \
UploaderModel.GetActiveNetworkInterfaces()
if len(activeNetworkInterfaces) == 0:
message = "No active network interfaces." \
"\n\n" \
"Please ensure that you have an active " \
"network interface (e.g. Ethernet or WiFi)."
def showDialog():
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
def endBusyCursorIfRequired():
try:
wx.EndBusyCursor()
except wx._core.PyAssertionError, e:
if "no matching wxBeginBusyCursor()" \
not in str(e):
logger.error(str(e))
raise
wx.CallAfter(endBusyCursorIfRequired)
self.frame.SetStatusMessage("")
self.frame.SetConnected(
self.settingsModel.GetMyTardisUrl(), False)
wx.CallAfter(showDialog)
return
self.settingsValidation = self.settingsModel.Validate()
settingsValidationForRefreshCompleteEvent = \
mde.MyDataEvent(
mde.EVT_SETTINGS_VALIDATION_FOR_REFRESH_COMPLETE,
needToValidateSettings=False)
wx.PostEvent(self.frame,
settingsValidationForRefreshCompleteEvent)
def endBusyCursorIfRequired():
try:
wx.EndBusyCursor()
except wx._core.PyAssertionError, e:
if "no matching wxBeginBusyCursor()" not in str(e):
logger.error(str(e))
raise
def CleanUp(self):
logger.debug("Joining FoldersModel's UploadDataThread...")
self.uploadDataThread.join()
logger.debug("Joined FoldersModel's UploadDataThread.")
logger.debug("Cleaning up each FolderModel record's threads...")
for row in range(0, self.GetRowCount()):
self.foldersData[row].CleanUp()
logger.debug("Cleaned up each FolderModel record's threads...")
def ScanForGroupFolders(self, incrementProgressDialog, shouldAbort):
"""
Scan for group folders.
"""
dataDir = self.settingsModel.GetDataDirectory()
userOrGroupFilterString = '*%s*' % self.settingsModel.GetUserFilter()
filesDepth1 = glob(os.path.join(dataDir, userOrGroupFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
groupFolderNames = [os.path.basename(d) for d in dirsDepth1]
for groupFolderName in groupFolderNames:
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
logger.debug("Found folder assumed to be user group name: " +
groupFolderName)
groupsDataViewId = self.groupsModel.GetMaxDataViewId() + 1
try:
groupName = self.settingsModel.GetGroupPrefix() + \
groupFolderName
groupRecord = \
GroupModel.GetGroupByName(self.settingsModel,
groupName)
except DoesNotExist:
groupRecord = None
message = "Didn't find a MyTardis user group record for " \
"folder \"%s\" in %s" % (groupFolderName,
dataDir)
logger.warning(message)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if groupRecord:
groupRecord.SetDataViewId(groupsDataViewId)
self.groupsModel.AddRow(groupRecord)
self.ImportGroupFolders(os.path.join(dataDir,
groupFolderName),
groupRecord)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if threading.current_thread().name == "MainThread":
incrementProgressDialog()
else:
wx.CallAfter(incrementProgressDialog)
def CountCompletedUploadsAndVerifications(self, event):
"""
Check if we have finished uploads and verifications,
and if so, call ShutDownUploadThreads
"""
# pylint: disable=unused-argument
if self.Completed() or self.Canceled():
return
numVerificationsCompleted = self.verificationsModel.GetCompletedCount()
uploadsToBePerformed = self.uploadsModel.GetRowCount() + \
self.uploadsQueue.qsize()
uploadsCompleted = self.uploadsModel.GetCompletedCount()
uploadsFailed = self.uploadsModel.GetFailedCount()
uploadsProcessed = uploadsCompleted + uploadsFailed
if hasattr(wx.GetApp(), "GetMainFrame"):
if numVerificationsCompleted == \
self.numVerificationsToBePerformed \
and uploadsToBePerformed > 0:
message = "Uploaded %d of %d files." % \
(uploadsCompleted, uploadsToBePerformed)
else:
message = "Looked up %d of %d files on server." % \
(numVerificationsCompleted,
self.numVerificationsToBePerformed)
wx.GetApp().GetMainFrame().SetStatusMessage(message)
finishedVerificationCounting = \
self.finishedScanningForDatasetFolders.isSet()
for folder in self.finishedCountingVerifications:
if not self.finishedCountingVerifications[folder]:
finishedVerificationCounting = False
break
if numVerificationsCompleted == \
self.numVerificationsToBePerformed \
and finishedVerificationCounting \
and (uploadsProcessed == uploadsToBePerformed or
self.testRun and
self.uploadsAcknowledged ==
uploadsToBePerformed):
logger.debug("All datafile verifications and uploads "
"have completed.")
logger.debug("Shutting down upload and verification threads.")
wx.PostEvent(self.notifyWindow,
self.shutdownUploadsEvent(completed=True))
def HandleUnresumableUpload(self, existingDatafile):
"""
We found an unverified datafile on the server for which
there is no point in checking for a resumable partial
upload.
This is usually because we are uploading using the POST upload method.
Or we could be using the STAGING method but failed to find any
DataFileObjects on the server for the datafile.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
logger.debug("Found unverified datafile record for \"%s\" "
"on MyTardis." % dataFilePath)
self.verificationModel.SetMessage("Found unverified datafile record.")
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE
else:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_NO_DFOS)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_NO_DFOS
self.verificationsModel.MessageUpdated(self.verificationModel)
if existingDatafile and not self.testRun:
if existingDatafile.GetMd5Sum() == \
self.settingsModel.GetFakeMd5Sum():
logger.warning("MD5(%s): %s" %
(dataFilePath, existingDatafile.GetMd5Sum()))
else:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=eventId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
class UploaderModel(object):
"""
Model class for MyTardis API v1's UploaderAppResource.
See: https://github.com/mytardis/mytardis-app-mydata/blob/master/api.py
"""
# pylint: disable=too-many-instance-attributes
def __init__(self, settingsModel):
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
self.settingsModel = settingsModel
self.interface = None
self.responseJson = None
self.id = None # pylint: disable=invalid-name
self.uploaderSettings = None
self.uuid = self.settingsModel.GetUuid()
if self.uuid is None:
self.GenerateUuid()
self.settingsModel.SetUuid(self.uuid)
self.osUsername = ""
self.cpus = 0
self.osPlatform = ""
self.hostname = ""
self.machine = ""
self.osVersion = ""
self.requestStagingAccessThreadLock = threading.Lock()
self.memory = ""
self.osSystem = ""
self.architecture = ""
self.osRelease = ""
self.processor = ""
self.architecture = ""
# Here we check connectivity even if we've already done so, because
# we need to ensure that we get the correct network interface for
# self.interface, otherwise if the active interface changes,
# we can get errors like this: KeyError: 'RTC'
# when accessing things like ipv4Address[self.interface]
activeInterfaces = UploaderModel.GetActiveNetworkInterfaces()
if len(activeInterfaces) == 0:
message = "No active network interfaces." \
"\n\n" \
"Please ensure that you have an active network interface " \
"(e.g. Ethernet or WiFi)."
raise NoActiveNetworkInterface(message)
# Sometimes on Windows XP, you can end up with multiple results
# from "netsh interface show interface"
# If there is one called "Local Area Connection",
# then that's the one we'll go with.
if "Local Area Connection" in activeInterfaces:
activeInterfaces = ["Local Area Connection"]
elif "Local Area Connection 2" in activeInterfaces:
activeInterfaces = ["Local Area Connection 2"]
elif "Ethernet" in activeInterfaces:
activeInterfaces = ["Ethernet"]
elif "Internet" in activeInterfaces:
activeInterfaces = ["Internet"]
elif "Wi-Fi" in activeInterfaces:
activeInterfaces = ["Wi-Fi"]
# For now, we're only dealing with one active network interface.
# It is possible to have more than one active network interface,
# but we hope that the code above has picked the best one.
# If there are no active interfaces, then we shouldn't have
# reached this point - we should have already raised an
# exception.
self.interface = activeInterfaces[0]
if sys.platform.startswith("win"):
proc = subprocess.Popen(["ipconfig", "/all"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=DEFAULT_STARTUP_INFO,
creationflags=DEFAULT_CREATION_FLAGS)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
macAddress = {}
ipv4Address = {}
ipv6Address = {}
subnetMask = {}
interface = ""
for row in stdout.split("\n"):
match = re.match(r"^\S.*adapter (.*):\s*$", row)
if match:
interface = match.groups()[0]
if interface == self.interface:
if ': ' in row:
key, value = row.split(': ')
if key.strip(' .') == "Physical Address":
macAddress[interface] = value.strip()
if "IPv4 Address" in key.strip(' .'):
ipv4Address[interface] = \
value.strip().replace("(Preferred)", "")
ipv4Address[interface] = \
ipv4Address[interface] \
.replace("(Tentative)", "")
if "IPv6 Address" in key.strip(' .'):
ipv6Address[interface] = \
value.strip().replace("(Preferred)", "")
ipv6Address[interface] = \
ipv6Address[interface] \
.replace("(Tentative)", "")
if "Subnet Mask" in key.strip(' .'):
subnetMask[interface] = value.strip()
elif sys.platform.startswith("darwin"):
proc = subprocess.Popen(["ifconfig", self.interface],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=DEFAULT_STARTUP_INFO,
creationflags=DEFAULT_CREATION_FLAGS)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
macAddress = {}
ipv4Address = {}
ipv6Address = {}
subnetMask = {}
for row in stdout.split("\n"):
if sys.platform.startswith("darwin"):
match = re.match(r"\s+ether (\S*)\s*$", row)
else:
match = re.match(r".*\s+HWaddr (\S*)\s*$", row)
if match:
macAddress[self.interface] = match.groups()[0]
match = re.match(r"\s+inet (\S*)\s+netmask\s+(\S*)\s+.*$", row)
if match:
ipv4Address[self.interface] = match.groups()[0]
subnetMask[self.interface] = match.groups()[1]
match = re.match(r"\s+inet6 (\S*)\s+.*$", row)
if match:
ipv6Address[self.interface] = match.groups()[0]
else:
macAddress = {}
ipv4Address = {}
ipv6Address = {}
subnetMask = {}
interface = self.interface
macAddress[interface] = \
netifaces.ifaddresses(interface)[netifaces.AF_LINK][0]['addr']
ipv4Addrs = netifaces.ifaddresses(interface)[netifaces.AF_INET]
ipv4Address[interface] = ipv4Addrs[0]['addr']
subnetMask[interface] = ipv4Addrs[0]['netmask']
ipv6Addrs = netifaces.ifaddresses(interface)[netifaces.AF_INET6]
for addr in ipv6Addrs:
match = re.match(r'(.+)%(.+)', addr['addr'])
if match and match.group(2) == interface:
ipv6Address[interface] = match.group(1)
self.macAddress = macAddress[self.interface]
if self.interface in ipv4Address:
self.ipv4Address = ipv4Address[self.interface]
else:
self.ipv4Address = ""
if self.interface in ipv6Address:
self.ipv6Address = ipv6Address[self.interface]
else:
self.ipv6Address = ""
if self.interface in subnetMask:
self.subnetMask = subnetMask[self.interface]
else:
self.subnetMask = ""
logger.debug("The active network interface is: " + str(self.interface))
self.name = self.settingsModel.GetInstrumentName()
self.contactName = self.settingsModel.GetContactName()
self.contactEmail = self.settingsModel.GetContactEmail()
self.userAgentName = "MyData"
self.userAgentVersion = VERSION
self.userAgentInstallLocation = ""
# pylint: disable=bare-except
if hasattr(sys, 'frozen'):
self.userAgentInstallLocation = os.path.dirname(sys.executable)
else:
try:
self.userAgentInstallLocation = \
os.path.dirname(pkgutil.get_loader("MyData").filename)
except:
self.userAgentInstallLocation = os.getcwd()
fmt = "%-17s %8s %8s %8s %5s%% %9s %s\n"
diskUsage = (
fmt % ("Device", "Total", "Used", "Free", "Use ", "Type", "Mount"))
for part in psutil.disk_partitions(all=False):
if os.name == 'nt':
if 'cdrom' in part.opts or part.fstype == '':
# skip cd-rom drives with no disk in it; they may raise
# ENOENT, pop-up a Windows GUI error for a non-ready
# partition or just hang.
continue
usage = psutil.disk_usage(part.mountpoint)
diskUsage = diskUsage + (
fmt % (part.device, BytesToHuman(usage.total),
BytesToHuman(usage.used), BytesToHuman(usage.free),
int(usage.percent), part.fstype, part.mountpoint))
self.diskUsage = diskUsage.strip()
self.dataPath = self.settingsModel.GetDataDirectory()
self.defaultUser = self.settingsModel.GetUsername()
def UploadUploaderInfo(self):
""" Uploads info about the instrument PC to MyTardis via HTTP POST """
# pylint: disable=too-many-statements
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/mydata_uploader/?format=json" + \
"&uuid=" + urllib.quote(self.uuid)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"
}
try:
response = requests.get(headers=headers, url=url)
except Exception, err:
logger.error(str(err))
raise
if response.status_code == 404:
message = "The MyData app is missing from the MyTardis server."
logger.error(url)
logger.error(message)
raise MissingMyDataAppOnMyTardisServer(message)
if response.status_code >= 200 and response.status_code < 300:
existingUploaderRecords = response.json()
else:
logger.error("An error occurred while retrieving uploader info.")
logger.error("Status code = " + str(response.status_code))
logger.error(response.text)
raise Exception(response.text)
numExistingUploaderRecords = \
existingUploaderRecords['meta']['total_count']
if numExistingUploaderRecords > 0:
self.id = existingUploaderRecords['objects'][0]['id']
if 'settings' in existingUploaderRecords['objects'][0]:
self.uploaderSettings = \
existingUploaderRecords['objects'][0]['settings']
logger.debug("Uploading uploader info to MyTardis...")
if numExistingUploaderRecords > 0:
url = myTardisUrl + "/api/v1/mydata_uploader/%d/" % self.id
else:
url = myTardisUrl + "/api/v1/mydata_uploader/"
self.osPlatform = sys.platform
self.osSystem = platform.system()
self.osRelease = platform.release()
self.osVersion = platform.version()
self.osUsername = getpass.getuser()
self.machine = platform.machine()
self.architecture = str(platform.architecture())
self.processor = platform.processor()
self.memory = BytesToHuman(psutil.virtual_memory().total)
self.cpus = psutil.cpu_count()
self.hostname = platform.node()
uploaderJson = {
"uuid": self.uuid,
"name": self.name,
"contact_name": self.contactName,
"contact_email": self.contactEmail,
"user_agent_name": self.userAgentName,
"user_agent_version": self.userAgentVersion,
"user_agent_install_location": self.userAgentInstallLocation,
"os_platform": self.osPlatform,
"os_system": self.osSystem,
"os_release": self.osRelease,
"os_version": self.osVersion,
"os_username": self.osUsername,
"machine": self.machine,
"architecture": self.architecture,
"processor": self.processor,
"memory": self.memory,
"cpus": self.cpus,
"disk_usage": self.diskUsage,
"data_path": self.dataPath,
"default_user": self.defaultUser,
"interface": self.interface,
"mac_address": self.macAddress,
"ipv4_address": self.ipv4Address,
"ipv6_address": self.ipv6Address,
"subnet_mask": self.subnetMask,
"hostname": self.hostname,
"instruments":
[self.settingsModel.GetInstrument().GetResourceUri()]
}
data = json.dumps(uploaderJson, indent=4)
logger.debug(data)
if numExistingUploaderRecords > 0:
response = requests.put(headers=headers, url=url, data=data)
else:
response = requests.post(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
logger.debug("Upload succeeded for uploader info.")
self.responseJson = response.json()
else:
logger.error("Upload failed for uploader info.")
logger.error("Status code = " + str(response.status_code))
logger.error(response.text)
raise Exception(response.text)
def Validate(settingsModel):
"""
Performs settings validation in separate thread.
"""
logger.debug("Starting run() method for thread %s" %
threading.current_thread().name)
try:
wx.CallAfter(BeginBusyCursorIfRequired)
if sys.platform.startswith("win"):
# BeginBusyCursor should update the cursor everywhere,
# but it doesn't always work on Windows.
if wx.version().startswith("3.0.3.dev"):
busyCursor = wx.Cursor(wx.CURSOR_WAIT)
else:
busyCursor = wx.StockCursor(wx.CURSOR_WAIT)
wx.CallAfter(event.settingsDialog.dialogPanel.SetCursor,
busyCursor)
wx.CallAfter(event.settingsDialog.okButton.Disable)
wx.CallAfter(event.settingsDialog.lockOrUnlockButton.Disable)
app = wx.GetApp()
if hasattr(app, "GetLastConnectivityCheckTime"):
intervalSinceLastCheck = datetime.now() - \
app.GetLastConnectivityCheckTime()
checkInterval = \
event.settingsModel.GetConnectivityCheckInterval()
if intervalSinceLastCheck.total_seconds() >= checkInterval \
or not app.GetLastConnectivityCheckSuccess():
settingsDialogValidationEvent = \
MyDataEvent(EVT_SETTINGS_DIALOG_VALIDATION,
settingsDialog=event.settingsDialog,
settingsModel=settingsModel,
okEvent=event)
checkConnectivityEvent = \
MyDataEvent(EVT_CHECK_CONNECTIVITY,
settingsDialog=event.settingsDialog,
settingsModel=settingsModel,
nextEvent=settingsDialogValidationEvent)
wx.PostEvent(app.GetMainFrame(),
checkConnectivityEvent)
return
try:
event.settingsModel.SaveFieldsFromDialog(
event.settingsDialog, saveToDisk=False)
except: # pylint: disable=bare-except
logger.error(traceback.format_exc())
def SetStatusMessage(message):
"""
Updates status bar.
"""
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
message)
settingsModel.Validate(SetStatusMessage)
wx.CallAfter(EndBusyCursorIfRequired, event)
if settingsModel.IsIncompatibleMyTardisVersion():
wx.CallAfter(event.settingsDialog.okButton.Enable)
wx.CallAfter(
event.settingsDialog.lockOrUnlockButton.Enable)
return
provideValidationResultsEvent = MyDataEvent(
EVT_PROVIDE_SETTINGS_VALIDATION_RESULTS,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel)
if hasattr(app, "GetMainFrame"):
wx.PostEvent(app.GetMainFrame(),
provideValidationResultsEvent)
except IncompatibleMyTardisVersion as err:
wx.CallAfter(EndBusyCursorIfRequired, event)
def ShowDialog(message):
"""
Show error dialog in main thread.
"""
logger.error(message)
# pylint: disable=no-member
# Otherwise pylint complains about PyAssertionError.
# pylint: disable=protected-access
try:
wx.EndBusyCursor()
if wx.version().startswith("3.0.3.dev"):
arrowCursor = wx.Cursor(wx.CURSOR_ARROW)
else:
arrowCursor = wx.StockCursor(wx.CURSOR_ARROW)
event.settingsDialog.dialogPanel.SetCursor(arrowCursor)
except wx._core.PyAssertionError, err:
if "no matching wxBeginBusyCursor()" \
not in str(err):
logger.error(str(err))
raise
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
message = str(err)
wx.CallAfter(ShowDialog, message)
def ProvideSettingsValidationResults(event):
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# Needs refactoring.
"""
Only called after settings dialog has been shown.
Not called if settings validation was triggered by
a background task.
"""
if event.GetEventId() != EVT_PROVIDE_SETTINGS_VALIDATION_RESULTS:
event.Skip()
return
settingsValidation = event.settingsModel.GetValidation()
if settingsValidation.Aborted():
wx.CallAfter(EndBusyCursorIfRequired, event)
if wx.version().startswith("3.0.3.dev"):
arrowCursor = wx.Cursor(wx.CURSOR_ARROW)
else:
arrowCursor = wx.StockCursor(wx.CURSOR_ARROW)
event.settingsDialog.dialogPanel.SetCursor(arrowCursor)
event.settingsModel.RollBack()
return
if settingsValidation is not None and \
not settingsValidation.IsValid():
wx.GetApp().GetMainFrame().SetStatusMessage("")
message = settingsValidation.GetMessage()
logger.error(message)
if settingsValidation.GetSuggestion():
currentValue = ""
if settingsValidation.GetField() == "instrument_name":
currentValue = event.settingsDialog.GetInstrumentName()
elif settingsValidation.GetField() == "facility_name":
currentValue = event.settingsDialog.GetFacilityName()
elif settingsValidation.GetField() == "mytardis_url":
currentValue = event.settingsDialog.GetMyTardisUrl()
message = message.strip()
if currentValue != "":
message += "\n\nMyData suggests that you replace \"%s\" " \
% currentValue
message += "with \"%s\"." \
% settingsValidation.GetSuggestion()
else:
message += "\n\nMyData suggests that you use \"%s\"." \
% settingsValidation.GetSuggestion()
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.CANCEL | wx.ICON_ERROR)
okToUseSuggestion = dlg.ShowModal()
if okToUseSuggestion == wx.ID_OK:
if settingsValidation.GetField() == "instrument_name":
event.settingsDialog\
.SetInstrumentName(settingsValidation
.GetSuggestion())
elif settingsValidation.GetField() == "facility_name":
event.settingsDialog.SetFacilityName(
settingsValidation.GetSuggestion())
elif settingsValidation.GetField() == "mytardis_url":
event.settingsDialog.SetMyTardisUrl(
settingsValidation.GetSuggestion())
else:
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
if settingsValidation.GetField() == "instrument_name":
event.settingsDialog.instrumentNameField.SetFocus()
event.settingsDialog.instrumentNameField.SelectAll()
elif settingsValidation.GetField() == "facility_name":
event.settingsDialog.facilityNameField.SetFocus()
event.settingsDialog.facilityNameField.SelectAll()
elif settingsValidation.GetField() == "data_directory":
event.settingsDialog.dataDirectoryField.SetFocus()
event.settingsDialog.dataDirectoryField.SelectAll()
elif settingsValidation.GetField() == "mytardis_url":
event.settingsDialog.myTardisUrlField.SetFocus()
event.settingsDialog.myTardisUrlField.SelectAll()
elif settingsValidation.GetField() == "contact_name":
event.settingsDialog.contactNameField.SetFocus()
event.settingsDialog.contactNameField.SelectAll()
elif settingsValidation.GetField() == "contact_email":
event.settingsDialog.contactEmailField.SetFocus()
event.settingsDialog.contactEmailField.SelectAll()
elif settingsValidation.GetField() == "username":
event.settingsDialog.usernameField.SetFocus()
event.settingsDialog.usernameField.SelectAll()
elif settingsValidation.GetField() == "api_key":
event.settingsDialog.apiKeyField.SetFocus()
event.settingsDialog.apiKeyField.SelectAll()
elif settingsValidation.GetField() == "scheduled_time":
event.settingsDialog.timeCtrl.SetFocus()
elif settingsValidation.GetField() == "includes_file":
event.settingsDialog.includesFileField.SetFocus()
event.settingsDialog.includesFileField.SelectAll()
elif settingsValidation.GetField() == "excludes_file":
event.settingsDialog.excludesFileField.SetFocus()
event.settingsDialog.excludesFileField.SelectAll()
logger.debug("Settings were not valid, so Settings dialog "
"should remain visible.")
if wx.version().startswith("3.0.3.dev"):
arrowCursor = wx.Cursor(wx.CURSOR_ARROW)
else:
arrowCursor = wx.StockCursor(wx.CURSOR_ARROW)
event.settingsDialog.dialogPanel.SetCursor(arrowCursor)
event.settingsModel.RollBack()
return
if event.settingsModel.IgnoreOldDatasets():
intervalIfUsed = " (created within the past %d %s)" \
% (event.settingsModel.GetIgnoreOldDatasetIntervalNumber(),
event.settingsModel.GetIgnoreOldDatasetIntervalUnit())
else:
intervalIfUsed = ""
numDatasets = settingsValidation.GetDatasetCount()
if numDatasets != -1:
message = "Assuming a folder structure of '%s', " \
"there %s %d %s in \"%s\"%s.\n\n" \
"Do you want to continue?" \
% (event.settingsModel.GetFolderStructure(),
"are" if numDatasets != 1 else "is",
settingsValidation.GetDatasetCount(),
"datasets" if numDatasets != 1 else "dataset",
event.settingsDialog.GetDataDirectory(),
intervalIfUsed)
confirmationDialog = \
wx.MessageDialog(None, message, "MyData",
wx.YES | wx.NO | wx.ICON_QUESTION)
okToContinue = confirmationDialog.ShowModal()
if okToContinue != wx.ID_YES:
return
logger.debug("Settings were valid, so we'll save the settings "
"to disk and close the Settings dialog.")
# pylint: disable=bare-except
try:
# Now is a good time to define the MyData instances's uploader
# model object, which will generate a UUID if necessary.
# The UUID will be saved to disk along with the settings from
# the settings dialog.
uploaderModel = UploaderModel(event.settingsModel)
event.settingsModel.SetUploaderModel(uploaderModel)
# Use the config path determined by appdirs, not the one
# determined by a user dragging and dropping a config
# file onto MyData's Settings dialog:
app = wx.GetApp()
if hasattr(app, "GetConfigPath"):
configPath = app.GetConfigPath()
else:
configPath = None
event.settingsModel.SaveFieldsFromDialog(event.settingsDialog,
configPath=configPath,
saveToDisk=True)
event.settingsDialog.EndModal(wx.ID_OK)
event.settingsDialog.Show(False)
logger.debug("Closed Settings dialog.")
if event.settingsModel.GetScheduleType() == "Manually":
message = \
"MyData's schedule type is currently " \
"set to 'manual', so you will need to click " \
"the Upload toolbar icon or the Sync Now " \
"menu item to begin the data scans and uploads."
title = "Manual Schedule"
dlg = wx.MessageDialog(None, message, title,
wx.OK | wx.ICON_WARNING)
dlg.ShowModal()
except:
logger.error(traceback.format_exc())
except ScpException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
message = str(err)
message += "\n\n" + err.command
logger.error(message)
except ValueError, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if str(err) == "read of closed file" or \
str(err) == "seek of closed file":
logger.debug("Aborting upload for \"%s\" because "
"file handle was closed." %
self.uploadModel.GetRelativePathToUpload())
return
else:
raise
except IncompatibleMyTardisVersion, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.shutdownUploadsEvent(
failed=True))
message = str(err)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
def Run(self):
# pylint: disable=too-many-locals
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
self.foldersController.uploadsThreadingLock.acquire()
uploadDataViewId = self.uploadsModel.GetMaxDataViewId() + 1
self.uploadModel = UploadModel(dataViewId=uploadDataViewId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex)
self.uploadsModel.AddRow(self.uploadModel)
self.foldersController.uploadsThreadingLock.release()
self.uploadModel.SetBytesUploadedPreviously(
self.bytesUploadedPreviously)
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
dataFileName = os.path.basename(dataFilePath)
dataFileDirectory = \
self.folderModel.GetDataFileDirectory(self.dataFileIndex)
ignoreNewFiles = self.settingsModel.IgnoreNewFiles()
ignoreNewFilesMinutes = self.settingsModel.GetIgnoreNewFilesMinutes()
ignoreNewFilesSeconds = 0
if ignoreNewFiles:
ignoreNewFilesSeconds = ignoreNewFilesMinutes * 60
if (time.time() - os.path.getmtime(dataFilePath)) <= \
ignoreNewFilesSeconds:
message = "Not uploading file, in case it is still being modified."
logger.warning(message.replace('file', dataFilePath))
self.uploadsModel.SetMessage(self.uploadModel, message)
self.uploadsModel.SetStatus(self.uploadModel, UploadStatus.FAILED)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.uploadCompleteEvent(
id=self.foldersController.EVT_UPLOAD_FAILED,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
uploadModel=self.uploadModel))
return
logger.debug("Uploading " +
self.folderModel.GetDataFileName(self.dataFileIndex) +
"...")
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
url = myTardisUrl + "/api/v1/mydata_dataset_file/"
else:
url = myTardisUrl + "/api/v1/dataset_file/"
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
if self.foldersController.IsShuttingDown():
return
message = "Getting data file size..."
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileSize = self.folderModel.GetDataFileSize(self.dataFileIndex)
self.uploadModel.SetFileSize(dataFileSize)
if self.foldersController.IsShuttingDown():
return
# The HTTP POST upload method doesn't support resuming uploads,
# so we always (re-)create the JSON to be POSTed when we find
# a file whose datafile record is unverified.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
message = "Calculating MD5 checksum..."
self.uploadsModel.SetMessage(self.uploadModel, message)
def Md5ProgressCallback(bytesProcessed):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
if dataFileSize > 0:
percentComplete = \
100.0 - ((dataFileSize - bytesProcessed) * 100.0) \
/ dataFileSize
else:
percentComplete = 100
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if dataFileSize >= (1024 * 1024 * 1024):
message = "%3.1f %% MD5 summed" % percentComplete
else:
message = "%3d %% MD5 summed" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileMd5Sum = \
self.CalculateMd5Sum(dataFilePath, dataFileSize,
self.uploadModel,
progressCallback=Md5ProgressCallback)
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
else:
dataFileSize = int(self.existingUnverifiedDatafile.GetSize())
self.uploadModel.SetProgress(0)
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if self.foldersController.IsShuttingDown():
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
message = "Checking MIME type..."
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileMimeType = self.mimeTypes.guess_type(dataFilePath)[0]
if self.foldersController.IsShuttingDown():
return
message = "Defining JSON data for POST..."
self.uploadsModel.SetMessage(self.uploadModel, message)
datasetUri = self.folderModel.GetDatasetModel().GetResourceUri()
dataFileCreatedTime = \
self.folderModel.GetDataFileCreatedTime(self.dataFileIndex)
dataFileModifiedTime = \
self.folderModel.GetDataFileModifiedTime(self.dataFileIndex)
dataFileJson = {
"dataset": datasetUri,
"filename": dataFileName,
"directory": dataFileDirectory,
"md5sum": dataFileMd5Sum,
"size": dataFileSize,
"mimetype": dataFileMimeType,
"created_time": dataFileCreatedTime,
"modification_time": dataFileModifiedTime,
}
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
dataFileJson['uploader_uuid'] = self.settingsModel.GetUuid()
dataFileJson['requester_key_fingerprint'] = \
self.settingsModel.GetSshKeyPair().GetFingerprint()
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
message = "Initializing buffered reader..."
self.uploadsModel.SetMessage(self.uploadModel, message)
datafileBufferedReader = io.open(dataFilePath, 'rb')
self.uploadModel.SetBufferedReader(datafileBufferedReader)
def ProgressCallback(current, total, message=None):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
if current is None:
# For a zero-sized file, current will be None
# before its upload, and 0 after is upload.
percentComplete = 0
current = 0
elif total > 0:
percentComplete = \
100.0 - ((total - current) * 100.0) / total
else:
percentComplete = 100
self.uploadModel.SetBytesUploaded(current)
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if message:
self.uploadsModel.SetMessage(self.uploadModel, message)
else:
if total >= (1024 * 1024 * 1024):
message = "%3.1f %% uploaded" % percentComplete
else:
message = "%3d %% uploaded" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
# The database interactions below should go in a model class.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
def PosterCallback(param, current, total):
# pylint: disable=unused-argument
ProgressCallback(current, total)
datagen, headers = poster.encode.multipart_encode(
{"json_data": json.dumps(dataFileJson),
"attached_file": datafileBufferedReader},
cb=PosterCallback)
opener = poster.streaminghttp.register_openers()
opener.addheaders = [("Authorization", "ApiKey " +
myTardisUsername +
":" + myTardisApiKey),
("Content-Type", "application/json"),
("Accept", "application/json")]
elif not self.existingUnverifiedDatafile:
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
data = json.dumps(dataFileJson)
message = "Uploading..."
self.uploadsModel.SetMessage(self.uploadModel, message)
postSuccess = False
uploadSuccess = False
request = None
response = None
# pylint: disable=broad-except
# pylint: disable=too-many-nested-blocks
try:
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
request = urllib2.Request(url, datagen, headers)
try:
if self.foldersController.uploadMethod == \
UploadMethod.HTTP_POST:
response = urllib2.urlopen(request)
postSuccess = True
uploadSuccess = True
else:
if not self.existingUnverifiedDatafile:
response = requests.post(headers=headers, url=url,
data=data)
postSuccess = response.status_code >= 200 and \
response.status_code < 300
logger.debug(response.text)
if postSuccess or self.existingUnverifiedDatafile:
uploadToStagingRequest = self.settingsModel\
.GetUploadToStagingRequest()
host = uploadToStagingRequest.GetScpHostname()
port = uploadToStagingRequest.GetScpPort()
location = uploadToStagingRequest.GetLocation()
username = uploadToStagingRequest.GetScpUsername()
privateKeyFilePath = self.settingsModel\
.GetSshKeyPair().GetPrivateKeyFilePath()
if self.existingUnverifiedDatafile:
uri = self.existingUnverifiedDatafile\
.GetReplicas()[0].GetUri()
remoteFilePath = "%s/%s" % (location.rstrip('/'),
uri)
else:
# DataFile creation via the MyTardis API doesn't
# return JSON, but if a DataFile record is created
# without specifying a storage location, then a
# temporary location is returned for the client
# to copy/upload the file to.
tempUrl = response.text
remoteFilePath = tempUrl
while True:
try:
UploadFile(dataFilePath,
dataFileSize,
username,
privateKeyFilePath,
host, port, remoteFilePath,
ProgressCallback,
self.foldersController,
self.uploadModel)
except IOError, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = "This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
except ScpException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = \
"This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
except SshException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = \
"This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
break
def CreateExperimentForFolder(folderModel, testRun=False):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
userFolderName = folderModel.GetUserFolderName()
hostname = settingsModel.GetUploaderModel().GetHostname()
location = folderModel.GetLocation()
groupFolderName = folderModel.GetGroupFolderName()
owner = folderModel.GetOwner()
ownerUsername = folderModel.GetOwner().GetUsername()
# pylint: disable=bare-except
try:
ownerUserId = folderModel.GetOwner().GetJson()['id']
except:
ownerUserId = None
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
experimentTitle = folderModel.GetExperimentTitle()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
if userFolderName:
message = "Creating experiment for uploader '%s', " \
"user folder '%s'." % (uploaderName, userFolderName)
if groupFolderName:
message += ", group folder : '%s'" % groupFolderName
elif groupFolderName:
message = "Creating experiment for uploader '%s', " \
"user group folder '%s'." % (uploaderName, groupFolderName)
else:
message = "Creating experiment for uploader '%s'" % uploaderName
logger.info(message)
if userFolderName:
description = ("Uploader: %s\n"
"User folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, userFolderName, hostname, location))
if groupFolderName:
description += "\nGroup folder name: %s" % groupFolderName
else:
description = ("Uploader: %s\n"
"Group folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, groupFolderName, hostname, location))
if testRun:
message = "CREATING NEW EXPERIMENT FOR FOLDER: %s\n" \
" Title: %s\n" \
" Description: \n" \
" Uploader: %s\n" \
" User folder name: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
experimentTitle, uploaderName, userFolderName,
ownerUsername)
logger.testrun(message)
return
experimentJson = {
"title":
experimentTitle,
"description":
description,
"immutable":
False,
"parameter_sets": [{
"schema":
"http://mytardis.org/schemas"
"/mydata/defaultexperiment",
"parameters": [{
"name": "uploader",
"value": uploaderUuid
}, {
"name": "user_folder_name",
"value": userFolderName
}]
}]
}
if groupFolderName:
experimentJson["parameter_sets"][0]["parameters"].append({
"name":
"group_folder_name",
"value":
groupFolderName
})
headers = {
"Authorization":
"ApiKey %s:%s" %
(myTardisDefaultUsername, myTardisDefaultUserApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/mydata_experiment/"
response = requests.post(headers=headers,
url=url,
data=json.dumps(experimentJson))
# pylint: disable=bare-except
try:
createdExperimentJson = response.json()
createdExperiment = ExperimentModel(settingsModel,
createdExperimentJson)
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the experiment.\n\n" \
"Please ask your MyTardis administrator to " \
"check that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message)
raise
if response.status_code == 201:
message = "Succeeded in creating experiment '%s' for uploader " \
"\"%s\" and user folder \"%s\"" \
% (experimentTitle, uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.debug(message)
facilityManagersGroup = settingsModel.GetFacility(
).GetManagerGroup()
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
facilityManagersGroup)
# Avoid creating a duplicate ObjectACL if the user folder's
# username matches the facility manager's username.
# Don't attempt to create an ObjectACL record for an
# invalid user (without a MyTardis user ID).
if myTardisDefaultUsername != ownerUsername and \
ownerUserId is not None:
ObjectAclModel.ShareExperimentWithUser(createdExperiment,
owner)
if folderModel.GetGroup() is not None and \
folderModel.GetGroup().GetId() != \
facilityManagersGroup.GetId():
ObjectAclModel.ShareExperimentWithGroup(
createdExperiment, folderModel.GetGroup())
else:
message = "Failed to create experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.error(message)
logger.error(headers)
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
return createdExperiment
def GetExperimentForFolder(folderModel):
"""
See also GetOrCreateExperimentForFolder
"""
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
userFolderName = folderModel.GetUserFolderName()
groupFolderName = folderModel.GetGroupFolderName()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
experimentTitle = folderModel.GetExperimentTitle()
if folderModel.ExperimentTitleSetManually():
expTitleEncoded = urllib2.quote(experimentTitle)
folderStructureEncoded = \
urllib2.quote(settingsModel.GetFolderStructure())
url = myTardisUrl + "/api/v1/mydata_experiment/?format=json" + \
"&title=" + expTitleEncoded + \
"&folder_structure=" + folderStructureEncoded
else:
url = myTardisUrl + "/api/v1/mydata_experiment/?format=json" + \
"&uploader=" + uploaderUuid
if userFolderName:
url += "&user_folder_name=" + urllib2.quote(userFolderName)
if groupFolderName:
url += "&group_folder_name=" + urllib2.quote(groupFolderName)
headers = {
"Authorization":
"ApiKey %s:%s" %
(myTardisDefaultUsername, myTardisDefaultUserApiKey)
}
response = requests.get(url=url, headers=headers)
numExperimentsFound = 0
experimentsJson = []
try:
experimentsJson = response.json()
numExperimentsFound = experimentsJson['meta']['total_count']
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 404:
message = "Failed to confirm existence of experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to retrieve the experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to retrieve the experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
raise
if numExperimentsFound == 0:
if folderModel.ExperimentTitleSetManually():
if userFolderName:
message = "Experiment not found for '%s', %s, '%s'" \
% (uploaderName, userFolderName, experimentTitle)
if groupFolderName:
message += ", '%s'" % groupFolderName
elif groupFolderName:
message = "Experiment not found for '%s', %s, '%s'" \
% (uploaderName, groupFolderName, experimentTitle)
else:
message = "Experiment not found for '%s', '%s'" \
% (uploaderName, experimentTitle)
else:
if userFolderName:
message = "Experiment not found for '%s', %s" \
% (uploaderName, userFolderName)
if groupFolderName:
message += ", '%s'" % groupFolderName
elif groupFolderName:
message = "Experiment not found for '%s', %s" \
% (uploaderName, groupFolderName)
else:
message = "Experiment not found for '%s'." % uploaderName
logger.debug(message)
raise DoesNotExist(message, modelClass=ExperimentModel)
elif numExperimentsFound == 1 or \
folderModel.ExperimentTitleSetManually():
# When an experiment is created for a single MyData
# uploader instance, we shouldn't find any duplicates,
# but when MyData is instructed to add datasets to an
# existing experiment (e.g. the "Experiment / Data"
# folder structure), we don't raise a critical error
# for duplicate experiments - instead we just use the
# first one we find.
if folderModel.ExperimentTitleSetManually():
if userFolderName:
message = "Found existing experiment with title '%s' " \
"and user folder '%s'" % (experimentTitle,
userFolderName)
if groupFolderName:
message += " and group folder '%s'." % groupFolderName
elif groupFolderName:
message = "Found existing experiment with title '%s' " \
"and user group folder '%s'" % (experimentTitle,
groupFolderName)
else:
message = "Found existing experiment with title '%s'." \
% experimentTitle
else:
if userFolderName:
message = "Found existing experiment for uploader '%s' " \
"and user folder '%s'." % (uploaderName,
userFolderName)
if groupFolderName:
message += " and group folder '%s'." % groupFolderName
elif groupFolderName:
message = "Found existing experiment for uploader '%s' " \
"and user group folder '%s'." % (uploaderName,
groupFolderName)
else:
message = "Found existing experiment for uploader '%s'." \
% uploaderName
logger.debug(message)
return ExperimentModel(settingsModel,
experimentsJson['objects'][0])
elif numExperimentsFound > 1:
message = "ERROR: Found multiple experiments matching " + \
"Uploader UUID for user '%s'" % userFolderName
if groupFolderName:
message += " and group '%s'" % groupFolderName
logger.error(message)
for expJson in experimentsJson['objects']:
logger.error("\t" + expJson['title'])
groupFolderString = ""
if groupFolderName:
groupFolderString = ", and group folder \"%s\"" \
% groupFolderName
message = "Multiple experiments were found matching " \
"uploader \"%s\" and user folder \"%s\"%s " \
"for folder \"%s\"." \
% (uploaderName, userFolderName, groupFolderString,
folderModel.GetFolder())
message += "\n\n"
message += "This shouldn't happen. Please ask your " \
"MyTardis administrator to investigate."
raise MultipleObjectsReturned(message)
def CreateDatasetIfNecessary(folderModel, testRun=False):
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
description = folderModel.GetFolder()
settingsModel = folderModel.settingsModel
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
experiment = folderModel.GetExperiment()
if experiment: # Could be None in test run
url = myTardisUrl + "/api/v1/dataset/?format=json" + \
"&experiments__id=" + str(experiment.GetId())
url = url + "&description=" + urllib.quote(description)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey)
}
response = requests.get(headers=headers, url=url)
existingMatchingDatasets = response.json()
numExistingMatchingDatasets = \
existingMatchingDatasets['meta']['total_count']
if numExistingMatchingDatasets == 1:
logger.debug("Found existing dataset for folder " +
description)
elif numExistingMatchingDatasets > 1:
logger.debug("WARNING: Found multiple datasets for folder " +
description)
else:
numExistingMatchingDatasets = 0
if numExistingMatchingDatasets == 0:
logger.debug("Creating dataset record for folder: " + description)
description = folderModel.GetFolder()
if experiment:
experimentUri = experiment.GetResourceUri()
else:
experimentUri = None
immutable = False
datasetJson = {
"instrument": settingsModel.GetInstrument().GetResourceUri(),
"description": description,
"experiments": [experimentUri],
"immutable": immutable
}
data = json.dumps(datasetJson)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/dataset/"
if testRun:
message = "CREATING NEW DATASET FOR FOLDER: %s\n" \
" Description: %s" \
% (folderModel.GetRelPath(),
description)
if experiment: # Could be None in test run.
message += "\n In Experiment: %s/%s" \
% (folderModel.settingsModel.GetMyTardisUrl(),
experiment.GetViewUri())
logger.testrun(message)
return
response = requests.post(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
newDatasetJson = response.json()
return DatasetModel(settingsModel, newDatasetJson)
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
if response.status_code == 401:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisUsername
raise Unauthorized(message)
elif response.status_code == 500:
message = "Couldn't create dataset \"%s\" " \
"for folder \"%s\"." \
% (description, folderModel.GetFolder())
message += "\n\n"
message += "An Internal Server Error occurred."
message += "\n\n"
message += "If running MyTardis in DEBUG mode, " \
"more information may be available below. " \
"Otherwise, please ask your MyTardis " \
"administrator to check in their logs " \
"for more information."
message += "\n\n"
try:
message += "ERROR: \"%s\"" \
% response.json()['error_message']
except: # pylint: disable=bare-except
message += response.text
raise InternalServerError(message)
raise Exception(response.text)
else:
existingDatasetJson = \
existingMatchingDatasets['objects'][0]
if testRun:
description = existingDatasetJson['description']
datasetId = existingDatasetJson['id']
viewUri = "dataset/%s" % datasetId
message = "ADDING TO EXISTING DATASET FOR FOLDER: %s\n" \
" URL: %s/%s\n" \
" Description: %s\n" \
" In Experiment: %s/%s" \
% (folderModel.GetRelPath(),
folderModel.settingsModel.GetMyTardisUrl(),
viewUri,
description,
folderModel.settingsModel.GetMyTardisUrl(),
folderModel.GetExperiment().GetViewUri())
logger.testrun(message)
return DatasetModel(settingsModel,
existingMatchingDatasets['objects'][0])
def GetDataFiles(self):
# pylint: disable=too-many-nested-blocks
if not self.datafiles:
try:
self.getDatasetFilesThreadingLock.acquire()
if self.datafiles:
return self.datafiles
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
# limit=0 can still encounter a limit of 1000 unless
# API_LIMIT_PER_PAGE is set to 0 in MyTardis's settings.py
limit = 0
url = "%s/api/v1/dataset/%d/files/?format=json&limit=%d" \
% (myTardisUrl, self.GetId(), limit)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey)
}
logger.debug(url)
response = requests.get(headers=headers, url=url)
if response.status_code >= 200 and response.status_code < 300:
from .datafile import DataFileModel
self.datafiles = []
datafilesJson = response.json()['objects']
for datafileJson in datafilesJson:
self.datafiles.append(
DataFileModel(self.settingsModel, self,
datafileJson))
offset = 0
while response.json()['meta']['next']:
# We should be able to use
# response.json()['meta']['next'] in the URL,
# instead of manually constructing the next
# URL using offset.
# But response.json()['meta']['next'] seems to give
# the wrong URL for /api/v1/dataset/%d/files/
offset += 1
url = "%s/api/v1/dataset/%d/files/?format=json" \
"&limit=%d&offset=%d" % (myTardisUrl, self.GetId(),
limit, offset)
logger.debug(url)
response = requests.get(headers=headers, url=url)
if response.status_code >= 200 and \
response.status_code < 300:
datafilesJson = response.json()['objects']
for datafileJson in datafilesJson:
self.datafiles\
.append(DataFileModel(self.settingsModel,
self, datafileJson))
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
else:
logger.error(url)
logger.error("response.status_code = " +
str(response.status_code))
logger.error(response.text)
if response.status_code == 401:
message = "Couldn't list files for dataset \"%s\". " \
% (self.GetDescription())
message += "\n\n"
message += "Please ask your MyTardis administrator " \
"to check the permissions of the \"%s\" " \
"user account." % myTardisUsername
self.getDatasetFilesThreadingLock.release()
raise Unauthorized(message)
self.getDatasetFilesThreadingLock.release()
raise Exception(response.text)
finally:
self.getDatasetFilesThreadingLock.release()
return self.datafiles
def CheckConnectivity(event):
"""
Checks network connectivity.
"""
if event.GetEventId() != EVT_CHECK_CONNECTIVITY:
event.Skip()
return
def CheckConnectivityWorker():
"""
Checks network connectivity in separate thread.
"""
wx.CallAfter(BeginBusyCursorIfRequired)
# pylint: disable=broad-except
try:
activeNetworkInterfaces = \
UploaderModel.GetActiveNetworkInterfaces()
except Exception, err:
logger.error(traceback.format_exc())
if type(err).__name__ == "WindowsError" and \
"The handle is invalid" in str(err):
message = "An error occurred, suggesting " \
"that you have launched MyData.exe from a " \
"Command Prompt window. Please launch it " \
"from a shortcut or from a Windows Explorer " \
"window instead.\n" \
"\n" \
"See: https://bugs.python.org/issue3905"
def ShowErrorDialog(message):
"""
Show error dialog in main thread.
"""
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.CallAfter(ShowErrorDialog, message)
wx.CallAfter(EndBusyCursorIfRequired, event)
if len(activeNetworkInterfaces) > 0:
logger.debug(
"Found at least one active network interface: %s." %
activeNetworkInterfaces[0])
app = wx.GetApp()
if hasattr(app, "GetMainFrame"):
app.SetLastConnectivityCheckSuccess(True)
app.SetLastConnectivityCheckTime(datetime.now())
app.SetActiveNetworkInterface(activeNetworkInterfaces[0])
if event.nextEvent:
wx.PostEvent(wx.GetApp().GetMainFrame(), event.nextEvent)
else:
wx.GetApp().SetLastConnectivityCheckSuccess(False)
wx.GetApp().SetLastConnectivityCheckTime(datetime.now())
wx.GetApp().SetActiveNetworkInterface(None)
message = "No active network interfaces." \
"\n\n" \
"Please ensure that you have an active " \
"network interface (e.g. Ethernet or WiFi)."
def ShowDialog():
"""
Show error dialog in main thread.
"""
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.GetApp().GetMainFrame().SetStatusMessage("")
wx.GetApp().GetMainFrame().SetConnected(
event.settingsModel.GetMyTardisUrl(), False)
wx.CallAfter(ShowDialog)
def __init__(self, settingsModel):
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
self.settingsModel = settingsModel
self.interface = None
self.responseJson = None
self.id = None # pylint: disable=invalid-name
self.uploaderSettings = None
self.uuid = self.settingsModel.GetUuid()
if self.uuid is None:
self.GenerateUuid()
self.settingsModel.SetUuid(self.uuid)
self.osUsername = ""
self.cpus = 0
self.osPlatform = ""
self.hostname = ""
self.machine = ""
self.osVersion = ""
self.requestStagingAccessThreadLock = threading.Lock()
self.memory = ""
self.osSystem = ""
self.architecture = ""
self.osRelease = ""
self.processor = ""
self.architecture = ""
# Here we check connectivity even if we've already done so, because
# we need to ensure that we get the correct network interface for
# self.interface, otherwise if the active interface changes,
# we can get errors like this: KeyError: 'RTC'
# when accessing things like ipv4Address[self.interface]
activeInterfaces = UploaderModel.GetActiveNetworkInterfaces()
if len(activeInterfaces) == 0:
message = "No active network interfaces." \
"\n\n" \
"Please ensure that you have an active network interface " \
"(e.g. Ethernet or WiFi)."
raise NoActiveNetworkInterface(message)
# Sometimes on Windows XP, you can end up with multiple results
# from "netsh interface show interface"
# If there is one called "Local Area Connection",
# then that's the one we'll go with.
if "Local Area Connection" in activeInterfaces:
activeInterfaces = ["Local Area Connection"]
elif "Local Area Connection 2" in activeInterfaces:
activeInterfaces = ["Local Area Connection 2"]
elif "Ethernet" in activeInterfaces:
activeInterfaces = ["Ethernet"]
elif "Internet" in activeInterfaces:
activeInterfaces = ["Internet"]
elif "Wi-Fi" in activeInterfaces:
activeInterfaces = ["Wi-Fi"]
# For now, we're only dealing with one active network interface.
# It is possible to have more than one active network interface,
# but we hope that the code above has picked the best one.
# If there are no active interfaces, then we shouldn't have
# reached this point - we should have already raised an
# exception.
self.interface = activeInterfaces[0]
if sys.platform.startswith("win"):
proc = subprocess.Popen(["ipconfig", "/all"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=DEFAULT_STARTUP_INFO,
creationflags=DEFAULT_CREATION_FLAGS)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
macAddress = {}
ipv4Address = {}
ipv6Address = {}
subnetMask = {}
interface = ""
for row in stdout.split("\n"):
match = re.match(r"^\S.*adapter (.*):\s*$", row)
if match:
interface = match.groups()[0]
if interface == self.interface:
if ': ' in row:
key, value = row.split(': ')
if key.strip(' .') == "Physical Address":
macAddress[interface] = value.strip()
if "IPv4 Address" in key.strip(' .'):
ipv4Address[interface] = \
value.strip().replace("(Preferred)", "")
ipv4Address[interface] = \
ipv4Address[interface] \
.replace("(Tentative)", "")
if "IPv6 Address" in key.strip(' .'):
ipv6Address[interface] = \
value.strip().replace("(Preferred)", "")
ipv6Address[interface] = \
ipv6Address[interface] \
.replace("(Tentative)", "")
if "Subnet Mask" in key.strip(' .'):
subnetMask[interface] = value.strip()
elif sys.platform.startswith("darwin"):
proc = subprocess.Popen(["ifconfig", self.interface],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=DEFAULT_STARTUP_INFO,
creationflags=DEFAULT_CREATION_FLAGS)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
macAddress = {}
ipv4Address = {}
ipv6Address = {}
subnetMask = {}
for row in stdout.split("\n"):
if sys.platform.startswith("darwin"):
match = re.match(r"\s+ether (\S*)\s*$", row)
else:
match = re.match(r".*\s+HWaddr (\S*)\s*$", row)
if match:
macAddress[self.interface] = match.groups()[0]
match = re.match(r"\s+inet (\S*)\s+netmask\s+(\S*)\s+.*$", row)
if match:
ipv4Address[self.interface] = match.groups()[0]
subnetMask[self.interface] = match.groups()[1]
match = re.match(r"\s+inet6 (\S*)\s+.*$", row)
if match:
ipv6Address[self.interface] = match.groups()[0]
else:
macAddress = {}
ipv4Address = {}
ipv6Address = {}
subnetMask = {}
interface = self.interface
macAddress[interface] = \
netifaces.ifaddresses(interface)[netifaces.AF_LINK][0]['addr']
ipv4Addrs = netifaces.ifaddresses(interface)[netifaces.AF_INET]
ipv4Address[interface] = ipv4Addrs[0]['addr']
subnetMask[interface] = ipv4Addrs[0]['netmask']
ipv6Addrs = netifaces.ifaddresses(interface)[netifaces.AF_INET6]
for addr in ipv6Addrs:
match = re.match(r'(.+)%(.+)', addr['addr'])
if match and match.group(2) == interface:
ipv6Address[interface] = match.group(1)
self.macAddress = macAddress[self.interface]
if self.interface in ipv4Address:
self.ipv4Address = ipv4Address[self.interface]
else:
self.ipv4Address = ""
if self.interface in ipv6Address:
self.ipv6Address = ipv6Address[self.interface]
else:
self.ipv6Address = ""
if self.interface in subnetMask:
self.subnetMask = subnetMask[self.interface]
else:
self.subnetMask = ""
logger.debug("The active network interface is: " + str(self.interface))
self.name = self.settingsModel.GetInstrumentName()
self.contactName = self.settingsModel.GetContactName()
self.contactEmail = self.settingsModel.GetContactEmail()
self.userAgentName = "MyData"
self.userAgentVersion = VERSION
self.userAgentInstallLocation = ""
# pylint: disable=bare-except
if hasattr(sys, 'frozen'):
self.userAgentInstallLocation = os.path.dirname(sys.executable)
else:
try:
self.userAgentInstallLocation = \
os.path.dirname(pkgutil.get_loader("MyData").filename)
except:
self.userAgentInstallLocation = os.getcwd()
fmt = "%-17s %8s %8s %8s %5s%% %9s %s\n"
diskUsage = (
fmt % ("Device", "Total", "Used", "Free", "Use ", "Type", "Mount"))
for part in psutil.disk_partitions(all=False):
if os.name == 'nt':
if 'cdrom' in part.opts or part.fstype == '':
# skip cd-rom drives with no disk in it; they may raise
# ENOENT, pop-up a Windows GUI error for a non-ready
# partition or just hang.
continue
usage = psutil.disk_usage(part.mountpoint)
diskUsage = diskUsage + (
fmt % (part.device, BytesToHuman(usage.total),
BytesToHuman(usage.used), BytesToHuman(usage.free),
int(usage.percent), part.fstype, part.mountpoint))
self.diskUsage = diskUsage.strip()
self.dataPath = self.settingsModel.GetDataDirectory()
self.defaultUser = self.settingsModel.GetUsername()
if "no matching wxBeginBusyCursor()" \
not in str(err):
logger.error(str(err))
raise
dlg = wx.MessageDialog(None, message, "MyData",
wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
message = str(err)
wx.CallAfter(ShowDialog, message)
finally:
wx.CallAfter(event.settingsDialog.okButton.Enable)
wx.CallAfter(event.settingsDialog.lockOrUnlockButton.Enable)
wx.CallAfter(EndBusyCursorIfRequired, event)
logger.debug("Finishing run() method for thread %s" %
threading.current_thread().name)
thread = threading.Thread(target=Validate,
args=(event.settingsModel, ),
name="SettingsModelValidationThread")
logger.debug("Starting thread %s" % thread.name)
thread.start()
logger.debug("Started thread %s" % thread.name)
@staticmethod
def ProvideSettingsValidationResults(event):
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
# Needs refactoring.
"""
Only called after settings dialog has been shown.
