def GetFacility(settingsModel, name):
"""
Get facility by name.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/facility/?format=json&name=" + \
urllib.quote(name)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"
}
response = requests.get(url=url, headers=headers, stream=False)
logger.debug(response.text)
if response.status_code != 200:
message = response.text
response.close()
raise Exception(message)
facilitiesJson = response.json()
response.close()
numFacilitiesFound = facilitiesJson['meta']['total_count']
if numFacilitiesFound == 0:
logger.warning("Facility \"%s\" was not found in MyTardis" % name)
return None
else:
logger.debug("Found facility record for name '" + name + "'.")
return FacilityModel(settingsModel=settingsModel,
name=name,
facilityJson=facilitiesJson['objects'][0])
def GetFacility(settingsModel, name):
"""
Get facility by name.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/facility/?format=json&name=" + urllib.quote(name)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json",
}
response = requests.get(url=url, headers=headers, stream=False)
logger.debug(response.text)
if response.status_code != 200:
message = response.text
response.close()
raise Exception(message)
facilitiesJson = response.json()
response.close()
numFacilitiesFound = facilitiesJson["meta"]["total_count"]
if numFacilitiesFound == 0:
logger.warning('Facility "%s" was not found in MyTardis' % name)
return None
else:
logger.debug("Found facility record for name '" + name + "'.")
return FacilityModel(settingsModel=settingsModel, name=name, facilityJson=facilitiesJson["objects"][0])
def CheckIfAlreadyRunning(self, appdirPath):
"""
Using wx.SingleInstanceChecker to check whether MyData is already
running.
Running MyData --version is allowed when MyData is already running.
A workaround for the 'Deleted stale lock file' issue with
SingleInstanceChecker on Mac OS X is to lower the wx logging level.
MyData doesn't use wx.Log
"""
wx.Log.SetLogLevel(wx.LOG_Error)
instance = wx.SingleInstanceChecker(self.name, path=appdirPath)
if instance.IsAnotherRunning():
logger.warning("Another MyData instance is already running.")
if sys.platform.startswith("darwin"):
applescript = (
'tell application "System Events"\n'
" set theprocs to every process whose name is "
'"MyData"\n'
" repeat with proc in theprocs\n"
" set the frontmost of proc to true\n"
" end repeat\n"
"end tell"
)
os.system("osascript -e '%s'" % applescript)
sys.exit(0)
else:
wx.MessageBox("MyData is already running!", "MyData", wx.ICON_ERROR)
sys.exit(1)
def ShowMessageDialog(self, event):
if self.IsShowingErrorDialog():
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we are already showing an error "
"dialog." % event.message)
return
elif event.message == self.GetLastErrorMessage():
# Sometimes multiple threads can encounter the same exception
# at around the same time. The first thread's exception leads
# to a modal error dialog, which blocks the events queue, so
# the next thread's (identical) show message dialog event doesn't
# get caught until after the first message dialog has been closed.
# In this case, the above check (to prevent two error dialogs
# from appearing at the same time) doesn't help.
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we already showed an error "
"dialog with the same message." % event.message)
return
self.SetLastErrorMessage(event.message)
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(True)
dlg = wx.MessageDialog(None, event.message, event.title,
wx.OK | event.icon)
# pylint: disable=bare-except
try:
wx.EndBusyCursor()
needToRestartBusyCursor = True
except:
needToRestartBusyCursor = False
dlg.ShowModal()
if needToRestartBusyCursor and not self.IsShuttingDown() \
and wx.GetApp().PerformingLookupsAndUploads():
BeginBusyCursorIfRequired()
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(False)
def ShowMessageDialog(self, event):
if self.IsShowingErrorDialog():
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we are already showing an error "
"dialog." % event.message)
return
elif event.message == self.GetLastErrorMessage():
# Sometimes multiple threads can encounter the same exception
# at around the same time. The first thread's exception leads
# to a modal error dialog, which blocks the events queue, so
# the next thread's (identical) show message dialog event doesn't
# get caught until after the first message dialog has been closed.
# In this case, the above check (to prevent two error dialogs
# from appearing at the same time) doesn't help.
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we already showed an error "
"dialog with the same message." % event.message)
return
self.SetLastErrorMessage(event.message)
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(True)
dlg = wx.MessageDialog(None, event.message, event.title,
wx.OK | event.icon)
# pylint: disable=bare-except
try:
wx.EndBusyCursor()
needToRestartBusyCursor = True
except:
needToRestartBusyCursor = False
dlg.ShowModal()
if needToRestartBusyCursor:
wx.BeginBusyCursor()
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(False)
def GetFacility(settingsModel, name):
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/facility/?format=json&name=" + \
urllib.quote(name)
headers = {'Authorization': 'ApiKey ' + myTardisUsername + ":" +
myTardisApiKey}
session = requests.Session()
response = session.get(url=url, headers=headers, stream=False)
logger.debug(response.text)
if response.status_code != 200:
message = response.text
response.close()
session.close()
raise Exception(message)
facilitiesJson = response.json()
response.close()
session.close()
numFacilitiesFound = facilitiesJson['meta']['total_count']
if numFacilitiesFound == 0:
logger.warning("Facility \"%s\" was not found in MyTardis" % name)
return None
else:
logger.debug("Found facility record for name '" + name + "'.")
return FacilityModel(
settingsModel=settingsModel, name=name,
facilityJson=facilitiesJson['objects'][0])
def TryRowDeleted(self, row):
try:
if row < self.GetCount():
self.RowDeleted(row)
else:
logger.warning("TryRowDeleted called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
except:
logger.debug(traceback.format_exc())
def TryRowDeleted(self, row):
# pylint: disable=bare-except
try:
if row < self.GetCount():
self.RowDeleted(row)
else:
logger.warning("TryRowDeleted called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
except:
logger.debug(traceback.format_exc())
def TryRowValueChanged(self, row, col):
try:
if row < self.GetCount():
self.RowValueChanged(row, col)
else:
logger.warning("TryRowValueChanged called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
self.RowValueChanged(row, col)
except wx.PyAssertionError:
logger.warning(traceback.format_exc())
def TryRowValueChanged(self, row, col):
try:
if row < self.GetCount():
self.RowValueChanged(row, col)
else:
logger.warning("TryRowValueChanged called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
self.RowValueChanged(row, col)
except wx.PyAssertionError:
logger.warning(traceback.format_exc())
def BeginBusyCursorIfRequired():
"""
Begin busy cursor if it's not already being displayed.
"""
# pylint: disable=no-member
# Otherwise pylint complains about PyAssertionError.
# pylint: disable=protected-access
try:
if not wx.IsBusy():
wx.BeginBusyCursor()
except wx._core.PyAssertionError, err:
logger.warning(err)
def TryRowValueChanged(self, row, col):
# pylint: disable=bare-except
try:
if row < self.GetCount():
self.RowValueChanged(row, col)
else:
logger.warning("TryRowValueChanged called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
self.RowValueChanged(row, col)
except:
logger.debug(traceback.format_exc())
def UploadDatafile(self, event):
"""
This method runs in the main thread, so it shouldn't do anything
time-consuming or blocking, unless it launches another thread.
Because this method adds upload tasks to a queue, it is important
to note that if the queue has a maxsize set, then an attempt to
add something to the queue could block the GUI thread, making the
application appear unresponsive.
"""
if self.IsShuttingDown():
return
before = datetime.now()
folderModel = event.folderModel
foldersController = event.foldersController
dfi = event.dataFileIndex
uploadsModel = foldersController.uploadsModel
if folderModel not in foldersController.uploadDatafileRunnable:
foldersController.uploadDatafileRunnable[folderModel] = {}
self.uploadsThreadingLock.acquire()
uploadDataViewId = uploadsModel.GetMaxDataViewId() + 1
uploadModel = UploadModel(dataViewId=uploadDataViewId,
folderModel=folderModel,
dataFileIndex=dfi)
uploadsModel.AddRow(uploadModel)
self.uploadsThreadingLock.release()
if hasattr(event, "bytesUploadedToStaging"):
uploadModel.SetBytesUploadedToStaging(event.bytesUploadedToStaging)
uploadModel.SetVerificationModel(event.verificationModel)
if self.IsShuttingDown():
return
existingUnverifiedDatafile = False
if hasattr(event, "existingUnverifiedDatafile"):
existingUnverifiedDatafile = event.existingUnverifiedDatafile
foldersController.uploadDatafileRunnable[folderModel][dfi] = \
UploadDatafileRunnable(self, self.foldersModel, folderModel,
dfi, self.uploadsModel, uploadModel,
self.settingsModel,
existingUnverifiedDatafile)
if self.IsShuttingDown():
return
self.uploadsQueue.put(foldersController
.uploadDatafileRunnable[folderModel][dfi])
after = datetime.now()
duration = after - before
if duration.total_seconds() >= 1:
logger.warning("UploadDatafile for " +
folderModel.GetDataFileName(dfi) +
" blocked the main GUI thread for %d seconds." +
duration.total_seconds())
def EndBusyCursorIfRequired():
"""
The built in wx.EndBusyCursor raises an ugly exception if the
busy cursor has already been stopped.
"""
# pylint: disable=no-member
# Otherwise pylint complains about PyAssertionError.
# pylint: disable=protected-access
try:
wx.EndBusyCursor()
except wx._core.PyAssertionError, err:
if "no matching wxBeginBusyCursor()" \
not in str(err):
logger.warning(str(err))
raise
def HandleUnresumableUpload(self, existingDatafile):
"""
We found an unverified datafile on the server for which
there is no point in checking for a resumable partial
upload.
This is usually because we are uploading using the POST upload method.
Or we could be using the STAGING method but failed to find any
DataFileObjects on the server for the datafile.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
logger.debug("Found unverified datafile record for \"%s\" "
"on MyTardis." % dataFilePath)
self.verificationModel.SetMessage("Found unverified datafile record.")
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE
else:
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_NO_DFOS)
eventId = self.foldersController\
.EVT_FOUND_UNVERIFIED_NO_DFOS
self.verificationsModel.MessageUpdated(self.verificationModel)
if existingDatafile and not self.testRun:
if existingDatafile.GetMd5Sum() == \
self.settingsModel.GetFakeMd5Sum():
logger.warning("MD5(%s): %s" %
(dataFilePath, existingDatafile.GetMd5Sum()))
else:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=eventId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def ScanForGroupFolders(self, incrementProgressDialog, shouldAbort):
"""
Scan for group folders.
"""
dataDir = self.settingsModel.GetDataDirectory()
userOrGroupFilterString = '*%s*' % self.settingsModel.GetUserFilter()
filesDepth1 = glob(os.path.join(dataDir, userOrGroupFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
groupFolderNames = [os.path.basename(d) for d in dirsDepth1]
for groupFolderName in groupFolderNames:
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
logger.debug("Found folder assumed to be user group name: " +
groupFolderName)
groupsDataViewId = self.groupsModel.GetMaxDataViewId() + 1
try:
groupName = self.settingsModel.GetGroupPrefix() + \
groupFolderName
groupRecord = \
GroupModel.GetGroupByName(self.settingsModel,
groupName)
except DoesNotExist:
groupRecord = None
message = "Didn't find a MyTardis user group record for " \
"folder \"%s\" in %s" % (groupFolderName,
dataDir)
logger.warning(message)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if groupRecord:
groupRecord.SetDataViewId(groupsDataViewId)
self.groupsModel.AddRow(groupRecord)
self.ImportGroupFolders(os.path.join(dataDir,
groupFolderName),
groupRecord)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if threading.current_thread().name == "MainThread":
incrementProgressDialog()
else:
wx.CallAfter(incrementProgressDialog)
def TryRowValueChanged(self, row, col):
"""
Use try/except when calling RowValueChanged, because
sometimes there are timing issues which raise wx
assertions suggesting that the row index we are trying
to report a change on is greater than or equal to the
total number of rows in the model.
"""
try:
if row < self.GetCount():
self.RowValueChanged(row, col)
else:
logger.warning("TryRowValueChanged called with "
"row=%d, self.GetRowCount()=%d" %
(row, self.GetRowCount()))
self.RowValueChanged(row, col)
except wx.PyAssertionError:
logger.warning(traceback.format_exc())
def ScanForExperimentFolders(self, pathToScan, owner, userFolderName):
"""
Scans for experiment folders.
"""
datasetFilterString = '*%s*' % self.settingsModel.GetDatasetFilter()
expFilterString = '*%s*' % self.settingsModel.GetExperimentFilter()
filesDepth1 = glob(os.path.join(pathToScan, expFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
expFolders = [os.path.basename(d) for d in dirsDepth1]
for expFolderName in expFolders:
expFolderPath = os.path.join(pathToScan, expFolderName)
filesDepth1 = glob(os.path.join(expFolderPath,
datasetFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
datasetFolders = [os.path.basename(d) for d in dirsDepth1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(expFolderPath,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath, self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
dataViewId = self.GetMaxDataViewId() + 1
folderModel = FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=expFolderPath,
userFolderName=userFolderName,
groupFolderName=None,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetCreatedDate()
folderModel.SetExperimentTitle(expFolderName)
self.AddRow(folderModel)
def Cancel(self):
try:
self.canceled = True
if self.verificationTimer:
try:
self.verificationTimer.cancel()
except: # pylint: disable=bare-except
logger.error(traceback.format_exc())
if self.bufferedReader is not None:
self.bufferedReader.close()
logger.debug("Closed buffered reader for \"" +
self.GetRelativePathToUpload() +
"\".")
if self.scpUploadProcessPid:
if sys.platform.startswith("win"):
os.kill(self.scpUploadProcessPid, signal.SIGABRT)
else:
os.kill(self.scpUploadProcessPid, signal.SIGKILL)
except: # pylint: disable=bare-except
logger.warning(traceback.format_exc())
def ScanForGroupFolders(self, incrementProgressDialog, shouldAbort):
dataDir = self.settingsModel.GetDataDirectory()
groupFolderNames = os.walk(dataDir).next()[1]
for groupFolderName in groupFolderNames:
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
logger.debug("Found folder assumed to be user group name: " +
groupFolderName)
groupsDataViewId = self.groupsModel.GetMaxDataViewId() + 1
try:
groupName = self.settingsModel.GetGroupPrefix() + \
groupFolderName
groupRecord = \
GroupModel.GetGroupByName(self.settingsModel,
groupName)
except DoesNotExist:
groupRecord = None
message = "Didn't find a MyTardis user group record for " \
"folder \"%s\" in %s" % (groupFolderName,
dataDir)
logger.warning(message)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if groupRecord:
groupRecord.SetDataViewId(groupsDataViewId)
self.groupsModel.AddRow(groupRecord)
self.ImportGroupFolders(os.path.join(dataDir,
groupFolderName),
groupRecord)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if threading.current_thread().name == "MainThread":
incrementProgressDialog()
else:
wx.CallAfter(incrementProgressDialog)
def GetInstrument(settingsModel, facility, name):
"""
Get instrument.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/instrument/?format=json" + \
"&facility__id=" + str(facility.GetId()) + \
"&name=" + urllib.quote(name)
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey)
}
session = requests.Session()
response = session.get(url=url, headers=headers)
if response.status_code != 200:
message = response.text
logger.error(message)
raise Exception(message)
instrumentsJson = response.json()
numInstrumentsFound = \
instrumentsJson['meta']['total_count']
if numInstrumentsFound == 0:
logger.warning("Instrument \"%s\" was not found in MyTardis" %
name)
logger.debug(url)
logger.debug(response.text)
response.close()
session.close()
return None
else:
logger.debug("Found instrument record for name \"%s\" "
"in facility \"%s\"" % (name, facility.GetName()))
instrumentJson = instrumentsJson['objects'][0]
response.close()
session.close()
return InstrumentModel(settingsModel=settingsModel,
name=name,
instrumentJson=instrumentJson)
def GetInstrument(settingsModel, facility, name):
"""
Get instrument.
"""
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisUsername = settingsModel.GetUsername()
myTardisApiKey = settingsModel.GetApiKey()
url = myTardisUrl + "/api/v1/instrument/?format=json" + \
"&facility__id=" + str(facility.GetId()) + \
"&name=" + urllib.quote(name)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
session = requests.Session()
response = session.get(url=url, headers=headers)
if response.status_code != 200:
message = response.text
logger.error(message)
raise Exception(message)
instrumentsJson = response.json()
numInstrumentsFound = \
instrumentsJson['meta']['total_count']
if numInstrumentsFound == 0:
logger.warning("Instrument \"%s\" was not found in MyTardis"
% name)
logger.debug(url)
logger.debug(response.text)
response.close()
session.close()
return None
else:
logger.debug("Found instrument record for name \"%s\" "
"in facility \"%s\"" %
(name, facility.GetName()))
instrumentJson = instrumentsJson['objects'][0]
response.close()
session.close()
return InstrumentModel(
settingsModel=settingsModel, name=name,
instrumentJson=instrumentJson)
def ScanForExperimentFolders(self, pathToScan, owner, userFolderName):
"""
Instead of looking for dataset folders as direct children of
the username folder, this method looks for dataset folders
structured in the following format:
<username>\mytardis\<experiment_title>\<dataset_name>
"""
expFolders = os.walk(pathToScan).next()[1]
for expFolderName in expFolders:
expFolderPath = os.path.join(pathToScan, expFolderName)
datasetFolders = os.walk(expFolderPath).next()[1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(expFolderPath,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath, self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
dataViewId = self.GetMaxDataViewId() + 1
folderModel = FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=expFolderPath,
userFolderName=userFolderName,
groupFolderName=None,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetCreatedDate()
folderModel.SetExperimentTitle(expFolderName)
self.AddRow(folderModel)
def HandleFullSizeResumableUpload(self, existingDatafile):
"""
If the existing unverified DataFile upload is the correct size
in staging, then we can request its verification, but no upload
is needed.
"""
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
self.verificationModel\
.SetMessage("Found unverified full-size datafile "
"on staging server.")
self.verificationModel.SetStatus(
VerificationStatus.FOUND_UNVERIFIED_FULL_SIZE)
self.verificationsModel.MessageUpdated(self.verificationModel)
self.folderModel.SetDataFileUploaded(self.dataFileIndex, True)
self.foldersModel.FolderStatusUpdated(self.folderModel)
if existingDatafile and not self.testRun:
if existingDatafile.GetMd5Sum() == \
self.settingsModel.GetFakeMd5Sum():
logger.warning("MD5(%s): %s" %
(dataFilePath, existingDatafile.GetMd5Sum()))
else:
DataFileModel.Verify(self.settingsModel, existingDatafile.GetId())
self.verificationsModel.SetComplete(self.verificationModel)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.foundUnverifiedDatafileEvent(
id=self.foldersController
.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
dataFilePath=dataFilePath))
if self.testRun:
message = "FOUND UNVERIFIED UPLOAD FOR: %s" \
% self.folderModel.GetDataFileRelPath(self.dataFileIndex)
logger.testrun(message)
def ScanForDatasetFolders(self, pathToScan, owner, userFolderName):
"""
Scan for dataset folders.
"""
# pylint: disable=bare-except
try:
logger.debug("Scanning " + pathToScan +
" for dataset folders...")
datasetFilterString = \
'*%s*' % self.settingsModel.GetDatasetFilter()
filesDepth1 = glob(os.path.join(pathToScan, datasetFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
datasetFolders = [os.path.basename(d) for d in dirsDepth1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(pathToScan,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > \
self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath,
self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
dataViewId = self.GetMaxDataViewId() + 1
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=pathToScan,
userFolderName=userFolderName,
groupFolderName=None,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetCreatedDate()
if not owner.UserNotFoundInMyTardis():
if owner.GetName().strip() != "":
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
owner.GetName())
else:
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
owner.GetUsername())
elif owner.GetName() != UserModel.userNotFoundString:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetName(),
UserModel.userNotFoundString)
elif owner.GetUsername() != UserModel.userNotFoundString:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetUsername(),
UserModel.userNotFoundString)
elif owner.GetEmail() != UserModel.userNotFoundString:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetEmail(),
UserModel.userNotFoundString)
else:
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
UserModel.userNotFoundString)
folderModel.SetExperimentTitle(experimentTitle)
self.AddRow(folderModel)
except:
print traceback.format_exc()
def ScanForUserFolders(self, incrementProgressDialog, shouldAbort):
"""
Scan for user folders.
"""
dataDir = self.settingsModel.GetDataDirectory()
userOrGroupFilterString = '*%s*' % self.settingsModel.GetUserFilter()
folderStructure = self.settingsModel.GetFolderStructure()
filesDepth1 = glob(os.path.join(dataDir, userOrGroupFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
userFolderNames = [os.path.basename(d) for d in dirsDepth1]
for userFolderName in userFolderNames:
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if folderStructure.startswith("Username"):
logger.debug("Found folder assumed to be username: "******"Email"):
logger.debug("Found folder assumed to be email: " +
userFolderName)
usersDataViewId = self.usersModel.GetMaxDataViewId() + 1
try:
if folderStructure.startswith("Username"):
userRecord = \
UserModel.GetUserByUsername(self.settingsModel,
userFolderName)
elif folderStructure.startswith("Email"):
userRecord = \
UserModel.GetUserByEmail(self.settingsModel,
userFolderName)
except DoesNotExist:
userRecord = None
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if userRecord is not None:
userRecord.SetDataViewId(usersDataViewId)
self.usersModel.AddRow(userRecord)
userFolderPath = os.path.join(dataDir, userFolderName)
if folderStructure == 'Username / Dataset' or \
folderStructure == 'Email / Dataset':
self.ScanForDatasetFolders(userFolderPath, userRecord,
userFolderName)
elif folderStructure == \
'Username / Experiment / Dataset' or \
folderStructure == 'Email / Experiment / Dataset':
self.ScanForExperimentFolders(userFolderPath, userRecord,
userFolderName)
elif folderStructure == \
'Username / "MyTardis" / Experiment / Dataset':
userFolderContents = os.listdir(userFolderPath)
myTardisFolderName = None
for item in userFolderContents:
if item.lower() == 'mytardis':
myTardisFolderName = item
if not myTardisFolderName:
message = 'Didn\'t find "MyTardis" folder in ' \
'"%s"' % userFolderPath
logger.error(message)
raise InvalidFolderStructure(message)
myTardisFolderPath = os.path.join(userFolderPath,
myTardisFolderName)
self.ScanForExperimentFolders(myTardisFolderPath,
userRecord,
userFolderName)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame()
.SetStatusMessage,
"Data uploads canceled")
return
else:
message = "Didn't find a MyTardis user record for folder " \
"\"%s\" in %s" % (userFolderName, dataDir)
logger.warning(message)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
if folderStructure.startswith("Username"):
userRecord = UserModel(settingsModel=self.settingsModel,
username=userFolderName,
userNotFoundInMyTardis=True)
elif folderStructure.startswith("Email"):
userRecord = \
UserModel(settingsModel=self.settingsModel,
email=userFolderName,
userNotFoundInMyTardis=True)
userRecord.SetDataViewId(usersDataViewId)
self.usersModel.AddRow(userRecord)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data uploads canceled")
return
self.ScanForDatasetFolders(os.path.join(dataDir,
userFolderName),
userRecord, userFolderName)
if threading.current_thread().name == "MainThread":
incrementProgressDialog()
else:
wx.CallAfter(incrementProgressDialog)
def ImportGroupFolders(self, groupFolderPath, groupModel):
"""
Scan folders within a user group folder,
e.g. D:\\Data\\Smith-Lab\\
"""
# pylint: disable=bare-except
try:
logger.debug("Scanning " + groupFolderPath +
" for instrument folders...")
datasetFilterString = \
'*%s*' % self.settingsModel.GetDatasetFilter()
instrumentName = self.settingsModel.GetInstrumentName()
filesDepth1 = glob(os.path.join(groupFolderPath, instrumentName))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
instrumentFolders = [os.path.basename(d) for d in dirsDepth1]
if len(instrumentFolders) > 1:
message = "Multiple instrument folders found in %s" \
% groupFolderPath
logger.warning(message)
elif len(instrumentFolders) == 0:
message = "No instrument folder was found in %s" \
% groupFolderPath
logger.warning(message)
return
# Rather than using any folder we happen to find at this level,
# we will use the instrument name specified in MyData's Settings
# dialog. That way, we can run MyData on a collection of data
# from multiple instruments, and just select one instrument at
# a time.
instrumentFolderPath = \
os.path.join(groupFolderPath,
self.settingsModel.GetInstrumentName())
if not os.path.exists(instrumentFolderPath):
logger.warning("Path %s doesn't exist." % instrumentFolderPath)
return
# For the User Group / Instrument / Researcher's Name / Dataset
# folder structure, the default owner in MyTardis will always
# by the user listed in MyData's settings dialog. An additional
# ObjectACL will be created in MyTardis to grant access to the
# User Group. The researcher's name in this folder structure is
# used to determine the default experiment name, but it is not
# used to determine access control.
owner = self.settingsModel.GetDefaultOwner()
logger.debug("Scanning " + instrumentFolderPath +
" for user folders...")
userFolders = os.walk(instrumentFolderPath).next()[1]
for userFolderName in userFolders:
userFolderPath = os.path.join(instrumentFolderPath,
userFolderName)
logger.debug("Scanning " + userFolderPath +
" for dataset folders...")
filesDepth1 = glob(os.path.join(userFolderPath,
datasetFilterString))
dirsDepth1 = [item for item in filesDepth1
if os.path.isdir(item)]
datasetFolders = [os.path.basename(d) for d in dirsDepth1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(userFolderPath,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath,
self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
groupFolderName = os.path.basename(groupFolderPath)
dataViewId = self.GetMaxDataViewId() + 1
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=userFolderPath,
userFolderName=userFolderName,
groupFolderName=groupFolderName,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetGroup(groupModel)
folderModel.SetCreatedDate()
folderModel.SetExperimentTitle(
"%s - %s" % (self.settingsModel.GetInstrumentName(),
userFolderName))
self.AddRow(folderModel)
except InvalidFolderStructure:
raise
except:
logger.error(traceback.format_exc())
class FoldersController(object):
# pylint: disable=too-many-public-methods
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-statements
"""
The main controller class for managing datafile verifications
and uploads from each of the folders in the Folders view.
"""
def __init__(self, notifyWindow, foldersModel, foldersView, usersModel,
verificationsModel, uploadsModel, settingsModel):
# pylint: disable=too-many-arguments
self.notifyWindow = notifyWindow
self.foldersModel = foldersModel
self.foldersView = foldersView
self.usersModel = usersModel
self.verificationsModel = verificationsModel
self.uploadsModel = uploadsModel
self.settingsModel = settingsModel
self.shuttingDown = threading.Event()
self.showingErrorDialog = threading.Event()
self.lastErrorMessage = None
self.showingWarningDialog = threading.Event()
self.canceled = threading.Event()
self.failed = threading.Event()
self.finishedCountingVerifications = dict()
self.finishedScanningForDatasetFolders = threading.Event()
self.verificationsQueue = None
self.threadingLock = threading.Lock()
self.uploadsThreadingLock = threading.Lock()
self.verifyDatafileRunnable = None
self.uploadsQueue = None
self.started = False
self.completed = False
self.uploadDatafileRunnable = None
self.numVerificationsToBePerformed = 0
self.numVerificationsToBePerformedLock = threading.Lock()
self.uploadsAcknowledged = 0
self.uploadMethod = UploadMethod.HTTP_POST
# These will get overwritten in InitForUploads, but we need
# to initialize them here, so that ShutDownUploadThreads()
# can be called.
self.numVerificationWorkerThreads = 0
self.verificationWorkerThreads = []
self.numUploadWorkerThreads = 0
self.uploadWorkerThreads = []
self.testRun = False
self.foldersView.Bind(wx.EVT_BUTTON, self.OnOpenFolder,
self.foldersView.GetOpenFolderButton())
self.foldersView.GetDataViewControl()\
.Bind(wx.dataview.EVT_DATAVIEW_ITEM_ACTIVATED, self.OnOpenFolder)
self.didntFindDatafileOnServerEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.EVT_DIDNT_FIND_FILE_ON_SERVER = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder, self.UploadDatafile)
self.unverifiedDatafileOnServerEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.EVT_INCOMPLETE_FILE_ON_STAGING = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder, self.UploadDatafile)
self.unverifiedNotFoundOnStagingEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.EVT_UNVERIFIED_NOT_FOUND_ON_STAGING = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder, self.UploadDatafile)
self.connectionStatusEvent, eventBinder = wx.lib.newevent.NewEvent()
self.notifyWindow.Bind(eventBinder, self.UpdateStatusBar)
self.showMessageDialogEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.notifyWindow.Bind(eventBinder, self.ShowMessageDialog)
self.shutdownUploadsEvent, eventBinder = wx.lib.newevent.NewEvent()
self.notifyWindow.Bind(eventBinder, self.ShutDownUploadThreads)
self.foundVerifiedDatafileEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_FOUND_VERIFIED_DATAFILE = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.foundUnverifiedDatafileEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.foundUnverifiedNoDfosDatafileEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_FOUND_UNVERIFIED_NO_DFOS = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.uploadCompleteEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_UPLOAD_COMPLETE = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.uploadFailedEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_UPLOAD_FAILED = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
def Started(self):
return self.started
def SetStarted(self, started=True):
self.started = started
def Canceled(self):
return self.canceled.isSet()
def SetCanceled(self, canceled=True):
if canceled:
self.canceled.set()
else:
self.canceled.clear()
def Failed(self):
return self.failed.isSet()
def SetFailed(self, failed=True):
if failed:
self.failed.set()
else:
self.failed.clear()
def Completed(self):
return self.completed
def SetCompleted(self, completed=True):
self.completed = completed
def IsShuttingDown(self):
return self.shuttingDown.isSet()
def SetShuttingDown(self, shuttingDown=True):
if shuttingDown:
self.shuttingDown.set()
else:
self.shuttingDown.clear()
def IsShowingErrorDialog(self):
return self.showingErrorDialog.isSet()
def SetShowingErrorDialog(self, showingErrorDialog=True):
if showingErrorDialog:
self.showingErrorDialog.set()
else:
self.showingErrorDialog.clear()
def GetLastErrorMessage(self):
return self.lastErrorMessage
def SetLastErrorMessage(self, message):
self.threadingLock.acquire()
self.lastErrorMessage = message
self.threadingLock.release()
def UpdateStatusBar(self, event):
if event.connectionStatus == ConnectionStatus.CONNECTED:
self.notifyWindow.SetConnected(event.myTardisUrl, True)
else:
self.notifyWindow.SetConnected(event.myTardisUrl, False)
def ShowMessageDialog(self, event):
if self.IsShowingErrorDialog():
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we are already showing an error "
"dialog." % event.message)
return
elif event.message == self.GetLastErrorMessage():
# Sometimes multiple threads can encounter the same exception
# at around the same time. The first thread's exception leads
# to a modal error dialog, which blocks the events queue, so
# the next thread's (identical) show message dialog event doesn't
# get caught until after the first message dialog has been closed.
# In this case, the above check (to prevent two error dialogs
# from appearing at the same time) doesn't help.
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we already showed an error "
"dialog with the same message." % event.message)
return
self.SetLastErrorMessage(event.message)
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(True)
dlg = wx.MessageDialog(None, event.message, event.title,
wx.OK | event.icon)
# pylint: disable=bare-except
try:
wx.EndBusyCursor()
needToRestartBusyCursor = True
except:
needToRestartBusyCursor = False
dlg.ShowModal()
if needToRestartBusyCursor and not self.IsShuttingDown() \
and wx.GetApp().PerformingLookupsAndUploads():
BeginBusyCursorIfRequired()
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(False)
def UploadDatafile(self, event):
"""
Called in response to didntFindDatafileOnServerEvent or
unverifiedDatafileOnServerEvent.
This method runs in the main thread, so it shouldn't do anything
time-consuming or blocking, unless it launches another thread.
Because this method adds upload tasks to a queue, it is important
to note that if the queue has a maxsize set, then an attempt to
add something to the queue could block the GUI thread, making the
application appear unresponsive.
"""
folderModel = event.folderModel
dfi = event.dataFileIndex
existingUnverifiedDatafile = \
getattr(event, "existingUnverifiedDatafile", False)
if self.testRun:
if existingUnverifiedDatafile:
message = "NEEDS RE-UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
else:
message = "NEEDS UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
self.uploadsAcknowledged += 1
logger.testrun(message)
self.CountCompletedUploadsAndVerifications(event=None)
return
if folderModel not in self.uploadDatafileRunnable:
self.uploadDatafileRunnable[folderModel] = {}
bytesUploadedPreviously = getattr(event, "bytesUploadedPreviously",
None)
verificationModel = getattr(event, "verificationModel", None)
self.uploadDatafileRunnable[folderModel][dfi] = \
UploadDatafileRunnable(self, self.foldersModel, folderModel,
dfi, self.uploadsModel,
self.settingsModel,
existingUnverifiedDatafile,
verificationModel,
bytesUploadedPreviously)
self.uploadsQueue.put(self.uploadDatafileRunnable[folderModel][dfi])
self.CountCompletedUploadsAndVerifications(event=None)
def InitForUploads(self):
fc = self # pylint: disable=invalid-name
app = wx.GetApp()
if hasattr(app, "TestRunRunning"):
fc.testRun = app.TestRunRunning()
else:
fc.testRun = False
fc.SetStarted()
settingsModel = fc.settingsModel
fc.SetCanceled(False)
fc.SetFailed(False)
fc.SetCompleted(False)
fc.verificationsModel.DeleteAllRows()
fc.uploadsModel.DeleteAllRows()
fc.verifyDatafileRunnable = {}
fc.verificationsQueue = Queue.Queue()
# For now, the max number of verification threads is hard-coded
# to 16:
fc.numVerificationWorkerThreads = 16
fc.verificationWorkerThreads = []
for i in range(fc.numVerificationWorkerThreads):
thread = threading.Thread(name="VerificationWorkerThread-%d" %
(i + 1),
target=fc.VerificationWorker)
fc.verificationWorkerThreads.append(thread)
thread.start()
fc.uploadDatafileRunnable = {}
fc.uploadsQueue = Queue.Queue()
fc.numUploadWorkerThreads = settingsModel.GetMaxUploadThreads()
fc.uploadMethod = UploadMethod.HTTP_POST
# pylint: disable=broad-except
try:
settingsModel.GetUploaderModel().RequestStagingAccess()
uploadToStagingRequest = settingsModel\
.GetUploadToStagingRequest()
except Exception, err:
# MyData app could be missing from MyTardis server.
logger.error(traceback.format_exc())
wx.PostEvent(
self.notifyWindow,
self.showMessageDialogEvent(title="MyData",
message=str(err),
icon=wx.ICON_ERROR))
return
message = None
if uploadToStagingRequest is None:
message = "Couldn't determine whether uploads to " \
"staging have been approved. " \
"Falling back to HTTP POST."
elif uploadToStagingRequest.IsApproved():
logger.info("Uploads to staging have been approved.")
fc.uploadMethod = UploadMethod.VIA_STAGING
else:
message = \
"Uploads to MyTardis's staging area require " \
"approval from your MyTardis administrator.\n\n" \
"A request has been sent, and you will be contacted " \
"once the request has been approved. Until then, " \
"MyData will upload files using HTTP POST, and will " \
"only upload one file at a time.\n\n" \
"HTTP POST is generally only suitable for small " \
"files (up to 100 MB each)."
if message:
logger.warning(message)
wx.PostEvent(
self.notifyWindow,
self.showMessageDialogEvent(title="MyData",
message=message,
icon=wx.ICON_WARNING))
fc.uploadMethod = UploadMethod.HTTP_POST
if fc.uploadMethod == UploadMethod.HTTP_POST and \
fc.numUploadWorkerThreads > 1:
logger.warning("Using HTTP POST, so setting "
"numUploadWorkerThreads to 1, "
"because urllib2 is not thread-safe.")
fc.numUploadWorkerThreads = 1
fc.uploadWorkerThreads = []
for i in range(fc.numUploadWorkerThreads):
thread = threading.Thread(name="UploadWorkerThread-%d" % (i + 1),
target=fc.UploadWorker,
args=())
fc.uploadWorkerThreads.append(thread)
thread.start()
# pylint: disable=bare-except
fc.finishedScanningForDatasetFolders = threading.Event()
fc.numVerificationsToBePerformed = 0
fc.finishedCountingVerifications = dict()
def Run(self):
# pylint: disable=too-many-locals
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
self.foldersController.uploadsThreadingLock.acquire()
uploadDataViewId = self.uploadsModel.GetMaxDataViewId() + 1
self.uploadModel = UploadModel(dataViewId=uploadDataViewId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex)
self.uploadsModel.AddRow(self.uploadModel)
self.foldersController.uploadsThreadingLock.release()
self.uploadModel.SetBytesUploadedPreviously(
self.bytesUploadedPreviously)
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
dataFileName = os.path.basename(dataFilePath)
dataFileDirectory = \
self.folderModel.GetDataFileDirectory(self.dataFileIndex)
ignoreNewFiles = self.settingsModel.IgnoreNewFiles()
ignoreNewFilesMinutes = self.settingsModel.GetIgnoreNewFilesMinutes()
ignoreNewFilesSeconds = 0
if ignoreNewFiles:
ignoreNewFilesSeconds = ignoreNewFilesMinutes * 60
if (time.time() - os.path.getmtime(dataFilePath)) <= \
ignoreNewFilesSeconds:
message = "Not uploading file, in case it is still being modified."
logger.warning(message.replace('file', dataFilePath))
self.uploadsModel.SetMessage(self.uploadModel, message)
self.uploadsModel.SetStatus(self.uploadModel, UploadStatus.FAILED)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.uploadCompleteEvent(
id=self.foldersController.EVT_UPLOAD_FAILED,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
uploadModel=self.uploadModel))
return
logger.debug("Uploading " +
self.folderModel.GetDataFileName(self.dataFileIndex) +
"...")
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
url = myTardisUrl + "/api/v1/mydata_dataset_file/"
else:
url = myTardisUrl + "/api/v1/dataset_file/"
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
if self.foldersController.IsShuttingDown():
return
message = "Getting data file size..."
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileSize = self.folderModel.GetDataFileSize(self.dataFileIndex)
self.uploadModel.SetFileSize(dataFileSize)
if self.foldersController.IsShuttingDown():
return
# The HTTP POST upload method doesn't support resuming uploads,
# so we always (re-)create the JSON to be POSTed when we find
# a file whose datafile record is unverified.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
message = "Calculating MD5 checksum..."
self.uploadsModel.SetMessage(self.uploadModel, message)
def Md5ProgressCallback(bytesProcessed):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
if dataFileSize > 0:
percentComplete = \
100.0 - ((dataFileSize - bytesProcessed) * 100.0) \
/ dataFileSize
else:
percentComplete = 100
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if dataFileSize >= (1024 * 1024 * 1024):
message = "%3.1f %% MD5 summed" % percentComplete
else:
message = "%3d %% MD5 summed" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileMd5Sum = \
self.CalculateMd5Sum(dataFilePath, dataFileSize,
self.uploadModel,
progressCallback=Md5ProgressCallback)
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
else:
dataFileSize = int(self.existingUnverifiedDatafile.GetSize())
self.uploadModel.SetProgress(0)
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if self.foldersController.IsShuttingDown():
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
message = "Checking MIME type..."
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileMimeType = self.mimeTypes.guess_type(dataFilePath)[0]
if self.foldersController.IsShuttingDown():
return
message = "Defining JSON data for POST..."
self.uploadsModel.SetMessage(self.uploadModel, message)
datasetUri = self.folderModel.GetDatasetModel().GetResourceUri()
dataFileCreatedTime = \
self.folderModel.GetDataFileCreatedTime(self.dataFileIndex)
dataFileModifiedTime = \
self.folderModel.GetDataFileModifiedTime(self.dataFileIndex)
dataFileJson = {
"dataset": datasetUri,
"filename": dataFileName,
"directory": dataFileDirectory,
"md5sum": dataFileMd5Sum,
"size": dataFileSize,
"mimetype": dataFileMimeType,
"created_time": dataFileCreatedTime,
"modification_time": dataFileModifiedTime,
}
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
dataFileJson['uploader_uuid'] = self.settingsModel.GetUuid()
dataFileJson['requester_key_fingerprint'] = \
self.settingsModel.GetSshKeyPair().GetFingerprint()
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
message = "Initializing buffered reader..."
self.uploadsModel.SetMessage(self.uploadModel, message)
datafileBufferedReader = io.open(dataFilePath, 'rb')
self.uploadModel.SetBufferedReader(datafileBufferedReader)
def ProgressCallback(current, total, message=None):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
if current is None:
# For a zero-sized file, current will be None
# before its upload, and 0 after is upload.
percentComplete = 0
current = 0
elif total > 0:
percentComplete = \
100.0 - ((total - current) * 100.0) / total
else:
percentComplete = 100
self.uploadModel.SetBytesUploaded(current)
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if message:
self.uploadsModel.SetMessage(self.uploadModel, message)
else:
if total >= (1024 * 1024 * 1024):
message = "%3.1f %% uploaded" % percentComplete
else:
message = "%3d %% uploaded" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
# The database interactions below should go in a model class.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
def PosterCallback(param, current, total):
# pylint: disable=unused-argument
ProgressCallback(current, total)
datagen, headers = poster.encode.multipart_encode(
{"json_data": json.dumps(dataFileJson),
"attached_file": datafileBufferedReader},
cb=PosterCallback)
opener = poster.streaminghttp.register_openers()
opener.addheaders = [("Authorization", "ApiKey " +
myTardisUsername +
":" + myTardisApiKey),
("Content-Type", "application/json"),
("Accept", "application/json")]
elif not self.existingUnverifiedDatafile:
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
data = json.dumps(dataFileJson)
message = "Uploading..."
self.uploadsModel.SetMessage(self.uploadModel, message)
postSuccess = False
uploadSuccess = False
request = None
response = None
# pylint: disable=broad-except
# pylint: disable=too-many-nested-blocks
try:
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
request = urllib2.Request(url, datagen, headers)
try:
if self.foldersController.uploadMethod == \
UploadMethod.HTTP_POST:
response = urllib2.urlopen(request)
postSuccess = True
uploadSuccess = True
else:
if not self.existingUnverifiedDatafile:
response = requests.post(headers=headers, url=url,
data=data)
postSuccess = response.status_code >= 200 and \
response.status_code < 300
logger.debug(response.text)
if postSuccess or self.existingUnverifiedDatafile:
uploadToStagingRequest = self.settingsModel\
.GetUploadToStagingRequest()
host = uploadToStagingRequest.GetScpHostname()
port = uploadToStagingRequest.GetScpPort()
location = uploadToStagingRequest.GetLocation()
username = uploadToStagingRequest.GetScpUsername()
privateKeyFilePath = self.settingsModel\
.GetSshKeyPair().GetPrivateKeyFilePath()
if self.existingUnverifiedDatafile:
uri = self.existingUnverifiedDatafile\
.GetReplicas()[0].GetUri()
remoteFilePath = "%s/%s" % (location.rstrip('/'),
uri)
else:
# DataFile creation via the MyTardis API doesn't
# return JSON, but if a DataFile record is created
# without specifying a storage location, then a
# temporary location is returned for the client
# to copy/upload the file to.
tempUrl = response.text
remoteFilePath = tempUrl
while True:
try:
UploadFile(dataFilePath,
dataFileSize,
username,
privateKeyFilePath,
host, port, remoteFilePath,
ProgressCallback,
self.foldersController,
self.uploadModel)
except IOError, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = "This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
except ScpException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = \
"This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
except SshException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = \
"This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
break
def Run(self):
# pylint: disable=too-many-locals
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
self.uploadsThreadingLock.acquire()
uploadDataViewId = self.uploadsModel.GetMaxDataViewId() + 1
self.uploadModel = UploadModel(dataViewId=uploadDataViewId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex)
self.uploadModel.SetExistingUnverifiedDatafile(
self.verificationModel.GetExistingUnverifiedDatafile())
self.uploadsModel.AddRow(self.uploadModel)
self.uploadsThreadingLock.release()
self.uploadModel.SetBytesUploadedPreviously(
self.bytesUploadedPreviously)
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
dataFileName = os.path.basename(dataFilePath)
dataFileDirectory = \
self.folderModel.GetDataFileDirectory(self.dataFileIndex)
ignoreNewFiles = self.settingsModel.IgnoreNewFiles()
ignoreNewFilesMinutes = self.settingsModel.GetIgnoreNewFilesMinutes()
ignoreNewFilesSeconds = 0
if ignoreNewFiles:
ignoreNewFilesSeconds = ignoreNewFilesMinutes * 60
if (time.time() - os.path.getmtime(dataFilePath)) <= \
ignoreNewFilesSeconds:
message = "Not uploading file, in case it is still being modified."
logger.warning(message.replace('file', dataFilePath))
self.uploadsModel.SetMessage(self.uploadModel, message)
self.uploadsModel.SetStatus(self.uploadModel, UploadStatus.FAILED)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.uploadCompleteEvent(
id=self.foldersController.EVT_UPLOAD_FAILED,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex,
uploadModel=self.uploadModel))
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
url = myTardisUrl + "/api/v1/mydata_dataset_file/"
else:
url = myTardisUrl + "/api/v1/dataset_file/"
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
if self.foldersController.IsShuttingDown():
return
message = "Getting data file size..."
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileSize = self.folderModel.GetDataFileSize(self.dataFileIndex)
self.uploadModel.SetFileSize(dataFileSize)
if self.foldersController.IsShuttingDown():
return
# The HTTP POST upload method doesn't support resuming uploads,
# so we always (re-)create the JSON to be POSTed when we find
# a file whose datafile record is unverified.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
message = "Calculating MD5 checksum..."
self.uploadsModel.SetMessage(self.uploadModel, message)
def Md5ProgressCallback(bytesProcessed):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
if dataFileSize > 0:
percentComplete = \
100.0 - ((dataFileSize - bytesProcessed) * 100.0) \
/ dataFileSize
else:
percentComplete = 100
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if dataFileSize >= (1024 * 1024 * 1024):
message = "%3.1f %% MD5 summed" % percentComplete
else:
message = "%3d %% MD5 summed" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
if self.settingsModel.FakeMd5Sum():
dataFileMd5Sum = self.settingsModel.GetFakeMd5Sum()
logger.warning("Faking MD5 sum for %s" % dataFilePath)
else:
dataFileMd5Sum = \
self.CalculateMd5Sum(dataFilePath, dataFileSize,
self.uploadModel,
progressCallback=Md5ProgressCallback)
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
else:
dataFileSize = int(self.existingUnverifiedDatafile.GetSize())
self.uploadModel.SetProgress(0)
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if self.foldersController.IsShuttingDown():
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
message = "Checking MIME type..."
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileMimeType = self.mimeTypes.guess_type(dataFilePath)[0]
if self.foldersController.IsShuttingDown():
return
message = "Defining JSON data for POST..."
self.uploadsModel.SetMessage(self.uploadModel, message)
datasetUri = self.folderModel.GetDatasetModel().GetResourceUri()
dataFileCreatedTime = \
self.folderModel.GetDataFileCreatedTime(self.dataFileIndex)
dataFileModifiedTime = \
self.folderModel.GetDataFileModifiedTime(self.dataFileIndex)
dataFileJson = {
"dataset": datasetUri,
"filename": dataFileName,
"directory": dataFileDirectory,
"md5sum": dataFileMd5Sum,
"size": dataFileSize,
"mimetype": dataFileMimeType,
"created_time": dataFileCreatedTime,
"modification_time": dataFileModifiedTime,
}
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
dataFileJson['uploader_uuid'] = self.settingsModel.GetUuid()
dataFileJson['requester_key_fingerprint'] = \
self.settingsModel.GetSshKeyPair().GetFingerprint()
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
message = "Initializing buffered reader..."
self.uploadsModel.SetMessage(self.uploadModel, message)
datafileBufferedReader = io.open(dataFilePath, 'rb')
self.uploadModel.SetBufferedReader(datafileBufferedReader)
def ProgressCallback(current, total, message=None):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
elif self.uploadModel.GetStatus() == UploadStatus.COMPLETED:
return
if current is None:
# For a zero-sized file, current will be None
# before its upload, and 0 after is upload.
percentComplete = 0
current = 0
elif total > 0:
percentComplete = \
100.0 - ((total - current) * 100.0) / total
else:
percentComplete = 100
self.uploadModel.SetBytesUploaded(current)
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if message:
self.uploadsModel.SetMessage(self.uploadModel, message)
else:
if total >= (1024 * 1024 * 1024):
message = "%3.1f %% uploaded" % percentComplete
else:
message = "%3d %% uploaded" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
# The database interactions below should go in a model class.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
def PosterCallback(param, current, total):
# pylint: disable=unused-argument
ProgressCallback(current, total)
datagen, headers = poster.encode.multipart_encode(
{"json_data": json.dumps(dataFileJson),
"attached_file": datafileBufferedReader},
cb=PosterCallback)
opener = poster.streaminghttp.register_openers()
opener.addheaders = [("Authorization", "ApiKey " +
myTardisUsername +
":" + myTardisApiKey),
("Content-Type", "application/json"),
("Accept", "application/json")]
elif not self.existingUnverifiedDatafile:
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
data = json.dumps(dataFileJson)
message = "Uploading..."
self.uploadsModel.SetMessage(self.uploadModel, message)
self.uploadModel.SetStartTime(datetime.now())
postSuccess = False
uploadSuccess = False
request = None
response = None
# pylint: disable=broad-except
# pylint: disable=too-many-nested-blocks
try:
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
request = urllib2.Request(url, datagen, headers)
try:
if self.foldersController.uploadMethod == \
UploadMethod.HTTP_POST:
response = urllib2.urlopen(request)
postSuccess = True
uploadSuccess = True
else:
if not self.existingUnverifiedDatafile:
response = requests.post(headers=headers, url=url,
data=data)
postSuccess = response.status_code >= 200 and \
response.status_code < 300
logger.debug(response.text)
if postSuccess or self.existingUnverifiedDatafile:
uploadToStagingRequest = self.settingsModel\
.GetUploadToStagingRequest()
host = uploadToStagingRequest.GetScpHostname()
port = uploadToStagingRequest.GetScpPort()
location = uploadToStagingRequest.GetLocation()
username = uploadToStagingRequest.GetScpUsername()
privateKeyFilePath = self.settingsModel\
.GetSshKeyPair().GetPrivateKeyFilePath()
if self.existingUnverifiedDatafile:
uri = self.existingUnverifiedDatafile\
.GetReplicas()[0].GetUri()
remoteFilePath = "%s/%s" % (location.rstrip('/'),
uri)
else:
# DataFile creation via the MyTardis API doesn't
# return JSON, but if a DataFile record is created
# without specifying a storage location, then a
# temporary location is returned for the client
# to copy/upload the file to.
tempUrl = response.text
remoteFilePath = tempUrl
dataFileId = \
response.headers['Location'].split('/')[-2]
self.uploadModel.SetDataFileId(dataFileId)
while True:
try:
UploadFile(dataFilePath,
dataFileSize,
username,
privateKeyFilePath,
host, port, remoteFilePath,
ProgressCallback,
self.foldersController,
self.uploadModel)
except IOError, err:
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
self.uploadModel.SetProgress(0)
continue
else:
raise
except ScpException, err:
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
self.uploadModel.SetProgress(0)
continue
else:
raise
except SshException, err:
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
self.uploadModel.SetProgress(0)
continue
else:
raise
break
def ImportGroupFolders(self, groupFolderPath, groupModel):
"""
Scan folders within a user group folder,
e.g. D:\\Data\\Smith-Lab\\
"""
# pylint: disable=bare-except
try:
logger.debug("Scanning " + groupFolderPath +
" for instrument folders...")
datasetFilterString = \
'*%s*' % self.settingsModel.GetDatasetFilter()
instrumentName = self.settingsModel.GetInstrumentName()
filesDepth1 = glob(os.path.join(groupFolderPath, instrumentName))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
instrumentFolders = [os.path.basename(d) for d in dirsDepth1]
if len(instrumentFolders) > 1:
message = "Multiple instrument folders found in %s" \
% groupFolderPath
logger.warning(message)
elif len(instrumentFolders) == 0:
message = "No instrument folder was found in %s" \
% groupFolderPath
logger.warning(message)
return
# Rather than using any folder we happen to find at this level,
# we will use the instrument name specified in MyData's Settings
# dialog. That way, we can run MyData on a collection of data
# from multiple instruments, and just select one instrument at
# a time.
instrumentFolderPath = \
os.path.join(groupFolderPath,
self.settingsModel.GetInstrumentName())
if not os.path.exists(instrumentFolderPath):
logger.warning("Path %s doesn't exist." % instrumentFolderPath)
return
# For the User Group / Instrument / Researcher's Name / Dataset
# folder structure, the default owner in MyTardis will always
# by the user listed in MyData's settings dialog. An additional
# ObjectACL will be created in MyTardis to grant access to the
# User Group. The researcher's name in this folder structure is
# used to determine the default experiment name, but it is not
# used to determine access control.
owner = self.settingsModel.GetDefaultOwner()
logger.debug("Scanning " + instrumentFolderPath +
" for user folders...")
userFolders = os.walk(instrumentFolderPath).next()[1]
for userFolderName in userFolders:
userFolderPath = os.path.join(instrumentFolderPath,
userFolderName)
logger.debug("Scanning " + userFolderPath +
" for dataset folders...")
filesDepth1 = glob(os.path.join(userFolderPath,
datasetFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
datasetFolders = [os.path.basename(d) for d in dirsDepth1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(userFolderPath,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath,
self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
groupFolderName = os.path.basename(groupFolderPath)
dataViewId = self.GetMaxDataViewId() + 1
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=userFolderPath,
userFolderName=userFolderName,
groupFolderName=groupFolderName,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetGroup(groupModel)
folderModel.SetCreatedDate()
folderModel.SetExperimentTitle(
"%s - %s" % (self.settingsModel.GetInstrumentName(),
userFolderName))
self.AddRow(folderModel)
except InvalidFolderStructure:
raise
except:
logger.error(traceback.format_exc())
def CreateWeeklyTask(self, event):
"""
Create and schedule task(s) according to the settings configured in
the Schedule tab of the Settings dialog.
"""
scheduleType = "Weekly"
logger.debug("Schedule type is Weekly.")
def RunTaskWeekly(event, jobId):
"""
Run a task on the days (of the week) and time specified
in the Schedule tab of the Settings dialog.
"""
app = wx.GetApp()
wx.CallAfter(app.DisableTestAndUploadToolbarButtons)
while not app.Processing():
time.sleep(0.01)
needToValidateSettings = False
wx.CallAfter(app.OnRefresh, event, needToValidateSettings, jobId)
# Sleep this thread until the job is really
# finished, so we can determine the job's
# finish time.
while app.Processing():
time.sleep(0.01)
jobDesc = "Scan folders and upload datafiles"
days = [
self.settingsModel.IsMondayChecked(),
self.settingsModel.IsTuesdayChecked(),
self.settingsModel.IsWednesdayChecked(),
self.settingsModel.IsThursdayChecked(),
self.settingsModel.IsFridayChecked(),
self.settingsModel.IsSaturdayChecked(),
self.settingsModel.IsSundayChecked()
]
if not max(days):
logger.warning("No days selected for weekly schedule.")
return
startTime = \
datetime.combine(datetime.date(datetime.now()),
self.settingsModel.GetScheduledTime())
while not days[startTime.weekday()]:
startTime = startTime + timedelta(days=1)
timeString = startTime.strftime("%I:%M %p")
dateString = \
"{d:%A} {d.day}/{d.month}/{d.year}".format(d=startTime)
wx.GetApp().GetMainFrame().SetStatusMessage(
"The \"%s\" task is scheduled "
"to run at %s on %s (recurring on specified days)" %
(jobDesc, timeString, dateString))
taskDataViewId = self.tasksModel.GetMaxDataViewId() + 1
jobArgs = [event, taskDataViewId]
task = TaskModel(taskDataViewId,
RunTaskWeekly,
jobArgs,
jobDesc,
startTime,
scheduleType=scheduleType,
days=days)
try:
self.tasksModel.AddRow(task)
except ValueError, err:
wx.MessageBox(str(err), "MyData", wx.ICON_ERROR)
return
def ScanForUserFolders(self, writeProgressUpdateToStatusBar, shouldAbort):
"""
Scan for user folders.
"""
dataDir = self.settingsModel.GetDataDirectory()
userOrGroupFilterString = '*%s*' % self.settingsModel.GetUserFilter()
folderStructure = self.settingsModel.GetFolderStructure()
filesDepth1 = glob(os.path.join(dataDir, userOrGroupFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
userFolderNames = [os.path.basename(d) for d in dirsDepth1]
for userFolderName in userFolderNames:
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
if folderStructure.startswith("Username"):
logger.debug("Found folder assumed to be username: "******"Email"):
logger.debug("Found folder assumed to be email: " +
userFolderName)
usersDataViewId = self.usersModel.GetMaxDataViewId() + 1
try:
if folderStructure.startswith("Username"):
userRecord = \
UserModel.GetUserByUsername(self.settingsModel,
userFolderName)
elif folderStructure.startswith("Email"):
userRecord = \
UserModel.GetUserByEmail(self.settingsModel,
userFolderName)
except DoesNotExist:
userRecord = None
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
if userRecord is not None:
userRecord.SetDataViewId(usersDataViewId)
self.usersModel.AddRow(userRecord)
userFolderPath = os.path.join(dataDir, userFolderName)
logger.debug("Folder structure: " + folderStructure)
if folderStructure == 'Username / Dataset' or \
folderStructure == 'Email / Dataset':
self.ScanForDatasetFolders(userFolderPath, userRecord,
userFolderName)
elif folderStructure == \
'Username / Experiment / Dataset' or \
folderStructure == 'Email / Experiment / Dataset':
self.ScanForExperimentFolders(userFolderPath, userRecord,
userFolderName)
elif folderStructure == \
'Username / "MyTardis" / Experiment / Dataset':
userFolderContents = os.listdir(userFolderPath)
myTardisFolderName = None
for item in userFolderContents:
if item.lower() == 'mytardis':
myTardisFolderName = item
if not myTardisFolderName:
message = 'Didn\'t find "MyTardis" folder in ' \
'"%s"' % userFolderPath
logger.error(message)
raise InvalidFolderStructure(message)
myTardisFolderPath = os.path.join(userFolderPath,
myTardisFolderName)
self.ScanForExperimentFolders(myTardisFolderPath,
userRecord,
userFolderName)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame()
.SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
else:
message = "Didn't find a MyTardis user record for folder " \
"\"%s\" in %s" % (userFolderName, dataDir)
logger.warning(message)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
if not self.settingsModel.UploadInvalidUserOrGroupFolders():
logger.warning("Skipping %s, because "
"'Upload invalid user folders' "
"setting is not checked." % userFolderName)
continue
if folderStructure.startswith("Username"):
userRecord = UserModel(settingsModel=self.settingsModel,
username=userFolderName,
userNotFoundInMyTardis=True)
elif folderStructure.startswith("Email"):
userRecord = \
UserModel(settingsModel=self.settingsModel,
email=userFolderName,
userNotFoundInMyTardis=True)
userRecord.SetDataViewId(usersDataViewId)
self.usersModel.AddRow(userRecord)
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
self.ScanForDatasetFolders(os.path.join(dataDir,
userFolderName),
userRecord, userFolderName)
if threading.current_thread().name == "MainThread":
writeProgressUpdateToStatusBar()
else:
wx.CallAfter(writeProgressUpdateToStatusBar)
def ScanForGroupFolders(self, writeProgressUpdateToStatusBar, shouldAbort):
"""
Scan for group folders.
"""
dataDir = self.settingsModel.GetDataDirectory()
userOrGroupFilterString = '*%s*' % self.settingsModel.GetUserFilter()
filesDepth1 = glob(os.path.join(dataDir, userOrGroupFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
groupFolderNames = [os.path.basename(d) for d in dirsDepth1]
folderStructure = self.settingsModel.GetFolderStructure()
for groupFolderName in groupFolderNames:
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
logger.debug("Found folder assumed to be user group name: " +
groupFolderName)
groupsDataViewId = self.groupsModel.GetMaxDataViewId() + 1
try:
groupName = self.settingsModel.GetGroupPrefix() + \
groupFolderName
groupRecord = \
GroupModel.GetGroupByName(self.settingsModel,
groupName)
except DoesNotExist:
groupRecord = None
message = "Didn't find a MyTardis user group record for " \
"folder \"%s\" in %s" % (groupFolderName,
dataDir)
logger.warning(message)
if not self.settingsModel.UploadInvalidUserOrGroupFolders():
logger.warning("Skipping %s, because "
"'Upload invalid user group folders' "
"setting is not checked." % groupFolderName)
continue
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
if groupRecord:
groupRecord.SetDataViewId(groupsDataViewId)
self.groupsModel.AddRow(groupRecord)
groupFolderPath = os.path.join(dataDir, groupFolderName)
if folderStructure == \
'User Group / Instrument / Full Name / Dataset':
self.ImportGroupFolders(groupFolderPath, groupRecord)
elif folderStructure == 'User Group / Experiment / Dataset':
defaultOwner = self.settingsModel.GetDefaultOwner()
self.ScanForExperimentFolders(groupFolderPath,
owner=defaultOwner,
groupRecord=groupRecord,
groupFolderName=groupFolderName)
else:
raise InvalidFolderStructure("Unknown folder structure.")
if shouldAbort():
wx.CallAfter(wx.GetApp().GetMainFrame().SetStatusMessage,
"Data scans and uploads were canceled.")
wx.CallAfter(EndBusyCursorIfRequired)
return
if threading.current_thread().name == "MainThread":
writeProgressUpdateToStatusBar()
else:
wx.CallAfter(writeProgressUpdateToStatusBar)
def ScanForDatasetFolders(self, pathToScan, owner, userFolderName):
"""
Scan for dataset folders.
"""
# pylint: disable=bare-except
try:
logger.debug("Scanning " + pathToScan +
" for dataset folders...")
datasetFilterString = \
'*%s*' % self.settingsModel.GetDatasetFilter()
filesDepth1 = glob(os.path.join(pathToScan, datasetFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
datasetFolders = [os.path.basename(d) for d in dirsDepth1]
for datasetFolderName in datasetFolders:
logger.debug("Found folder assumed to be dataset: " +
datasetFolderName)
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(pathToScan,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > \
self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath,
self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
dataViewId = self.GetMaxDataViewId() + 1
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=pathToScan,
userFolderName=userFolderName,
groupFolderName=None,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetCreatedDate()
if not owner.UserNotFoundInMyTardis():
if owner.GetName().strip() != "":
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
owner.GetName())
else:
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
owner.GetUsername())
elif owner.GetName() != UserModel.userNotFoundString:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetName(),
UserModel.userNotFoundString)
elif owner.GetUsername() != UserModel.userNotFoundString:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetUsername(),
UserModel.userNotFoundString)
elif owner.GetEmail() != UserModel.userNotFoundString:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetEmail(),
UserModel.userNotFoundString)
else:
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
UserModel.userNotFoundString)
folderModel.SetExperimentTitle(experimentTitle)
self.AddRow(folderModel)
except:
logger.error(traceback.format_exc())
def Run(self):
# pylint: disable=too-many-locals
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
if self.uploadModel.Canceled():
# self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
dataFilePath = self.folderModel.GetDataFilePath(self.dataFileIndex)
dataFileName = os.path.basename(dataFilePath)
dataFileDirectory = \
self.folderModel.GetDataFileDirectory(self.dataFileIndex)
thirtySeconds = 30
if (time.time() - os.path.getmtime(dataFilePath)) <= thirtySeconds:
message = "Not uploading file, in case it is still being modified."
self.uploadModel.SetMessage(message)
self.uploadsModel.UploadMessageUpdated(self.uploadModel)
self.uploadModel.SetStatus(UploadStatus.FAILED)
self.uploadsModel.UploadStatusUpdated(self.uploadModel)
return
logger.debug("Uploading " +
self.folderModel.GetDataFileName(self.dataFileIndex) +
"...")
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
if self.foldersController.uploadMethod == \
UploadMethod.VIA_STAGING:
url = myTardisUrl + "/api/v1/mydata_dataset_file/"
else:
url = myTardisUrl + "/api/v1/dataset_file/"
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey)}
if self.foldersController.IsShuttingDown():
return
self.uploadModel.SetMessage("Getting data file size...")
dataFileSize = self.folderModel.GetDataFileSize(self.dataFileIndex)
self.uploadModel.SetFileSize(dataFileSize)
if self.foldersController.IsShuttingDown():
return
# The HTTP POST upload method doesn't support resuming uploads,
# so we always (re-)create the JSON to be POSTed when we find
# a file whose datafile record is unverified.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
self.uploadModel.SetMessage("Calculating MD5 checksum...")
def Md5ProgressCallback(bytesProcessed):
if self.uploadModel.Canceled():
# self.foldersController.SetCanceled()
return
percentComplete = \
100.0 - ((dataFileSize - bytesProcessed) * 100.0) \
/ dataFileSize
# self.uploadModel.SetProgress(float(percentComplete))
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if dataFileSize >= (1024 * 1024 * 1024):
self.uploadModel.SetMessage("%3.1f %% MD5 summed"
% percentComplete)
else:
self.uploadModel.SetMessage("%3d %% MD5 summed"
% int(percentComplete))
self.uploadsModel.UploadMessageUpdated(self.uploadModel)
myTardisUrl = self.settingsModel.GetMyTardisUrl()
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.connectionStatusEvent(
myTardisUrl=myTardisUrl,
connectionStatus=ConnectionStatus.CONNECTED))
dataFileMd5Sum = \
self.foldersController\
.CalculateMd5Sum(dataFilePath, dataFileSize,
self.uploadModel,
progressCallback=Md5ProgressCallback)
if self.uploadModel.Canceled():
# self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
else:
dataFileSize = int(self.existingUnverifiedDatafile.GetSize())
self.uploadModel.SetProgress(0)
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if dataFileSize == 0:
self.uploadsModel.UploadFileSizeUpdated(self.uploadModel)
self.uploadModel.SetMessage("MyTardis will not accept a "
"data file with a size of zero.")
self.uploadsModel.UploadMessageUpdated(self.uploadModel)
self.uploadModel.SetStatus(UploadStatus.FAILED)
self.uploadsModel.UploadStatusUpdated(self.uploadModel)
return
if self.foldersController.IsShuttingDown():
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
self.uploadModel.SetMessage("Checking MIME type...")
# mimetypes.guess_type(...) is not thread-safe!
mimeTypes = mimetypes.MimeTypes()
dataFileMimeType = mimeTypes.guess_type(dataFilePath)[0]
if self.foldersController.IsShuttingDown():
return
self.uploadModel.SetMessage("Defining JSON data for POST...")
datasetUri = self.folderModel.GetDatasetModel().GetResourceUri()
dataFileCreatedTime = \
self.folderModel.GetDataFileCreatedTime(self.dataFileIndex)
dataFileJson = {"dataset": datasetUri,
"filename": dataFileName,
"directory": dataFileDirectory,
"md5sum": dataFileMd5Sum,
"size": dataFileSize,
"mimetype": dataFileMimeType,
"created_time": dataFileCreatedTime}
if self.uploadModel.Canceled():
# self.foldersController.SetCanceled()
logger.debug("Upload for \"%s\" was canceled "
"before it began uploading." %
self.uploadModel.GetRelativePathToUpload())
return
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
self.uploadModel.SetMessage("Initializing buffered reader...")
datafileBufferedReader = io.open(dataFilePath, 'rb')
self.uploadModel.SetBufferedReader(datafileBufferedReader)
def ProgressCallback(current, total, message=None):
if self.uploadModel.Canceled():
# self.foldersController.SetCanceled()
return
percentComplete = \
100.0 - ((total - current) * 100.0) / total
self.uploadModel.SetBytesUploaded(current)
# self.uploadModel.SetProgress(float(percentComplete))
self.uploadModel.SetProgress(int(percentComplete))
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
if message:
self.uploadModel.SetMessage(message)
else:
if total >= (1024 * 1024 * 1024):
self.uploadModel.SetMessage("%3.1f %% uploaded"
% percentComplete)
else:
self.uploadModel.SetMessage("%3d %% uploaded"
% int(percentComplete))
self.uploadsModel.UploadMessageUpdated(self.uploadModel)
myTardisUrl = self.settingsModel.GetMyTardisUrl()
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.connectionStatusEvent(
myTardisUrl=myTardisUrl,
connectionStatus=ConnectionStatus.CONNECTED))
# The database interactions below should go in a model class.
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
def PosterCallback(param, current, total):
# pylint: disable=unused-argument
ProgressCallback(current, total)
datagen, headers = poster.encode.multipart_encode(
{"json_data": json.dumps(dataFileJson),
"attached_file": datafileBufferedReader},
cb=PosterCallback)
opener = poster.streaminghttp.register_openers()
opener.addheaders = [("Authorization", "ApiKey " +
myTardisUsername +
":" + myTardisApiKey),
("Content-Type", "application/json"),
("Accept", "application/json")]
elif not self.existingUnverifiedDatafile:
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
data = json.dumps(dataFileJson)
self.uploadModel.SetMessage("Uploading...")
postSuccess = False
uploadSuccess = False
request = None
response = None
# pylint: disable=broad-except
try:
if self.foldersController.uploadMethod == UploadMethod.HTTP_POST:
request = urllib2.Request(url, datagen, headers)
try:
if self.foldersController.uploadMethod == \
UploadMethod.HTTP_POST:
response = urllib2.urlopen(request)
postSuccess = True
uploadSuccess = True
else:
if not self.existingUnverifiedDatafile:
response = requests.post(headers=headers, url=url,
data=data)
postSuccess = response.status_code >= 200 and \
response.status_code < 300
logger.debug(response.text)
if postSuccess or self.existingUnverifiedDatafile:
uploadToStagingRequest = self.settingsModel\
.GetUploadToStagingRequest()
host = uploadToStagingRequest.GetScpHostname()
port = uploadToStagingRequest.GetScpPort()
location = uploadToStagingRequest.GetLocation()
username = uploadToStagingRequest.GetScpUsername()
privateKeyFilePath = self.settingsModel\
.GetSshKeyPair().GetPrivateKeyFilePath()
if self.existingUnverifiedDatafile:
uri = self.existingUnverifiedDatafile\
.GetReplicas()[0].GetUri()
remoteFilePath = "%s/%s" % (location.rstrip('/'),
uri)
else:
# DataFile creation via the MyTardis API doesn't
# return JSON, but if a DataFile record is created
# without specifying a storage location, then a
# temporary location is returned for the client
# to copy/upload the file to.
tempUrl = response.text
remoteFilePath = tempUrl
while True:
try:
UploadFile(dataFilePath,
dataFileSize,
username,
privateKeyFilePath,
host, port, remoteFilePath,
ProgressCallback,
self.foldersController,
self.uploadModel)
except IOError, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
self.uploadModel.SetMessage(
"This file will be re-uploaded...")
self.uploadModel.SetProgress(0)
continue
else:
raise
except ScpException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
self.uploadModel.SetMessage(
"This file will be re-uploaded...")
self.uploadModel.SetProgress(0)
continue
else:
raise
except SshException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
self.settingsModel.GetMaxUploadRetries():
logger.warning(str(err))
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
self.uploadModel.SetMessage(
"This file will be re-uploaded...")
self.uploadModel.SetProgress(0)
continue
else:
raise
break
def ScanForExperimentFolders(self, pathToScan, owner,
userFolderName=None,
groupRecord=None, groupFolderName=None):
# pylint: disable=too-many-arguments
"""
Scans for experiment folders.
The MyTardis role account specified in the Settings dialog will
automatically be given access (and ownership) to every experiment
created. If the experiment folder is found within a user folder,
then that user will be given access, and similarly, if it is
found within a user group folder, then the user group will be
given access.
"""
datasetFilterString = '*%s*' % self.settingsModel.GetDatasetFilter()
expFilterString = '*%s*' % self.settingsModel.GetExperimentFilter()
filesDepth1 = glob(os.path.join(pathToScan, expFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
expFolders = [os.path.basename(d) for d in dirsDepth1]
folderStructure = self.settingsModel.GetFolderStructure()
for expFolderName in expFolders:
expFolderPath = os.path.join(pathToScan, expFolderName)
filesDepth1 = glob(os.path.join(expFolderPath,
datasetFilterString))
dirsDepth1 = [item for item in filesDepth1 if os.path.isdir(item)]
datasetFolders = [os.path.basename(d) for d in dirsDepth1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(expFolderPath,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath, self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
dataViewId = self.GetMaxDataViewId() + 1
if folderStructure.startswith("Username") or \
folderStructure.startswith("Email") or \
folderStructure.startswith("Experiment"):
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=expFolderPath,
userFolderName=userFolderName,
groupFolderName=None,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetExperimentTitle(expFolderName)
elif folderStructure.startswith("User Group"):
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=expFolderPath,
userFolderName=None,
groupFolderName=groupFolderName,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetGroup(groupRecord)
if groupRecord:
groupName = groupRecord.GetShortName()
else:
groupName = groupFolderName
folderModel.SetExperimentTitle(
"%s - %s" % (groupName, expFolderName))
else:
raise InvalidFolderStructure("Unknown folder structure.")
folderModel.SetCreatedDate()
self.AddRow(folderModel)
# finished, so we can determine the job's
# finish time.
while self.toolbar.GetToolEnabled(self.stopTool.GetId()):
time.sleep(0.01)
jobArgs = [self, event, False]
jobDesc = "Scan folders and upload datafiles"
days = [self.settingsModel.IsMondayChecked(),
self.settingsModel.IsTuesdayChecked(),
self.settingsModel.IsWednesdayChecked(),
self.settingsModel.IsThursdayChecked(),
self.settingsModel.IsFridayChecked(),
self.settingsModel.IsSaturdayChecked(),
self.settingsModel.IsSundayChecked()]
if not max(days):
logger.warning("No days selected for weekly schedule.")
return
startTime = \
datetime.combine(datetime.date(datetime.now()),
self.settingsModel.GetScheduledTime())
while not days[startTime.weekday()]:
startTime = startTime + timedelta(days=1)
timeString = startTime.strftime("%I:%M %p")
dateString = \
"{d:%A} {d.day}/{d.month}/{d.year}".format(d=startTime)
self.frame.SetStatusMessage(
"The \"%s\" task is scheduled "
"to run at %s on %s (recurring on specified days)"
% (jobDesc, timeString, dateString))
taskDataViewId = self.tasksModel.GetMaxDataViewId() + 1
task = TaskModel(taskDataViewId, jobFunc, jobArgs, jobDesc,
def ScanForDatasetFolders(self, pathToScan, owner, userFolderName):
try:
logger.debug("Scanning " + pathToScan +
" for dataset folders...")
datasetFolders = os.walk(pathToScan).next()[1]
for datasetFolderName in datasetFolders:
if self.ignoreOldDatasets:
datasetFolderPath = os.path.join(pathToScan,
datasetFolderName)
ctimestamp = os.path.getctime(datasetFolderPath)
ctime = datetime.fromtimestamp(ctimestamp)
age = datetime.now() - ctime
if age.total_seconds() > \
self.ignoreIntervalSeconds:
message = "Ignoring \"%s\", because it is " \
"older than %d %s" \
% (datasetFolderPath,
self.ignoreIntervalNumber,
self.ignoreIntervalUnit)
logger.warning(message)
continue
dataViewId = self.GetMaxDataViewId() + 1
folderModel = \
FolderModel(dataViewId=dataViewId,
folder=datasetFolderName,
location=pathToScan,
userFolderName=userFolderName,
groupFolderName=None,
owner=owner,
foldersModel=self,
usersModel=self.usersModel,
settingsModel=self.settingsModel)
folderModel.SetCreatedDate()
if not owner.UserNotFoundInMyTardis():
if owner.GetName().strip() != "":
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
owner.GetName())
else:
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
owner.GetUsername())
elif owner.GetName() != UserModel.USER_NOT_FOUND_STRING:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetName(),
UserModel.USER_NOT_FOUND_STRING)
elif owner.GetUsername() != UserModel.USER_NOT_FOUND_STRING:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetUsername(),
UserModel.USER_NOT_FOUND_STRING)
elif owner.GetEmail() != UserModel.USER_NOT_FOUND_STRING:
experimentTitle = "%s - %s (%s)" \
% (self.settingsModel.GetInstrumentName(),
owner.GetEmail(),
UserModel.USER_NOT_FOUND_STRING)
else:
experimentTitle = "%s - %s" \
% (self.settingsModel.GetInstrumentName(),
UserModel.USER_NOT_FOUND_STRING)
folderModel.SetExperimentTitle(experimentTitle)
self.AddRow(folderModel)
except:
print traceback.format_exc()
