def VerificationWorker(self):
# pylint: disable=fixme
# FIXME: Should this be in verifications (not folders) controller?
"""
One worker per thread.
By default, up to 5 threads can run simultaneously
for verifying whether local data files exist on
the MyTardis server.
"""
while True:
if self.IsShuttingDown():
return
task = self.verificationsQueue.get()
if task is None:
break
# pylint: disable=bare-except
try:
task.Run()
except ValueError, err:
if str(err) == "I/O operation on closed file":
logger.info(
"Ignoring closed file exception - it is normal "
"to encounter these exceptions while canceling "
"uploads.")
self.verificationsQueue.task_done()
return
else:
logger.error(traceback.format_exc())
self.verificationsQueue.task_done()
return
except:
def ScanDataDirs():
"""
Scan data folders, looking for datafiles to look up on MyTardis
and upload if necessary.
"""
logger.debug("Starting run() method for thread %s" % threading.current_thread().name)
wx.CallAfter(self.frame.SetStatusMessage, "Scanning data folders...")
try:
self.scanningFoldersThreadingLock.acquire()
self.SetScanningFolders(True)
logger.info("Just set ScanningFolders to True")
self.toolbar.EnableTool(self.stopTool.GetId(), True)
self.foldersModel.ScanFolders(WriteProgressUpdateToStatusBar, self.ShouldAbort)
self.SetScanningFolders(False)
self.scanningFoldersThreadingLock.release()
logger.info("Just set ScanningFolders to False")
except InvalidFolderStructure, ifs:
def ShowMessageDialog():
"""
Needs to run in the main thread.
"""
dlg = wx.MessageDialog(None, str(ifs), "MyData", wx.OK | wx.ICON_ERROR)
dlg.ShowModal()
wx.CallAfter(ShowMessageDialog)
self.frame.SetStatusMessage(str(ifs))
return
def Rename(self, name):
"""
Rename instrument.
"""
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
logger.info("Renaming instrument \"%s\" to \"%s\"."
% (str(self), name))
url = myTardisUrl + "/api/v1/instrument/%d/" % self.GetId()
uploaderJson = {"name": name}
data = json.dumps(uploaderJson)
response = requests.put(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
logger.info("Renaming instrument succeeded.")
else:
logger.info("Renaming instrument failed.")
logger.info("Status code = " + str(response.status_code))
logger.info(response.text)
response.close()
def Rename(self, name):
"""
Rename instrument.
"""
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
headers = {
"Authorization":
"ApiKey %s:%s" % (myTardisUsername, myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"
}
logger.info("Renaming instrument \"%s\" to \"%s\"." %
(str(self), name))
url = myTardisUrl + "/api/v1/instrument/%d/" % self.GetId()
uploaderJson = {"name": name}
data = json.dumps(uploaderJson)
response = requests.put(headers=headers, url=url, data=data)
if response.status_code >= 200 and response.status_code < 300:
logger.info("Renaming instrument succeeded.")
else:
logger.info("Renaming instrument failed.")
logger.info("Status code = " + str(response.status_code))
logger.info(response.text)
response.close()
def VerificationWorker(self):
# pylint: disable=fixme
# FIXME: Should this be in verifications (not folders) controller?
"""
One worker per thread.
By default, up to 5 threads can run simultaneously
for verifying whether local data files exist on
the MyTardis server.
"""
while True:
if self.IsShuttingDown():
return
task = self.verificationsQueue.get()
if task is None:
break
# pylint: disable=bare-except
try:
task.Run()
except ValueError, err:
if str(err) == "I/O operation on closed file":
logger.info(
"Ignoring closed file exception - it is normal "
"to encounter these exceptions while canceling "
"uploads.")
self.verificationsQueue.task_done()
return
else:
logger.error(traceback.format_exc())
self.verificationsQueue.task_done()
return
except:
def OnCloseFrame(self, event): # pylint: disable=unused-argument
"""
Don't actually destroy the frame, just hide it.
"""
if wx.GetApp().TestRunRunning():
logger.info("Closing Test Run Window and calling OnStop.")
wx.GetApp().OnStop(None)
else:
logger.info("Closing Test Run Window.")
self.Hide()
def OnCloseFrame(self, event): # pylint: disable=unused-argument
"""
Don't actually destroy the frame, just hide it.
"""
if wx.GetApp().TestRunRunning():
logger.info("Closing Test Run Window and calling OnStop.")
wx.GetApp().OnStop(None)
else:
logger.info("Closing Test Run Window.")
self.Hide()
def ExistingUploadToStagingRequest(self):
try:
# The private key file path must be ~/.ssh/MyData
keyPair = OpenSSH.FindKeyPair("MyData")
except PrivateKeyDoesNotExist:
keyPair = OpenSSH.NewKeyPair("MyData")
self.settingsModel.SetSshKeyPair(keyPair)
myTardisUrl = self.settingsModel.GetMyTardisUrl()
myTardisUsername = self.settingsModel.GetUsername()
myTardisApiKey = self.settingsModel.GetApiKey()
url = myTardisUrl + \
"/api/v1/mydata_uploaderregistrationrequest/?format=json" + \
"&uploader__uuid=" + self.uuid + \
"&requester_key_fingerprint=" + \
urllib.quote(keyPair.GetFingerprint())
logger.debug(url)
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisUsername,
myTardisApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
response = requests.get(headers=headers, url=url)
if response.status_code < 200 or response.status_code >= 300:
if response.status_code == 404:
response.close()
raise DoesNotExist("HTTP 404 (Not Found) received for: " + url)
message = response.text
response.close()
raise Exception(message)
logger.debug(response.text)
existingUploaderRecords = response.json()
numExistingUploaderRecords = \
existingUploaderRecords['meta']['total_count']
if numExistingUploaderRecords > 0:
approvalJson = existingUploaderRecords['objects'][0]
logger.info("A request already exists for this uploader.")
response.close()
return UploaderRegistrationRequest(
settingsModel=self.settingsModel,
uploaderRegRequestJson=approvalJson)
else:
message = "This uploader hasn't requested uploading " \
"via staging yet."
logger.info(message)
response.close()
raise DoesNotExist(message)
def StartDataUploadsForFolderWorker():
"""
Start the data uploads in a dedicated thread.
"""
logger.debug("Starting run() method for thread %s" %
threading.current_thread().name)
logger.debug("StartDataUploadsForFolderWorker")
wx.CallAfter(BeginBusyCursorIfRequired)
message = "Checking for data files on MyTardis and uploading " \
"if necessary for folder: %s" % event.folderModel.GetFolder()
logger.info(message)
app = wx.GetApp()
if app.TestRunRunning():
logger.testrun(message)
app.DisableTestAndUploadToolbarButtons()
wx.GetApp().SetPerformingLookupsAndUploads(True)
app.foldersController.StartUploadsForFolder(event.folderModel)
wx.CallAfter(EndBusyCursorIfRequired, event)
def StartDataUploadsForFolderWorker():
"""
Start the data uploads in a dedicated thread.
"""
logger.debug("Starting run() method for thread %s"
% threading.current_thread().name)
logger.debug("StartDataUploadsForFolderWorker")
wx.CallAfter(BeginBusyCursorIfRequired)
message = "Checking for data files on MyTardis and uploading " \
"if necessary for folder: %s" % event.folderModel.GetFolder()
logger.info(message)
app = wx.GetApp()
if app.TestRunRunning():
logger.testrun(message)
app.DisableTestAndUploadToolbarButtons()
wx.GetApp().SetPerformingLookupsAndUploads(True)
app.foldersController.StartUploadsForFolder(
event.folderModel)
wx.CallAfter(EndBusyCursorIfRequired, event)
def RequestStagingAccess(self):
"""
This could be called from multiple threads simultaneously,
so it requires locking.
"""
if not hasattr(self, "requestStagingAccessThreadingLock"):
self.requestStagingAccessThreadingLock = threading.Lock()
if self.requestStagingAccessThreadingLock.acquire(False):
try:
try:
self.UploadUploaderInfo()
except:
print traceback.format_exc()
logger.error(traceback.format_exc())
raise
uploadToStagingRequest = None
try:
uploadToStagingRequest = \
self.ExistingUploadToStagingRequest()
except DoesNotExist:
uploadToStagingRequest = \
self.RequestUploadToStagingApproval()
logger.info("Uploader registration request created.")
except PrivateKeyDoesNotExist:
logger.info("Generating new uploader registration request, "
"because private key was moved or deleted.")
uploadToStagingRequest = \
self.RequestUploadToStagingApproval()
logger.info("Generated new uploader registration request, "
"because private key was moved or deleted.")
if uploadToStagingRequest.IsApproved():
logger.info("Uploads to staging have been approved!")
else:
logger.info("Uploads to staging haven't been approved yet.")
self.settingsModel\
.SetUploadToStagingRequest(uploadToStagingRequest)
except:
logger.error(traceback.format_exc())
raise
finally:
self.requestStagingAccessThreadingLock.release()
def CreateExperimentForFolder(folderModel):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
userFolderName = folderModel.GetUserFolderName()
hostname = settingsModel.GetUploaderModel().GetHostname()
location = folderModel.GetLocation()
groupFolderName = folderModel.GetGroupFolderName()
owner = folderModel.GetOwner()
ownerUsername = folderModel.GetOwner().GetUsername()
# pylint: disable=bare-except
try:
ownerUserId = folderModel.GetOwner().GetJson()['id']
except:
ownerUserId = None
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
experimentTitle = folderModel.GetExperimentTitle()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
message = "Creating experiment for uploader \"" + \
uploaderName + ", user folder " + userFolderName
if groupFolderName:
message += ", group folder : " + groupFolderName
logger.info(message)
description = ("Uploader: %s\n"
"User folder name: %s\n"
"Uploaded from: %s:%s"
% (uploaderName, userFolderName, hostname, location))
if groupFolderName:
description += "\nGroup folder name: %s" % groupFolderName
experimentJson = {
"title": experimentTitle,
"description": description,
"immutable": False,
"parameter_sets": [{
"schema": "http://mytardis.org/schemas"
"/mydata/defaultexperiment",
"parameters": [{"name": "uploader",
"value": uploaderUuid},
{"name": "user_folder_name",
"value": userFolderName}]}]}
if groupFolderName:
experimentJson["parameter_sets"][0]["parameters"].append(
{"name": "group_folder_name", "value": groupFolderName})
headers = {
"Authorization": "ApiKey %s:%s" % (myTardisDefaultUsername,
myTardisDefaultUserApiKey),
"Content-Type": "application/json",
"Accept": "application/json"}
url = myTardisUrl + "/api/v1/mydata_experiment/"
response = requests.post(headers=headers, url=url,
data=json.dumps(experimentJson))
# pylint: disable=bare-except
try:
createdExperimentJson = response.json()
createdExperiment = ExperimentModel(settingsModel,
createdExperimentJson)
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " +
str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the experiment.\n\n" \
"Please ask your MyTardis administrator to " \
"check that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message)
raise
if response.status_code == 201:
message = "Succeeded in creating experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.debug(message)
facilityManagersGroup = settingsModel.GetFacility().GetManagerGroup()
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
facilityManagersGroup)
# Avoid creating a duplicate ObjectACL if the user folder's
# username matches the facility manager's username.
# Don't attempt to create an ObjectACL record for an
# invalid user (without a MyTardis user ID).
if myTardisDefaultUsername != ownerUsername and \
ownerUserId is not None:
ObjectAclModel.ShareExperimentWithUser(createdExperiment,
owner)
if folderModel.GetGroup() is not None and \
folderModel.GetGroup().GetId() != \
facilityManagersGroup.GetId():
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
folderModel.GetGroup())
else:
message = "Failed to create experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.error(message)
logger.error(headers)
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " +
str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
raise
return createdExperiment
def GetActiveNetworkInterfaces():
logger.info("Determining the active network interface...")
activeInterfaces = []
if sys.platform.startswith("win"):
proc = subprocess.Popen(["netsh", "interface",
"show", "interface"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=defaultStartupInfo,
creationflags=defaultCreationFlags)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
for row in stdout.split("\n"):
m = re.match(r"^(Enabled|Disabled)\s*(Connected|Disconnected)"
"\s*(Dedicated|Internal|Loopback)\s*(.*)\s*$",
row)
if m:
adminState = m.groups()[0]
state = m.groups()[1]
interfaceType = m.groups()[2]
interface = m.groups()[3].strip()
if adminState == "Enabled" and state == "Connected" \
and interfaceType == "Dedicated":
activeInterfaces.append(interface)
# On Windows XP, the state may be blank:
m = re.match(r"^(Enabled|Disabled)\s*"
"(Dedicated|Internal|Loopback)\s*(.*)\s*$", row)
if m:
adminState = m.groups()[0]
interfaceType = m.groups()[1]
interface = m.groups()[2].strip()
if adminState == "Enabled" and \
interfaceType == "Dedicated":
activeInterfaces.append(interface)
elif sys.platform.startswith("darwin"):
# Was using "route get default" here, but for VPN, that can
# return "utun0" which doesn't have a corresponding MAC address,
# and there may be other missing network-related fields in the
# ifconfig entry for "utun0". For now, we will instead find
# the physical network device which is active, but in future
# we can relax the requirement of needing the MAC address etc.
# because we are now using a UUID as our Uploader record's
# unique identifier instead of a MAC address.
proc = subprocess.Popen(["ifconfig"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=defaultStartupInfo,
creationflags=defaultCreationFlags)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
currentInterface = None
for line in stdout.split("\n"):
m = re.match(r"^(\S+): flags=.*", line)
if m:
currentInterface = m.groups()[0].strip()
m = re.match(r"^\s+status: active", line)
if m and currentInterface:
activeInterfaces.append(currentInterface)
elif sys.platform.startswith("linux"):
proc = subprocess.Popen(["route"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=defaultStartupInfo,
creationflags=defaultCreationFlags)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
for line in stdout.split("\n"):
m = re.match(r"^default.*\s+(\S+)\s*$", line)
if m:
interface = m.groups()[0].strip()
activeInterfaces.append(interface)
return activeInterfaces
def __init__(self, settingsModel):
self.settingsModel = settingsModel
self.interface = None
self.responseJson = None
self.uuid = self.settingsModel.GetUuid()
if self.uuid is None:
self.GenerateUuid()
self.settingsModel.SetUuid(self.uuid)
intervalSinceLastConnectivityCheck = \
datetime.now() - wx.GetApp().GetLastNetworkConnectivityCheckTime()
# Here we check connectivity even if we've already done so, because
# we need to ensure that we get the correct network interface for
# self.interface, otherwise if the active interface changes,
# we can get errors like this: KeyError: 'RTC'
# when accessing things like ipv4_address[self.interface]
activeInterfaces = UploaderModel.GetActiveNetworkInterfaces()
if len(activeInterfaces) == 0:
message = "No active network interfaces." \
"\n\n" \
"Please ensure that you have an active network interface " \
"(e.g. Ethernet or WiFi)."
raise NoActiveNetworkInterface(message)
# Sometimes on Windows XP, you can end up with multiple results
# from "netsh interface show interface"
# If there is one called "Local Area Connection",
# then that's the one we'll go with.
if "Local Area Connection" in activeInterfaces:
activeInterfaces = ["Local Area Connection"]
elif "Local Area Connection 2" in activeInterfaces:
activeInterfaces = ["Local Area Connection 2"]
elif "Ethernet" in activeInterfaces:
activeInterfaces = ["Ethernet"]
elif "Wi-Fi" in activeInterfaces:
activeInterfaces = ["Wi-Fi"]
# For now, we're only dealing with one active network interface.
# It is possible to have more than one active network interface,
# but we hope that the code above has picked the best one.
# If there are no active interfaces, then we shouldn't have
# reached this point - we should have already raised an
# exception.
self.interface = activeInterfaces[0]
if sys.platform.startswith("win"):
proc = subprocess.Popen(["ipconfig", "/all"],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=defaultStartupInfo,
creationflags=defaultCreationFlags)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
mac_address = {}
ipv4_address = {}
ipv6_address = {}
subnet_mask = {}
interface = ""
for row in stdout.split("\n"):
m = re.match(r"^\S.*adapter (.*):\s*$", row)
if m:
interface = m.groups()[0]
if interface == self.interface:
if ': ' in row:
key, value = row.split(': ')
if key.strip(' .') == "Physical Address":
mac_address[interface] = value.strip()
if "IPv4 Address" in key.strip(' .'):
ipv4_address[interface] = \
value.strip().replace("(Preferred)", "")
if "IPv6 Address" in key.strip(' .'):
ipv6_address[interface] = \
value.strip().replace("(Preferred)", "")
if "Subnet Mask" in key.strip(' .'):
subnet_mask[interface] = value.strip()
else:
proc = subprocess.Popen(["ifconfig", self.interface],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
startupinfo=defaultStartupInfo,
creationflags=defaultCreationFlags)
stdout, _ = proc.communicate()
if proc.returncode != 0:
raise Exception(stdout)
mac_address = {}
ipv4_address = {}
ipv6_address = {}
subnet_mask = {}
for row in stdout.split("\n"):
m = re.match(r"\s+ether (\S*)\s*$", row)
if m:
mac_address[self.interface] = m.groups()[0]
m = re.match(r"\s+inet (\S*)\s+netmask\s+(\S*)\s+.*$", row)
if m:
ipv4_address[self.interface] = m.groups()[0]
subnet_mask[self.interface] = m.groups()[1]
m = re.match(r"\s+inet6 (\S*)\s+.*$", row)
if m:
ipv6_address[self.interface] = m.groups()[0]
self.mac_address = mac_address[self.interface]
if self.interface in ipv4_address:
self.ipv4_address = ipv4_address[self.interface]
else:
self.ipv4_address = ""
if self.interface in ipv6_address:
self.ipv6_address = ipv6_address[self.interface]
else:
self.ipv6_address = ""
if self.interface in subnet_mask:
self.subnet_mask = subnet_mask[self.interface]
else:
self.subnet_mask = ""
logger.info("The active network interface is: " + str(self.interface))
self.name = self.settingsModel.GetInstrumentName()
self.contact_name = self.settingsModel.GetContactName()
self.contact_email = self.settingsModel.GetContactEmail()
self.user_agent_name = "MyData"
self.user_agent_version = VERSION
self.user_agent_install_location = ""
if hasattr(sys, 'frozen'):
self.user_agent_install_location = os.path.dirname(sys.executable)
else:
try:
self.user_agent_install_location = \
os.path.dirname(pkgutil.get_loader("MyData").filename)
except:
self.user_agent_install_location = os.getcwd()
fmt = "%-17s %8s %8s %8s %5s%% %9s %s\n"
disk_usage = (fmt % ("Device", "Total", "Used", "Free", "Use ", "Type",
"Mount"))
for part in psutil.disk_partitions(all=False):
if os.name == 'nt':
if 'cdrom' in part.opts or part.fstype == '':
# skip cd-rom drives with no disk in it; they may raise
# ENOENT, pop-up a Windows GUI error for a non-ready
# partition or just hang.
continue
usage = psutil.disk_usage(part.mountpoint)
disk_usage = disk_usage + (fmt % (
part.device,
self._bytes2human(usage.total),
self._bytes2human(usage.used),
self._bytes2human(usage.free),
int(usage.percent),
part.fstype,
part.mountpoint))
self.disk_usage = disk_usage.strip()
self.data_path = self.settingsModel.GetDataDirectory()
self.default_user = self.settingsModel.GetUsername()
def JobFunc(taskModel, tasksModel, row, col):
# pylint: disable=too-many-statements
def TaskJobFunc():
assert callable(taskModel.GetJobFunc())
title = "Starting"
message = taskModel.GetJobDesc()
Notification.Notify(message, title=title)
taskModel.GetJobFunc()(*taskModel.GetJobArgs())
taskModel.SetFinishTime(datetime.now())
title = "Finished"
message = taskModel.GetJobDesc()
Notification.Notify(message, title=title)
wx.CallAfter(tasksModel.TryRowValueChanged, row, col)
scheduleType = taskModel.GetScheduleType()
if scheduleType == "Timer":
intervalMinutes = taskModel.GetIntervalMinutes()
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(minutes=intervalMinutes)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Timer",
intervalMinutes=intervalMinutes)
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = "{d:%A} {d.day}/{d.month}/{d.year}"\
.format(d=newStartTime)
wx.CallAfter(
wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring every %d minutes)" %
(taskModel.GetJobDesc(), timeString, dateString,
intervalMinutes))
tasksModel.AddRow(newTaskModel)
elif scheduleType == "Daily":
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(days=1)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Daily")
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = "{d:%A} {d.day}/{d.month}/{d.year}"\
.format(d=newStartTime)
wx.CallAfter(
wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring daily)" %
(taskModel.GetJobDesc(), timeString, dateString))
tasksModel.AddRow(newTaskModel)
elif scheduleType == "Weekly":
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(days=1)
days = taskModel.GetDays()
while not days[newStartTime.weekday()]:
newStartTime = newStartTime + timedelta(days=1)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Weekly",
days=days)
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = "{d:%A} {d.day}/{d.month}/{d.year}"\
.format(d=newStartTime)
wx.CallAfter(
wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring on specified days)" %
(taskModel.GetJobDesc(), timeString, dateString))
tasksModel.AddRow(newTaskModel)
app = wx.GetApp()
if not app.ShouldAbort():
thread = threading.Thread(target=TaskJobFunc)
logger.debug("Starting task %s" % taskModel.GetJobDesc())
thread.start()
else:
logger.info("Not starting task because we are aborting.")
app.EnableTestAndUploadToolbarButtons()
EndBusyCursorIfRequired()
app.SetShouldAbort(False)
message = "Data scans and uploads were canceled."
wx.GetApp().GetMainFrame().SetStatusMessage(message)
return
def InstrumentNameMismatch(event):
"""
Responds to instrument name mismatch in Settings dialog.
"""
if event.GetEventId() != EVT_INSTRUMENT_NAME_MISMATCH:
event.Skip()
return
message = "A previous instrument name of \"%s\" " \
"has been associated with this MyData instance.\n" \
"Please choose how you would like the new \"%s\" " \
"instrument name to be applied." \
% (event.oldInstrumentName, event.newInstrumentName)
renameChoice = "Rename the existing instrument record to " \
"\"%s\"." % event.newInstrumentName
discardChoice = "Discard the new instrument name and revert " \
"to \"%s\"." % event.oldInstrumentName
createChoice = "Use a separate instrument record for \"%s\", " \
"creating it if necessary." \
% event.newInstrumentName
dlg = wx.SingleChoiceDialog(
event.settingsDialog, message, "MyData - Instrument Name Changed",
[renameChoice, discardChoice, createChoice], wx.CHOICEDLG_STYLE)
if dlg.ShowModal() == wx.ID_OK:
if dlg.GetStringSelection() == renameChoice:
logger.info("OK, we will rename the "
"existing instrument record.")
settingsDialogValidationEvent = \
MyDataEvent(EVT_SETTINGS_DIALOG_VALIDATION,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel)
renameInstrumentEvent = MyDataEvent(
EVT_RENAME_INSTRUMENT,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel,
facilityName=event.settingsDialog.GetFacilityName(),
oldInstrumentName=event.oldInstrumentName,
newInstrumentName=event.newInstrumentName,
nextEvent=settingsDialogValidationEvent)
wx.PostEvent(wx.GetApp().GetMainFrame(), renameInstrumentEvent)
return
elif dlg.GetStringSelection() == discardChoice:
logger.info("OK, we will discard the new instrument name.")
event.settingsDialog.SetInstrumentName(
event.settingsModel.GetInstrumentName())
event.settingsDialog.instrumentNameField.SetFocus()
event.settingsDialog.instrumentNameField.SelectAll()
elif dlg.GetStringSelection() == createChoice:
logger.info("OK, we will create a new instrument record.")
settingsDialogValidationEvent = \
MyDataEvent(EVT_SETTINGS_DIALOG_VALIDATION,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel)
intervalSinceLastCheck = \
datetime.now() - \
wx.GetApp().GetLastConnectivityCheckTime()
checkInterval = \
event.settingsModel.GetConnectivityCheckInterval()
if intervalSinceLastCheck.total_seconds() >= checkInterval \
or not wx.GetApp()\
.GetLastConnectivityCheckSuccess():
checkConnectivityEvent = \
MyDataEvent(EVT_CHECK_CONNECTIVITY,
settingsModel=event.settingsModel,
nextEvent=settingsDialogValidationEvent)
wx.PostEvent(wx.GetApp().GetMainFrame(),
checkConnectivityEvent)
else:
wx.PostEvent(wx.GetApp().GetMainFrame(),
settingsDialogValidationEvent)
def CreateExperimentForFolder(folderModel, testRun=False):
# pylint: disable=too-many-branches
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
settingsModel = folderModel.GetSettingsModel()
userFolderName = folderModel.GetUserFolderName()
hostname = settingsModel.GetUploaderModel().GetHostname()
location = folderModel.GetLocation()
groupFolderName = folderModel.GetGroupFolderName()
owner = folderModel.GetOwner()
ownerUsername = folderModel.GetOwner().GetUsername()
# pylint: disable=bare-except
try:
ownerUserId = folderModel.GetOwner().GetJson()['id']
except:
ownerUserId = None
uploaderName = settingsModel.GetUploaderModel().GetName()
uploaderUuid = settingsModel.GetUploaderModel().GetUuid()
experimentTitle = folderModel.GetExperimentTitle()
myTardisUrl = settingsModel.GetMyTardisUrl()
myTardisDefaultUsername = settingsModel.GetUsername()
myTardisDefaultUserApiKey = settingsModel.GetApiKey()
if userFolderName:
message = "Creating experiment for uploader '%s', " \
"user folder '%s'." % (uploaderName, userFolderName)
if groupFolderName:
message += ", group folder : '%s'" % groupFolderName
elif groupFolderName:
message = "Creating experiment for uploader '%s', " \
"user group folder '%s'." % (uploaderName, groupFolderName)
else:
message = "Creating experiment for uploader '%s'" % uploaderName
logger.info(message)
if userFolderName:
description = ("Uploader: %s\n"
"User folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, userFolderName, hostname, location))
if groupFolderName:
description += "\nGroup folder name: %s" % groupFolderName
else:
description = ("Uploader: %s\n"
"Group folder name: %s\n"
"Uploaded from: %s:%s" %
(uploaderName, groupFolderName, hostname, location))
if testRun:
message = "CREATING NEW EXPERIMENT FOR FOLDER: %s\n" \
" Title: %s\n" \
" Description: \n" \
" Uploader: %s\n" \
" User folder name: %s\n" \
" Owner: %s" \
% (folderModel.GetRelPath(),
experimentTitle, uploaderName, userFolderName,
ownerUsername)
logger.testrun(message)
return
experimentJson = {
"title":
experimentTitle,
"description":
description,
"immutable":
False,
"parameter_sets": [{
"schema":
"http://mytardis.org/schemas"
"/mydata/defaultexperiment",
"parameters": [{
"name": "uploader",
"value": uploaderUuid
}, {
"name": "user_folder_name",
"value": userFolderName
}]
}]
}
if groupFolderName:
experimentJson["parameter_sets"][0]["parameters"].append({
"name":
"group_folder_name",
"value":
groupFolderName
})
headers = {
"Authorization":
"ApiKey %s:%s" %
(myTardisDefaultUsername, myTardisDefaultUserApiKey),
"Content-Type":
"application/json",
"Accept":
"application/json"
}
url = myTardisUrl + "/api/v1/mydata_experiment/"
response = requests.post(headers=headers,
url=url,
data=json.dumps(experimentJson))
# pylint: disable=bare-except
try:
createdExperimentJson = response.json()
createdExperiment = ExperimentModel(settingsModel,
createdExperimentJson)
logger.debug(url)
except:
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "A 404 (Not Found) error occurred while " \
"attempting to create the experiment.\n\n" \
"Please ask your MyTardis administrator to " \
"check that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
raise DoesNotExist(message)
raise
if response.status_code == 201:
message = "Succeeded in creating experiment '%s' for uploader " \
"\"%s\" and user folder \"%s\"" \
% (experimentTitle, uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.debug(message)
facilityManagersGroup = settingsModel.GetFacility(
).GetManagerGroup()
ObjectAclModel.ShareExperimentWithGroup(createdExperiment,
facilityManagersGroup)
# Avoid creating a duplicate ObjectACL if the user folder's
# username matches the facility manager's username.
# Don't attempt to create an ObjectACL record for an
# invalid user (without a MyTardis user ID).
if myTardisDefaultUsername != ownerUsername and \
ownerUserId is not None:
ObjectAclModel.ShareExperimentWithUser(createdExperiment,
owner)
if folderModel.GetGroup() is not None and \
folderModel.GetGroup().GetId() != \
facilityManagersGroup.GetId():
ObjectAclModel.ShareExperimentWithGroup(
createdExperiment, folderModel.GetGroup())
else:
message = "Failed to create experiment for uploader " \
"\"%s\" and user folder \"%s\"" \
% (uploaderName, userFolderName)
if groupFolderName:
message += " and group folder \"%s\"" % groupFolderName
logger.error(message)
logger.error(headers)
logger.error(url)
logger.error(response.text)
logger.error("response.status_code = " + str(response.status_code))
if response.status_code == 401:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
message += "Please ask your MyTardis administrator to " \
"check the permissions of the \"%s\" user " \
"account." % myTardisDefaultUsername
raise Unauthorized(message)
elif response.status_code == 404:
message = "Couldn't create experiment \"%s\" " \
"for folder \"%s\"." \
% (experimentTitle, folderModel.GetFolder())
message += "\n\n"
modelClassOfObjectNotFound = None
# pylint: disable=bare-except
try:
errorResponse = response.json()
if errorResponse['error_message'] == \
"UserProfile matching query does not exist.":
modelClassOfObjectNotFound = UserProfileModel
elif errorResponse['error_message'] == \
"Schema matching query does not exist.":
modelClassOfObjectNotFound = SchemaModel
elif errorResponse['error_message'] == \
"Sorry, this request could not be processed. " \
"Please try again later.":
raise Exception("TASTYPIE_CANNED_ERROR")
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record:\n\n" \
" %s\n\n" % errorResponse['error_message']
except:
message += "A 404 (Not Found) error occurred while " \
"attempting to create an experiment " \
"record. This could be caused by a missing " \
"UserProfile record for user \"%s\" or it " \
"could be caused by a missing Schema record " \
"(see https://github.com/wettenhj/" \
"mytardis-app-mydata/blob/master/README.md)" \
"\n\n" \
"Turning on DEBUG mode on the MyTardis " \
"server could help to isolate the problem." \
% myTardisDefaultUsername
if modelClassOfObjectNotFound == UserProfileModel:
message += "Please ask your MyTardis administrator to " \
"ensure that a User Profile record exists " \
"for the \"%s\" user account." \
% myTardisDefaultUsername
elif modelClassOfObjectNotFound == SchemaModel:
message += "Please ask your MyTardis administrator to " \
"create the experiment metadata schema " \
"described in the \"MyTardis Prerequisites\" " \
"section of the MyData documentation:\n\n" \
"http://mydata.readthedocs.org/en/latest/" \
"mytardis-prerequisites.html"
raise DoesNotExist(message,
modelClass=modelClassOfObjectNotFound)
return createdExperiment
def ShutDownUploadThreads(self, event=None):
# pylint: disable=too-many-branches
if self.IsShuttingDown():
return
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
if not wx.GetApp().PerformingLookupsAndUploads():
EndBusyCursorIfRequired()
return
self.SetShuttingDown(True)
message = "Shutting down upload threads..."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if hasattr(event, "failed") and event.failed:
self.SetFailed()
self.uploadsModel.CancelRemaining()
elif hasattr(event, "completed") and event.completed:
self.SetCompleted()
else:
self.SetCanceled()
self.uploadsModel.CancelRemaining()
logger.debug("Shutting down FoldersController upload worker threads.")
for _ in range(self.numUploadWorkerThreads):
self.uploadsQueue.put(None)
if self.uploadMethod == UploadMethod.VIA_STAGING:
# SCP can leave orphaned SSH processes which need to be
# cleaned up.
CleanUpSshProcesses(self.settingsModel)
for thread in self.uploadWorkerThreads:
thread.join()
logger.debug("Shutting down FoldersController verification "
"worker threads.")
for _ in range(self.numVerificationWorkerThreads):
self.verificationsQueue.put(None)
for thread in self.verificationWorkerThreads:
thread.join()
self.verifyDatafileRunnable = {}
self.uploadDatafileRunnable = {}
if self.testRun:
numVerificationsCompleted = \
self.verificationsModel.GetCompletedCount()
numVerifiedUploads = \
self.verificationsModel.GetFoundVerifiedCount()
numFilesNotFoundOnServer = \
self.verificationsModel.GetNotFoundCount()
numFullSizeUnverifiedUploads = \
self.verificationsModel.GetFoundUnverifiedFullSizeCount()
numIncompleteUploads = \
self.verificationsModel.GetFoundUnverifiedNotFullSizeCount()
numFailedLookups = self.verificationsModel.GetFailedCount()
logger.testrun("")
logger.testrun("SUMMARY")
logger.testrun("")
logger.testrun("Files looked up on server: %s"
% numVerificationsCompleted)
logger.testrun("Files verified on server: %s" % numVerifiedUploads)
logger.testrun("Files not found on server: %s"
% numFilesNotFoundOnServer)
logger.testrun("Files unverified (but full size) on server: %s"
% numFullSizeUnverifiedUploads)
logger.testrun("Files unverified (and incomplete) on server: %s"
% numIncompleteUploads)
logger.testrun("Failed lookups: %s" % numFailedLookups)
logger.testrun("")
if self.Failed():
message = "Data scans and uploads failed."
elif self.Canceled():
message = "Data scans and uploads were canceled."
elif self.uploadsModel.GetFailedCount() > 0:
message = \
"Data scans and uploads completed with " \
"%d failed upload(s)." % self.uploadsModel.GetFailedCount()
elif self.Completed():
message = "Data scans and uploads completed successfully."
elapsedTime = self.uploadsModel.GetElapsedTime()
if elapsedTime and not self.testRun:
averageSpeed = "%3.1f MB/s" % \
(float(self.uploadsModel.GetCompletedSize()) / 1000000.0 \
/ elapsedTime.total_seconds())
message += " Average speed: %s" % averageSpeed
else:
message = "Data scans and uploads appear to have " \
"completed successfully."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if self.testRun:
logger.testrun(message)
app = wx.GetApp()
if hasattr(app, "toolbar"):
app.EnableTestAndUploadToolbarButtons()
app.SetShouldAbort(False)
if self.testRun:
app.testRunFrame.saveButton.Enable()
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
wx.GetApp().SetPerformingLookupsAndUploads(False)
self.SetShuttingDown(False)
if hasattr(app, "SetTestRunRunning"):
app.SetTestRunRunning(False)
EndBusyCursorIfRequired()
logger.debug("")
def InstrumentNameMismatch(event):
"""
Responds to instrument name mismatch in Settings dialog.
"""
if event.GetEventId() != EVT_INSTRUMENT_NAME_MISMATCH:
event.Skip()
return
message = "A previous instrument name of \"%s\" " \
"has been associated with this MyData instance.\n" \
"Please choose how you would like the new \"%s\" " \
"instrument name to be applied." \
% (event.oldInstrumentName, event.newInstrumentName)
renameChoice = "Rename the existing instrument record to " \
"\"%s\"." % event.newInstrumentName
discardChoice = "Discard the new instrument name and revert " \
"to \"%s\"." % event.oldInstrumentName
createChoice = "Use a separate instrument record for \"%s\", " \
"creating it if necessary." \
% event.newInstrumentName
dlg = wx.SingleChoiceDialog(event.settingsDialog, message,
"MyData - Instrument Name Changed",
[renameChoice, discardChoice,
createChoice], wx.CHOICEDLG_STYLE)
if dlg.ShowModal() == wx.ID_OK:
if dlg.GetStringSelection() == renameChoice:
logger.info("OK, we will rename the "
"existing instrument record.")
settingsDialogValidationEvent = \
MyDataEvent(EVT_SETTINGS_DIALOG_VALIDATION,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel)
renameInstrumentEvent = MyDataEvent(
EVT_RENAME_INSTRUMENT,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel,
facilityName=event.settingsDialog.GetFacilityName(),
oldInstrumentName=event.oldInstrumentName,
newInstrumentName=event.newInstrumentName,
nextEvent=settingsDialogValidationEvent)
wx.PostEvent(wx.GetApp().GetMainFrame(), renameInstrumentEvent)
return
elif dlg.GetStringSelection() == discardChoice:
logger.info("OK, we will discard the new instrument name.")
event.settingsDialog.SetInstrumentName(
event.settingsModel.GetInstrumentName())
event.settingsDialog.instrumentNameField.SetFocus()
event.settingsDialog.instrumentNameField.SelectAll()
elif dlg.GetStringSelection() == createChoice:
logger.info("OK, we will create a new instrument record.")
settingsDialogValidationEvent = \
MyDataEvent(EVT_SETTINGS_DIALOG_VALIDATION,
settingsDialog=event.settingsDialog,
settingsModel=event.settingsModel)
intervalSinceLastCheck = \
datetime.now() - \
wx.GetApp().GetLastConnectivityCheckTime()
checkInterval = \
event.settingsModel.GetConnectivityCheckInterval()
if intervalSinceLastCheck.total_seconds() >= checkInterval \
or not wx.GetApp()\
.GetLastConnectivityCheckSuccess():
checkConnectivityEvent = \
MyDataEvent(EVT_CHECK_CONNECTIVITY,
settingsModel=event.settingsModel,
nextEvent=settingsDialogValidationEvent)
wx.PostEvent(wx.GetApp().GetMainFrame(),
checkConnectivityEvent)
else:
wx.PostEvent(wx.GetApp().GetMainFrame(),
settingsDialogValidationEvent)
class FoldersController(object):
# pylint: disable=too-many-public-methods
# pylint: disable=too-many-instance-attributes
# pylint: disable=too-many-statements
"""
The main controller class for managing datafile verifications
and uploads from each of the folders in the Folders view.
"""
def __init__(self, notifyWindow, foldersModel, foldersView, usersModel,
verificationsModel, uploadsModel, settingsModel):
# pylint: disable=too-many-arguments
self.notifyWindow = notifyWindow
self.foldersModel = foldersModel
self.foldersView = foldersView
self.usersModel = usersModel
self.verificationsModel = verificationsModel
self.uploadsModel = uploadsModel
self.settingsModel = settingsModel
self.shuttingDown = threading.Event()
self.showingErrorDialog = threading.Event()
self.lastErrorMessage = None
self.showingWarningDialog = threading.Event()
self.canceled = threading.Event()
self.failed = threading.Event()
self.finishedCountingVerifications = dict()
self.finishedScanningForDatasetFolders = threading.Event()
self.verificationsQueue = None
self.threadingLock = threading.Lock()
self.uploadsThreadingLock = threading.Lock()
self.verifyDatafileRunnable = None
self.uploadsQueue = None
self.started = False
self.completed = False
self.uploadDatafileRunnable = None
self.numVerificationsToBePerformed = 0
self.numVerificationsToBePerformedLock = threading.Lock()
self.uploadsAcknowledged = 0
self.uploadMethod = UploadMethod.HTTP_POST
# These will get overwritten in InitForUploads, but we need
# to initialize them here, so that ShutDownUploadThreads()
# can be called.
self.numVerificationWorkerThreads = 0
self.verificationWorkerThreads = []
self.numUploadWorkerThreads = 0
self.uploadWorkerThreads = []
self.testRun = False
self.foldersView.Bind(wx.EVT_BUTTON, self.OnOpenFolder,
self.foldersView.GetOpenFolderButton())
self.foldersView.GetDataViewControl()\
.Bind(wx.dataview.EVT_DATAVIEW_ITEM_ACTIVATED, self.OnOpenFolder)
self.didntFindDatafileOnServerEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.EVT_DIDNT_FIND_FILE_ON_SERVER = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder, self.UploadDatafile)
self.unverifiedDatafileOnServerEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.EVT_INCOMPLETE_FILE_ON_STAGING = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder, self.UploadDatafile)
self.unverifiedNotFoundOnStagingEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.EVT_UNVERIFIED_NOT_FOUND_ON_STAGING = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder, self.UploadDatafile)
self.connectionStatusEvent, eventBinder = wx.lib.newevent.NewEvent()
self.notifyWindow.Bind(eventBinder, self.UpdateStatusBar)
self.showMessageDialogEvent, eventBinder = \
wx.lib.newevent.NewEvent()
self.notifyWindow.Bind(eventBinder, self.ShowMessageDialog)
self.shutdownUploadsEvent, eventBinder = wx.lib.newevent.NewEvent()
self.notifyWindow.Bind(eventBinder, self.ShutDownUploadThreads)
self.foundVerifiedDatafileEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_FOUND_VERIFIED_DATAFILE = wx.NewId() # pylint: disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.foundUnverifiedDatafileEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_FOUND_UNVERIFIED_BUT_FULL_SIZE_DATAFILE = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.foundUnverifiedNoDfosDatafileEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_FOUND_UNVERIFIED_NO_DFOS = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.uploadCompleteEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_UPLOAD_COMPLETE = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
self.uploadFailedEvent, eventBinder = \
wx.lib.newevent.NewCommandEvent()
self.EVT_UPLOAD_FAILED = wx.NewId() # pylint:disable=invalid-name
self.notifyWindow.Bind(eventBinder,
self.CountCompletedUploadsAndVerifications)
def Started(self):
return self.started
def SetStarted(self, started=True):
self.started = started
def Canceled(self):
return self.canceled.isSet()
def SetCanceled(self, canceled=True):
if canceled:
self.canceled.set()
else:
self.canceled.clear()
def Failed(self):
return self.failed.isSet()
def SetFailed(self, failed=True):
if failed:
self.failed.set()
else:
self.failed.clear()
def Completed(self):
return self.completed
def SetCompleted(self, completed=True):
self.completed = completed
def IsShuttingDown(self):
return self.shuttingDown.isSet()
def SetShuttingDown(self, shuttingDown=True):
if shuttingDown:
self.shuttingDown.set()
else:
self.shuttingDown.clear()
def IsShowingErrorDialog(self):
return self.showingErrorDialog.isSet()
def SetShowingErrorDialog(self, showingErrorDialog=True):
if showingErrorDialog:
self.showingErrorDialog.set()
else:
self.showingErrorDialog.clear()
def GetLastErrorMessage(self):
return self.lastErrorMessage
def SetLastErrorMessage(self, message):
self.threadingLock.acquire()
self.lastErrorMessage = message
self.threadingLock.release()
def UpdateStatusBar(self, event):
if event.connectionStatus == ConnectionStatus.CONNECTED:
self.notifyWindow.SetConnected(event.myTardisUrl, True)
else:
self.notifyWindow.SetConnected(event.myTardisUrl, False)
def ShowMessageDialog(self, event):
if self.IsShowingErrorDialog():
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we are already showing an error "
"dialog." % event.message)
return
elif event.message == self.GetLastErrorMessage():
# Sometimes multiple threads can encounter the same exception
# at around the same time. The first thread's exception leads
# to a modal error dialog, which blocks the events queue, so
# the next thread's (identical) show message dialog event doesn't
# get caught until after the first message dialog has been closed.
# In this case, the above check (to prevent two error dialogs
# from appearing at the same time) doesn't help.
logger.warning("Refusing to show message dialog for message "
"\"%s\" because we already showed an error "
"dialog with the same message." % event.message)
return
self.SetLastErrorMessage(event.message)
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(True)
dlg = wx.MessageDialog(None, event.message, event.title,
wx.OK | event.icon)
# pylint: disable=bare-except
try:
wx.EndBusyCursor()
needToRestartBusyCursor = True
except:
needToRestartBusyCursor = False
dlg.ShowModal()
if needToRestartBusyCursor and not self.IsShuttingDown() \
and wx.GetApp().PerformingLookupsAndUploads():
BeginBusyCursorIfRequired()
if event.icon == wx.ICON_ERROR:
self.SetShowingErrorDialog(False)
def UploadDatafile(self, event):
"""
Called in response to didntFindDatafileOnServerEvent or
unverifiedDatafileOnServerEvent.
This method runs in the main thread, so it shouldn't do anything
time-consuming or blocking, unless it launches another thread.
Because this method adds upload tasks to a queue, it is important
to note that if the queue has a maxsize set, then an attempt to
add something to the queue could block the GUI thread, making the
application appear unresponsive.
"""
folderModel = event.folderModel
dfi = event.dataFileIndex
existingUnverifiedDatafile = \
getattr(event, "existingUnverifiedDatafile", False)
if self.testRun:
if existingUnverifiedDatafile:
message = "NEEDS RE-UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
else:
message = "NEEDS UPLOADING: %s" \
% folderModel.GetDataFileRelPath(dfi)
self.uploadsAcknowledged += 1
logger.testrun(message)
self.CountCompletedUploadsAndVerifications(event=None)
return
if folderModel not in self.uploadDatafileRunnable:
self.uploadDatafileRunnable[folderModel] = {}
bytesUploadedPreviously = getattr(event, "bytesUploadedPreviously",
None)
verificationModel = getattr(event, "verificationModel", None)
self.uploadDatafileRunnable[folderModel][dfi] = \
UploadDatafileRunnable(self, self.foldersModel, folderModel,
dfi, self.uploadsModel,
self.settingsModel,
existingUnverifiedDatafile,
verificationModel,
bytesUploadedPreviously)
self.uploadsQueue.put(self.uploadDatafileRunnable[folderModel][dfi])
self.CountCompletedUploadsAndVerifications(event=None)
def InitForUploads(self):
fc = self # pylint: disable=invalid-name
app = wx.GetApp()
if hasattr(app, "TestRunRunning"):
fc.testRun = app.TestRunRunning()
else:
fc.testRun = False
fc.SetStarted()
settingsModel = fc.settingsModel
fc.SetCanceled(False)
fc.SetFailed(False)
fc.SetCompleted(False)
fc.verificationsModel.DeleteAllRows()
fc.uploadsModel.DeleteAllRows()
fc.verifyDatafileRunnable = {}
fc.verificationsQueue = Queue.Queue()
# For now, the max number of verification threads is hard-coded
# to 16:
fc.numVerificationWorkerThreads = 16
fc.verificationWorkerThreads = []
for i in range(fc.numVerificationWorkerThreads):
thread = threading.Thread(name="VerificationWorkerThread-%d" %
(i + 1),
target=fc.VerificationWorker)
fc.verificationWorkerThreads.append(thread)
thread.start()
fc.uploadDatafileRunnable = {}
fc.uploadsQueue = Queue.Queue()
fc.numUploadWorkerThreads = settingsModel.GetMaxUploadThreads()
fc.uploadMethod = UploadMethod.HTTP_POST
# pylint: disable=broad-except
try:
settingsModel.GetUploaderModel().RequestStagingAccess()
uploadToStagingRequest = settingsModel\
.GetUploadToStagingRequest()
except Exception, err:
# MyData app could be missing from MyTardis server.
logger.error(traceback.format_exc())
wx.PostEvent(
self.notifyWindow,
self.showMessageDialogEvent(title="MyData",
message=str(err),
icon=wx.ICON_ERROR))
return
message = None
if uploadToStagingRequest is None:
message = "Couldn't determine whether uploads to " \
"staging have been approved. " \
"Falling back to HTTP POST."
elif uploadToStagingRequest.IsApproved():
logger.info("Uploads to staging have been approved.")
fc.uploadMethod = UploadMethod.VIA_STAGING
else:
message = \
"Uploads to MyTardis's staging area require " \
"approval from your MyTardis administrator.\n\n" \
"A request has been sent, and you will be contacted " \
"once the request has been approved. Until then, " \
"MyData will upload files using HTTP POST, and will " \
"only upload one file at a time.\n\n" \
"HTTP POST is generally only suitable for small " \
"files (up to 100 MB each)."
if message:
logger.warning(message)
wx.PostEvent(
self.notifyWindow,
self.showMessageDialogEvent(title="MyData",
message=message,
icon=wx.ICON_WARNING))
fc.uploadMethod = UploadMethod.HTTP_POST
if fc.uploadMethod == UploadMethod.HTTP_POST and \
fc.numUploadWorkerThreads > 1:
logger.warning("Using HTTP POST, so setting "
"numUploadWorkerThreads to 1, "
"because urllib2 is not thread-safe.")
fc.numUploadWorkerThreads = 1
fc.uploadWorkerThreads = []
for i in range(fc.numUploadWorkerThreads):
thread = threading.Thread(name="UploadWorkerThread-%d" % (i + 1),
target=fc.UploadWorker,
args=())
fc.uploadWorkerThreads.append(thread)
thread.start()
# pylint: disable=bare-except
fc.finishedScanningForDatasetFolders = threading.Event()
fc.numVerificationsToBePerformed = 0
fc.finishedCountingVerifications = dict()
def ShutDownUploadThreads(self, event=None):
# pylint: disable=too-many-branches
if self.IsShuttingDown():
return
self.SetShuttingDown(True)
message = "Shutting down upload threads..."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if hasattr(event, "failed") and event.failed:
self.SetFailed()
self.uploadsModel.CancelRemaining()
elif hasattr(event, "completed") and event.completed:
self.SetCompleted()
else:
self.SetCanceled()
logger.debug("Shutting down FoldersController upload worker threads.")
for _ in range(self.numUploadWorkerThreads):
self.uploadsQueue.put(None)
for thread in self.uploadWorkerThreads:
thread.join()
logger.debug("Shutting down FoldersController verification "
"worker threads.")
for _ in range(self.numVerificationWorkerThreads):
self.verificationsQueue.put(None)
for thread in self.verificationWorkerThreads:
thread.join()
self.verifyDatafileRunnable = {}
self.uploadDatafileRunnable = {}
if sys.platform == 'darwin':
sshControlMasterPool = \
OPENSSH.GetSshControlMasterPool(createIfMissing=False)
if sshControlMasterPool:
sshControlMasterPool.ShutDown()
if self.Failed():
message = "Data scans and uploads failed."
elif self.Canceled():
message = "Data scans and uploads were canceled."
elif self.uploadsModel.GetFailedCount() > 0:
message = \
"Data scans and uploads completed with " \
"%d failed upload(s)." % self.uploadsModel.GetFailedCount()
elif self.Completed():
message = "Data scans and uploads completed successfully."
else:
message = "Data scans and uploads appear to have " \
"completed successfully."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
app = wx.GetApp()
if hasattr(app, "toolbar"):
app.toolbar.EnableTool(app.stopTool.GetId(), False)
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
wx.GetApp().SetPerformingLookupsAndUploads(False)
self.SetShuttingDown(False)
# pylint: disable=bare-except
try:
wx.EndBusyCursor()
except:
pass
logger.debug("")
def ShutDownUploadThreads(self, event=None):
# pylint: disable=too-many-branches
if self.IsShuttingDown():
return
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
if not wx.GetApp().PerformingLookupsAndUploads():
EndBusyCursorIfRequired()
return
self.SetShuttingDown(True)
message = "Shutting down upload threads..."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if hasattr(event, "failed") and event.failed:
self.SetFailed()
self.uploadsModel.CancelRemaining()
elif hasattr(event, "completed") and event.completed:
self.SetCompleted()
else:
self.SetCanceled()
self.uploadsModel.CancelRemaining()
logger.debug("Shutting down FoldersController upload worker threads.")
for _ in range(self.numUploadWorkerThreads):
self.uploadsQueue.put(None)
for thread in self.uploadWorkerThreads:
thread.join()
logger.debug("Shutting down FoldersController verification "
"worker threads.")
for _ in range(self.numVerificationWorkerThreads):
self.verificationsQueue.put(None)
for thread in self.verificationWorkerThreads:
thread.join()
self.verifyDatafileRunnable = {}
self.uploadDatafileRunnable = {}
if sys.platform == 'darwin':
sshControlMasterPool = \
OPENSSH.GetSshControlMasterPool(createIfMissing=False)
if sshControlMasterPool:
sshControlMasterPool.ShutDown()
if self.testRun:
numVerificationsCompleted = \
self.verificationsModel.GetCompletedCount()
numVerifiedUploads = \
self.verificationsModel.GetFoundVerifiedCount()
numFilesNotFoundOnServer = \
self.verificationsModel.GetNotFoundCount()
numFullSizeUnverifiedUploads = \
self.verificationsModel.GetFoundUnverifiedFullSizeCount()
numIncompleteUploads = \
self.verificationsModel.GetFoundUnverifiedNotFullSizeCount()
numFailedLookups = self.verificationsModel.GetFailedCount()
logger.testrun("")
logger.testrun("SUMMARY")
logger.testrun("")
logger.testrun("Files looked up on server: %s" %
numVerificationsCompleted)
logger.testrun("Files verified on server: %s" % numVerifiedUploads)
logger.testrun("Files not found on server: %s" %
numFilesNotFoundOnServer)
logger.testrun("Files unverified (but full size) on server: %s" %
numFullSizeUnverifiedUploads)
logger.testrun("Files unverified (and incomplete) on server: %s" %
numIncompleteUploads)
logger.testrun("Failed lookups: %s" % numFailedLookups)
logger.testrun("")
if self.Failed():
message = "Data scans and uploads failed."
elif self.Canceled():
message = "Data scans and uploads were canceled."
elif self.uploadsModel.GetFailedCount() > 0:
message = \
"Data scans and uploads completed with " \
"%d failed upload(s)." % self.uploadsModel.GetFailedCount()
elif self.Completed():
message = "Data scans and uploads completed successfully."
else:
message = "Data scans and uploads appear to have " \
"completed successfully."
logger.info(message)
if hasattr(wx.GetApp(), "GetMainFrame"):
wx.GetApp().GetMainFrame().SetStatusMessage(message)
if self.testRun:
logger.testrun(message)
app = wx.GetApp()
if hasattr(app, "toolbar"):
app.EnableTestAndUploadToolbarButtons()
app.SetShouldAbort(False)
if self.testRun:
app.testRunFrame.saveButton.Enable()
if hasattr(wx.GetApp(), "SetPerformingLookupsAndUploads"):
wx.GetApp().SetPerformingLookupsAndUploads(False)
self.SetShuttingDown(False)
if hasattr(app, "SetTestRunRunning"):
app.SetTestRunRunning(False)
EndBusyCursorIfRequired()
logger.debug("")
def JobFunc(taskModel, tasksModel, row, col):
# pylint: disable=too-many-statements
def TaskJobFunc():
assert callable(taskModel.GetJobFunc())
title = "Starting"
message = taskModel.GetJobDesc()
Notification.Notify(message, title=title)
taskModel.GetJobFunc()(*taskModel.GetJobArgs())
taskModel.SetFinishTime(datetime.now())
title = "Finished"
message = taskModel.GetJobDesc()
Notification.Notify(message, title=title)
wx.CallAfter(tasksModel.TryRowValueChanged, row, col)
scheduleType = taskModel.GetScheduleType()
if scheduleType == "Timer":
intervalMinutes = taskModel.GetIntervalMinutes()
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(minutes=intervalMinutes)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Timer",
intervalMinutes=intervalMinutes)
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = "{d:%A} {d.day}/{d.month}/{d.year}"\
.format(d=newStartTime)
wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring every %d minutes)"
% (taskModel.GetJobDesc(),
timeString, dateString, intervalMinutes))
tasksModel.AddRow(newTaskModel)
elif scheduleType == "Daily":
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(days=1)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Daily")
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = "{d:%A} {d.day}/{d.month}/{d.year}"\
.format(d=newStartTime)
wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring daily)"
% (taskModel.GetJobDesc(),
timeString, dateString))
tasksModel.AddRow(newTaskModel)
elif scheduleType == "Weekly":
newTaskDataViewId = tasksModel.GetMaxDataViewId() + 1
newStartTime = taskModel.GetStartTime() + \
timedelta(days=1)
days = taskModel.GetDays()
while not days[newStartTime.weekday()]:
newStartTime = newStartTime + timedelta(days=1)
newTaskModel = TaskModel(newTaskDataViewId,
taskModel.GetJobFunc(),
taskModel.GetJobArgs(),
taskModel.GetJobDesc(),
newStartTime,
scheduleType="Weekly",
days=days)
timeString = newStartTime.strftime("%I:%M:%S %p")
dateString = "{d:%A} {d.day}/{d.month}/{d.year}"\
.format(d=newStartTime)
wx.CallAfter(wx.GetApp().frame.SetStatusMessage,
"The \"%s\" task is scheduled "
"to run at %s on %s "
"(recurring on specified days)"
% (taskModel.GetJobDesc(),
timeString, dateString))
tasksModel.AddRow(newTaskModel)
app = wx.GetApp()
if not app.ShouldAbort():
thread = threading.Thread(target=TaskJobFunc)
logger.debug("Starting task %s" % taskModel.GetJobDesc())
thread.start()
else:
logger.info("Not starting task because we are aborting.")
app.EnableTestAndUploadToolbarButtons()
EndBusyCursorIfRequired()
app.SetShouldAbort(False)
message = "Data scans and uploads were canceled."
wx.GetApp().GetMainFrame().SetStatusMessage(message)
return
