def SetLabel(self, label, do_cmd=False):
self.index = self.labels.index(label)
QuiskCheckbutton.SetLabel(self, label)
QuiskCheckbutton.SetValue(self, self.index)
if do_cmd and self.command:
event = wx.PyEvent()
event.SetEventObject(self)
self.command(event)
def ChangeSlider(self, slider_value):
self.second_data[2] = slider_value
command = self.second_data[1]
if command:
event = wx.PyEvent()
self.first_button.slider_value = slider_value
event.SetEventObject(self.first_button)
command(event)
def SetLabel(self, label, do_cmd=False):
self.first_button.SetLabel(label)
self.AddSecondButton(label)
self.RbDialog.RbGroup.SetLabel(label, False)
if do_cmd and self.pop_command:
event = wx.PyEvent()
event.SetEventObject(self.first_button)
self.pop_command(event)
def test_mydata_test_run(self):
"""
Test ability to use MyData's Test Run.
"""
ValidateSettings()
self.mydataApp = MyData(argv=['MyData', '--loglevel', 'DEBUG'])
popupMenu = self.mydataApp.frame.taskBarIcon.CreatePopupMenu()
# Just for test coverage:
LogStartScansAndUploadsCaller(event=None, jobId=1)
LogStartScansAndUploadsCaller(event=None, jobId=None)
pyEvent = wx.PyEvent()
jobId = None
toolbar = self.mydataApp.frame.toolbar
pyEvent.SetId(toolbar.settingsTool.GetId())
LogStartScansAndUploadsCaller(pyEvent, jobId)
pyEvent.SetId(toolbar.uploadTool.GetId())
LogStartScansAndUploadsCaller(pyEvent, jobId)
# Requires popupMenu (defined above):
pyEvent.SetId(
self.mydataApp.frame.taskBarIcon.GetSyncNowMenuItem().GetId())
LogStartScansAndUploadsCaller(pyEvent, jobId)
mydataEvent = MYDATA_EVENTS.ValidateSettingsForRefreshEvent()
LogStartScansAndUploadsCaller(mydataEvent, jobId)
mydataEvent = MYDATA_EVENTS.SettingsValidationCompleteEvent()
LogStartScansAndUploadsCaller(mydataEvent, jobId)
mydataEvent = MYDATA_EVENTS.ShutdownForRefreshCompleteEvent()
LogStartScansAndUploadsCaller(mydataEvent, jobId)
mydataEvent = MYDATA_EVENTS.SettingsValidationCompleteEvent()
LogStartScansAndUploadsCaller(mydataEvent, jobId)
pyEvent = wx.PyEvent()
pyEvent.SetEventType(12345)
LogStartScansAndUploadsCaller(pyEvent, jobId)
popupMenu.Destroy()
# Test opening webpages using fake MyTardis URL.
pyEvent = wx.PyEvent()
self.mydataApp.frame.OnMyTardis(pyEvent)
self.mydataApp.frame.OnAbout(pyEvent)
# When running MyData without an event loop, this will block until complete:
OnTestRunFromToolbar(event=wx.PyEvent())
def OnLCSMRightDown(self, event):
selectBeforePopup(event)
event.Skip()
menu = self.getPopupMenu()
if menu:
evt = wx.PyEvent()
evt.SetEventType(wxEVT_DOPOPUPMENU)
evt.menu = menu
evt.pos = event.GetPosition()
wx.PostEvent(self, evt)
def OnGroupButton(self, event):
btn = event.GetEventObject()
label = btn.GetLabel()
self.first_button.SetLabel(label)
self.first_button.Refresh()
self.RbDialog.Hide()
self.AddSecondButton(label)
if self.pop_command:
event = wx.PyEvent()
event.SetEventObject(self.first_button)
self.pop_command(event)
def SetValue(self, value, do_cmd=False):
wx.lib.buttons.GenToggleButton.SetValue(self, value)
self.button_down = value
if value:
self.SetBackgroundColour(self.color)
else:
self.SetBackgroundColour(conf.color_btn)
if do_cmd and self.command:
event = wx.PyEvent()
event.SetEventObject(self)
self.command(event)
def _run_evaluators_later(evaluator=None): if evaluator is not None: self._post_any_code_evaluators += [evaluator] try: de = wx.PyEvent() de.SetEventType(wxEVT_AnyCode) wx.PostEvent(self.GetWin(), de) except TypeError: pass except AttributeError: # FIXME print "Cannot post message"
def tearDown(self):
"""
Destroy the app
"""
if self.app:
self.app.frame.Hide()
self.app.OnCloseFrame(wx.PyEvent())
self.app.frame.Destroy()
del os.environ['PYUPDATER_FILESERVER_DIR']
del os.environ['WXUPDATEDEMO_TESTING']
shutil.rmtree(self.fileServerDir)
def SetLabel(self, label, do_cmd=False, direction=None):
direc = self.first_button.direction
if direction is not None:
self.first_button.direction = direction
self.first_button.SetLabel(label)
self.AddSecondButton(label)
self.RbDialog.RbGroup.SetLabel(label, False)
if do_cmd and self.pop_command:
event = wx.PyEvent()
event.SetEventObject(self.first_button)
self.pop_command(event)
self.first_button.direction = direc
def test_PyEvent(self):
id = wx.NewId()
typ = wx.NewEventType()
evt = wx.PyEvent(id, typ)
evt.newAttr = "hello"
evt2 = evt.Clone()
self.assertTrue(type(evt2) == wx.PyEvent)
self.assertTrue(evt is not evt2)
self.assertTrue(getattr(evt2, 'newAttr'))
self.assertTrue(evt.newAttr == evt2.newAttr)
self.assertTrue(evt.Id == evt2.Id)
self.assertTrue(evt.EventType == evt2.EventType)
def test_PyEvtCloneRefCounts(self):
# Since we're doing some funky stuff under the covers with Clone, make
# sure that the reference counts on everything (before and after)
# still make sense
evt1 = wx.PyEvent()
rc1 = sys.getrefcount(evt1)
evt1.attr = 'Howdy!'
evt2 = evt1.Clone()
rc2 = sys.getrefcount(evt2)
rc3 = sys.getrefcount(evt1)
self.assertTrue(rc1 == rc2 == rc3)
self.assertTrue(evt1.attr == evt2.attr)
def ChangeSlider(self, value):
if not self.dual:
self.button.slider_value = value
elif self.button.GetValue():
self.button.slider_value_on = value
else:
self.button.slider_value_off = value
if self.wintype == 'filter':
self.button.SetLabel(str(value))
self.button.Refresh()
if self.command:
event = wx.PyEvent()
event.SetEventObject(self.button)
self.command(event)
def run(self):
'''Run the thread. Internal function.'''
interval_in_seconds = self.parent.interval / 1000.0
def sleep():
time.sleep(interval_in_seconds)
sleep()
try:
while self.parent.alive is True and self.retired is False:
event = wx.PyEvent(self.parent.wx_id)
event.SetEventType(wxEVT_THREAD_TIMER)
wx.PostEvent(self.parent.parent, event)
sleep()
except:
return # Just so it wouldn't raise an error when `wx` is shutting
def OnLCSMRightDown(self, event):
selectBeforePopup(event)
menu = self.getPopupMenu()
#event.Skip()
if menu:
# XXX
self.PopupMenu(menu, event.GetPosition())
self.afterPopupMenu(menu)
return
evt = wx.PyEvent()
evt.SetEventType(self.wxEVT_DOPOPUPMENU)
evt.menu = menu
evt.pos = event.GetPosition()
wx.PostEvent(self, evt)
def Shortcut(self, event):
# Multiple buttons can have the same shortcut, so move to the next one.
index = self.last_shortcut + 1
length = len(self.shortcuts)
if index >= length:
index = 0
for i in range(length):
shortcut = self.shortcuts[index]
if shortcut and wx.GetKeyState(ord(shortcut)):
break
index += 1
if index >= length:
index = 0
else:
return
self.last_shortcut = index
button = self.buttons[index]
event = wx.PyEvent()
event.SetEventObject(button)
button.OnButton(event)
def test_on_settings_saved_schedule(self):
"""
Test On Settings Saved schedule type.
"""
ValidateSettings()
self.mydataApp = MyData(argv=['MyData', '--loglevel', 'DEBUG'])
self.mydataApp.frame.taskBarIcon.CreatePopupMenu()
pyEvent = wx.PyEvent()
SETTINGS.lastSettingsUpdateTrigger = \
LastSettingsUpdateTrigger.UI_RESPONSE
self.mydataApp.scheduleController.ApplySchedule(pyEvent)
SETTINGS.lastSettingsUpdateTrigger = \
LastSettingsUpdateTrigger.READ_FROM_DISK
# testdataUsernameDataset_POST.cfg has upload_invalid_user_folders = True,
# so INVALID_USER/InvalidUserDataset1/InvalidUserFile1.txt is included
# in the uploads completed count:
uploadsModel = DATAVIEW_MODELS['uploads']
self.assertEqual(uploadsModel.GetCompletedCount(), 8)
self.assertIn(("CreateOnSettingsSavedTask - MainThread - DEBUG - "
"Schedule type is On Settings Saved"),
logger.loggerOutput.getvalue())
def OnInit(self):
self.running = True
if profile:
try:
os.unlink(prof_file_name)
except:
pass
self.prof = Profiler()
self.prof.enable()
wx.the_app = self
self._DoIterationId = wx.NewEventType()
self.Connect(-1, -1, self._DoIterationId, self._doIteration)
self.evt = wx.PyEvent()
self.evt.SetEventType(self._DoIterationId)
self.event_queue = []
# this breaks TreeListCtrl, and I'm too lazy to figure out why
#wx.IdleEvent_SetMode(wx.IDLE_PROCESS_SPECIFIED)
# this fixes 24bit-color toolbar buttons
wx.SystemOptions_SetOptionInt("msw.remap", 0)
icon_path = os.path.join(image_root, 'bittorrent.ico')
self.icon = wx.Icon(icon_path, wx.BITMAP_TYPE_ICO)
return True
def SetLabel(self, label, do_cmd=False):
self.button = None
for b in self.buttons:
if self.button is not None:
b.SetValue(False)
elif isinstance(b, QuiskCycleCheckbutton):
try:
index = b.labels.index(label)
except ValueError:
b.SetValue(False)
continue
else:
b.SetIndex(index)
self.button = b
b.SetValue(True)
elif b.GetLabel() == label:
b.SetValue(True)
self.button = b
else:
b.SetValue(False)
if do_cmd and self.command and self.button:
event = wx.PyEvent()
event.SetEventObject(self.button)
self.command(event)
def _post_close_event(self):
'''Post a close event to the frame.'''
event = wx.PyEvent(self.Id)
event.SetEventType(wx.wxEVT_CLOSE_WINDOW)
wx.PostEvent(self, event)
def test_scan_folders(self):
"""
Test scanning the Username / Dataset structure and upload using POST.
"""
# pylint: disable=too-many-locals
self.UpdateSettingsFromCfg("testdataUsernameDataset_POST",
dataFolderName="testdataUsernameDataset")
ValidateSettings()
InitializeModels()
self.assertTrue(SETTINGS.advanced.uploadInvalidUserOrGroupFolders)
foldersModel = DATAVIEW_MODELS['folders']
foldersModel.ScanFolders(MyDataScanFoldersTester.ProgressCallback)
usersModel = DATAVIEW_MODELS['users']
self.assertEqual(sorted(usersModel.GetValuesForColname("Username")),
['INVALID_USER', 'testuser1', 'testuser2'])
folders = []
for row in range(foldersModel.GetRowCount()):
folders.append(foldersModel.GetFolderRecord(row).folderName)
self.assertEqual(
sorted(folders),
['Birds', 'Dataset with spaces', 'Flowers', 'InvalidUserDataset1'])
numFiles = 0
for row in range(foldersModel.GetRowCount()):
numFiles += foldersModel.GetFolderRecord(row).numFiles
self.assertEqual(numFiles, 12)
uploadsModel = UploadsModel()
verificationsModel = VerificationsModel()
DATAVIEW_MODELS['verifications'] = verificationsModel
DATAVIEW_MODELS['uploads'] = uploadsModel
self.app.foldersController = FoldersController(self.app.frame)
self.app.foldersController.InitForUploads()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
self.app.foldersController.StartUploadsForFolder(folderModel)
self.app.foldersController.FinishedScanningForDatasetFolders()
numVerificationsCompleted = verificationsModel.GetCompletedCount()
uploadsCompleted = uploadsModel.GetCompletedCount()
uploadsFailed = uploadsModel.GetFailedCount()
uploadsProcessed = uploadsCompleted + uploadsFailed
self.assertEqual(numVerificationsCompleted, numFiles)
sys.stderr.write("\n")
sys.stderr.write("%d/%d uploads completed.\n" %
(uploadsCompleted, uploadsProcessed))
if uploadsFailed > 0:
sys.stderr.write("%d/%d uploads failed.\n" %
(uploadsFailed, uploadsProcessed))
self.assertEqual(uploadsCompleted, 8)
# Now let's test canceling the uploads:
loggerOutput = logger.GetValue()
def StartUploads():
"""
Start Uploads worker
"""
self.app.foldersController.InitForUploads()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
self.app.foldersController.StartUploadsForFolder(folderModel)
startUploadsThread = threading.Thread(target=StartUploads,
name="StartUploads")
# Do this synchronously to ensure that the completed flag is reset:
self.app.foldersController.InitializeStatusFlags()
startUploadsThread.start()
sys.stderr.write("Waiting for uploads to start...\n")
while uploadsModel.GetCount() == 0:
time.sleep(0.05)
while uploadsModel.rowsData[0].status == UploadStatus.NOT_STARTED:
time.sleep(0.05)
sys.stderr.write("\nCanceling uploads...\n")
FLAGS.shouldAbort = True
self.app.foldersController.ShutDownUploadThreads(event=wx.PyEvent())
startUploadsThread.join()
FLAGS.shouldAbort = False
newLogs = Subtract(logger.GetValue(), loggerOutput)
self.assertIn("Data scans and uploads were canceled.", newLogs)
# Simulate ConnectionError while trying to access MyTardis URL:
sys.stderr.write(
"\nAsking fake MyTardis server to shut down abruptly...\n")
loggerOutput = logger.GetValue()
SETTINGS.general.myTardisUrl = \
"%s/request/connectionerror/" % self.fakeMyTardisUrl
event = wx.PyEvent()
event.folderModel = foldersModel.GetFolderRecord(0)
event.dataFileIndex = 0
self.app.foldersController.UploadDatafile(event)
newLogs = Subtract(logger.GetValue(), loggerOutput)
# We should see some sort of connection error in the log, but we don't
# know which one it will be.
# Errno 10053 is a Winsock error: "Software caused connection abort"
self.assertTrue("urlopen error [Errno 32] Broken pipe" in newLogs
or "[Errno 54] Connection reset by peer" in newLogs
or "BadStatusLine" in newLogs
or "urlopen error [Errno 10053]" in newLogs)
def _clear_idle_block_and_do(self):
self.idle_block = False
event = wx.PyEvent()
event.SetEventType(wx.wxEVT_IDLE)
wx.PostEvent(self, event)
def SendCommand(self, command):
if command:
event = wx.PyEvent()
event.SetEventObject(self)
command(event)
def test_scan_folders(self):
"""
Test scanning the Username / Dataset structure and uploading with SCP.
"""
# pylint: disable=too-many-statements
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
self.UpdateSettingsFromCfg("testdataUsernameDataset")
self.assertEqual(SETTINGS.miscellaneous.uuid, "1234567890")
self.assertEqual(SETTINGS.general.instrumentName, "Test Instrument")
ValidateSettings()
# Reset global settings' uploader model, so we when we next call
# the SETTINGS.uploaderModel property method, we'll generate a
# new UploaderModel instance, using the up-to-date
# SETTINGS.general.instrumentName:
SETTINGS.uploaderModel = None
SETTINGS.uploaderModel.UploadUploaderInfo()
self.assertEqual(SETTINGS.uploaderModel.name, "Test Instrument")
SETTINGS.uploaderModel.sshKeyPair = self.keyPair
InitializeModels()
foldersModel = DATAVIEW_MODELS['folders']
foldersModel.ScanFolders(MyDataScanFoldersTester.ProgressCallback)
# testdataUsernameDataset.cfg has upload_invalid_user_folders = False,
# so the "INVALID_USER" folder is not included:
usersModel = DATAVIEW_MODELS['users']
self.assertEqual(
sorted(usersModel.GetValuesForColname("Username")),
["testuser1", "testuser2"])
folders = []
for row in range(foldersModel.GetRowCount()):
folders.append(foldersModel.GetFolderRecord(row).folderName)
self.assertEqual(sorted(folders),
["Birds", "Dataset with spaces", "Flowers"])
numFiles = 0
for row in range(foldersModel.GetRowCount()):
numFiles += foldersModel.GetFolderRecord(row).numFiles
self.assertEqual(numFiles, 11)
numExistingVerifiedFiles = 1
numUnverifiedFullSizeFiles = 1
numTriggeringMissingApiEndpoint = 1
DATAVIEW_MODELS['verifications'] = VerificationsModel()
verificationsModel = DATAVIEW_MODELS['verifications']
DATAVIEW_MODELS['uploads'] = UploadsModel()
uploadsModel = DATAVIEW_MODELS['uploads']
foldersController = FoldersController(self.app.frame)
# This helps with PostEvent's logging in mydata/events/__init__.py:
self.app.foldersController = foldersController
username = "******"
privateKeyFilePath = self.keyPair.privateKeyFilePath
host = "127.0.0.1"
sys.stderr.write("Waiting for fake SSH server to start up...\n")
attempts = 0
while not OpenSSH.SshServerIsReady(username, privateKeyFilePath,
host, self.scpPort):
attempts += 1
if attempts > 10:
raise Exception(
"Couldn't connect to SSH server at 127.0.0.1:%s"
% self.scpPort)
time.sleep(0.25)
foldersController.InitForUploads()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
foldersController.StartUploadsForFolder(folderModel)
foldersController.FinishedScanningForDatasetFolders()
while True:
numVerificationsCompleted = verificationsModel.GetCompletedCount()
uploadsToBePerformed = uploadsModel.GetRowCount()
uploadsCompleted = uploadsModel.GetCompletedCount()
uploadsFailed = uploadsModel.GetFailedCount()
uploadsProcessed = uploadsCompleted + uploadsFailed
finishedVerificationCounting = True
for folder in foldersController.finishedCountingVerifications:
if not foldersController.finishedCountingVerifications[folder]:
finishedVerificationCounting = False
if numVerificationsCompleted == numFiles \
and finishedVerificationCounting \
and uploadsProcessed == uploadsToBePerformed:
break
time.sleep(0.1)
self.assertEqual(numVerificationsCompleted, numFiles)
sys.stderr.write("Waiting for uploads to complete...\n")
while True:
uploadsCompleted = uploadsModel.GetCompletedCount()
uploadsFailed = uploadsModel.GetFailedCount()
uploadsProcessed = uploadsCompleted + uploadsFailed
if uploadsProcessed == uploadsToBePerformed:
break
if uploadsProcessed > uploadsToBePerformed:
raise Exception("Processed %s/%s uploads!"
% (uploadsProcessed, uploadsToBePerformed))
time.sleep(0.1)
foldersController.ShutDownUploadThreads()
sys.stderr.write("\n")
sys.stderr.write("%d/%d uploads completed.\n" % (uploadsCompleted,
uploadsProcessed))
if uploadsFailed > 0:
sys.stderr.write("%d/%d uploads failed.\n" % (uploadsFailed,
uploadsProcessed))
for i in range(uploadsProcessed):
uploadModel = uploadsModel.rowsData[i]
if uploadModel.status == UploadStatus.FAILED:
sys.stderr.write(
"Upload failed for %s: %s\n" %
(uploadModel.filename, uploadModel.message.strip()))
if uploadModel.traceback:
sys.stderr.write(uploadModel.traceback)
sys.stderr.write("\n")
self.assertEqual(uploadsCompleted,
numFiles - numExistingVerifiedFiles -
numUnverifiedFullSizeFiles -
numTriggeringMissingApiEndpoint)
sys.stderr.write("Testing canceling uploads...\n")
loggerOutput = logger.GetValue()
def StartUploads():
"""
Start Uploads worker
"""
foldersController.InitForUploads()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
foldersController.StartUploadsForFolder(folderModel)
startUploadsThread = threading.Thread(
target=StartUploads, name="StartUploads")
# Do this synchronously to ensure that the completed flag is reset:
foldersController.InitializeStatusFlags()
startUploadsThread.start()
sys.stderr.write("Waiting for uploads to start...\n")
# We don't need to have an active SCP process during the Cancel for
# the test to pass, but if we do, that will improve test coverage,
# because UploadModel.Cancel() will need to terminate the SCP process.
# If we wait too long to request canceling, the cancel request might
# not be processed before the uploads are completed, hence the short
# sleep times below. Ideally all races conditions should be removed
# from the "Cancel" test, but a slightly "racy" test is better then
# not having any test!
while uploadsModel.GetCount() == 0:
time.sleep(0.01)
while uploadsModel.rowsData[0].status == UploadStatus.NOT_STARTED:
time.sleep(0.01)
sys.stderr.write("Canceling uploads...\n")
FLAGS.shouldAbort = True
foldersController.ShutDownUploadThreads(event=wx.PyEvent())
startUploadsThread.join()
FLAGS.shouldAbort = False
newLogs = Subtract(logger.GetValue(), loggerOutput)
self.assertIn("Data scans and uploads were canceled.", newLogs)
sys.stderr.write(
"\nTesting with missing 'scp_username' storage box attribute...\n")
# This UUID tells our fake MyTardis server to simulate a
# missing storage box attribute:
SETTINGS.miscellaneous.uuid = "1234567891"
loggerOutput = logger.GetValue()
foldersController.InitForUploads()
self.assertEqual(uploadsModel.GetCompletedCount(), 0)
self.assertTrue(foldersController.failed)
newLogs = Subtract(logger.GetValue(), loggerOutput)
self.assertIn(
"Key 'scp_username' not found in attributes for storage box",
newLogs)
self.assertEqual(uploadsModel.GetCompletedCount(), 0)
SETTINGS.miscellaneous.uuid = "1234567890"
sys.stderr.write(
"\nTesting handling of invalid path to SCP binary...\n")
OpenSSH.OPENSSH.scp += "_INVALID"
loggerOutput = logger.GetValue()
foldersController.InitForUploads()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
foldersController.StartUploadsForFolder(folderModel)
foldersController.FinishedScanningForDatasetFolders()
OpenSSH.OPENSSH.scp = OpenSSH.OPENSSH.scp.rstrip("_INVALID")
newLogs = Subtract(logger.GetValue(), loggerOutput)
if sys.platform.startswith("win"):
self.assertRegexpMatches(
newLogs, (".*The system cannot find the file specified.*"))
else:
self.assertRegexpMatches(
newLogs, (".*No such file or directory.*"))
sys.stderr.write(
"\nTesting handling of invalid path to SSH binary...\n")
# SSH would normally be called to run "mkdir -p" on the staging server,
# but it won't be if that has already been done in the current session
# for the given directory, due to the OpenSSH.REMOTE_DIRS_CREATED
# cache. We'll clear the cache here to force the remote mkdir (via ssh)
# command to run.
OpenSSH.REMOTE_DIRS_CREATED = dict()
OpenSSH.OPENSSH.ssh += "_INVALID"
loggerOutput = logger.GetValue()
foldersController.InitForUploads()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
foldersController.StartUploadsForFolder(folderModel)
foldersController.FinishedScanningForDatasetFolders()
OpenSSH.OPENSSH.ssh = OpenSSH.OPENSSH.ssh.rstrip("_INVALID")
newLogs = Subtract(logger.GetValue(), loggerOutput)
if sys.platform.startswith("win"):
self.assertRegexpMatches(
newLogs, (".*The system cannot find the file specified.*"))
else:
self.assertRegexpMatches(
newLogs, (".*No such file or directory.*"))
sys.stderr.write(
"\nTesting attempted uploads with invalid file paths...\n")
foldersController.InitForUploads()
loggerOutput = logger.GetValue()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
for dataFileIndex in range(folderModel.numFiles):
folderModel.dataFilePaths['files'][dataFileIndex] += "_INVALID"
foldersController.StartUploadsForFolder(folderModel)
foldersController.FinishedScanningForDatasetFolders()
newLogs = Subtract(logger.GetValue(), loggerOutput)
self.assertRegexpMatches(
newLogs, (".*Not uploading .+, because it has been "
"moved, renamed or deleted.*"))
self.assertEqual(uploadsModel.GetCompletedCount(), 0)
# Now let's try to upload without a functioning SCP server:
sys.stderr.write(
"\nTesting attempted uploads while SCP server is offline...\n")
self.sshd.shutdown()
self.fakeSshServerThread.join()
self.fakeSshServerStopped = True
defaultTimeout = SETTINGS.miscellaneous.connectionTimeout
SETTINGS.miscellaneous.connectionTimeout = 1
sys.stderr.write(
"\tSSH ConnectionTimeout: %s second(s)\n"
% SETTINGS.miscellaneous.connectionTimeout)
sys.stderr.write(
"\tMax upload retries: %s\n" % SETTINGS.advanced.maxUploadRetries)
sys.stderr.write("\tNumber of uploads: %s\n\n" % uploadsProcessed)
foldersController.InitForUploads()
loggerOutput = logger.GetValue()
for row in range(foldersModel.GetRowCount()):
folderModel = foldersModel.GetFolderRecord(row)
for dataFileIndex in range(folderModel.numFiles):
folderModel.dataFilePaths['files'][dataFileIndex] = \
folderModel.dataFilePaths['files'][dataFileIndex] \
.rstrip("_INVALID")
foldersController.StartUploadsForFolder(folderModel)
foldersController.FinishedScanningForDatasetFolders()
newLogs = Subtract(logger.GetValue(), loggerOutput)
# Instead of allowing multiple SCP failures and asserting the expected
# number of failures here, MyData now raises a critical exception
# when one SCP upload fails (after SETTINGS.advanced.maxUploadRetries
# has been exceeded for that upload), because if we allow MyData to
# continue creating DataFile records via the MyTardis API while SCP
# uploads are failing, it could create a large number of empty DataFile
# records very quickly in the case of a "Disk quota exceeded" error.
self.assertTrue(foldersController.failed)
self.assertRegexpMatches(
newLogs,
".*ssh: connect to host localhost port %s: Connection refused.*"
"|"
".*ssh: connect to host localhost port %s: Operation timed out.*"
"|"
".*Connection timed out during banner exchange.*"
% (self.scpPort, self.scpPort))
self.assertEqual(uploadsModel.GetCompletedCount(), 0)
SETTINGS.miscellaneous.connectionTimeout = defaultTimeout
