def loadState(self):
if os.path.exists(self.treeFilePath) :
Logger.log("Reading state from %s" % self.treeFilePath)
return self.serializer.unserialize(self.treeFilePath)
else :
Logger.log("Creating new state object")
default = State()
default.lastCheck = 0
default.lastFileId = 0
default.trees = {}
default.treeFilePathEncoded = self.nameManager.encodeName(self.treeFilePath, default)
return default
def runRestore(self, backupFolder, outputFolder) :
if self.lock.acquireLock():
Logger.log("Lock acquired")
#try:
state = self.loadState()
self.restoreFolders(state, backupFolder, outputFolder)
#except Exception, e:
# Logger.log("Error: %s" % e)
# raise e
self.lock.releaseLock()
else:
Logger.log('Could not acquire lock')
def restore(self, srcEncryptedPath, dstPath) :
try:
fd=os.open(self.passphrasePath, os.O_RDONLY)
cmd='gpg --no-tty --passphrase-fd {fd}'.format(fd=fd)
self._createDirsForFile(dstPath)
with open(os.path.join(self.backupFolder, srcEncryptedPath), 'r') as stdin_fh:
with open(dstPath, 'w') as stdout_fh:
proc=subprocess.Popen(shlex.split(cmd), stdin=stdin_fh, stdout=stdout_fh)
proc.communicate()
if proc.returncode != 0:
raise IOError('gpg return code is non-zero ({code})'.format(code=proc.returncode))
if (not os.path.exists(dstPath)) or (os.path.getsize(dstPath) == 0):
raise IOError('File {src} was not encrypted properly as {dst}'.format(src=srcEncryptedPath, dst=dstPath))
except:
raise
finally:
os.close(fd)
Logger.log('Decrypted {src} and saved as {dst}'.format(src=srcEncryptedPath, dst=dstPath))
def backup(self, srcName, dstName) :
try:
# TODO: remove os-specific /
fd=os.open(self.passphrasePath, os.O_RDONLY)
dst = os.path.join(self.backupFolder, dstName)
cmd='gpg --no-tty --force-mdc --passphrase-fd {fd} -c'.format(fd=fd)
self._createDirsForFile(dst)
with open(srcName, 'r') as stdin_fh:
with open(dst, 'w') as stdout_fh:
proc=subprocess.Popen(shlex.split(cmd), stdin=stdin_fh, stdout=stdout_fh)
proc.communicate()
if proc.returncode != 0:
raise IOError('gpg return code is non-zero ({code})'.format(code=proc.returncode))
if (not os.path.exists(dst)) or (os.path.getsize(dst) == 0):
raise IOError('File {src} was not encrypted properly as {dst}'.format(src=srcName, dst=dst))
except:
raise
finally:
os.close(fd)
assert os.path.exists(dst) and os.path.getsize(dst) > 0, 'File {0} not saved correctly'.format(dst)
Logger.log('+ Encrypted {src} and saved as {dst}'.format(src=srcName, dst=dst))
def backupFolders(self, state, inputFolders, outputFolder, excludePatterns, stats):
totalNumberOfFiles = 0
totalSize = 0
for folder in inputFolders:
Logger.log("Listing files in %s" % folder)
tree = TreeNodeDirectory.createTreeFromFilesystem(folder, os.path.basename(folder), excludePatterns)
if folder in state.trees:
state.trees[folder].ensureDstPathOnAll()
Logger.log("Comparing files in %s" % folder)
(listOfRemoved, listOfAdded, listOfModified) = tree.merge(state.trees[folder])
else:
Logger.log("Adding all files from new folder %s" % folder)
listOfAdded = tree.getAllFiles()
listOfRemoved = []
listOfModified = []
upFolder = os.path.dirname(folder)
for f in listOfAdded:
self.addFile(upFolder, f, state, stats)
if f.dstPath is None: raise Exception('error')
for f in listOfModified:
self.updateFile(upFolder, f, stats)
if f.dstPath is None: raise Exception('error')
for f in listOfRemoved:
self.removeFile(upFolder, f, stats)
if f.dstPath is None: raise Exception('error')
tree.ensureDstPathOnAll()
state.trees[folder] = tree
totalNumberOfFiles += tree.numberOfFiles
totalSize += tree.size
state.lastCheck = int(time.time())
Logger.log("folder %s:" % folder)
Logger.log("Number of files: %d, size: %.2f MB" % (totalNumberOfFiles, totalSize/1024/1024))
def runBackup(self, inputFolders, outputFolder, excludePatterns, updateEvery) :
if self.lock.acquireLock():
Logger.log("Lock acquired")
#try:
state = self.loadState()
if self.isUpdatePending(state, updateEvery):
Logger.log("update pending")
#Logger.log('last successful backup at {0}'.format(datetime.datetime.fromtimestamp(stats['lastSearch']).strftime('%Y-%m-%d %H:%M:%S')))
stats = Stats()
self.backupFolders(state, inputFolders, outputFolder, excludePatterns, stats)
self.printStats(stats)
self.saveState(state)
else:
Logger.log('backup not necessary at this moment')
#Logger.log('next no sooner than {0}'.format(datetime.datetime.fromtimestamp(stats['lastSearch'] + updateEvery).strftime('%Y-%m-%d %H:%M:%S')))
#except Exception, e:
# Logger.log("Error: %s" % e)
# raise e
self.lock.releaseLock()
else:
Logger.log('Could not acquire lock')
def tearDown(self):
print("Test script: " + self._testMethodName +
" is complete. See script logs for additional details.")
# get the test case id decorator and log it
errors_result_log = ''
failures_result_log = ''
# get the actual test outcome
result = self.defaultTestResult(
) # these 2 methods have no side effects
self._feedErrorsToResult(result, self._outcome.errors)
error = self._list2reason(result.errors)
failure = self._list2reason(result.failures)
if error:
errors_result_log = self._get_all_test_method_exceptions_result_text(
error)
if failure:
failures_result_log = self._get_all_test_method_failures_result_text(
failure)
# status: [1= passed, 2= blocked, 4= retest, 5= failed ]
screen_shot_text = self._testMethodName + str(random.randint(9, 99999))
screen_shot = FileHelper.resolveAgnosticPath(
TestRun.screenshots_unc_path, screen_shot_text + '.png')
# str.format(
# r'{}\{}.png', TestRun.screenshots_unc_path, screen_shot_text)
try:
self.driver.save_screenshot(str(screen_shot))
except:
print('There may have been an issue with taking the screenshot')
# job_url = ''
# if TestRun.use_sauce_labs == True:
# sauce_status = True
# if failure:
# sauce_status = False
# if error:
# sauce_status = False
# try:
# self.sauce = SauceClient(SauceLabs.user, SauceLabs.access_key)
# self.sauce.jobs.update_job(self.driver.session_id, passed=sauce_status)
# job = self.sauce.jobs.get_job(self.driver.session_id)
# import hmac
# from hashlib import md5
# a = hmac.new(
# bytes("{}:{}".format(
# SauceLabs.user,
# SauceLabs.access_key),
# 'latin-1'),
# bytes(job['id'], 'latin-1'),
# md5)
#
# auth_token = a.hexdigest()
# video_url = 'https://assets.saucelabs.com/jobs/{}/video.mp4?auth={}'.format(job['id'], auth_token)
# # video_name = job['id'] + '.mp4'
# # video_path = FileHelper.resolveAgnosticPath(TestRun.screenshots_unc_path, video_name)
# #
# # try:
# # import urllib.request
# # urllib.request.urlretrieve(video_url, video_path)
# # except:
# # pass
#
# job_url = '[CLICK TO VIEW VIDEO]({})'.format(video_url)
#
# # job_url = '<script src = "https://saucelabs.com/video-embed/{}.js?auth={}"></script>'.format(
# # job['id'],
# # str(auth_token)
# # )
#
# # # Logger.log_and_debug(99, 'info', str(assets))
# except Exception as e:
# print('SAUCE EXCEPTION: ' + str(e))
# pass
if TestRun.toggle_test_rail == True:
test_rail = TestRailApi()
status = 1
if failure:
Logger.log_exception(failure)
status = 5
elif error:
Logger.log_exception(error)
status = 4
file = ''
try:
file = open(Logger.current_log_file)
except:
print('unable to open log file!\nLOG FILE: ' +
Logger.current_log_file)
with file as f:
print('adding test result')
result_comment = '\n'.join(f.readlines(
)) + self.get_screen_shot_log_text(screen_shot_text)
current_test_results = []
for line in result_comment.split('\n'):
# test logs
if line.__contains__(self._testMethodName):
if line == self._get_exception_identifier():
continue
current_test_results.append(line)
result = '\n'.join(current_test_results)
result = result + errors_result_log
result = result + failures_result_log
# if TestRun.use_sauce_labs == True:
# result = result + '\n\n#' + job_url + '#\n\n'
# This assumes since you have test rail toggled - you've created a test run
test_case_id = self.test_case_id
if test_case_id is None:
test_case_id = test_rail.get_test_id_by_title(
self._testMethodName, self.test_type.test_section_id)
try:
test_rail.add_test_case_result_for_test_run(
TestRun.test_run_id, test_case_id, status, str(result))
except:
test_case_id = test_rail.get_test_id_by_title(
self._testMethodName, self.test_type.test_section_id)
test_rail.add_test_case_result_for_test_run(
TestRun.test_run_id, test_case_id, status, str(result))
try:
self.driver.close()
self.driver.quit()
except:
print(
'there may have been an issue closing or quitting the driver. Killing all driver processes'
)
try:
nameManager = HexNumbersNameManager()
backupProvider = EncryptedBackupProvider(config.storeBackupFolder, config.passphrasePath)
serializer = PickleSerializer()
if len(sys.argv) < 2:
print 'expected at least one command line argument (e.g. backup, restore or ls)'
sys.exit()
if sys.argv[1] == 'backup' :
controller = TreeStoreBackupController(config.dataPath, lock, serializer, nameManager, backupProvider)
controller.runBackup(config.foldersToBackup, config.storeBackupFolder, config.excludePatterns, config.updateBackupEvery)
elif sys.argv[1] == 'ls' :
state = serializer.unserialize(os.path.join(config.dataPath, 'tree.dat'))
for tree in state.trees.values():
for f in tree.getAllFiles():
print f.path
elif sys.argv[1] == 'restore' :
if len(sys.argv) == 5:
pass
else:
print 'you need to specify input and output folder and passphrase file path for restore'
else:
print 'unknown command line option (expected backup or ls)'
except KeyboardInterrupt:
Logger.log('Action aborted by a user')
lock.releaseLock()
TestRun.log_file_base_path = 'test_modules/test_output/script_logs/' # have to change log file base path when running from test controller
# if server_address == 'stage.env.com' or \
# server_address == 'stg2.env.com' or \
# server_address == 'qa2.env.com':
# TestRun.use_sauce_labs = True
#####################
#### UNITTEST ##############
loader = unittest.TestLoader()
test_suite_by_category = unittest.TestSuite()
#####################
####### LOGGING ##########
dt_string = datetime.now()
log_title = __test_type + '_' + str(dt_string.strftime("%Y%m%d-%H%M%S"))
log_name, log_path = Logger.logging_setup(log_title)
BaseSeleniumTest.logs.append(log_name)
#####################
def _load_tests_of_type(test_class, test_type: TestType, categories=None):
for name, method in test_class.__dict__.items():
if hasattr(method, 'test_type'):
tp = method.test_type
if tp == test_type:
if categories is not None:
if hasattr(method, 'test_category'):
for c in method.test_category:
item = [cat for cat in categories if cat.category is c.category]
if item.__len__() != 0:
test_suite_by_category.addTest(test_class(name))
else:
def updateFile(self, folder, f, stats):
assert f.dstPath is not None, "missing dstPath for updated file: " + str(f)
Logger.log("Updating or adding file: %s" % f.path)
stats.updatedFilesSize += f.size
stats.updatedFilesCount += 1
self.backupProvider.backup(os.path.join(folder, f.path), f.dstPath)
def addFile(self, folder, f, state, stats):
Logger.log("Updating or adding file: %s" % f.path)
stats.addedFilesSize += f.size
stats.addedFilesCount += 1
f.dstPath = self.nameManager.encodeName(f.path, state)
self.backupProvider.backup(os.path.join(folder, f.path), f.dstPath)
def saveState(self, state):
Logger.log("Serializing state to %s" % self.treeFilePath)
self.serializer.serialize(self.treeFilePath, state)
self.backupProvider.backup(self.treeFilePath, state.treeFilePathEncoded)
def printStats(self, stats):
Logger.log("Added %d files (%.2f MB)" % (stats.addedFilesCount, stats.addedFilesSize/1024/1024))
Logger.log("Updated %d files (%.2f MB)" % (stats.updatedFilesCount, stats.updatedFilesSize/1024/1024))
Logger.log("Removed %d files (%.2f MB)" % (stats.removedFilesCount, stats.removedFilesSize/1024/1024))
def removeFile(self, folder, f, stats):
assert f.dstPath is not None, "missing dstPath for removed file: " + str(f)
Logger.log("Removing file: %s" % f.path)
stats.removedFilesSize += f.size
stats.removedFilesCount += 1
self.backupProvider.remove(f.dstPath)
