def unpacketize(**kwargs):
fileNameList = os.listdir(kwargs["inDirectoryFullPath"])
t = [item for item in fileNameList if item.split(".")[-1] == "0"]
if len(t) != 1:
return {"retCode": 1, "errMsg": "Unable to determine packet zero"}
packetZeroFileName = t[0]
packetZeroFullPath = os.path.join(kwargs["inDirectoryFullPath"], packetZeroFileName)
tmpZipFileName = "tmpZip.zip"
tmpZipFileFullPath = os.path.join(kwargs["outDirectoryFullPath"], tmpZipFileName)
pUtils.createDirectory(kwargs["outDirectoryFullPath"])
t = pUtils.pUnSlice(packetZeroFullPath, tmpZipFileFullPath)
if t["retCode"] != 0:
return {"retCode": 2, "errMsg": "Unable to unslice", "debug": t}
t = pUtils.unzipFile(tmpZipFileFullPath, kwargs["outDirectoryFullPath"])
if t != 0:
return {"retCode": 3, "errMsg": "Unable to unzip", "debug": t}
os.remove(tmpZipFileFullPath)
manifestFileFullPath = os.path.join(kwargs["outDirectoryFullPath"], MANIFEST_FILE_NAME)
t = verify(manifestFileFullPath=manifestFileFullPath)
if t["retCode"] != 0:
return {"retCode": 4, "errMsg": "Checksum verification failed!", "debug": t}
return {"retCode": 0, "errMsg": None}
def test_fileOperations_1(self):
testDirFullPath = tempfile.mkdtemp('','unitTest_pUtils_')
self.assertEqual(os.path.exists(testDirFullPath),True)
string = 'print \'Hello World!\''
pUtils.quickFileWrite(os.path.join(testDirFullPath,'hw.py'),string)
data = pUtils.quickFileRead(os.path.join(testDirFullPath,'hw.py'))
self.assertEqual(string,data)
self.assertEqual(pUtils.getFileSha1(os.path.join(testDirFullPath,'hw.py')),'a849bee4b303051f907d64b6c461ee6c699c3e79')
pUtils.pSlice(os.path.join(testDirFullPath,'hw.py'),testDirFullPath,1)
self.assertEqual(len(pUtils.filterListByRegex(os.listdir(testDirFullPath),r'hw\.py\.[0-9]+')),21)
os.remove(os.path.join(testDirFullPath,'hw.py'))
self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),False)
pUtils.pUnSlice(os.path.join(testDirFullPath,'hw.py.0'),os.path.join(testDirFullPath,'hw.py'))
self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),True)
self.assertEqual(pUtils.quickFileRead(os.path.join(testDirFullPath,'hw.py')),string)
pUtils.createZipFile(testDirFullPath,['hw.py'],os.path.join(testDirFullPath,'aFile.zip'))
self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'aFile.zip')),True)
os.remove(os.path.join(testDirFullPath,'hw.py'))
self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),False)
pUtils.unzipFile(os.path.join(testDirFullPath,'aFile.zip'),testDirFullPath)
self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),True)
self.assertEqual(pUtils.quickFileRead(os.path.join(testDirFullPath,'hw.py')),string)
pUtils.emptyDirectory(testDirFullPath)
self.assertEqual(len(os.listdir(testDirFullPath)),0)
pUtils.createDirectory(os.path.join(testDirFullPath,'ttt','ttt2'))
self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'ttt','ttt2')),True)
pUtils.removeDirectory(testDirFullPath)
self.assertEqual(os.path.exists(testDirFullPath),False)
def initTestRunFolder(self, startTimestamp):
"""
| Creates the directory to be used for the test run.
| It uses the value of self.SN .
| It uses the value of self.testRunID
| The full path remains stored as an object variable.
Args:
* startTimestamp (str): The timestamp from the start of the test
Returns:
A string with the full path to the folder for the test run
"""
if self.getIsMemoryOnly():
raise Exception("Attempting to create TestRunFolder while isMemoryOnly flag is set")
testSequenceID = self.configData["testSequenceID"]
self.testRunFolderName = testSequenceID + "_" + startTimestamp + "_" + self.SN + "_" + self.testRunID
print "testRunFolderName=" + self.testRunFolderName
self.testRunFolderFullPath = os.path.join(
self.getTestStationRootFolder(), "TestRunDataStorage", self.testRunFolderName
)
pUtils.createDirectory(self.testRunFolderFullPath)
return self.testRunFolderFullPath
def transport(**kwargs):
srcPath = os.path.dirname(kwargs["packetZeroPath"])
baseFileName = os.path.basename(kwargs["packetZeroPath"])[:-2]
def scpStr(index):
s = "scp -P " + kwargs["port"] + " " + kwargs["user"] + "@" + kwargs["host"] + ":"
s += os.path.join(srcPath, baseFileName + "." + str(index))
s += " " + kwargs["dstPath"]
return s
if not os.path.exists(os.path.join(kwargs["dstPath"], baseFileName + ".0")):
pUtils.createDirectory(kwargs["dstPath"])
cmd = scpStr(0)
t = pUtils.runProgram(cmd, True)
if t["returnCode"] != 0:
return {"retCode": 2, "errMsg": "Unable to retrieve packet zero", "debug": t, "cmd": cmd}
fileFullPath = os.path.join(kwargs["dstPath"], baseFileName + ".0")
manifestData = json.loads(pUtils.quickFileRead(fileFullPath))
sliceAmount = manifestData["sliceAmount"]
checksumDict = manifestData["checksumDict"]
for i in range(1, sliceAmount + 1):
fileFullPath = os.path.join(kwargs["dstPath"], baseFileName + "." + str(i))
if os.path.exists(fileFullPath):
fileSha1 = pUtils.getFileSha1(fileFullPath)
if fileSha1 == checksumDict[baseFileName + "." + str(i)]:
continue
cmd = scpStr(i)
t = None
retryCounter = 0
while True:
retryCounter += 1
if retryCounter > int(kwargs["try"]):
return {"retCode": 1, "errMsg": "Max retry reached", "debug": t, "cmd": cmd}
t = pUtils.runProgram(cmd, True)
if t["returnCode"] != 0:
continue
fileSha1 = pUtils.getFileSha1(fileFullPath)
if fileSha1 != checksumDict[baseFileName + "." + str(i)]:
continue
break
return {"retCode": 0, "errMsg": None}
def packetize(**kwargs):
tmpZipFileName = "tmpZip.zip"
tmpZipFileFullPath = os.path.join(kwargs["outDirectoryFullPath"], tmpZipFileName)
pUtils.createDirectory(kwargs["outDirectoryFullPath"])
fileNameList = os.listdir(kwargs["inDirectoryFullPath"])
pUtils.createZipFile(kwargs["inDirectoryFullPath"], fileNameList, tmpZipFileFullPath)
t = pUtils.pSlice(tmpZipFileFullPath, kwargs["outDirectoryFullPath"], kwargs["sliceSize"])
if t["retCode"] != 0:
return {"retCode": 1, "errMsg": "Unable to slice file", "debug": t}
os.remove(tmpZipFileFullPath)
return {"retCode": 0, "errMsg": None}
def genConfig(dirPath, configManager, **kwargs):
pUtils.createDirectory(os.path.abspath(dirPath))
filePath = os.path.join(dirPath, 'config1.json')
if configManager.saveFullConfig(filePath) != 0:
pprint('Error: ', color=COLOR.RED, endLine=False)
pprint('File: %s already exists' % filePath)
exit(1)
filePath = os.path.join(dirPath, 'configMenu.json')
if configManager.genMenuConfigFile(filePath,
configName='config1',
configPath='config1.json'):
pprint('Error: ', color=COLOR.RED, endLine=False)
pprint('File: %s already exists' % filePath)
exit(1)
pprint('DONE', color=COLOR.TEAL)
def dump(**kwargs):
if kwargs["configFileFullPath"]:
DATAMINING_CONFIG = json.loads(pUtils.quickFileRead(kwargs["configFileFullPath"]))
else:
DATAMINING_CONFIG = json.loads(
pUtils.quickFileRead(
os.path.join(os.environ["MTP_TESTSTATION"], "MTP", "config", "miselu", "database", kwargs["config"])
)
)
sql = SQL(**DATAMINING_CONFIG)
sql.conn()
### Create virtual table of testRunID's ###
SNlist = None
v = []
s = "SELECT testRunID INTO TEMP TABLE SelectedTestRunID FROM TestRun"
s += "\n WHERE creationTimestamp>=%s"
v.append(kwargs["creationTimestamp_rangeStart"])
if kwargs["creationTimestamp_rangeEnd"] != None:
s += "\n AND creationTimestamp<=%s"
if kwargs["startTimestamp_rangeStart"] != None:
s += "\n AND startTimestamp>=%s"
v.append(kwargs["startTimestamp_rangeStart"])
if kwargs["startTimestamp_rangeEnd"] != None:
s += "\n AND startTimestamp<=%s"
v.append(kwargs["startTimestamp_rangeEnd"])
if kwargs["endTimestamp_rangeStart"] != None:
s += "\n AND endTimestamp>%s"
v.append(kwargs["endTimestamp_rangeStart"])
if kwargs["endTimestamp_rangeEnd"] != None:
s += "\n AND endTimestamp<%s"
v.append(kwargs["endTimestamp_rangeEnd"])
if SNlist != None:
s += "\n AND ( SN=%s"
s += "\n OR SN=%s" * (len(SNlist) - 1)
s += "\n )"
v += SNlist
if kwargs["siteID"] != None:
s += "\n AND siteID=%s"
v.append(kwargs["siteID"])
if kwargs["stationID"] != None:
s += "\n AND stationID=%s"
v.append(kwargs["stationID"])
if kwargs["testSequenceID"] != None:
s += "\n AND testSequenceID=%s"
v.append(kwargs["testSequenceID"])
if kwargs["isPass"] != None:
s += "\n AND isPass=%s"
v.append(kwargs["isPass"])
# s+=' limit 10'
s += "\n ;"
sql.execute(s, v) ##TODO check if it is blocking or not
# For every table, filtered with the TestRunID's table and write it to a file
directoryFullPath = kwargs["directoryFullPath"]
pUtils.createDirectory(directoryFullPath)
if len(os.listdir(directoryFullPath)) != 0:
raise Exception("Directory specified for dump is not empty")
for tableName in tableNameList:
v = []
s = "COPY"
s += "\n (SELECT %s.* FROM %s,SelectedTestRunID"
s += "\n WHERE SelectedTestRunID.testRunID = %s.testRunID)"
s += "\n TO STDOUT"
s += "\n WITH CSV "
s += "\n HEADER "
s = s % ((tableName,) * 3)
fileFullPath = os.path.join(directoryFullPath, fileBaseName + "_" + tableName + "." + extension)
with open(fileFullPath, "wt") as f:
sql.cur.copy_expert(s, f)
sql.close()
createManifest(MANIFEST_FILE_NAME, directoryFullPath)
return {"retCode": 0, "errMsg": None}
