def unpacketize(**kwargs): fileNameList = os.listdir(kwargs["inDirectoryFullPath"]) t = [item for item in fileNameList if item.split(".")[-1] == "0"] if len(t) != 1: return {"retCode": 1, "errMsg": "Unable to determine packet zero"} packetZeroFileName = t[0] packetZeroFullPath = os.path.join(kwargs["inDirectoryFullPath"], packetZeroFileName) tmpZipFileName = "tmpZip.zip" tmpZipFileFullPath = os.path.join(kwargs["outDirectoryFullPath"], tmpZipFileName) pUtils.createDirectory(kwargs["outDirectoryFullPath"]) t = pUtils.pUnSlice(packetZeroFullPath, tmpZipFileFullPath) if t["retCode"] != 0: return {"retCode": 2, "errMsg": "Unable to unslice", "debug": t} t = pUtils.unzipFile(tmpZipFileFullPath, kwargs["outDirectoryFullPath"]) if t != 0: return {"retCode": 3, "errMsg": "Unable to unzip", "debug": t} os.remove(tmpZipFileFullPath) manifestFileFullPath = os.path.join(kwargs["outDirectoryFullPath"], MANIFEST_FILE_NAME) t = verify(manifestFileFullPath=manifestFileFullPath) if t["retCode"] != 0: return {"retCode": 4, "errMsg": "Checksum verification failed!", "debug": t} return {"retCode": 0, "errMsg": None}
def test_fileOperations_1(self): testDirFullPath = tempfile.mkdtemp('','unitTest_pUtils_') self.assertEqual(os.path.exists(testDirFullPath),True) string = 'print \'Hello World!\'' pUtils.quickFileWrite(os.path.join(testDirFullPath,'hw.py'),string) data = pUtils.quickFileRead(os.path.join(testDirFullPath,'hw.py')) self.assertEqual(string,data) self.assertEqual(pUtils.getFileSha1(os.path.join(testDirFullPath,'hw.py')),'a849bee4b303051f907d64b6c461ee6c699c3e79') pUtils.pSlice(os.path.join(testDirFullPath,'hw.py'),testDirFullPath,1) self.assertEqual(len(pUtils.filterListByRegex(os.listdir(testDirFullPath),r'hw\.py\.[0-9]+')),21) os.remove(os.path.join(testDirFullPath,'hw.py')) self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),False) pUtils.pUnSlice(os.path.join(testDirFullPath,'hw.py.0'),os.path.join(testDirFullPath,'hw.py')) self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),True) self.assertEqual(pUtils.quickFileRead(os.path.join(testDirFullPath,'hw.py')),string) pUtils.createZipFile(testDirFullPath,['hw.py'],os.path.join(testDirFullPath,'aFile.zip')) self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'aFile.zip')),True) os.remove(os.path.join(testDirFullPath,'hw.py')) self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),False) pUtils.unzipFile(os.path.join(testDirFullPath,'aFile.zip'),testDirFullPath) self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'hw.py')),True) self.assertEqual(pUtils.quickFileRead(os.path.join(testDirFullPath,'hw.py')),string) pUtils.emptyDirectory(testDirFullPath) self.assertEqual(len(os.listdir(testDirFullPath)),0) pUtils.createDirectory(os.path.join(testDirFullPath,'ttt','ttt2')) self.assertEqual(os.path.exists(os.path.join(testDirFullPath,'ttt','ttt2')),True) pUtils.removeDirectory(testDirFullPath) self.assertEqual(os.path.exists(testDirFullPath),False)
def initTestRunFolder(self, startTimestamp): """ | Creates the directory to be used for the test run. | It uses the value of self.SN . | It uses the value of self.testRunID | The full path remains stored as an object variable. Args: * startTimestamp (str): The timestamp from the start of the test Returns: A string with the full path to the folder for the test run """ if self.getIsMemoryOnly(): raise Exception("Attempting to create TestRunFolder while isMemoryOnly flag is set") testSequenceID = self.configData["testSequenceID"] self.testRunFolderName = testSequenceID + "_" + startTimestamp + "_" + self.SN + "_" + self.testRunID print "testRunFolderName=" + self.testRunFolderName self.testRunFolderFullPath = os.path.join( self.getTestStationRootFolder(), "TestRunDataStorage", self.testRunFolderName ) pUtils.createDirectory(self.testRunFolderFullPath) return self.testRunFolderFullPath
def transport(**kwargs): srcPath = os.path.dirname(kwargs["packetZeroPath"]) baseFileName = os.path.basename(kwargs["packetZeroPath"])[:-2] def scpStr(index): s = "scp -P " + kwargs["port"] + " " + kwargs["user"] + "@" + kwargs["host"] + ":" s += os.path.join(srcPath, baseFileName + "." + str(index)) s += " " + kwargs["dstPath"] return s if not os.path.exists(os.path.join(kwargs["dstPath"], baseFileName + ".0")): pUtils.createDirectory(kwargs["dstPath"]) cmd = scpStr(0) t = pUtils.runProgram(cmd, True) if t["returnCode"] != 0: return {"retCode": 2, "errMsg": "Unable to retrieve packet zero", "debug": t, "cmd": cmd} fileFullPath = os.path.join(kwargs["dstPath"], baseFileName + ".0") manifestData = json.loads(pUtils.quickFileRead(fileFullPath)) sliceAmount = manifestData["sliceAmount"] checksumDict = manifestData["checksumDict"] for i in range(1, sliceAmount + 1): fileFullPath = os.path.join(kwargs["dstPath"], baseFileName + "." + str(i)) if os.path.exists(fileFullPath): fileSha1 = pUtils.getFileSha1(fileFullPath) if fileSha1 == checksumDict[baseFileName + "." + str(i)]: continue cmd = scpStr(i) t = None retryCounter = 0 while True: retryCounter += 1 if retryCounter > int(kwargs["try"]): return {"retCode": 1, "errMsg": "Max retry reached", "debug": t, "cmd": cmd} t = pUtils.runProgram(cmd, True) if t["returnCode"] != 0: continue fileSha1 = pUtils.getFileSha1(fileFullPath) if fileSha1 != checksumDict[baseFileName + "." + str(i)]: continue break return {"retCode": 0, "errMsg": None}
def packetize(**kwargs): tmpZipFileName = "tmpZip.zip" tmpZipFileFullPath = os.path.join(kwargs["outDirectoryFullPath"], tmpZipFileName) pUtils.createDirectory(kwargs["outDirectoryFullPath"]) fileNameList = os.listdir(kwargs["inDirectoryFullPath"]) pUtils.createZipFile(kwargs["inDirectoryFullPath"], fileNameList, tmpZipFileFullPath) t = pUtils.pSlice(tmpZipFileFullPath, kwargs["outDirectoryFullPath"], kwargs["sliceSize"]) if t["retCode"] != 0: return {"retCode": 1, "errMsg": "Unable to slice file", "debug": t} os.remove(tmpZipFileFullPath) return {"retCode": 0, "errMsg": None}
def genConfig(dirPath, configManager, **kwargs): pUtils.createDirectory(os.path.abspath(dirPath)) filePath = os.path.join(dirPath, 'config1.json') if configManager.saveFullConfig(filePath) != 0: pprint('Error: ', color=COLOR.RED, endLine=False) pprint('File: %s already exists' % filePath) exit(1) filePath = os.path.join(dirPath, 'configMenu.json') if configManager.genMenuConfigFile(filePath, configName='config1', configPath='config1.json'): pprint('Error: ', color=COLOR.RED, endLine=False) pprint('File: %s already exists' % filePath) exit(1) pprint('DONE', color=COLOR.TEAL)
def dump(**kwargs): if kwargs["configFileFullPath"]: DATAMINING_CONFIG = json.loads(pUtils.quickFileRead(kwargs["configFileFullPath"])) else: DATAMINING_CONFIG = json.loads( pUtils.quickFileRead( os.path.join(os.environ["MTP_TESTSTATION"], "MTP", "config", "miselu", "database", kwargs["config"]) ) ) sql = SQL(**DATAMINING_CONFIG) sql.conn() ### Create virtual table of testRunID's ### SNlist = None v = [] s = "SELECT testRunID INTO TEMP TABLE SelectedTestRunID FROM TestRun" s += "\n WHERE creationTimestamp>=%s" v.append(kwargs["creationTimestamp_rangeStart"]) if kwargs["creationTimestamp_rangeEnd"] != None: s += "\n AND creationTimestamp<=%s" if kwargs["startTimestamp_rangeStart"] != None: s += "\n AND startTimestamp>=%s" v.append(kwargs["startTimestamp_rangeStart"]) if kwargs["startTimestamp_rangeEnd"] != None: s += "\n AND startTimestamp<=%s" v.append(kwargs["startTimestamp_rangeEnd"]) if kwargs["endTimestamp_rangeStart"] != None: s += "\n AND endTimestamp>%s" v.append(kwargs["endTimestamp_rangeStart"]) if kwargs["endTimestamp_rangeEnd"] != None: s += "\n AND endTimestamp<%s" v.append(kwargs["endTimestamp_rangeEnd"]) if SNlist != None: s += "\n AND ( SN=%s" s += "\n OR SN=%s" * (len(SNlist) - 1) s += "\n )" v += SNlist if kwargs["siteID"] != None: s += "\n AND siteID=%s" v.append(kwargs["siteID"]) if kwargs["stationID"] != None: s += "\n AND stationID=%s" v.append(kwargs["stationID"]) if kwargs["testSequenceID"] != None: s += "\n AND testSequenceID=%s" v.append(kwargs["testSequenceID"]) if kwargs["isPass"] != None: s += "\n AND isPass=%s" v.append(kwargs["isPass"]) # s+=' limit 10' s += "\n ;" sql.execute(s, v) ##TODO check if it is blocking or not # For every table, filtered with the TestRunID's table and write it to a file directoryFullPath = kwargs["directoryFullPath"] pUtils.createDirectory(directoryFullPath) if len(os.listdir(directoryFullPath)) != 0: raise Exception("Directory specified for dump is not empty") for tableName in tableNameList: v = [] s = "COPY" s += "\n (SELECT %s.* FROM %s,SelectedTestRunID" s += "\n WHERE SelectedTestRunID.testRunID = %s.testRunID)" s += "\n TO STDOUT" s += "\n WITH CSV " s += "\n HEADER " s = s % ((tableName,) * 3) fileFullPath = os.path.join(directoryFullPath, fileBaseName + "_" + tableName + "." + extension) with open(fileFullPath, "wt") as f: sql.cur.copy_expert(s, f) sql.close() createManifest(MANIFEST_FILE_NAME, directoryFullPath) return {"retCode": 0, "errMsg": None}