def downloadScanFile(Framework): targetScanFile = downloadRemoteScanFile(Framework) if not targetScanFile: logger.debug('Remote scan file was not downloaded, will try previously downloaded scan file(if exists)') targetScanFile = useTempScanFile(Framework) if not targetScanFile: logger.debug('No scan file downloaded from previous execution, download file step failed') Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_REMOTE_SCANFILE_NOT_FOUND, None) return logger.debug('Scan file was successfully downloaded to ', targetScanFile.getCanonicalPath()) #set download time to current time Framework.setProperty(InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILE_DOWNLOAD_TIME, Date()) #Check the drity files in sending folder sendingFolder = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.SENDING_FOLDER_NAME deleteDirtyFile(File(sendingFolder, targetScanFile.getName())) #delete temporary scan file from previous execution (if any) tempScanFileFolder = CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId() + CollectorsParameters.FILE_SEPARATOR tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_EXTENTION) deleteDirtyFile(File(tempScanFileFolder, tempScanFileName)) tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_DELTA_EXTENTION) deleteDirtyFile(File(tempScanFileFolder, tempScanFileName)) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def downloadScanFile(Framework): targetScanFile = downloadRemoteScanFile(Framework) if not targetScanFile: logger.debug("Remote scan file was not downloaded, will try previously downloaded scan file(if exists)") targetScanFile = useTempScanFile(Framework) if not targetScanFile: logger.debug("No scan file downloaded from previous execution, download file step failed") Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE) Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_REMOTE_SCANFILE_NOT_FOUND, None) return logger.debug("Scan file was successfully downloaded to ", targetScanFile.getCanonicalPath()) # set download time to current time Framework.setProperty(InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILE_DOWNLOAD_TIME, Date()) # Check the drity files in sending folder sendingFolder = ( CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.SENDING_FOLDER_NAME ) deleteDirtyFile(File(sendingFolder, targetScanFile.getName())) # delete temporary scan file from previous execution (if any) tempScanFileFolder = ( CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId() + CollectorsParameters.FILE_SEPARATOR ) tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_EXTENTION) deleteDirtyFile(File(tempScanFileFolder, tempScanFileName)) tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_DELTA_EXTENTION) deleteDirtyFile(File(tempScanFileFolder, tempScanFileName)) Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def downloadRemoteScanFile(Framework): remoteScanFileLocation = Framework.getProperty(InventoryUtils.STATE_PROPERTY_REMOTE_SCAN_FILE_LOCATION) # download scanner log file before downloading scan file downloadScanLogFile(Framework) if remoteScanFileLocation is None: logger.debug("No scan file to downloaded from current execution") return None logger.debug("About to download scan file from current execution:", remoteScanFileLocation) extension = InventoryUtils.getFileExtension(remoteScanFileLocation) localScanFileName = InventoryUtils.generateScanFileName(Framework, extension) # folder for scan files localScanFileFolderPath = ( CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME ) downloadedScanFilesDir = File(localScanFileFolderPath) downloadedScanFilesDir.mkdirs() # this scan file will be created after downloading from remote machine targetScanFile = File(downloadedScanFilesDir, localScanFileName) # get file to the local machine logger.debug("Scan file to be downloaded to location:", targetScanFile.getCanonicalPath()) if not InventoryUtils.copyRemoteFileToLocal(Framework, remoteScanFileLocation, targetScanFile.getCanonicalPath()): return None return targetScanFile
def useTempScanFile(Framework): tempScanFilePath = Framework.getProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE) if tempScanFilePath is None: logger.debug("No scan file found from previous scanner execution") return None logger.debug("Using scan file from previous execution:", tempScanFilePath) extension = InventoryUtils.getFileExtension(tempScanFilePath) localScanFileName = InventoryUtils.generateScanFileName(Framework, extension) # folder for scan files localScanFileFolderPath = ( CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME ) downloadedScanFilesDir = File(localScanFileFolderPath) downloadedScanFilesDir.mkdirs() # this scan file will be created after downloading from remote machine targetScanFile = File(downloadedScanFilesDir, localScanFileName) logger.debug("Scan file from previous execution will be moved to ", targetScanFile.getCanonicalPath()) tempScanFile = File(tempScanFilePath) if not tempScanFile.renameTo(targetScanFile): return None return targetScanFile
def downloadRemoteScanFile(Framework): remoteScanFileLocation = Framework.getProperty(InventoryUtils.STATE_PROPERTY_REMOTE_SCAN_FILE_LOCATION) # download scanner log file before downloading scan file downloadScanLogFile(Framework) if remoteScanFileLocation is None: logger.debug('No scan file to downloaded from current execution') return None logger.debug('About to download scan file from current execution:', remoteScanFileLocation) extension = InventoryUtils.getFileExtension(remoteScanFileLocation) localScanFileName = InventoryUtils.generateScanFileName(Framework, extension) #folder for scan files localScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME downloadedScanFilesDir = File(localScanFileFolderPath) downloadedScanFilesDir.mkdirs() #this scan file will be created after downloading from remote machine targetScanFile = File(downloadedScanFilesDir, localScanFileName) #get file to the local machine logger.debug('Scan file to be downloaded to location:', targetScanFile.getCanonicalPath()) if not InventoryUtils.copyRemoteFileToLocal(Framework, remoteScanFileLocation, targetScanFile.getCanonicalPath()): return None return targetScanFile
def updateCmdForDeltaScanning(commandLine, Framework): originalScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.ORIGINAL_FOLDER_NAME originalScanFile = File(originalScanFileFolderPath, InventoryUtils.generateScanFileName(Framework)) if originalScanFile.exists(): scan = None try: try: buffer = jarray.zeros(0x24, 'b') fileSize = originalScanFile.length() if fileSize > 0x24: scan = RandomAccessFile(originalScanFile, "r") scan.readFully(buffer) if (buffer[0] == 0x1F) and ((buffer[1] & 0xFF) == 0x8B) and (buffer[2] == 0x08): scan.seek(fileSize - 8) scan.readFully(buffer, 0, 8) crc32 = getInt(buffer, 0) size = getInt(buffer, 4) deltaParams = ' -oldscanid:' + str(crc32) + ' -oldscansize:' + str(size) + ' ' index = String(commandLine).indexOf(ENTERPRISE_MODE) + String(ENTERPRISE_MODE).length() commandLine = commandLine[0:index] + deltaParams + commandLine[index + 1:] logger.debug('Scanner execution command updated to ', commandLine) except: logger.debugException("Failed to calculate CRC32 and size of zipped scan file " + originalScanFile.getAbsolutePath()) finally: if scan is not None: try: scan.close() except: pass return commandLine
def checkEnrichedFileExisted(Framework): localScanFileName = InventoryUtils.generateScanFileName(Framework) localScanFileSendingFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.SENDING_FOLDER_NAME targetScanFile = File(localScanFileSendingFolderPath, localScanFileName) if not targetScanFile.exists(): logger.debug('No processed scan file yet. XML-Enricher is still running.') Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_ENRICHED_SCANFILE_NOTREADY, [localScanFileName]) Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE) else: logger.debug('find processed scan file, goto next step') Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def checkScanFileExistance(Framework): # this step is always finished with success since we DON'T require scan file from previous execution, just nice to have Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) DownloadScanFileBeforeExecution = Boolean.parseBoolean(Framework.getParameter('DownloadScanFileBeforeExecution')) if DownloadScanFileBeforeExecution: try: client = Framework.getConnectedClient() options = LockUtils.getClientOptionsMap(client) previousExecutionStarted = options.get(InventoryUtils.STATE_PROPERTY_EXECUTION_STARTED) if (previousExecutionStarted is None) or (len(previousExecutionStarted.strip()) == 0): logger.debug('Previous execution timestamp no found, continuing with workflow') return remoteScanFileLocation = options.get(InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILENAME) if not InventoryUtils.isPathValid(remoteScanFileLocation): logger.debug('No scan file path found on remote machine, continuing with workflow') return lastSuccessExecuton = Framework.getState().getJobLastSuccessfulRun() logger.debug('Last success execution ' + str(lastSuccessExecuton)) logger.debug('Remote scan file execution ' + str(previousExecutionStarted)) if long(lastSuccessExecuton) > long(previousExecutionStarted): logger.debug('Scan file on probe side is newer than on remote machine, skipping downloading') return logger.debug('Last success execution ' + str(lastSuccessExecuton) + ' older than scan file on remote machine ' + str(remoteScanFileLocation) + '. Going to download scan file:' + str(remoteScanFileLocation)) tempScanFileFolder = CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId() + CollectorsParameters.FILE_SEPARATOR File(tempScanFileFolder).mkdirs() extension = InventoryUtils.getFileExtension(remoteScanFileLocation) tempScanFileName = InventoryUtils.generateScanFileName(Framework, extension) tempScanFile = File(tempScanFileFolder, tempScanFileName) tempScanFilePath = tempScanFile.getCanonicalPath() logger.debug('Try to download scan file to the:', tempScanFilePath) if not InventoryUtils.copyRemoteFileToLocal(Framework, remoteScanFileLocation, tempScanFilePath, 0): logger.debug('Failed to download scan file before current execution') Framework.setProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE, tempScanFilePath) except: reason = str(sys.exc_info()[1]) logger.debug('Failed to check/download scan file from previous execution. Reason:', reason) else: logger.debug('Even not checking whether scan file exists on remote machine or not.')
def updateCmdForDeltaScanning(commandLine, Framework): originalScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.ORIGINAL_FOLDER_NAME originalScanFile = File(originalScanFileFolderPath, InventoryUtils.generateScanFileName(Framework)) if originalScanFile.exists(): scan = None try: try: buffer = jarray.zeros(0x24, 'b') fileSize = originalScanFile.length() if fileSize > 0x24: scan = RandomAccessFile(originalScanFile, "r") scan.readFully(buffer) if (buffer[0] == 0x1F) and ( (buffer[1] & 0xFF) == 0x8B) and (buffer[2] == 0x08): scan.seek(fileSize - 8) scan.readFully(buffer, 0, 8) crc32 = getInt(buffer, 0) size = getInt(buffer, 4) deltaParams = ' -oldscanid:' + str( crc32) + ' -oldscansize:' + str(size) + ' ' index = String(commandLine).indexOf( ENTERPRISE_MODE) + String( ENTERPRISE_MODE).length() commandLine = commandLine[ 0:index] + deltaParams + commandLine[index + 1:] logger.debug('Scanner execution command updated to ', commandLine) except: logger.debugException( "Failed to calculate CRC32 and size of zipped scan file " + originalScanFile.getAbsolutePath()) finally: if scan is not None: try: scan.close() except: pass return commandLine
def useTempScanFile(Framework): tempScanFilePath = Framework.getProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE) if tempScanFilePath is None: logger.debug('No scan file found from previous scanner execution') return None logger.debug('Using scan file from previous execution:', tempScanFilePath) extension = InventoryUtils.getFileExtension(tempScanFilePath) localScanFileName = InventoryUtils.generateScanFileName(Framework, extension) #folder for scan files localScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME downloadedScanFilesDir = File(localScanFileFolderPath) downloadedScanFilesDir.mkdirs() #this scan file will be created after downloading from remote machine targetScanFile = File(downloadedScanFilesDir, localScanFileName) logger.debug('Scan file from previous execution will be moved to ', targetScanFile.getCanonicalPath()) tempScanFile = File(tempScanFilePath) if not tempScanFile.renameTo(targetScanFile): return None return targetScanFile
def checkScanFileExistance(Framework): # this step is always finished with success since we DON'T require scan file from previous execution, just nice to have Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS) DownloadScanFileBeforeExecution = Boolean.parseBoolean( Framework.getParameter('DownloadScanFileBeforeExecution')) if DownloadScanFileBeforeExecution: try: client = Framework.getConnectedClient() options = LockUtils.getClientOptionsMap(client) previousExecutionStarted = options.get( InventoryUtils.STATE_PROPERTY_EXECUTION_STARTED) if (previousExecutionStarted is None) or (len( previousExecutionStarted.strip()) == 0): logger.debug( 'Previous execution timestamp no found, continuing with workflow' ) return remoteScanFileLocation = options.get( InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILENAME) if not InventoryUtils.isPathValid(remoteScanFileLocation): logger.debug( 'No scan file path found on remote machine, continuing with workflow' ) return lastSuccessExecuton = Framework.getState().getJobLastSuccessfulRun( ) logger.debug('Last success execution ' + str(lastSuccessExecuton)) logger.debug('Remote scan file execution ' + str(previousExecutionStarted)) if long(lastSuccessExecuton) > long(previousExecutionStarted): logger.debug( 'Scan file on probe side is newer than on remote machine, skipping downloading' ) return logger.debug('Last success execution ' + str(lastSuccessExecuton) + ' older than scan file on remote machine ' + str(remoteScanFileLocation) + '. Going to download scan file:' + str(remoteScanFileLocation)) tempScanFileFolder = CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId( ) + CollectorsParameters.FILE_SEPARATOR File(tempScanFileFolder).mkdirs() extension = InventoryUtils.getFileExtension(remoteScanFileLocation) tempScanFileName = InventoryUtils.generateScanFileName( Framework, extension) tempScanFile = File(tempScanFileFolder, tempScanFileName) tempScanFilePath = tempScanFile.getCanonicalPath() logger.debug('Try to download scan file to the:', tempScanFilePath) if not InventoryUtils.copyRemoteFileToLocal( Framework, remoteScanFileLocation, tempScanFilePath, 0): logger.debug( 'Failed to download scan file before current execution') Framework.setProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE, tempScanFilePath) except: reason = str(sys.exc_info()[1]) logger.debug( 'Failed to check/download scan file from previous execution. Reason:', reason) else: logger.debug( 'Even not checking whether scan file exists on remote machine or not.' )