def downloadScanFile(Framework):
targetScanFile = downloadRemoteScanFile(Framework)
if not targetScanFile:
logger.debug('Remote scan file was not downloaded, will try previously downloaded scan file(if exists)')
targetScanFile = useTempScanFile(Framework)
if not targetScanFile:
logger.debug('No scan file downloaded from previous execution, download file step failed')
Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE)
Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_REMOTE_SCANFILE_NOT_FOUND, None)
return
logger.debug('Scan file was successfully downloaded to ', targetScanFile.getCanonicalPath())
#set download time to current time
Framework.setProperty(InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILE_DOWNLOAD_TIME, Date())
#Check the drity files in sending folder
sendingFolder = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.SENDING_FOLDER_NAME
deleteDirtyFile(File(sendingFolder, targetScanFile.getName()))
#delete temporary scan file from previous execution (if any)
tempScanFileFolder = CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId() + CollectorsParameters.FILE_SEPARATOR
tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_EXTENTION)
deleteDirtyFile(File(tempScanFileFolder, tempScanFileName))
tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_DELTA_EXTENTION)
deleteDirtyFile(File(tempScanFileFolder, tempScanFileName))
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def downloadScanFile(Framework):
targetScanFile = downloadRemoteScanFile(Framework)
if not targetScanFile:
logger.debug("Remote scan file was not downloaded, will try previously downloaded scan file(if exists)")
targetScanFile = useTempScanFile(Framework)
if not targetScanFile:
logger.debug("No scan file downloaded from previous execution, download file step failed")
Framework.setStepExecutionStatus(WorkflowStepStatus.FATAL_FAILURE)
Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_REMOTE_SCANFILE_NOT_FOUND, None)
return
logger.debug("Scan file was successfully downloaded to ", targetScanFile.getCanonicalPath())
# set download time to current time
Framework.setProperty(InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILE_DOWNLOAD_TIME, Date())
# Check the drity files in sending folder
sendingFolder = (
CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.SENDING_FOLDER_NAME
)
deleteDirtyFile(File(sendingFolder, targetScanFile.getName()))
# delete temporary scan file from previous execution (if any)
tempScanFileFolder = (
CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD
+ Framework.getDiscoveryJobId()
+ CollectorsParameters.FILE_SEPARATOR
)
tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_EXTENTION)
deleteDirtyFile(File(tempScanFileFolder, tempScanFileName))
tempScanFileName = InventoryUtils.generateScanFileName(Framework, InventoryUtils.SCANFILE_DELTA_EXTENTION)
deleteDirtyFile(File(tempScanFileFolder, tempScanFileName))
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def downloadRemoteScanFile(Framework):
remoteScanFileLocation = Framework.getProperty(InventoryUtils.STATE_PROPERTY_REMOTE_SCAN_FILE_LOCATION)
# download scanner log file before downloading scan file
downloadScanLogFile(Framework)
if remoteScanFileLocation is None:
logger.debug("No scan file to downloaded from current execution")
return None
logger.debug("About to download scan file from current execution:", remoteScanFileLocation)
extension = InventoryUtils.getFileExtension(remoteScanFileLocation)
localScanFileName = InventoryUtils.generateScanFileName(Framework, extension)
# folder for scan files
localScanFileFolderPath = (
CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME
)
downloadedScanFilesDir = File(localScanFileFolderPath)
downloadedScanFilesDir.mkdirs()
# this scan file will be created after downloading from remote machine
targetScanFile = File(downloadedScanFilesDir, localScanFileName)
# get file to the local machine
logger.debug("Scan file to be downloaded to location:", targetScanFile.getCanonicalPath())
if not InventoryUtils.copyRemoteFileToLocal(Framework, remoteScanFileLocation, targetScanFile.getCanonicalPath()):
return None
return targetScanFile
def useTempScanFile(Framework):
tempScanFilePath = Framework.getProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE)
if tempScanFilePath is None:
logger.debug("No scan file found from previous scanner execution")
return None
logger.debug("Using scan file from previous execution:", tempScanFilePath)
extension = InventoryUtils.getFileExtension(tempScanFilePath)
localScanFileName = InventoryUtils.generateScanFileName(Framework, extension)
# folder for scan files
localScanFileFolderPath = (
CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME
)
downloadedScanFilesDir = File(localScanFileFolderPath)
downloadedScanFilesDir.mkdirs()
# this scan file will be created after downloading from remote machine
targetScanFile = File(downloadedScanFilesDir, localScanFileName)
logger.debug("Scan file from previous execution will be moved to ", targetScanFile.getCanonicalPath())
tempScanFile = File(tempScanFilePath)
if not tempScanFile.renameTo(targetScanFile):
return None
return targetScanFile
def downloadRemoteScanFile(Framework):
remoteScanFileLocation = Framework.getProperty(InventoryUtils.STATE_PROPERTY_REMOTE_SCAN_FILE_LOCATION)
# download scanner log file before downloading scan file
downloadScanLogFile(Framework)
if remoteScanFileLocation is None:
logger.debug('No scan file to downloaded from current execution')
return None
logger.debug('About to download scan file from current execution:', remoteScanFileLocation)
extension = InventoryUtils.getFileExtension(remoteScanFileLocation)
localScanFileName = InventoryUtils.generateScanFileName(Framework, extension)
#folder for scan files
localScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME
downloadedScanFilesDir = File(localScanFileFolderPath)
downloadedScanFilesDir.mkdirs()
#this scan file will be created after downloading from remote machine
targetScanFile = File(downloadedScanFilesDir, localScanFileName)
#get file to the local machine
logger.debug('Scan file to be downloaded to location:', targetScanFile.getCanonicalPath())
if not InventoryUtils.copyRemoteFileToLocal(Framework, remoteScanFileLocation, targetScanFile.getCanonicalPath()):
return None
return targetScanFile
def updateCmdForDeltaScanning(commandLine, Framework):
originalScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.ORIGINAL_FOLDER_NAME
originalScanFile = File(originalScanFileFolderPath, InventoryUtils.generateScanFileName(Framework))
if originalScanFile.exists():
scan = None
try:
try:
buffer = jarray.zeros(0x24, 'b')
fileSize = originalScanFile.length()
if fileSize > 0x24:
scan = RandomAccessFile(originalScanFile, "r")
scan.readFully(buffer)
if (buffer[0] == 0x1F) and ((buffer[1] & 0xFF) == 0x8B) and (buffer[2] == 0x08):
scan.seek(fileSize - 8)
scan.readFully(buffer, 0, 8)
crc32 = getInt(buffer, 0)
size = getInt(buffer, 4)
deltaParams = ' -oldscanid:' + str(crc32) + ' -oldscansize:' + str(size) + ' '
index = String(commandLine).indexOf(ENTERPRISE_MODE) + String(ENTERPRISE_MODE).length()
commandLine = commandLine[0:index] + deltaParams + commandLine[index + 1:]
logger.debug('Scanner execution command updated to ', commandLine)
except:
logger.debugException("Failed to calculate CRC32 and size of zipped scan file " + originalScanFile.getAbsolutePath())
finally:
if scan is not None:
try:
scan.close()
except:
pass
return commandLine
def checkEnrichedFileExisted(Framework):
localScanFileName = InventoryUtils.generateScanFileName(Framework)
localScanFileSendingFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.SENDING_FOLDER_NAME
targetScanFile = File(localScanFileSendingFolderPath, localScanFileName)
if not targetScanFile.exists():
logger.debug('No processed scan file yet. XML-Enricher is still running.')
Framework.reportError(inventoryerrorcodes.INVENTORY_DISCOVERY_ENRICHED_SCANFILE_NOTREADY, [localScanFileName])
Framework.setStepExecutionStatus(WorkflowStepStatus.FAILURE)
else:
logger.debug('find processed scan file, goto next step')
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
def checkScanFileExistance(Framework):
# this step is always finished with success since we DON'T require scan file from previous execution, just nice to have
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
DownloadScanFileBeforeExecution = Boolean.parseBoolean(Framework.getParameter('DownloadScanFileBeforeExecution'))
if DownloadScanFileBeforeExecution:
try:
client = Framework.getConnectedClient()
options = LockUtils.getClientOptionsMap(client)
previousExecutionStarted = options.get(InventoryUtils.STATE_PROPERTY_EXECUTION_STARTED)
if (previousExecutionStarted is None) or (len(previousExecutionStarted.strip()) == 0):
logger.debug('Previous execution timestamp no found, continuing with workflow')
return
remoteScanFileLocation = options.get(InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILENAME)
if not InventoryUtils.isPathValid(remoteScanFileLocation):
logger.debug('No scan file path found on remote machine, continuing with workflow')
return
lastSuccessExecuton = Framework.getState().getJobLastSuccessfulRun()
logger.debug('Last success execution ' + str(lastSuccessExecuton))
logger.debug('Remote scan file execution ' + str(previousExecutionStarted))
if long(lastSuccessExecuton) > long(previousExecutionStarted):
logger.debug('Scan file on probe side is newer than on remote machine, skipping downloading')
return
logger.debug('Last success execution ' + str(lastSuccessExecuton) + ' older than scan file on remote machine ' + str(remoteScanFileLocation) + '. Going to download scan file:' + str(remoteScanFileLocation))
tempScanFileFolder = CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId() + CollectorsParameters.FILE_SEPARATOR
File(tempScanFileFolder).mkdirs()
extension = InventoryUtils.getFileExtension(remoteScanFileLocation)
tempScanFileName = InventoryUtils.generateScanFileName(Framework, extension)
tempScanFile = File(tempScanFileFolder, tempScanFileName)
tempScanFilePath = tempScanFile.getCanonicalPath()
logger.debug('Try to download scan file to the:', tempScanFilePath)
if not InventoryUtils.copyRemoteFileToLocal(Framework, remoteScanFileLocation, tempScanFilePath, 0):
logger.debug('Failed to download scan file before current execution')
Framework.setProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE, tempScanFilePath)
except:
reason = str(sys.exc_info()[1])
logger.debug('Failed to check/download scan file from previous execution. Reason:', reason)
else:
logger.debug('Even not checking whether scan file exists on remote machine or not.')
def updateCmdForDeltaScanning(commandLine, Framework):
originalScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.ORIGINAL_FOLDER_NAME
originalScanFile = File(originalScanFileFolderPath,
InventoryUtils.generateScanFileName(Framework))
if originalScanFile.exists():
scan = None
try:
try:
buffer = jarray.zeros(0x24, 'b')
fileSize = originalScanFile.length()
if fileSize > 0x24:
scan = RandomAccessFile(originalScanFile, "r")
scan.readFully(buffer)
if (buffer[0] == 0x1F) and (
(buffer[1] & 0xFF) == 0x8B) and (buffer[2] == 0x08):
scan.seek(fileSize - 8)
scan.readFully(buffer, 0, 8)
crc32 = getInt(buffer, 0)
size = getInt(buffer, 4)
deltaParams = ' -oldscanid:' + str(
crc32) + ' -oldscansize:' + str(size) + ' '
index = String(commandLine).indexOf(
ENTERPRISE_MODE) + String(
ENTERPRISE_MODE).length()
commandLine = commandLine[
0:index] + deltaParams + commandLine[index + 1:]
logger.debug('Scanner execution command updated to ',
commandLine)
except:
logger.debugException(
"Failed to calculate CRC32 and size of zipped scan file " +
originalScanFile.getAbsolutePath())
finally:
if scan is not None:
try:
scan.close()
except:
pass
return commandLine
def useTempScanFile(Framework):
tempScanFilePath = Framework.getProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE)
if tempScanFilePath is None:
logger.debug('No scan file found from previous scanner execution')
return None
logger.debug('Using scan file from previous execution:', tempScanFilePath)
extension = InventoryUtils.getFileExtension(tempScanFilePath)
localScanFileName = InventoryUtils.generateScanFileName(Framework, extension)
#folder for scan files
localScanFileFolderPath = CollectorsParameters.PROBE_MGR_INVENTORY_XMLENRICHER_FILES_FOLDER + XmlEnricherConstants.INCOMING_FOLDER_NAME
downloadedScanFilesDir = File(localScanFileFolderPath)
downloadedScanFilesDir.mkdirs()
#this scan file will be created after downloading from remote machine
targetScanFile = File(downloadedScanFilesDir, localScanFileName)
logger.debug('Scan file from previous execution will be moved to ', targetScanFile.getCanonicalPath())
tempScanFile = File(tempScanFilePath)
if not tempScanFile.renameTo(targetScanFile):
return None
return targetScanFile
def checkScanFileExistance(Framework):
# this step is always finished with success since we DON'T require scan file from previous execution, just nice to have
Framework.setStepExecutionStatus(WorkflowStepStatus.SUCCESS)
DownloadScanFileBeforeExecution = Boolean.parseBoolean(
Framework.getParameter('DownloadScanFileBeforeExecution'))
if DownloadScanFileBeforeExecution:
try:
client = Framework.getConnectedClient()
options = LockUtils.getClientOptionsMap(client)
previousExecutionStarted = options.get(
InventoryUtils.STATE_PROPERTY_EXECUTION_STARTED)
if (previousExecutionStarted is None) or (len(
previousExecutionStarted.strip()) == 0):
logger.debug(
'Previous execution timestamp no found, continuing with workflow'
)
return
remoteScanFileLocation = options.get(
InventoryUtils.AGENT_OPTION_DISCOVERY_SCANFILENAME)
if not InventoryUtils.isPathValid(remoteScanFileLocation):
logger.debug(
'No scan file path found on remote machine, continuing with workflow'
)
return
lastSuccessExecuton = Framework.getState().getJobLastSuccessfulRun(
)
logger.debug('Last success execution ' + str(lastSuccessExecuton))
logger.debug('Remote scan file execution ' +
str(previousExecutionStarted))
if long(lastSuccessExecuton) > long(previousExecutionStarted):
logger.debug(
'Scan file on probe side is newer than on remote machine, skipping downloading'
)
return
logger.debug('Last success execution ' + str(lastSuccessExecuton) +
' older than scan file on remote machine ' +
str(remoteScanFileLocation) +
'. Going to download scan file:' +
str(remoteScanFileLocation))
tempScanFileFolder = CollectorsParameters.PROBE_MGR_TEMPDOWNLOAD + Framework.getDiscoveryJobId(
) + CollectorsParameters.FILE_SEPARATOR
File(tempScanFileFolder).mkdirs()
extension = InventoryUtils.getFileExtension(remoteScanFileLocation)
tempScanFileName = InventoryUtils.generateScanFileName(
Framework, extension)
tempScanFile = File(tempScanFileFolder, tempScanFileName)
tempScanFilePath = tempScanFile.getCanonicalPath()
logger.debug('Try to download scan file to the:', tempScanFilePath)
if not InventoryUtils.copyRemoteFileToLocal(
Framework, remoteScanFileLocation, tempScanFilePath, 0):
logger.debug(
'Failed to download scan file before current execution')
Framework.setProperty(InventoryUtils.STATE_PROPERTY_TEMP_SCAN_FILE,
tempScanFilePath)
except:
reason = str(sys.exc_info()[1])
logger.debug(
'Failed to check/download scan file from previous execution. Reason:',
reason)
else:
logger.debug(
'Even not checking whether scan file exists on remote machine or not.'
)
