def getFullDataSet(runtime, jobPath, query):
"""Pull requested dataset from ITDM."""
## Load the file containing the query
queryFile = os.path.join(jobPath, 'input', query + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'JSON query file does not exist: {}'.format(queryFile))
queryContent = None
with open(queryFile) as fp:
queryContent = json.load(fp)
## Get chunked query results from the API; best to use when the data
## set can be quite large, but the second version can be used if you
## want to gather everything in one call or in a non-Flat format:
#queryResults = getApiQueryResultsInChunks(runtime, queryContent)
queryResults = getApiQueryResultsFull(
runtime,
queryContent,
resultsFormat='Nested-Simple',
headers={'removeEmptyAttributes': False})
if queryResults is None or len(queryResults) <= 0:
raise EnvironmentError(
'No results found from database; nothing to send.')
## Convert the API results into a desired layout for our needs here? No,
## we would only need to do this if using chunked/flat result format.
#transformResultsFromApi(runtime, queryResults)
## end getFullDataSet
return queryResults
def getCurrentURLs(runtime, urlList):
"""Pull URLs from the database."""
targetQuery = runtime.parameters.get('targetQuery')
queryFile = os.path.join(runtime.env.contentGatheringPkgPath,
'checkUrlResponse', 'input',
targetQuery + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'JSON query file does not exist: {}'.format(queryFile))
## Load the file containing the query for IPs
queryContent = None
with open(queryFile) as fp:
queryContent = json.load(fp)
## Get the results from the query
queryResults = utilities.getApiQueryResultsFull(runtime,
queryContent,
resultsFormat='Flat')
if queryResults is None or len(queryResults) <= 0:
raise EnvironmentError('No URLs found from database; nothing to do.')
## Convert the API results into a desired layout for our needs here
for entry in queryResults.get('objects', []):
url = entry.get('data').get('name')
urlList.append(url)
queryResults = None
## end getCurrentURLs
return
def startJob(runtime):
"""Standard job entry point.
Arguments:
runtime (dict) : object used for providing input into jobs and tracking
the job thread through the life of its runtime.
"""
try:
## Set the corresponding directories for input
jobScriptPath = os.path.dirname(os.path.realpath(__file__))
basePath = os.path.abspath(os.path.join(jobScriptPath, '..'))
orderFile = os.path.join(basePath, 'conf', 'orderedList.json')
inputPath = os.path.join(basePath, 'input')
transformPath = os.path.join(basePath, 'transform')
## Read in the conf/orderedList.json file
with open(orderFile) as fp:
orderedList = json.load(fp)
for task in orderedList:
try:
inputName = task.get('inputName')
transformName = task.get('transformName')
runtime.logger.report('Working on input query {inputName!r}', inputName=inputName)
## Read the input query and transformation descriptor files
inputFile = loadJsonFile(os.path.join(inputPath, inputName + '.json'), 'input')
transformFile = loadJsonFile(os.path.join(transformPath, transformName + '.json'), 'transform')
## Query API for the query result
queryResults = getApiQueryResultsFull(runtime, inputFile, resultsFormat='Nested-Simple', headers={'removeEmptyAttributes': False}, verify=runtime.ocpCertFile)
if queryResults is None or len(queryResults) <= 0:
runtime.logger.report('No results found for input query {inputName!r}; skipping.', inputName=inputName)
continue
runtime.logger.report(' -- queryResults: {queryResults!r}', queryResults=queryResults)
## Special case since we need Nested format; if query was only
## looking for a single object, the nested format drops down to
## a Flat format, with a list of objects.
if 'objects' in queryResults:
queryResults = queryResults['objects']
## Start the work
processThisEntry(runtime, queryResults, transformFile, transformName)
except:
runtime.logger.error('Failure with query {inputName!r}: {stacktrace!r}', inputName=inputName, stacktrace=str(sys.exc_info()[1]))
## Update the runtime status to success
if runtime.getStatus() == 'UNKNOWN':
runtime.status(1)
except:
runtime.setError(__name__)
## end startJob
return
def getQueryResults(runtime, queryName, resultsFormat):
"""Run a query to get all matching objects."""
queryFile = os.path.join(runtime.env.universalJobPkgPath, 'logicalModels',
'input', queryName + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'Missing query file specified in the parameters: {queryFile!r}',
queryFile=queryFile)
queryContent = None
with open(queryFile) as fp:
queryContent = json.load(fp)
queryResults = getApiQueryResultsFull(runtime,
queryContent,
resultsFormat=resultsFormat,
verify=runtime.ocpCertFile)
## end getQueryResults
return queryResults
def getItdmDataSet(runtime, jobPath):
"""Pull requested dataset from ITDM."""
targetQuery = runtime.parameters.get('targetQuery')
queryFile = os.path.join(jobPath, 'input', targetQuery + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError('JSON query file does not exist: {}'.format(queryFile))
## Load the file containing the query
queryContent = None
with open(queryFile) as fp:
queryContent = json.load(fp)
## Get chunked query results from the API; best to use when the data
## set can be quite large, but the second version can be used if you
## want to gather everything in one call or in a non-Flat format:
#queryResults = getApiQueryResultsInChunks(runtime, queryContent)
queryResults = getApiQueryResultsFull(runtime, queryContent, resultsFormat='Nested', headers={'removeEmptyAttributes': False})
if queryResults is None or len(queryResults) <= 0:
raise EnvironmentError('No results found from ITDM; nothing to update.')
## end getItdmDataSet
return queryResults
def getDataSet(runtime, jobPath, queryName):
"""Pull requested dataset from ITDM."""
queryFile = os.path.join(jobPath, 'input', queryName + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'JSON query file does not exist: {}'.format(queryFile))
## Load the file containing the query
queryContent = None
with open(queryFile) as fp:
queryContent = json.load(fp)
queryResults = getApiQueryResultsFull(
runtime,
queryContent,
resultsFormat='Nested-Simple',
headers={'removeEmptyAttributes': False})
if queryResults is None or len(queryResults) <= 0:
raise EnvironmentError(
'No IPs found from database; nothing to update.')
## end getDataSet
return queryResults
def startJob(runtime):
"""Standard job entry point.
Arguments:
runtime (dict) : object used for providing input into jobs and tracking
the job thread through the life of its runtime.
"""
try:
## Establish our runtime working directory
jobRuntimePath = verifyJobRuntimePath(__file__)
runtime.logger.report('path jobRuntimePath: {jobRuntimePath!r}',
jobRuntimePath=jobRuntimePath)
## Initialize a workbook
wb = Workbook()
worksheets = runtime.parameters.get('worksheets')
firstSheet = True
## Go through each worksheet definition set in the job
for definition in worksheets:
try:
queryName = definition.get('queryName')
sheetName = definition.get('sheetName')
worksheet = wb.active
if firstSheet:
worksheet.title = sheetName
firstSheet = False
else:
wb.create_sheet(sheetName)
worksheet = wb.get_sheet_by_name(sheetName)
## Read the query definition
queryFile = os.path.join(runtime.env.universalJobPkgPath,
'reportWorkbook', 'input',
queryName + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'JSON query file does not exist: {}'.format(queryFile))
queryContent = None
with open(queryFile) as fp:
queryContent = json.load(fp)
runtime.logger.report(' Requesting queryName {queryName!r}',
queryName=queryName)
queryResults = getApiQueryResultsFull(
runtime,
queryContent,
resultsFormat='Nested-Simple',
headers={'removeEmptyAttributes': False},
verify=runtime.ocpCertFile)
if queryResults is None or len(queryResults) <= 0:
runtime.logger.debug(
'No results found for queryName {queryName!r}',
queryName=queryName)
continue
## Special case since we need Nested format; if query was only
## looking for a single object, the nested format drops down to
## a Flat format, with a list of objects.
if 'objects' in queryResults:
queryResults = queryResults['objects']
## Convert the API results into a desired layout for our needs here
transformResults(runtime, queryResults, worksheet)
except:
stacktrace = traceback.format_exception(
sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
runtime.logger.error(
'Failure parsing queryName {queryName!r}: {stacktrace!r}',
queryName=queryName,
stacktrace=stacktrace)
## Save the workbook as a file
reportName = '{}.xlsx'.format(runtime.parameters.get('reportFile'))
reportFile = os.path.join(jobRuntimePath, reportName)
wb.save(reportFile)
## Update the runtime status to success
if runtime.getStatus() == 'UNKNOWN':
runtime.status(1)
except:
runtime.setError(__name__)
## end startJob
return
def processResults(runtime, mappingEntries, queryContent):
"""Get all nodes with snmp_oid values; update any that match a mapping."""
try:
queryResults = getApiQueryResultsFull(runtime,
queryContent,
resultsFormat='Flat',
verify=runtime.ocpCertFile)
nodeResults = queryResults.get('objects', [])
runtime.logger.info('Found {} base nodes with SNMP OID values'.format(
len(nodeResults)))
## Loop through the target nodes
for result in nodeResults:
## Get node attributes
identifier = result.get('identifier')
nodeData = result.get('data', {})
nodeName = nodeData.get('hostname')
deviceOID = nodeData.get('snmp_oid')
runtime.logger.report(
' node: {name!r} with OID {oid!r} has ID: {identifier!r}',
name=nodeName,
oid=deviceOID,
identifier=identifier)
## Loop through the mapping entries
foundMatch = False
for entry in mappingEntries:
ref = entry.get('deviceTypeForReferenceOnly')
matchingSection = entry.get('matchingSection', {})
## Matching section
snmpOID = matchingSection.get('snmpOID')
compareType = matchingSection.get('compareType')
compareValue = matchingSection.get('compareValue')
if compareType == '==':
if deviceOID == snmpOID:
foundMatch = True
elif compareType.lower() == 'regex':
escapedOID = snmpOID.replace('.', '[.]')
value = compareValue.replace('snmpOID', escapedOID, re.I)
if re.search(value, deviceOID):
foundMatch = True
else:
runtime.logger.info(
'Unknown compare type for mapping {}. Received {} and expected either "==" or "regEx".'
.format(ref, compareType))
## If no match, continue to the next mapping definition
if not foundMatch:
continue
## Mapping section
mappingSection = entry.get('mappingSection', {})
nodeType = mappingSection.get('type')
attributeValueOverrides = mappingSection.get(
'attributeValueOverrides', {})
attributeValuesToUseWhenEmpty = mappingSection.get(
'attributeValuesToUseWhenEmpty', {})
attrs = {}
## Explicitly set domain to null if it was null before
nodeDomain = nodeData.get('domain')
attrs['hostname'] = nodeName
attrs['domain'] = nodeDomain
## Update static attribute values
for key, value in attributeValuesToUseWhenEmpty.items():
## The data section by default will not remove null values
if key not in nodeData:
attrs[key] = value
for key, value in attributeValueOverrides.items():
attrs[key] = value
runtime.logger.report(
' updating subtype for {}'.format(nodeName))
runtime.logger.report(' ==================')
runtime.logger.report(' previous data:')
for key, value in nodeData.items():
runtime.logger.report(' {}: {}'.format(key, value))
runtime.logger.report(' attribute updates:')
for key, value in attrs.items():
runtime.logger.report(' {}: {}'.format(key, value))
runtime.logger.report(' ==================')
## Update the object and subtype accordingly
runtime.results.addObject(nodeType,
uniqueId=identifier,
**attrs)
break
except:
runtime.setError(__name__)
## end processResults
return
def getDeltaDataSet(runtime, jobPath, query, fullQuery,
previousRuntimeStatistics, deltaSyncExpirationInDays):
"""Pull partial/delta dataset from ITDM, based on last successful run."""
## First check the previous runtime stats to direct the type of job. The job
## params asked for delta, but if we didn't have a successful in the past,
## then this needs to redirect to a full sync instead.
runtime.logger.report('SNOW JOB Stats: {stats!r}',
stats=previousRuntimeStatistics)
lastSuccess = previousRuntimeStatistics.get('date_last_success')
lastSuccessDate = None
redirectToFull = False
if lastSuccess is None:
redirectToFull = True
runtime.logger.info(
'Redirecting this delta-sync to full-sync; previous runtime statistics do not hold a value for date_last_success.'
)
else:
## The deltaSyncExpirationInDays variable holds the number of days since
## the last successful delta sync, before the job redirects to full sync
#lastSuccessDate = datetime.datetime.strptime(lastSuccess, '%Y-%m-%d %H:%M:%S')
lastSuccessDate = arrow.get(lastSuccess, 'YYYY-MM-DD HH:mm:ss')
expirationDate = arrow.utcnow().shift(
days=-(deltaSyncExpirationInDays)).datetime
if expirationDate > lastSuccessDate:
redirectToFull = True
runtime.logger.info(
'Redirecting this delta-sync to full-sync; value for \'date_last_success\'={lastSuccess!r} is past the number of days allowed in the user provided parameter \'deltaSyncExpirationInDays\'={deltaSyncExpirationInDays!r}.',
lastSuccess=lastSuccess,
deltaSyncExpirationInDays=deltaSyncExpirationInDays)
if redirectToFull or lastSuccessDate is None:
runtime.logger.info('Redirecting Delta sync to Full sync')
return (getFullDataSet(runtime, jobPath, fullQuery))
## Load the file containing the query
queryFile = os.path.join(jobPath, 'input', query + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'JSON query file does not exist: {}'.format(queryFile))
queryContent = None
with open(queryFile, 'r') as fp:
queryContent = fp.read()
## Replace the <VALUE1> and <VALUE2> placeholders in the in betweendate
## operation, with the proper formats:
## VALUE1 = the last successful runtime of the job, which should be in UTC
## VALUE2 = current UTC time sent in the expected string format
utcNow = arrow.utcnow().format('YYYY-MM-DD HH:mm:ss')
runtime.logger.info(
'Using this time comparison for deta-sync query: time_created is between {lastSuccess!r} and {utcNow!r}...',
lastSuccess=lastSuccess,
utcNow=utcNow)
# lastSuccess = "2019-07-26 09:39:00"
# utcNow = "2019-07-10 09:39:00"
queryContent = queryContent.replace('<VALUE1>',
'"{} UTC"'.format(lastSuccess))
queryContent = queryContent.replace('<VALUE2>', '"{} UTC"'.format(utcNow))
queryResults = getApiQueryResultsFull(
runtime,
queryContent,
resultsFormat='Nested-Simple',
headers={'removeEmptyAttributes': False})
if queryResults is None or len(queryResults) <= 0:
raise EnvironmentError(
'No results found from database; nothing to send.')
## end getDeltaDataSet
return queryResults
def issueApiCall(runtime):
endpoint = None
## Get the parameters
inputQuery = runtime.parameters.get('inputQuery')
targetIp = runtime.parameters.get('endpointIp')
## Corresponding directory for our input query
jobScriptPath = os.path.dirname(os.path.realpath(__file__))
jobPath = os.path.abspath(os.path.join(jobScriptPath, '..'))
## Load the file containing the query
queryFile = os.path.join(jobPath, 'input', inputQuery + '.json')
if not os.path.isfile(queryFile):
raise EnvironmentError(
'JSON query file does not exist: {}'.format(queryFile))
queryContent = None
with open(queryFile, 'r') as fp:
queryContent = fp.read()
## We do not know shell type, and in the future there may be more. So we
## will loop through each one, trying until we find it.
## Note: previously this was sent in by the endpoint query, but when we
## switched over to make the IP a parameter instead... the shell details are
## no longer sent into the job. Hence this step to request from the API.
## The idea was to simplify user experience by adding more code.
queryResults = None
for shellType in validShellTypes:
thisQuery = queryContent
## Replace <VALUE1> and <VALUE2> placeholders:
## VALUE1 = the shell type: PowerShell or SSH
## VALUE2 = the IP address of the target endpoint you wish to template
thisQuery = thisQuery.replace('<VALUE1>', '"{}"'.format(shellType))
thisQuery = thisQuery.replace('<VALUE2>', '"{}"'.format(targetIp))
queryResults = getApiQueryResultsFull(
runtime,
thisQuery,
resultsFormat='Nested',
headers={'removeEmptyAttributes': False},
verify=runtime.ocpCertFile)
if queryResults is not None and len(queryResults[shellType]) > 0:
break
if queryResults is None or len(queryResults[shellType]) <= 0:
raise EnvironmentError(
'Could not find endpoint {} with a correspoinding shell. Please make sure to run the corresponding "Find" job first.'
.format(targetIp))
runtime.logger.report('queryResults: {queryResults!r}...',
queryResults=queryResults)
## Set the protocol data on runtime, as though it was sent in regularly.
## And don't use 'get'; allow exceptions here if endpoint values are missing
endpoint = queryResults[shellType][0]
runtime.endpoint = {
"class_name": endpoint["class_name"],
"identifier": endpoint["identifier"],
"data": endpoint["data"]
}
runtime.setParameters()
return endpoint