def getFirstToken():
configuration, configuration_settings = readJSON()
if (configuration):
token = generateToken_pw_flow(configuration_settings)
return token
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def checkStatus(configuration_settings, token, analyzerId):
# Make request
headers = {'Authorization': 'Bearer {}'.format(token)}
get_url = configuration_settings['aca_main_url'] + "/{0}".format(
analyzerId)
try:
response = requests.request(
"GET",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
if "result" in response_json:
return True, response_json["result"][0]
else:
return False, {}
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
def deleteFile(configuration_settings, analyzerId):
# Make requests
headers = {'apiKey': configuration_settings['api_key']}
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
headers['Authorization'] = 'Basic %s' % encoded_credentials.decode(
"utf-8")
get_url = configuration_settings['main_url'] + "/{0}".format(analyzerId)
try:
response = requests.request(
"DELETE",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
return True, response_json
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
def generateToken_pw_flow(config):
zen_token_url = 'https://' + config['zen_host'].strip(
) + '/v1/preauth/validateAuth'
zen_username = config['zen_username'].strip()
zen_pw = config['zen_password'].strip()
try:
encoding = base64.b64encode('{}:{}'.format(zen_username,
zen_pw).encode())
headers = {
"authorization": "Basic " + encoding.decode("utf-8"),
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("GET",
zen_token_url,
headers=headers,
verify=config['ssl_verification'])
if response.status_code == 200:
# token is generated
# {
# "accessToken": "xxxxxxxxx"
# }
content = getContent(response)
# print(content)
token = content.get('accessToken')
logger.info('Successfully got the token')
json.dump({"zen_token": token}, open("output.json", 'w'), indent=4)
return token
else:
logger.error("Failed to generate a token")
content = getContent(response)
logger.debug(content)
return None
except Exception as ex:
logger.error("Failed to generate a token due to exception")
logger.debug(str(ex))
return None
def checkTokenValid(token, config):
if token:
if config:
zen_check_token_url = 'https://' + config['zen_host'].strip(
) + '/usermgmt/v1/user/currentUserInfo'
try:
headers = {
"authorization": "Bearer " + token,
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("GET",
zen_check_token_url,
headers=headers,
verify=config['ssl_verification'])
if response.status_code == 200:
# content = getContent(response)
# res = content.get('uid')
logger.info('zen token provided is valid')
return token
elif response.status_code == 401:
logger.info(
'zen token provided is invalid or expired, trying to generate a new one'
)
token = generateToken_pw_flow(config)
return token
else:
logger.error("Failed to check the token valid or not")
content = getContent(response)
logger.debug(content)
return None
except Exception as ex:
logger.error(
"Failed to check the token valid or not due to exception")
logger.debug(str(ex))
return None
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return None
else:
logger.error("Zen token is required, please provide a valid token")
return None
def downloadFile(configuration_settings, analyzerId, output, output_path, filename,bearerToken):
# Make request
headers = {
'apiKey': configuration_settings['api_key']
}
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (
configuration_settings["function_id"], configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
#headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8")
#headers['Authorization'] = configuration_settings["bear_token"]
headers['Authorization'] = bearerToken
extra_path = "{0}/{1}/".format(analyzerId, output.lower())
get_url = urlparse.urljoin(configuration_settings['main_url'], extra_path)
try:
response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
file_name = copy.copy(filename)
filename_output = "{0}.{1}".format(file_name, output.lower()) if output.lower() != "utf8" else "{0}.txt".format(file_name)
filename_output = filename_output if output.lower() != "pdf" else "New_{0}.{1}".format(file_name, output.lower())
file_output_path = os.path.join(configuration_settings["output_directory_path"], output_path, output.lower())
filename_output_path = os.path.join(file_output_path, filename_output)
if( not os.path.exists(file_output_path)):
os.makedirs(file_output_path)
logger.info("Created new output directory, "+ file_output_path)
if(output.lower() == "json"):
response_output = json.loads(response.text)
if "data" in response_output:
json.dump(response_output["data"], open(filename_output_path, "w"), indent=4)
elif output.lower() == "pdf":
output_write = open(filename_output_path, "wb")
output_write.write(response.content)
output_write.close()
else:
output_write = open(filename_output_path, "w")
try:
output_write.write(response.text)
except:
output_write.write(response.text.encode('utf-8'))
output_write.close()
return True, ""
except SSLError as sslerror:
logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.")
logger.debug(sslerror, exc_info=True)
except Exception as ex:
logger.error("An error occurred when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, ""
def downloadFile(configuration_settings, token, analyzerId, output,
output_path, filename):
# Make request
headers = {'Authorization': 'Bearer {}'.format(token)}
extra_path = "/{0}/{1}".format(analyzerId, output.lower())
get_url = configuration_settings['aca_main_url'] + extra_path
try:
response = requests.request(
"GET",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
# print(response.text)
if response.status_code >= 400:
logger.error("An error occurred while trying to download output.")
return False, response.text
else:
file_name = copy.copy(filename)
filename_output = "{0}.{1}".format(file_name, output.lower(
)) if output.lower() != "utf8" else "{0}.txt".format(file_name)
filename_output = filename_output if output.lower(
) != "pdf" else "New_{0}.{1}".format(file_name, output.lower())
file_output_path = os.path.join(
configuration_settings["output_directory_path"], output_path,
output.lower())
filename_output_path = os.path.join(file_output_path,
filename_output)
if (not os.path.exists(file_output_path)):
os.makedirs(file_output_path)
logger.info("Created new output directory, " +
file_output_path)
if (output.lower() == "json"):
response_output = json.loads(response.text)
if "result" in response_output and "data" in response_output[
"result"][0]:
json.dump(response_output["result"][0]["data"],
open(filename_output_path, "w"),
indent=4)
elif output.lower() == "pdf":
output_write = open(filename_output_path, "wb")
output_write.write(response.content)
output_write.close()
return True, ""
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
logger.error("An error occurred when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, ""
def deleteFiles(bearerToken):
configuration, configuration_settings = readJSON()
if (configuration):
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "data" in json.loads(result["response"]) and "analyzerId" in \
json.loads(result["response"])["data"]:
if ("deleted" in result and result["deleted"]
== False) or "deleted" not in result:
response = json.loads(result["response"])
analyzerId = response["data"]["analyzerId"]
status, result_response = deleteFile(
configuration_settings, analyzerId,
bearerToken)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
def checkStatus(configuration_settings, analyzerId,bearerToken):
logger.info("Config check")
# Make request
headers = {
'apiKey': configuration_settings['api_key']
}
logger.info("Config check" + configuration_settings['api_key'])
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (
configuration_settings["function_id"], configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
#headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8")
#headers['Authorization'] = configuration_settings["bear_token"]
headers['Authorization'] = bearerToken
get_url = urlparse.urljoin(configuration_settings['main_url'], "{0}/".format(analyzerId))
logger.info("Config check " + get_url)
try:
response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification'])
logger.info(response.status_code)
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
return True, response_json
except SSLError as sslerror:
logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.")
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
def deleteFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "result" in json.loads(
result["response"]
) and "data" in json.loads(
result["response"]
)["result"][0] and "analyzerId" in json.loads(
result["response"])["result"][0]["data"]:
if ("deleted" in result and result["deleted"]
== False) or "deleted" not in result:
response = json.loads(result["response"])
# print(response)
analyzerId = response["result"][0]["data"][
"analyzerId"]
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
status, result_response = deleteFile(
configuration_settings, token,
analyzerId)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
else:
token = generateToken_pw_flow(
configuration_settings)
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json,
open("output.json", 'w'),
indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
else:
logger.error("Zen token is required to delete files")
def executeRules():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["output_directory_path"]
count = 0
errors = []
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
#print(file)
file_split = new_file.rsplit(".")
if (file_split[1] == "json"):
odm_file_value = str(file_split[0]) + "ODM.json"
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
#print(new_file_path)
f = open(new_file_path, "r")
# deserializes into dict
# and returns dict.
y = json.loads(f.read())
#json.append()
odmData = {
'__DecisionID__': "myworld",
"document_in": y,
"metaData_inout": {
"dynamicParams": "string"
}
}
#var rex = new RegExp('Sensitivity":false', 'g');
#var BACA_STRING_RESPONSE = JSON.stringify(res.data)
#BACA_STRING_RESPONSE = BACA_STRING_RESPONSE.replace(rex, 'Sensitivity":0');
odmJson = json.dumps(odmData)
#print (odmJson)
odmJson = odmJson.replace('"Sensitivity": false',
'"Sensitivity":0')
odmJson = odmJson.replace('Language": ""', 'Language":[]')
#odmpayload = {"__DecisionID__": "string","document_in": y, "metaData_inout": {"dynamicParams": "" }}
authPayload = HTTPBasicAuth(
configuration_settings["odmUser"],
configuration_settings["odmPassword"])
ODM_ProcessBacaDocument_ServiceURL = configuration_settings[
"odm_url"]
#JsonPayload = odmpayload
headers = {'content-type': 'application/json'}
#print(odmData)
try:
response = requests.request(
"POST",
ODM_ProcessBacaDocument_ServiceURL,
headers=headers,
data=odmJson,
auth=authPayload,
verify=False)
print(response.text)
jsonPayloadODM = json.loads(response.text)
fileOutPutPath = configuration_settings[
"odm_output_directory_path"] + "/" + odm_file_value
with open(fileOutPutPath, 'w') as outfile:
json.dump(jsonPayloadODM, outfile)
#return response
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
#dict_object.update({"error": str(sslerror)})
#errors.append(dict_object)
return False
except Exception as ex:
#dict_object.update({"error": str(ex)})
#errors.append(dict_object)
#logger.error("An error occurred when trying to upload file " + file_path)
logger.debug(ex, exc_info=True)
pass
f.close()
return True
def downloadFiles():
pending_completion = True
configuration, configuration_settings = readJSON()
if (configuration):
loop = 0
failed_download = []
completed_download = []
completed_count = 0
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
while pending_completion and loop < 1000:
output_json = json.load(open(output_json_path, "r"))
loop += 1
logger.info("Loop " + str(loop))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "data" in json.loads(
result["response"]
) and "analyzerId" in json.loads(
result["response"])["data"]:
if "download_success" not in result:
response = json.loads(result["response"])
path = result["path"]
filename = result["filename"]
analyzerId = response["data"]["analyzerId"]
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
if ("download_completed" not in result):
completed = checkCompleted(
latest_output_outputs, result)
if (completed):
result["download_completed"] = True
else:
logger.info(
"Checking status of analyzerId: "
+ analyzerId)
status, result_response = checkStatus(
configuration_settings,
analyzerId)
if (status):
if ("data" in result_response
and "statusDetails" in
result_response["data"]
):
status_result_response = result_response[
"data"][
"statusDetails"]
done_output = []
for output in status_result_response:
if (output["type"] in
latest_output_outputs
and output[
"status"]
== "Completed"
and
output["type"]
not in result):
logger.info(
"Downloading {0} of analyzerId: {1}"
.format(
output[
"type"],
analyzerId
))
response, reason = downloadFile(
configuration_settings,
analyzerId,
output["type"],
path,
filename.
rsplit(".")[0])
result[output[
"type"]] = response
if (not response):
result[
output[
"type"]
+
"_error"] = reason
done_output.append(
output["type"])
elif (output["type"] in
latest_output_outputs
and
output["status"]
== "Failed" and
output["type"]
not in result):
result[output[
"type"]] = False
result[
output["type"]
+
"_error"] = output[
"status"]
done_output.append(
output["type"])
elif output[
"type"] in result:
done_output.append(
output["type"])
if (len(
done_output
) == len(
latest_output_outputs
)):
completed_download.append(
True)
completed = checkCompleted(
latest_output_outputs,
result)
if (completed):
result[
"download_completed"] = True
else:
result[
"download_success"] = False
result[
"download_failure_reason"] = result_response
failed_download.append(True)
else:
logger.error(
"We could not find any information to download files from."
)
result["download_success"] = False
result[
"download_failure_reason"] = "No available data to download from"
failed_download.append(True)
except:
result["download_success"] = False
result[
"download_failure_reason"] = "No analyzerID available to download results"
logger.error(
"No analyzerID available to download results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
failed_download.append(True)
# new_output_json_result.append(result)
# print(len(new_output_json_result))
endtime = dt.datetime.now()
output_json["output_results"] = output_json_result
output_json["endtime"] = str(endtime)
json.dump(output_json,
open("output.json", 'w'),
indent=4)
completed_count = len(failed_download) + len(
completed_download)
if (completed_count >= len(output_json_result)):
pending_completion = False
else:
if (loop >= 1000):
pending_completion = False
logger.error(
"Reached maximum number of download retries.")
else:
time.sleep(5)
else:
pending_completion = False
logger.error("No results available to download.")
return True
else:
logger.info(
"Done downloading all output files to your output_directory_path"
)
logger.info("Download status reported in output.json")
if (loop < 999):
return True
else:
return False
else:
logger.error(
"output.json file does not exist. No results available to download."
)
return False
return None
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return None
else:
logger.error("Zen token is required, please provide a valid token")
return None
def getFirstToken():
configuration, configuration_settings = readJSON()
if (configuration):
token = generateToken_pw_flow(configuration_settings)
return token
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
if __name__ == '__main__':
logger.info("Logs can be found in current directory (processing.log)")
token = None
try:
getFirstToken()
except Exception as ex:
logger.error("An error occurred, please check logs for more details")
logger.debug(ex, exc_info=True)
def updateReport():
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
new_output_json_result = []
unfinished_files = []
unfinished_files_with_path = []
new_file_upload_errors = [
output["filename"] for output in output_json["upload_errors"]
] if "upload_errors" in output_json else []
new_file_errors = [
output["filename"] for output in output_json["errors"]
] if "errors" in output_json else []
unfinished_files.extend(new_file_upload_errors)
unfinished_files.extend(new_file_errors)
new_file_upload_errors_with_path = [
{
"filename": output["filename"],
"path": output["path"]
} for output in output_json["upload_errors"]
] if "upload_errors" in output_json else []
new_file_errors_with_path = [{
"filename": output["filename"],
"path": output["path"]
} for output in output_json["errors"]
] if "errors" in output_json else []
unfinished_files_with_path.extend(new_file_upload_errors_with_path)
unfinished_files_with_path.extend(new_file_errors_with_path)
endtime = dt.datetime.now()
starttime = parser.parse(output_json["starttime"])
if ("output_results" in output_json):
output_json_result = output_json["output_results"]
if (len(output_json["output_results"]) > 0):
for outresult in output_json_result:
result = outresult
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
unfinished_object = {
"filename":
result["filename"],
"path":
result["path"] if "path" in result else "",
"full_path":
result["full_path"] if "full_path" in result else ""
}
try:
if ("download_success" in result
and result["download_success"] == False):
if (result["filename"] not in unfinished_files):
unfinished_files.append(result["filename"])
unfinished_files_with_path.append(
unfinished_object)
else:
output_count = 0
for output in latest_output_outputs:
if (output not in result):
if (result["filename"]
not in unfinished_files):
unfinished_files.append(
result["filename"])
unfinished_files_with_path.append(
unfinished_object)
if output in result:
output_count += 1
if result[output] == False:
if (result["filename"]
not in unfinished_files):
unfinished_files.append(
result["filename"])
unfinished_files_with_path.append(
unfinished_object)
if output_count == 0:
if (result["filename"]
not in unfinished_files):
unfinished_files.append(result["filename"])
unfinished_files_with_path.append(
unfinished_object)
except:
result["deleted"] = False
# new_output_json_result.append(result)
seconds = (endtime - starttime).total_seconds()
output_json["endtime"] = str(endtime)
output_json["total_seconds"] = str(seconds)
output_json["unfinished_files"] = unfinished_files_with_path
output_json["no_of_unfinished_files"] = len(unfinished_files)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info(
"Done updating report in the current directory, output.json")
else:
logger.error("No results available for reporting")
else:
logger.error(
"output.json file does not exist. No results available for reporting."
)
def uploadFiles():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['main_url']
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file': (file_name, open(file_path,
'rb'), "multipart/form-data")
}
headers = {'apiKey': configuration_settings['api_key']}
if "function_id" in configuration_settings:
credentials = ('%s:%s' %
(configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(
credentials.encode('ascii'))
headers[
'Authorization'] = 'Basic %s' % encoded_credentials.decode(
"utf-8")
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings['json_options'],
'responseType':
configuration_settings['output_options']
},
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code), file_path))
dict_object.update({"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings['output_options'].split(
",")
})
output_results.append(dict_object)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file " +
file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"), 'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def uploadFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['aca_main_url']
logger.info("ACA main url is: {0} ".format(upload_url))
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file':
(file_name, open(file_path,
'rb'), "multipart/form-data")
}
# print(files)
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(
os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(token)
}
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings[
'json_options'],
'responseType':
configuration_settings[
'output_options']
},
headers=headers,
verify=configuration_settings[
'ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code),
file_path))
print(response.text)
error = response.text if response.status_code == 500 else json.loads(
response.text)
logger.error(
"Error details: {}".format(error))
dict_object.update(
{"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings[
'output_options'].split(",")
})
output_results.append(dict_object)
else:
message = "Zen token is required to upload the files, filename {}".format(
file_name)
logger.error(message)
error.append({'error': message})
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file "
+ file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"zen_token": token,
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"),
'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error("Zen token is required to upload the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def downloadFiles(token):
pending_completion = True
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
starttime = dt.datetime.now()
if token:
loop = 0
failed_download = []
completed_download = []
compelted_check = []
completed_count = 0
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
while pending_completion and loop < 1000:
output_json = json.load(open(output_json_path, "r"))
loop += 1
logger.info("Loop " + str(loop))
# print(output_json)
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "result" in json.loads(
result["response"]
) and "data" in json.loads(
result["response"]
)["result"][0] and "analyzerId" in json.loads(
result["response"]
)["result"][0]["data"]:
if "download_success" not in result:
response = json.loads(
result["response"])
path = result["path"]
filename = result["filename"]
analyzerId = response["result"][0][
"data"]["analyzerId"]
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
if ("download_completed"
not in result):
completed = checkCompleted(
latest_output_outputs, result)
if (completed):
result[
"download_completed"] = True
else:
current_time = dt.datetime.now(
)
seconds = (
current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(
token)
}
logger.info(
"Checking status of analyzerId: {}, filename: {}"
.format(
analyzerId,
filename))
status, result_response = checkStatus(
configuration_settings,
token, analyzerId)
if (status):
if ("data" in
result_response
and
"statusDetails"
in
result_response[
"data"]
):
status_result_response = result_response[
"data"][
"statusDetails"]
done_output = []
for output in status_result_response:
# print(output)
logger.info(
"status: {}, type {}, analyzerId: {}, filename: {}"
.
format(
output
.
get(
"status"
),
output
.
get(
"type"
),
analyzerId,
filename
))
if (output[
"type"]
in
latest_output_outputs
and
output[
"status"]
==
"Completed"
and
output[
"type"]
not in
result
):
logger.info(
"Downloading {0} of analyzerId: {1}"
.
format(
output[
"type"],
analyzerId
))
response, reason = downloadFile(
configuration_settings,
token,
analyzerId,
output[
"type"],
path,
filename
.
rsplit(
"."
)
[0]
)
result[output[
"type"]] = response
if (not response
):
result[
output[
"type"]
+
"_error"] = reason
done_output.append(
output[
"type"]
)
elif (output[
"type"]
in
latest_output_outputs
and
output[
"status"]
==
"Failed"
and
output[
"type"]
not in
result
):
result[output[
"type"]] = False
result[
output[
"type"]
+
"_error"] = output[
"status"]
logger.info(
"Processing failed of analyzerId: {}, filename: {}"
.
format(
analyzerId,
filename
))
done_output.append(
output[
"type"]
)
elif output[
"type"] in result:
logger.info(
"Processing of analyzerId: {}, filename: {}"
.
format(
analyzerId,
filename
))
done_output.append(
output[
"type"]
)
if (len(
done_output
) == len(
latest_output_outputs
)):
completed_download.append(
True)
compelted_check.append(
filename
)
completed = checkCompleted(
latest_output_outputs,
result)
if (completed):
result[
"download_completed"] = True
else:
logger.error(
"Cannot get status of analyzerId: {}, filename: {}"
.format(
analyzerId,
filename))
result[
"download_success"] = False
result[
"download_failure_reason"] = result_response
failed_download.append(
True)
else:
message = "Zen token is required to check file status, filename {}".format(
filename)
logger.error(message)
failed_download.append(
True)
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
else:
logger.error(
"We could not find any information to download files from."
)
result["download_success"] = False
result[
"download_failure_reason"] = "No available data to download from"
failed_download.append(True)
except Exception as err:
result["download_success"] = False
result[
"download_failure_reason"] = "No analyzerID available to download results"
logger.error(
"No analyzerID available to download results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
failed_download.append(True)
logger.error(err)
# new_output_json_result.append(result)
# print(len(new_output_json_result))
endtime = dt.datetime.now()
output_json["output_results"] = output_json_result
output_json["endtime"] = str(endtime)
json.dump(output_json,
open("output.json", 'w'),
indent=4)
completed_count = len(failed_download) + len(
completed_download)
logger.info("count, failed {}, completed {}".format(
len(failed_download), len(completed_download)))
if (completed_count >= len(output_json_result)):
print(compelted_check)
pending_completion = False
else:
if (loop >= 1000):
pending_completion = False
logger.error(
"Reached maximum number of download retries."
)
else:
time.sleep(10)
else:
pending_completion = False
logger.error("No results available to download.")
return True
logger.info(
"Done downloading all output files to your output_directory_path"
)
logger.info("Download status reported in output.json")
if (loop < 999):
return True
else:
return False
else:
logger.error(
"output.json file does not exist. No results available to download."
)
return False
else:
logger.error("Zen token is required to download the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
token = getFirstToken()
if token:
token_time = dt.datetime.now()
logger.info("Successfully generate the token, continue...")
logger.info("Uploading files")
uploadSuccess = uploadFiles(token)
if (uploadSuccess):
logger.info("Ready to download output files...")
complete = downloadFiles(token)
if (complete):
logger.info("Deleting files on the server")
deleteFiles(token)
else:
logger.info(
"Could not delete at this time because download has not been completed yet, please run deleteFiles.py at a later time"
)
logger.info("Updating report")
updateReport()
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
logger.info(
"ADP API Sample tool ended. Processing time took {0} seconds, Disclaimer: This includes time to upload, download and delete and has nothing to do with actual processing time."
.format(int(seconds)))
else:
logger.error("Failed to generate token to continue")
