def oauth():
configuration, configuration_settings = readJSON()
username = configuration_settings["function_id"]
password = configuration_settings["password"]
scopes = ['opendid']
url = configuration_settings["ums_url"]
credentials = ('%s:%s' % (configuration_settings["client_id"],
configuration_settings["client_secret"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
payload = 'username='******'&password='******'Authorization': 'Basic %s' % encoded_credentials.decode("utf-8"),
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("POST",
url,
headers=headers,
data=payload,
verify=False)
j = json.loads(response.text)
return j['access_token']
def basicAuth():
configuration, configuration_settings = readJSON()
credentials = ('%s:%s' % (configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
basicToken = 'Basic %s' % encoded_credentials.decode("utf-8")
return basicToken
def getFirstToken():
configuration, configuration_settings = readJSON()
if (configuration):
token = generateToken_pw_flow(configuration_settings)
return token
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def deleteFiles(bearerToken):
configuration, configuration_settings = readJSON()
if (configuration):
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "data" in json.loads(result["response"]) and "analyzerId" in \
json.loads(result["response"])["data"]:
if ("deleted" in result and result["deleted"]
== False) or "deleted" not in result:
response = json.loads(result["response"])
analyzerId = response["data"]["analyzerId"]
status, result_response = deleteFile(
configuration_settings, analyzerId,
bearerToken)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
def uploadFiles():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['main_url']
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file': (file_name, open(file_path,
'rb'), "multipart/form-data")
}
headers = {'apiKey': configuration_settings['api_key']}
if "function_id" in configuration_settings:
credentials = ('%s:%s' %
(configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(
credentials.encode('ascii'))
headers[
'Authorization'] = 'Basic %s' % encoded_credentials.decode(
"utf-8")
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings['json_options'],
'responseType':
configuration_settings['output_options']
},
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code), file_path))
dict_object.update({"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings['output_options'].split(
",")
})
output_results.append(dict_object)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file " +
file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"), 'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def uploadFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['aca_main_url']
logger.info("ACA main url is: {0} ".format(upload_url))
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file':
(file_name, open(file_path,
'rb'), "multipart/form-data")
}
# print(files)
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(
os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(token)
}
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings[
'json_options'],
'responseType':
configuration_settings[
'output_options']
},
headers=headers,
verify=configuration_settings[
'ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code),
file_path))
print(response.text)
error = response.text if response.status_code == 500 else json.loads(
response.text)
logger.error(
"Error details: {}".format(error))
dict_object.update(
{"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings[
'output_options'].split(",")
})
output_results.append(dict_object)
else:
message = "Zen token is required to upload the files, filename {}".format(
file_name)
logger.error(message)
error.append({'error': message})
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file "
+ file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"zen_token": token,
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"),
'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error("Zen token is required to upload the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def downloadFiles(token):
pending_completion = True
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
starttime = dt.datetime.now()
if token:
loop = 0
failed_download = []
completed_download = []
compelted_check = []
completed_count = 0
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
while pending_completion and loop < 1000:
output_json = json.load(open(output_json_path, "r"))
loop += 1
logger.info("Loop " + str(loop))
# print(output_json)
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "result" in json.loads(
result["response"]
) and "data" in json.loads(
result["response"]
)["result"][0] and "analyzerId" in json.loads(
result["response"]
)["result"][0]["data"]:
if "download_success" not in result:
response = json.loads(
result["response"])
path = result["path"]
filename = result["filename"]
analyzerId = response["result"][0][
"data"]["analyzerId"]
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
if ("download_completed"
not in result):
completed = checkCompleted(
latest_output_outputs, result)
if (completed):
result[
"download_completed"] = True
else:
current_time = dt.datetime.now(
)
seconds = (
current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(
token)
}
logger.info(
"Checking status of analyzerId: {}, filename: {}"
.format(
analyzerId,
filename))
status, result_response = checkStatus(
configuration_settings,
token, analyzerId)
if (status):
if ("data" in
result_response
and
"statusDetails"
in
result_response[
"data"]
):
status_result_response = result_response[
"data"][
"statusDetails"]
done_output = []
for output in status_result_response:
# print(output)
logger.info(
"status: {}, type {}, analyzerId: {}, filename: {}"
.
format(
output
.
get(
"status"
),
output
.
get(
"type"
),
analyzerId,
filename
))
if (output[
"type"]
in
latest_output_outputs
and
output[
"status"]
==
"Completed"
and
output[
"type"]
not in
result
):
logger.info(
"Downloading {0} of analyzerId: {1}"
.
format(
output[
"type"],
analyzerId
))
response, reason = downloadFile(
configuration_settings,
token,
analyzerId,
output[
"type"],
path,
filename
.
rsplit(
"."
)
[0]
)
result[output[
"type"]] = response
if (not response
):
result[
output[
"type"]
+
"_error"] = reason
done_output.append(
output[
"type"]
)
elif (output[
"type"]
in
latest_output_outputs
and
output[
"status"]
==
"Failed"
and
output[
"type"]
not in
result
):
result[output[
"type"]] = False
result[
output[
"type"]
+
"_error"] = output[
"status"]
logger.info(
"Processing failed of analyzerId: {}, filename: {}"
.
format(
analyzerId,
filename
))
done_output.append(
output[
"type"]
)
elif output[
"type"] in result:
logger.info(
"Processing of analyzerId: {}, filename: {}"
.
format(
analyzerId,
filename
))
done_output.append(
output[
"type"]
)
if (len(
done_output
) == len(
latest_output_outputs
)):
completed_download.append(
True)
compelted_check.append(
filename
)
completed = checkCompleted(
latest_output_outputs,
result)
if (completed):
result[
"download_completed"] = True
else:
logger.error(
"Cannot get status of analyzerId: {}, filename: {}"
.format(
analyzerId,
filename))
result[
"download_success"] = False
result[
"download_failure_reason"] = result_response
failed_download.append(
True)
else:
message = "Zen token is required to check file status, filename {}".format(
filename)
logger.error(message)
failed_download.append(
True)
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
else:
logger.error(
"We could not find any information to download files from."
)
result["download_success"] = False
result[
"download_failure_reason"] = "No available data to download from"
failed_download.append(True)
except Exception as err:
result["download_success"] = False
result[
"download_failure_reason"] = "No analyzerID available to download results"
logger.error(
"No analyzerID available to download results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
failed_download.append(True)
logger.error(err)
# new_output_json_result.append(result)
# print(len(new_output_json_result))
endtime = dt.datetime.now()
output_json["output_results"] = output_json_result
output_json["endtime"] = str(endtime)
json.dump(output_json,
open("output.json", 'w'),
indent=4)
completed_count = len(failed_download) + len(
completed_download)
logger.info("count, failed {}, completed {}".format(
len(failed_download), len(completed_download)))
if (completed_count >= len(output_json_result)):
print(compelted_check)
pending_completion = False
else:
if (loop >= 1000):
pending_completion = False
logger.error(
"Reached maximum number of download retries."
)
else:
time.sleep(10)
else:
pending_completion = False
logger.error("No results available to download.")
return True
logger.info(
"Done downloading all output files to your output_directory_path"
)
logger.info("Download status reported in output.json")
if (loop < 999):
return True
else:
return False
else:
logger.error(
"output.json file does not exist. No results available to download."
)
return False
else:
logger.error("Zen token is required to download the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def downloadFiles():
pending_completion = True
configuration, configuration_settings = readJSON()
if (configuration):
loop = 0
failed_download = []
completed_download = []
completed_count = 0
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
while pending_completion and loop < 1000:
output_json = json.load(open(output_json_path, "r"))
loop += 1
logger.info("Loop " + str(loop))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "data" in json.loads(
result["response"]
) and "analyzerId" in json.loads(
result["response"])["data"]:
if "download_success" not in result:
response = json.loads(result["response"])
path = result["path"]
filename = result["filename"]
analyzerId = response["data"]["analyzerId"]
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
if ("download_completed" not in result):
completed = checkCompleted(
latest_output_outputs, result)
if (completed):
result["download_completed"] = True
else:
logger.info(
"Checking status of analyzerId: "
+ analyzerId)
status, result_response = checkStatus(
configuration_settings,
analyzerId)
if (status):
if ("data" in result_response
and "statusDetails" in
result_response["data"]
):
status_result_response = result_response[
"data"][
"statusDetails"]
done_output = []
for output in status_result_response:
if (output["type"] in
latest_output_outputs
and output[
"status"]
== "Completed"
and
output["type"]
not in result):
logger.info(
"Downloading {0} of analyzerId: {1}"
.format(
output[
"type"],
analyzerId
))
response, reason = downloadFile(
configuration_settings,
analyzerId,
output["type"],
path,
filename.
rsplit(".")[0])
result[output[
"type"]] = response
if (not response):
result[
output[
"type"]
+
"_error"] = reason
done_output.append(
output["type"])
elif (output["type"] in
latest_output_outputs
and
output["status"]
== "Failed" and
output["type"]
not in result):
result[output[
"type"]] = False
result[
output["type"]
+
"_error"] = output[
"status"]
done_output.append(
output["type"])
elif output[
"type"] in result:
done_output.append(
output["type"])
if (len(
done_output
) == len(
latest_output_outputs
)):
completed_download.append(
True)
completed = checkCompleted(
latest_output_outputs,
result)
if (completed):
result[
"download_completed"] = True
else:
result[
"download_success"] = False
result[
"download_failure_reason"] = result_response
failed_download.append(True)
else:
logger.error(
"We could not find any information to download files from."
)
result["download_success"] = False
result[
"download_failure_reason"] = "No available data to download from"
failed_download.append(True)
except:
result["download_success"] = False
result[
"download_failure_reason"] = "No analyzerID available to download results"
logger.error(
"No analyzerID available to download results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
failed_download.append(True)
# new_output_json_result.append(result)
# print(len(new_output_json_result))
endtime = dt.datetime.now()
output_json["output_results"] = output_json_result
output_json["endtime"] = str(endtime)
json.dump(output_json,
open("output.json", 'w'),
indent=4)
completed_count = len(failed_download) + len(
completed_download)
if (completed_count >= len(output_json_result)):
pending_completion = False
else:
if (loop >= 1000):
pending_completion = False
logger.error(
"Reached maximum number of download retries.")
else:
time.sleep(5)
else:
pending_completion = False
logger.error("No results available to download.")
return True
else:
logger.info(
"Done downloading all output files to your output_directory_path"
)
logger.info("Download status reported in output.json")
if (loop < 999):
return True
else:
return False
else:
logger.error(
"output.json file does not exist. No results available to download."
)
return False
def deleteFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "result" in json.loads(
result["response"]
) and "data" in json.loads(
result["response"]
)["result"][0] and "analyzerId" in json.loads(
result["response"])["result"][0]["data"]:
if ("deleted" in result and result["deleted"]
== False) or "deleted" not in result:
response = json.loads(result["response"])
# print(response)
analyzerId = response["result"][0]["data"][
"analyzerId"]
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
status, result_response = deleteFile(
configuration_settings, token,
analyzerId)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
else:
token = generateToken_pw_flow(
configuration_settings)
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json,
open("output.json", 'w'),
indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
else:
logger.error("Zen token is required to delete files")
def executeRules():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["output_directory_path"]
count = 0
errors = []
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
#print(file)
file_split = new_file.rsplit(".")
if (file_split[1] == "json"):
odm_file_value = str(file_split[0]) + "ODM.json"
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
#print(new_file_path)
f = open(new_file_path, "r")
# deserializes into dict
# and returns dict.
y = json.loads(f.read())
#json.append()
odmData = {
'__DecisionID__': "myworld",
"document_in": y,
"metaData_inout": {
"dynamicParams": "string"
}
}
#var rex = new RegExp('Sensitivity":false', 'g');
#var BACA_STRING_RESPONSE = JSON.stringify(res.data)
#BACA_STRING_RESPONSE = BACA_STRING_RESPONSE.replace(rex, 'Sensitivity":0');
odmJson = json.dumps(odmData)
#print (odmJson)
odmJson = odmJson.replace('"Sensitivity": false',
'"Sensitivity":0')
odmJson = odmJson.replace('Language": ""', 'Language":[]')
#odmpayload = {"__DecisionID__": "string","document_in": y, "metaData_inout": {"dynamicParams": "" }}
authPayload = HTTPBasicAuth(
configuration_settings["odmUser"],
configuration_settings["odmPassword"])
ODM_ProcessBacaDocument_ServiceURL = configuration_settings[
"odm_url"]
#JsonPayload = odmpayload
headers = {'content-type': 'application/json'}
#print(odmData)
try:
response = requests.request(
"POST",
ODM_ProcessBacaDocument_ServiceURL,
headers=headers,
data=odmJson,
auth=authPayload,
verify=False)
print(response.text)
jsonPayloadODM = json.loads(response.text)
fileOutPutPath = configuration_settings[
"odm_output_directory_path"] + "/" + odm_file_value
with open(fileOutPutPath, 'w') as outfile:
json.dump(jsonPayloadODM, outfile)
#return response
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
#dict_object.update({"error": str(sslerror)})
#errors.append(dict_object)
return False
except Exception as ex:
#dict_object.update({"error": str(ex)})
#errors.append(dict_object)
#logger.error("An error occurred when trying to upload file " + file_path)
logger.debug(ex, exc_info=True)
pass
f.close()
return True
