def deleteFile(configuration_settings, analyzerId):
# Make requests
headers = {'apiKey': configuration_settings['api_key']}
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
headers['Authorization'] = 'Basic %s' % encoded_credentials.decode(
"utf-8")
get_url = configuration_settings['main_url'] + "/{0}".format(analyzerId)
try:
response = requests.request(
"DELETE",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
return True, response_json
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
def checkStatus(configuration_settings, token, analyzerId):
# Make request
headers = {'Authorization': 'Bearer {}'.format(token)}
get_url = configuration_settings['aca_main_url'] + "/{0}".format(
analyzerId)
try:
response = requests.request(
"GET",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
if "result" in response_json:
return True, response_json["result"][0]
else:
return False, {}
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
def downloadFile(configuration_settings, token, analyzerId, output,
output_path, filename):
# Make request
headers = {'Authorization': 'Bearer {}'.format(token)}
extra_path = "/{0}/{1}".format(analyzerId, output.lower())
get_url = configuration_settings['aca_main_url'] + extra_path
try:
response = requests.request(
"GET",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
# print(response.text)
if response.status_code >= 400:
logger.error("An error occurred while trying to download output.")
return False, response.text
else:
file_name = copy.copy(filename)
filename_output = "{0}.{1}".format(file_name, output.lower(
)) if output.lower() != "utf8" else "{0}.txt".format(file_name)
filename_output = filename_output if output.lower(
) != "pdf" else "New_{0}.{1}".format(file_name, output.lower())
file_output_path = os.path.join(
configuration_settings["output_directory_path"], output_path,
output.lower())
filename_output_path = os.path.join(file_output_path,
filename_output)
if (not os.path.exists(file_output_path)):
os.makedirs(file_output_path)
logger.info("Created new output directory, " +
file_output_path)
if (output.lower() == "json"):
response_output = json.loads(response.text)
if "result" in response_output and "data" in response_output[
"result"][0]:
json.dump(response_output["result"][0]["data"],
open(filename_output_path, "w"),
indent=4)
elif output.lower() == "pdf":
output_write = open(filename_output_path, "wb")
output_write.write(response.content)
output_write.close()
return True, ""
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
logger.error("An error occurred when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, ""
def downloadFile(configuration_settings, analyzerId, output, output_path, filename,bearerToken):
# Make request
headers = {
'apiKey': configuration_settings['api_key']
}
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (
configuration_settings["function_id"], configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
#headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8")
#headers['Authorization'] = configuration_settings["bear_token"]
headers['Authorization'] = bearerToken
extra_path = "{0}/{1}/".format(analyzerId, output.lower())
get_url = urlparse.urljoin(configuration_settings['main_url'], extra_path)
try:
response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
file_name = copy.copy(filename)
filename_output = "{0}.{1}".format(file_name, output.lower()) if output.lower() != "utf8" else "{0}.txt".format(file_name)
filename_output = filename_output if output.lower() != "pdf" else "New_{0}.{1}".format(file_name, output.lower())
file_output_path = os.path.join(configuration_settings["output_directory_path"], output_path, output.lower())
filename_output_path = os.path.join(file_output_path, filename_output)
if( not os.path.exists(file_output_path)):
os.makedirs(file_output_path)
logger.info("Created new output directory, "+ file_output_path)
if(output.lower() == "json"):
response_output = json.loads(response.text)
if "data" in response_output:
json.dump(response_output["data"], open(filename_output_path, "w"), indent=4)
elif output.lower() == "pdf":
output_write = open(filename_output_path, "wb")
output_write.write(response.content)
output_write.close()
else:
output_write = open(filename_output_path, "w")
try:
output_write.write(response.text)
except:
output_write.write(response.text.encode('utf-8'))
output_write.close()
return True, ""
except SSLError as sslerror:
logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.")
logger.debug(sslerror, exc_info=True)
except Exception as ex:
logger.error("An error occurred when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, ""
def checkTokenValid(token, config):
if token:
if config:
zen_check_token_url = 'https://' + config['zen_host'].strip(
) + '/usermgmt/v1/user/currentUserInfo'
try:
headers = {
"authorization": "Bearer " + token,
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("GET",
zen_check_token_url,
headers=headers,
verify=config['ssl_verification'])
if response.status_code == 200:
# content = getContent(response)
# res = content.get('uid')
logger.info('zen token provided is valid')
return token
elif response.status_code == 401:
logger.info(
'zen token provided is invalid or expired, trying to generate a new one'
)
token = generateToken_pw_flow(config)
return token
else:
logger.error("Failed to check the token valid or not")
content = getContent(response)
logger.debug(content)
return None
except Exception as ex:
logger.error(
"Failed to check the token valid or not due to exception")
logger.debug(str(ex))
return None
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return None
else:
logger.error("Zen token is required, please provide a valid token")
return None
def generateToken_pw_flow(config):
zen_token_url = 'https://' + config['zen_host'].strip(
) + '/v1/preauth/validateAuth'
zen_username = config['zen_username'].strip()
zen_pw = config['zen_password'].strip()
try:
encoding = base64.b64encode('{}:{}'.format(zen_username,
zen_pw).encode())
headers = {
"authorization": "Basic " + encoding.decode("utf-8"),
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("GET",
zen_token_url,
headers=headers,
verify=config['ssl_verification'])
if response.status_code == 200:
# token is generated
# {
# "accessToken": "xxxxxxxxx"
# }
content = getContent(response)
# print(content)
token = content.get('accessToken')
logger.info('Successfully got the token')
json.dump({"zen_token": token}, open("output.json", 'w'), indent=4)
return token
else:
logger.error("Failed to generate a token")
content = getContent(response)
logger.debug(content)
return None
except Exception as ex:
logger.error("Failed to generate a token due to exception")
logger.debug(str(ex))
return None
def checkStatus(configuration_settings, analyzerId,bearerToken):
logger.info("Config check")
# Make request
headers = {
'apiKey': configuration_settings['api_key']
}
logger.info("Config check" + configuration_settings['api_key'])
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (
configuration_settings["function_id"], configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
#headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8")
#headers['Authorization'] = configuration_settings["bear_token"]
headers['Authorization'] = bearerToken
get_url = urlparse.urljoin(configuration_settings['main_url'], "{0}/".format(analyzerId))
logger.info("Config check " + get_url)
try:
response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification'])
logger.info(response.status_code)
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
return True, response_json
except SSLError as sslerror:
logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.")
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return None
else:
logger.error("Zen token is required, please provide a valid token")
return None
def getFirstToken():
configuration, configuration_settings = readJSON()
if (configuration):
token = generateToken_pw_flow(configuration_settings)
return token
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
if __name__ == '__main__':
logger.info("Logs can be found in current directory (processing.log)")
token = None
try:
getFirstToken()
except Exception as ex:
logger.error("An error occurred, please check logs for more details")
logger.debug(ex, exc_info=True)
def uploadFiles():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['main_url']
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file': (file_name, open(file_path,
'rb'), "multipart/form-data")
}
headers = {'apiKey': configuration_settings['api_key']}
if "function_id" in configuration_settings:
credentials = ('%s:%s' %
(configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(
credentials.encode('ascii'))
headers[
'Authorization'] = 'Basic %s' % encoded_credentials.decode(
"utf-8")
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings['json_options'],
'responseType':
configuration_settings['output_options']
},
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code), file_path))
dict_object.update({"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings['output_options'].split(
",")
})
output_results.append(dict_object)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file " +
file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"), 'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def uploadFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['aca_main_url']
logger.info("ACA main url is: {0} ".format(upload_url))
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file':
(file_name, open(file_path,
'rb'), "multipart/form-data")
}
# print(files)
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(
os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(token)
}
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings[
'json_options'],
'responseType':
configuration_settings[
'output_options']
},
headers=headers,
verify=configuration_settings[
'ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code),
file_path))
print(response.text)
error = response.text if response.status_code == 500 else json.loads(
response.text)
logger.error(
"Error details: {}".format(error))
dict_object.update(
{"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings[
'output_options'].split(",")
})
output_results.append(dict_object)
else:
message = "Zen token is required to upload the files, filename {}".format(
file_name)
logger.error(message)
error.append({'error': message})
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file "
+ file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"zen_token": token,
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"),
'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error("Zen token is required to upload the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def executeRules():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["output_directory_path"]
count = 0
errors = []
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
#print(file)
file_split = new_file.rsplit(".")
if (file_split[1] == "json"):
odm_file_value = str(file_split[0]) + "ODM.json"
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
#print(new_file_path)
f = open(new_file_path, "r")
# deserializes into dict
# and returns dict.
y = json.loads(f.read())
#json.append()
odmData = {
'__DecisionID__': "myworld",
"document_in": y,
"metaData_inout": {
"dynamicParams": "string"
}
}
#var rex = new RegExp('Sensitivity":false', 'g');
#var BACA_STRING_RESPONSE = JSON.stringify(res.data)
#BACA_STRING_RESPONSE = BACA_STRING_RESPONSE.replace(rex, 'Sensitivity":0');
odmJson = json.dumps(odmData)
#print (odmJson)
odmJson = odmJson.replace('"Sensitivity": false',
'"Sensitivity":0')
odmJson = odmJson.replace('Language": ""', 'Language":[]')
#odmpayload = {"__DecisionID__": "string","document_in": y, "metaData_inout": {"dynamicParams": "" }}
authPayload = HTTPBasicAuth(
configuration_settings["odmUser"],
configuration_settings["odmPassword"])
ODM_ProcessBacaDocument_ServiceURL = configuration_settings[
"odm_url"]
#JsonPayload = odmpayload
headers = {'content-type': 'application/json'}
#print(odmData)
try:
response = requests.request(
"POST",
ODM_ProcessBacaDocument_ServiceURL,
headers=headers,
data=odmJson,
auth=authPayload,
verify=False)
print(response.text)
jsonPayloadODM = json.loads(response.text)
fileOutPutPath = configuration_settings[
"odm_output_directory_path"] + "/" + odm_file_value
with open(fileOutPutPath, 'w') as outfile:
json.dump(jsonPayloadODM, outfile)
#return response
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
#dict_object.update({"error": str(sslerror)})
#errors.append(dict_object)
return False
except Exception as ex:
#dict_object.update({"error": str(ex)})
#errors.append(dict_object)
#logger.error("An error occurred when trying to upload file " + file_path)
logger.debug(ex, exc_info=True)
pass
f.close()
return True
