def deleteFile(configuration_settings, analyzerId): # Make requests headers = {'apiKey': configuration_settings['api_key']} if "function_id" in configuration_settings: credentials = ('%s:%s' % (configuration_settings["function_id"], configuration_settings["password"])) encoded_credentials = b64encode(credentials.encode('ascii')) headers['Authorization'] = 'Basic %s' % encoded_credentials.decode( "utf-8") get_url = configuration_settings['main_url'] + "/{0}".format(analyzerId) try: response = requests.request( "DELETE", get_url, headers=headers, verify=configuration_settings['ssl_verification']) if response.status_code >= 400: logger.error("An error occurred while trying to get file status.") return False, response.text else: response_json = json.loads(response.text) return True, response_json except SSLError as sslerror: logger.error( "SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file." ) logger.debug(sslerror, exc_info=True) except Exception as ex: print(ex) logger.error("An error occured when trying to get file") logger.debug(ex, exc_info=True) pass return False, {}
def checkStatus(configuration_settings, token, analyzerId): # Make request headers = {'Authorization': 'Bearer {}'.format(token)} get_url = configuration_settings['aca_main_url'] + "/{0}".format( analyzerId) try: response = requests.request( "GET", get_url, headers=headers, verify=configuration_settings['ssl_verification']) if response.status_code >= 400: logger.error("An error occurred while trying to get file status.") return False, response.text else: response_json = json.loads(response.text) if "result" in response_json: return True, response_json["result"][0] else: return False, {} except SSLError as sslerror: logger.error( "SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file." ) logger.debug(sslerror, exc_info=True) except Exception as ex: print(ex) logger.error("An error occured when trying to get file") logger.debug(ex, exc_info=True) pass return False, {}
def downloadFile(configuration_settings, token, analyzerId, output, output_path, filename): # Make request headers = {'Authorization': 'Bearer {}'.format(token)} extra_path = "/{0}/{1}".format(analyzerId, output.lower()) get_url = configuration_settings['aca_main_url'] + extra_path try: response = requests.request( "GET", get_url, headers=headers, verify=configuration_settings['ssl_verification']) # print(response.text) if response.status_code >= 400: logger.error("An error occurred while trying to download output.") return False, response.text else: file_name = copy.copy(filename) filename_output = "{0}.{1}".format(file_name, output.lower( )) if output.lower() != "utf8" else "{0}.txt".format(file_name) filename_output = filename_output if output.lower( ) != "pdf" else "New_{0}.{1}".format(file_name, output.lower()) file_output_path = os.path.join( configuration_settings["output_directory_path"], output_path, output.lower()) filename_output_path = os.path.join(file_output_path, filename_output) if (not os.path.exists(file_output_path)): os.makedirs(file_output_path) logger.info("Created new output directory, " + file_output_path) if (output.lower() == "json"): response_output = json.loads(response.text) if "result" in response_output and "data" in response_output[ "result"][0]: json.dump(response_output["result"][0]["data"], open(filename_output_path, "w"), indent=4) elif output.lower() == "pdf": output_write = open(filename_output_path, "wb") output_write.write(response.content) output_write.close() return True, "" except SSLError as sslerror: logger.error( "SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file." ) logger.debug(sslerror, exc_info=True) except Exception as ex: logger.error("An error occurred when trying to get file") logger.debug(ex, exc_info=True) pass return False, ""
def downloadFile(configuration_settings, analyzerId, output, output_path, filename,bearerToken): # Make request headers = { 'apiKey': configuration_settings['api_key'] } if "function_id" in configuration_settings: credentials = ('%s:%s' % ( configuration_settings["function_id"], configuration_settings["password"])) encoded_credentials = b64encode(credentials.encode('ascii')) #headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8") #headers['Authorization'] = configuration_settings["bear_token"] headers['Authorization'] = bearerToken extra_path = "{0}/{1}/".format(analyzerId, output.lower()) get_url = urlparse.urljoin(configuration_settings['main_url'], extra_path) try: response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification']) if response.status_code >= 400: logger.error("An error occurred while trying to get file status.") return False, response.text else: file_name = copy.copy(filename) filename_output = "{0}.{1}".format(file_name, output.lower()) if output.lower() != "utf8" else "{0}.txt".format(file_name) filename_output = filename_output if output.lower() != "pdf" else "New_{0}.{1}".format(file_name, output.lower()) file_output_path = os.path.join(configuration_settings["output_directory_path"], output_path, output.lower()) filename_output_path = os.path.join(file_output_path, filename_output) if( not os.path.exists(file_output_path)): os.makedirs(file_output_path) logger.info("Created new output directory, "+ file_output_path) if(output.lower() == "json"): response_output = json.loads(response.text) if "data" in response_output: json.dump(response_output["data"], open(filename_output_path, "w"), indent=4) elif output.lower() == "pdf": output_write = open(filename_output_path, "wb") output_write.write(response.content) output_write.close() else: output_write = open(filename_output_path, "w") try: output_write.write(response.text) except: output_write.write(response.text.encode('utf-8')) output_write.close() return True, "" except SSLError as sslerror: logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.") logger.debug(sslerror, exc_info=True) except Exception as ex: logger.error("An error occurred when trying to get file") logger.debug(ex, exc_info=True) pass return False, ""
def checkTokenValid(token, config): if token: if config: zen_check_token_url = 'https://' + config['zen_host'].strip( ) + '/usermgmt/v1/user/currentUserInfo' try: headers = { "authorization": "Bearer " + token, 'Content-Type': 'application/x-www-form-urlencoded' } response = requests.request("GET", zen_check_token_url, headers=headers, verify=config['ssl_verification']) if response.status_code == 200: # content = getContent(response) # res = content.get('uid') logger.info('zen token provided is valid') return token elif response.status_code == 401: logger.info( 'zen token provided is invalid or expired, trying to generate a new one' ) token = generateToken_pw_flow(config) return token else: logger.error("Failed to check the token valid or not") content = getContent(response) logger.debug(content) return None except Exception as ex: logger.error( "Failed to check the token valid or not due to exception") logger.debug(str(ex)) return None else: logger.error( "Check your configuration file (config.json) for correct format and valid parameters" ) return None else: logger.error("Zen token is required, please provide a valid token") return None
def generateToken_pw_flow(config): zen_token_url = 'https://' + config['zen_host'].strip( ) + '/v1/preauth/validateAuth' zen_username = config['zen_username'].strip() zen_pw = config['zen_password'].strip() try: encoding = base64.b64encode('{}:{}'.format(zen_username, zen_pw).encode()) headers = { "authorization": "Basic " + encoding.decode("utf-8"), 'Content-Type': 'application/x-www-form-urlencoded' } response = requests.request("GET", zen_token_url, headers=headers, verify=config['ssl_verification']) if response.status_code == 200: # token is generated # { # "accessToken": "xxxxxxxxx" # } content = getContent(response) # print(content) token = content.get('accessToken') logger.info('Successfully got the token') json.dump({"zen_token": token}, open("output.json", 'w'), indent=4) return token else: logger.error("Failed to generate a token") content = getContent(response) logger.debug(content) return None except Exception as ex: logger.error("Failed to generate a token due to exception") logger.debug(str(ex)) return None
def checkStatus(configuration_settings, analyzerId,bearerToken): logger.info("Config check") # Make request headers = { 'apiKey': configuration_settings['api_key'] } logger.info("Config check" + configuration_settings['api_key']) if "function_id" in configuration_settings: credentials = ('%s:%s' % ( configuration_settings["function_id"], configuration_settings["password"])) encoded_credentials = b64encode(credentials.encode('ascii')) #headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8") #headers['Authorization'] = configuration_settings["bear_token"] headers['Authorization'] = bearerToken get_url = urlparse.urljoin(configuration_settings['main_url'], "{0}/".format(analyzerId)) logger.info("Config check " + get_url) try: response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification']) logger.info(response.status_code) if response.status_code >= 400: logger.error("An error occurred while trying to get file status.") return False, response.text else: response_json = json.loads(response.text) return True, response_json except SSLError as sslerror: logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.") logger.debug(sslerror, exc_info=True) except Exception as ex: print(ex) logger.error("An error occured when trying to get file") logger.debug(ex, exc_info=True) pass return False, {}
else: logger.error( "Check your configuration file (config.json) for correct format and valid parameters" ) return None else: logger.error("Zen token is required, please provide a valid token") return None def getFirstToken(): configuration, configuration_settings = readJSON() if (configuration): token = generateToken_pw_flow(configuration_settings) return token else: logger.error( "Check your configuration file (config.json) for correct format and valid parameters" ) return False if __name__ == '__main__': logger.info("Logs can be found in current directory (processing.log)") token = None try: getFirstToken() except Exception as ex: logger.error("An error occurred, please check logs for more details") logger.debug(ex, exc_info=True)
def uploadFiles(): configuration, configuration_settings = readJSON() if (configuration): starttime = dt.datetime.now() dir_path = configuration_settings["directory_path"] count = 0 errors = [] upload_url = configuration_settings['main_url'] accepted_extensions = configuration_settings["accepted_extensions"] file_types = configuration_settings[ 'file_type'] if 'file_type' in configuration_settings and type( configuration_settings['file_type'] is list) and len( configuration_settings['file_type'] ) > 0 else accepted_extensions file_types = [f_type.lower() for f_type in file_types] output_results = [] for subdir, dirs, files in os.walk(dir_path): for file in files: file_path = os.path.join(subdir, file) new_file = copy.copy(file) file_split = new_file.rsplit(".") file_extension = str(file_split[-1].strip()) old_file_name = new_file.replace("." + file_extension, '').strip() file_name = re.sub( '[^A-Za-z0-9 _]+', ' ', old_file_name).strip() + "." + str(file_extension) new_file_path = os.path.join(subdir, file_name) if (file_extension != "" and file_extension.lower() in accepted_extensions and file_extension.lower() in file_types): count += 1 try: logger.info("Uploading {0} ".format(new_file_path)) except: pass files = { 'file': (file_name, open(file_path, 'rb'), "multipart/form-data") } headers = {'apiKey': configuration_settings['api_key']} if "function_id" in configuration_settings: credentials = ('%s:%s' % (configuration_settings["function_id"], configuration_settings["password"])) encoded_credentials = b64encode( credentials.encode('ascii')) headers[ 'Authorization'] = 'Basic %s' % encoded_credentials.decode( "utf-8") dict_object = { "filename": file_name, "path": os.path.basename(subdir), "full_path": os.path.join(os.path.abspath(subdir), old_file_name + "." + str(file_extension)) } # Make request try: response = requests.request( "POST", upload_url, files=files, data={ 'jsonOptions': configuration_settings['json_options'], 'responseType': configuration_settings['output_options'] }, headers=headers, verify=configuration_settings['ssl_verification']) if response.status_code >= 400: logger.error( "HTTP error {0} occurred when uploading file: {1} " .format(str(response.status_code), file_path)) dict_object.update({"error": response.text}) errors.append(dict_object) else: dict_object.update({ "response": response.text, "output_type": configuration_settings['output_options'].split( ",") }) output_results.append(dict_object) except SSLError as sslerror: logger.error( "SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file." ) dict_object.update({"error": str(sslerror)}) errors.append(dict_object) except Exception as ex: dict_object.update({"error": str(ex)}) errors.append(dict_object) logger.error( "An error occurred when trying to upload file " + file_path) logger.debug(ex, exc_info=True) pass endtime = dt.datetime.now() seconds = (endtime - starttime).total_seconds() result = { "starttime": str(starttime), "endtime": str(endtime), "no_of_files": count, "output_results": output_results, "no_output_results": len(output_results), "total_upload_seconds": seconds, "upload_errors": errors, "no_of_upload_errors": len(errors) } json.dump(result, open(os.path.join(os.getcwd(), "output.json"), 'w'), indent=4) logger.info("Upload status reported in output.json") if count == 0: logger.error("No files found in directory_path, " + dir_path) return False elif len(errors) > 0: logger.error("Errors occurred during upload.") return True else: logger.info("Done uploading {0} files".format(count)) return True else: logger.error( "Check your configuration file (config.json) for correct format and valid parameters" ) return False
def uploadFiles(token): configuration, configuration_settings = readJSON() if (configuration): token = checkTokenValid(token, configuration_settings) if token: starttime = dt.datetime.now() dir_path = configuration_settings["directory_path"] count = 0 errors = [] upload_url = configuration_settings['aca_main_url'] logger.info("ACA main url is: {0} ".format(upload_url)) accepted_extensions = configuration_settings["accepted_extensions"] file_types = configuration_settings[ 'file_type'] if 'file_type' in configuration_settings and type( configuration_settings['file_type'] is list) and len( configuration_settings['file_type'] ) > 0 else accepted_extensions file_types = [f_type.lower() for f_type in file_types] output_results = [] for subdir, dirs, files in os.walk(dir_path): for file in files: file_path = os.path.join(subdir, file) new_file = copy.copy(file) file_split = new_file.rsplit(".") file_extension = str(file_split[-1].strip()) old_file_name = new_file.replace("." + file_extension, '').strip() file_name = re.sub( '[^A-Za-z0-9 _]+', ' ', old_file_name).strip() + "." + str(file_extension) new_file_path = os.path.join(subdir, file_name) if (file_extension != "" and file_extension.lower() in accepted_extensions and file_extension.lower() in file_types): count += 1 try: logger.info("Uploading {0} ".format(new_file_path)) except: pass files = { 'file': (file_name, open(file_path, 'rb'), "multipart/form-data") } # print(files) dict_object = { "filename": file_name, "path": os.path.basename(subdir), "full_path": os.path.join( os.path.abspath(subdir), old_file_name + "." + str(file_extension)) } # Make request try: current_time = dt.datetime.now() seconds = (current_time - starttime).total_seconds() if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours) if token: headers = { 'Authorization': 'Bearer {}'.format(token) } response = requests.request( "POST", upload_url, files=files, data={ 'jsonOptions': configuration_settings[ 'json_options'], 'responseType': configuration_settings[ 'output_options'] }, headers=headers, verify=configuration_settings[ 'ssl_verification']) if response.status_code >= 400: logger.error( "HTTP error {0} occurred when uploading file: {1} " .format(str(response.status_code), file_path)) print(response.text) error = response.text if response.status_code == 500 else json.loads( response.text) logger.error( "Error details: {}".format(error)) dict_object.update( {"error": response.text}) errors.append(dict_object) else: dict_object.update({ "response": response.text, "output_type": configuration_settings[ 'output_options'].split(",") }) output_results.append(dict_object) else: message = "Zen token is required to upload the files, filename {}".format( file_name) logger.error(message) error.append({'error': message}) else: token, checked_time = generateToken_pw_flow( configuration_settings) except SSLError as sslerror: logger.error( "SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file." ) dict_object.update({"error": str(sslerror)}) errors.append(dict_object) except Exception as ex: dict_object.update({"error": str(ex)}) errors.append(dict_object) logger.error( "An error occurred when trying to upload file " + file_path) logger.debug(ex, exc_info=True) pass endtime = dt.datetime.now() seconds = (endtime - starttime).total_seconds() result = { "zen_token": token, "starttime": str(starttime), "endtime": str(endtime), "no_of_files": count, "output_results": output_results, "no_output_results": len(output_results), "total_upload_seconds": seconds, "upload_errors": errors, "no_of_upload_errors": len(errors) } json.dump(result, open(os.path.join(os.getcwd(), "output.json"), 'w'), indent=4) logger.info("Upload status reported in output.json") if count == 0: logger.error("No files found in directory_path, " + dir_path) return False elif len(errors) > 0: logger.error("Errors occurred during upload.") return True else: logger.info("Done uploading {0} files".format(count)) return True else: logger.error("Zen token is required to upload the files") return False else: logger.error( "Check your configuration file (config.json) for correct format and valid parameters" ) return False
def executeRules(): configuration, configuration_settings = readJSON() if (configuration): starttime = dt.datetime.now() dir_path = configuration_settings["output_directory_path"] count = 0 errors = [] output_results = [] for subdir, dirs, files in os.walk(dir_path): for file in files: file_path = os.path.join(subdir, file) new_file = copy.copy(file) #print(file) file_split = new_file.rsplit(".") if (file_split[1] == "json"): odm_file_value = str(file_split[0]) + "ODM.json" file_extension = str(file_split[-1].strip()) old_file_name = new_file.replace("." + file_extension, '').strip() file_name = re.sub( '[^A-Za-z0-9 _]+', ' ', old_file_name).strip() + "." + str(file_extension) new_file_path = os.path.join(subdir, file_name) #print(new_file_path) f = open(new_file_path, "r") # deserializes into dict # and returns dict. y = json.loads(f.read()) #json.append() odmData = { '__DecisionID__': "myworld", "document_in": y, "metaData_inout": { "dynamicParams": "string" } } #var rex = new RegExp('Sensitivity":false', 'g'); #var BACA_STRING_RESPONSE = JSON.stringify(res.data) #BACA_STRING_RESPONSE = BACA_STRING_RESPONSE.replace(rex, 'Sensitivity":0'); odmJson = json.dumps(odmData) #print (odmJson) odmJson = odmJson.replace('"Sensitivity": false', '"Sensitivity":0') odmJson = odmJson.replace('Language": ""', 'Language":[]') #odmpayload = {"__DecisionID__": "string","document_in": y, "metaData_inout": {"dynamicParams": "" }} authPayload = HTTPBasicAuth( configuration_settings["odmUser"], configuration_settings["odmPassword"]) ODM_ProcessBacaDocument_ServiceURL = configuration_settings[ "odm_url"] #JsonPayload = odmpayload headers = {'content-type': 'application/json'} #print(odmData) try: response = requests.request( "POST", ODM_ProcessBacaDocument_ServiceURL, headers=headers, data=odmJson, auth=authPayload, verify=False) print(response.text) jsonPayloadODM = json.loads(response.text) fileOutPutPath = configuration_settings[ "odm_output_directory_path"] + "/" + odm_file_value with open(fileOutPutPath, 'w') as outfile: json.dump(jsonPayloadODM, outfile) #return response except SSLError as sslerror: logger.error( "SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file." ) #dict_object.update({"error": str(sslerror)}) #errors.append(dict_object) return False except Exception as ex: #dict_object.update({"error": str(ex)}) #errors.append(dict_object) #logger.error("An error occurred when trying to upload file " + file_path) logger.debug(ex, exc_info=True) pass f.close() return True