def downloadFile(configuration_settings, analyzerId, output, output_path, filename,bearerToken):
# Make request
headers = {
'apiKey': configuration_settings['api_key']
}
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (
configuration_settings["function_id"], configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
#headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8")
#headers['Authorization'] = configuration_settings["bear_token"]
headers['Authorization'] = bearerToken
extra_path = "{0}/{1}/".format(analyzerId, output.lower())
get_url = urlparse.urljoin(configuration_settings['main_url'], extra_path)
try:
response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
file_name = copy.copy(filename)
filename_output = "{0}.{1}".format(file_name, output.lower()) if output.lower() != "utf8" else "{0}.txt".format(file_name)
filename_output = filename_output if output.lower() != "pdf" else "New_{0}.{1}".format(file_name, output.lower())
file_output_path = os.path.join(configuration_settings["output_directory_path"], output_path, output.lower())
filename_output_path = os.path.join(file_output_path, filename_output)
if( not os.path.exists(file_output_path)):
os.makedirs(file_output_path)
logger.info("Created new output directory, "+ file_output_path)
if(output.lower() == "json"):
response_output = json.loads(response.text)
if "data" in response_output:
json.dump(response_output["data"], open(filename_output_path, "w"), indent=4)
elif output.lower() == "pdf":
output_write = open(filename_output_path, "wb")
output_write.write(response.content)
output_write.close()
else:
output_write = open(filename_output_path, "w")
try:
output_write.write(response.text)
except:
output_write.write(response.text.encode('utf-8'))
output_write.close()
return True, ""
except SSLError as sslerror:
logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.")
logger.debug(sslerror, exc_info=True)
except Exception as ex:
logger.error("An error occurred when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, ""
def downloadFile(configuration_settings, token, analyzerId, output,
output_path, filename):
# Make request
headers = {'Authorization': 'Bearer {}'.format(token)}
extra_path = "/{0}/{1}".format(analyzerId, output.lower())
get_url = configuration_settings['aca_main_url'] + extra_path
try:
response = requests.request(
"GET",
get_url,
headers=headers,
verify=configuration_settings['ssl_verification'])
# print(response.text)
if response.status_code >= 400:
logger.error("An error occurred while trying to download output.")
return False, response.text
else:
file_name = copy.copy(filename)
filename_output = "{0}.{1}".format(file_name, output.lower(
)) if output.lower() != "utf8" else "{0}.txt".format(file_name)
filename_output = filename_output if output.lower(
) != "pdf" else "New_{0}.{1}".format(file_name, output.lower())
file_output_path = os.path.join(
configuration_settings["output_directory_path"], output_path,
output.lower())
filename_output_path = os.path.join(file_output_path,
filename_output)
if (not os.path.exists(file_output_path)):
os.makedirs(file_output_path)
logger.info("Created new output directory, " +
file_output_path)
if (output.lower() == "json"):
response_output = json.loads(response.text)
if "result" in response_output and "data" in response_output[
"result"][0]:
json.dump(response_output["result"][0]["data"],
open(filename_output_path, "w"),
indent=4)
elif output.lower() == "pdf":
output_write = open(filename_output_path, "wb")
output_write.write(response.content)
output_write.close()
return True, ""
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
logger.debug(sslerror, exc_info=True)
except Exception as ex:
logger.error("An error occurred when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, ""
def checkTokenValid(token, config):
if token:
if config:
zen_check_token_url = 'https://' + config['zen_host'].strip(
) + '/usermgmt/v1/user/currentUserInfo'
try:
headers = {
"authorization": "Bearer " + token,
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("GET",
zen_check_token_url,
headers=headers,
verify=config['ssl_verification'])
if response.status_code == 200:
# content = getContent(response)
# res = content.get('uid')
logger.info('zen token provided is valid')
return token
elif response.status_code == 401:
logger.info(
'zen token provided is invalid or expired, trying to generate a new one'
)
token = generateToken_pw_flow(config)
return token
else:
logger.error("Failed to check the token valid or not")
content = getContent(response)
logger.debug(content)
return None
except Exception as ex:
logger.error(
"Failed to check the token valid or not due to exception")
logger.debug(str(ex))
return None
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return None
else:
logger.error("Zen token is required, please provide a valid token")
return None
def deleteFiles(bearerToken):
configuration, configuration_settings = readJSON()
if (configuration):
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "data" in json.loads(result["response"]) and "analyzerId" in \
json.loads(result["response"])["data"]:
if ("deleted" in result and result["deleted"]
== False) or "deleted" not in result:
response = json.loads(result["response"])
analyzerId = response["data"]["analyzerId"]
status, result_response = deleteFile(
configuration_settings, analyzerId,
bearerToken)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
def generateToken_pw_flow(config):
zen_token_url = 'https://' + config['zen_host'].strip(
) + '/v1/preauth/validateAuth'
zen_username = config['zen_username'].strip()
zen_pw = config['zen_password'].strip()
try:
encoding = base64.b64encode('{}:{}'.format(zen_username,
zen_pw).encode())
headers = {
"authorization": "Basic " + encoding.decode("utf-8"),
'Content-Type': 'application/x-www-form-urlencoded'
}
response = requests.request("GET",
zen_token_url,
headers=headers,
verify=config['ssl_verification'])
if response.status_code == 200:
# token is generated
# {
# "accessToken": "xxxxxxxxx"
# }
content = getContent(response)
# print(content)
token = content.get('accessToken')
logger.info('Successfully got the token')
json.dump({"zen_token": token}, open("output.json", 'w'), indent=4)
return token
else:
logger.error("Failed to generate a token")
content = getContent(response)
logger.debug(content)
return None
except Exception as ex:
logger.error("Failed to generate a token due to exception")
logger.debug(str(ex))
return None
def checkStatus(configuration_settings, analyzerId,bearerToken):
logger.info("Config check")
# Make request
headers = {
'apiKey': configuration_settings['api_key']
}
logger.info("Config check" + configuration_settings['api_key'])
if "function_id" in configuration_settings:
credentials = ('%s:%s' % (
configuration_settings["function_id"], configuration_settings["password"]))
encoded_credentials = b64encode(credentials.encode('ascii'))
#headers['Authorization'] = 'Basic %s' % encoded_credentials.decode("utf-8")
#headers['Authorization'] = configuration_settings["bear_token"]
headers['Authorization'] = bearerToken
get_url = urlparse.urljoin(configuration_settings['main_url'], "{0}/".format(analyzerId))
logger.info("Config check " + get_url)
try:
response = requests.request("GET", get_url, headers=headers, verify=configuration_settings['ssl_verification'])
logger.info(response.status_code)
if response.status_code >= 400:
logger.error("An error occurred while trying to get file status.")
return False, response.text
else:
response_json = json.loads(response.text)
return True, response_json
except SSLError as sslerror:
logger.error("SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file.")
logger.debug(sslerror, exc_info=True)
except Exception as ex:
print(ex)
logger.error("An error occured when trying to get file")
logger.debug(ex, exc_info=True)
pass
return False, {}
def uploadFiles():
configuration, configuration_settings = readJSON()
if (configuration):
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['main_url']
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file': (file_name, open(file_path,
'rb'), "multipart/form-data")
}
headers = {'apiKey': configuration_settings['api_key']}
if "function_id" in configuration_settings:
credentials = ('%s:%s' %
(configuration_settings["function_id"],
configuration_settings["password"]))
encoded_credentials = b64encode(
credentials.encode('ascii'))
headers[
'Authorization'] = 'Basic %s' % encoded_credentials.decode(
"utf-8")
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings['json_options'],
'responseType':
configuration_settings['output_options']
},
headers=headers,
verify=configuration_settings['ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code), file_path))
dict_object.update({"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings['output_options'].split(
",")
})
output_results.append(dict_object)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file " +
file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"), 'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
return None
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return None
else:
logger.error("Zen token is required, please provide a valid token")
return None
def getFirstToken():
configuration, configuration_settings = readJSON()
if (configuration):
token = generateToken_pw_flow(configuration_settings)
return token
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
if __name__ == '__main__':
logger.info("Logs can be found in current directory (processing.log)")
token = None
try:
getFirstToken()
except Exception as ex:
logger.error("An error occurred, please check logs for more details")
logger.debug(ex, exc_info=True)
else:
logger.info(
"output.json file does not exist. No results available to reupload."
)
else:
logger.error("Zen token is required to reUpload the files")
return False
else:
logger.info(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
if __name__ == '__main__':
logger.info("**************************************")
logger.info("API Sample tool reUploadUnfinished starting...")
logger.info("Logs can be found in the current directory, processing.log")
try:
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
token = output_json.get('zen_token') if output_json.get(
'zen_token') else "token"
print(token)
else:
token = "token"
reUploadFiles(token)
except Exception as ex:
logger.error("An error occurred, please check logs for more details")
logger.debug(ex, exc_info=True)
Copyright IBM Corp. 2019 All Rights Reserved.
To run, see README.md
'''
import datetime as dt
from loggingHandler import logger
from downloadFiles import downloadFiles
from reUploadUnfinished import reUploadFiles
from deleteFiles import deleteFiles
from updateReport import updateReport
starttime = dt.datetime.now()
if __name__ == '__main__':
logger.info("**************************************")
logger.info("API Sample tool reupload starting...")
logger.info("Logs can be found in the current directory, processing.log")
logger.info("Uploading files")
uploadSuccess = reUploadFiles()
if (uploadSuccess):
logger.info("Ready to download output files...")
complete = downloadFiles()
if (complete):
logger.info("Deleting files on the server")
deleteFiles()
else:
logger.info(
configuration_settings, analyzerId)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
if __name__ == '__main__':
logger.info("**************************************")
logger.info("API Sample tool deleteFiles starting...")
logger.info("Logs can be found in the current directory, processing.log")
deleteFiles()
def updateReport():
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
new_output_json_result = []
unfinished_files = []
unfinished_files_with_path = []
new_file_upload_errors = [
output["filename"] for output in output_json["upload_errors"]
] if "upload_errors" in output_json else []
new_file_errors = [
output["filename"] for output in output_json["errors"]
] if "errors" in output_json else []
unfinished_files.extend(new_file_upload_errors)
unfinished_files.extend(new_file_errors)
new_file_upload_errors_with_path = [
{
"filename": output["filename"],
"path": output["path"]
} for output in output_json["upload_errors"]
] if "upload_errors" in output_json else []
new_file_errors_with_path = [{
"filename": output["filename"],
"path": output["path"]
} for output in output_json["errors"]
] if "errors" in output_json else []
unfinished_files_with_path.extend(new_file_upload_errors_with_path)
unfinished_files_with_path.extend(new_file_errors_with_path)
endtime = dt.datetime.now()
starttime = parser.parse(output_json["starttime"])
if ("output_results" in output_json):
output_json_result = output_json["output_results"]
if (len(output_json["output_results"]) > 0):
for outresult in output_json_result:
result = outresult
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
unfinished_object = {
"filename":
result["filename"],
"path":
result["path"] if "path" in result else "",
"full_path":
result["full_path"] if "full_path" in result else ""
}
try:
if ("download_success" in result
and result["download_success"] == False):
if (result["filename"] not in unfinished_files):
unfinished_files.append(result["filename"])
unfinished_files_with_path.append(
unfinished_object)
else:
output_count = 0
for output in latest_output_outputs:
if (output not in result):
if (result["filename"]
not in unfinished_files):
unfinished_files.append(
result["filename"])
unfinished_files_with_path.append(
unfinished_object)
if output in result:
output_count += 1
if result[output] == False:
if (result["filename"]
not in unfinished_files):
unfinished_files.append(
result["filename"])
unfinished_files_with_path.append(
unfinished_object)
if output_count == 0:
if (result["filename"]
not in unfinished_files):
unfinished_files.append(result["filename"])
unfinished_files_with_path.append(
unfinished_object)
except:
result["deleted"] = False
# new_output_json_result.append(result)
seconds = (endtime - starttime).total_seconds()
output_json["endtime"] = str(endtime)
output_json["total_seconds"] = str(seconds)
output_json["unfinished_files"] = unfinished_files_with_path
output_json["no_of_unfinished_files"] = len(unfinished_files)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info(
"Done updating report in the current directory, output.json")
else:
logger.error("No results available for reporting")
else:
logger.error(
"output.json file does not exist. No results available for reporting."
)
Copyright IBM Corp. 2019 All Rights Reserved.
To run, see README.md
'''
import datetime as dt
from loggingHandler import logger
from downloadFiles import downloadFiles
from uploadFiles import uploadFiles
from deleteFiles import deleteFiles
from updateReport import updateReport
starttime = dt.datetime.now()
if __name__ == '__main__':
logger.info("**************************************")
logger.info("API Sample tool starting...")
logger.info("Logs can be found in the current directory, processing.log")
logger.info("Uploading files")
uploadSuccess = uploadFiles()
if (uploadSuccess):
logger.info("Ready to download output files...")
complete = downloadFiles()
if (complete):
logger.info("Deleting files on the server")
deleteFiles()
else:
logger.info(
unfinished_files_with_path.append(
unfinished_object)
except:
result["deleted"] = False
# new_output_json_result.append(result)
seconds = (endtime - starttime).total_seconds()
output_json["endtime"] = str(endtime)
output_json["total_seconds"] = str(seconds)
output_json["unfinished_files"] = unfinished_files_with_path
output_json["no_of_unfinished_files"] = len(unfinished_files)
output_json["output_results"] = output_json_result
json.dump(output_json, open("output.json", 'w'), indent=4)
logger.info(
"Done updating report in the current directory, output.json")
else:
logger.error("No results available for reporting")
else:
logger.error(
"output.json file does not exist. No results available for reporting."
)
if __name__ == '__main__':
logger.info("**************************************")
logger.info("API Sample tool updateReport starting...")
logger.info("Logs can be found in the current directory, processing.log")
updateReport()
def deleteFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
output_json = json.load(open(output_json_path, "r"))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "result" in json.loads(
result["response"]
) and "data" in json.loads(
result["response"]
)["result"][0] and "analyzerId" in json.loads(
result["response"])["result"][0]["data"]:
if ("deleted" in result and result["deleted"]
== False) or "deleted" not in result:
response = json.loads(result["response"])
# print(response)
analyzerId = response["result"][0]["data"][
"analyzerId"]
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
status, result_response = deleteFile(
configuration_settings, token,
analyzerId)
if (status):
result["deleted"] = True
else:
result["deleted"] = False
else:
token = generateToken_pw_flow(
configuration_settings)
except:
result["deleted"] = False
logger.error(
"No analyzerID available to delete results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
# new_output_json_result.append(result)
output_json["output_results"] = output_json_result
json.dump(output_json,
open("output.json", 'w'),
indent=4)
logger.info("Done deleting files on the server")
logger.info("Delete status reported in output.json")
else:
logger.error("No results found that needs deleting.")
else:
logger.error(
"output.json file does not exist. No results available to delete."
)
else:
logger.error("Zen token is required to delete files")
def uploadFiles(token):
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
if token:
starttime = dt.datetime.now()
dir_path = configuration_settings["directory_path"]
count = 0
errors = []
upload_url = configuration_settings['aca_main_url']
logger.info("ACA main url is: {0} ".format(upload_url))
accepted_extensions = configuration_settings["accepted_extensions"]
file_types = configuration_settings[
'file_type'] if 'file_type' in configuration_settings and type(
configuration_settings['file_type'] is list) and len(
configuration_settings['file_type']
) > 0 else accepted_extensions
file_types = [f_type.lower() for f_type in file_types]
output_results = []
for subdir, dirs, files in os.walk(dir_path):
for file in files:
file_path = os.path.join(subdir, file)
new_file = copy.copy(file)
file_split = new_file.rsplit(".")
file_extension = str(file_split[-1].strip())
old_file_name = new_file.replace("." + file_extension,
'').strip()
file_name = re.sub(
'[^A-Za-z0-9 _]+', ' ',
old_file_name).strip() + "." + str(file_extension)
new_file_path = os.path.join(subdir, file_name)
if (file_extension != ""
and file_extension.lower() in accepted_extensions
and file_extension.lower() in file_types):
count += 1
try:
logger.info("Uploading {0} ".format(new_file_path))
except:
pass
files = {
'file':
(file_name, open(file_path,
'rb'), "multipart/form-data")
}
# print(files)
dict_object = {
"filename":
file_name,
"path":
os.path.basename(subdir),
"full_path":
os.path.join(
os.path.abspath(subdir),
old_file_name + "." + str(file_extension))
}
# Make request
try:
current_time = dt.datetime.now()
seconds = (current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(token)
}
response = requests.request(
"POST",
upload_url,
files=files,
data={
'jsonOptions':
configuration_settings[
'json_options'],
'responseType':
configuration_settings[
'output_options']
},
headers=headers,
verify=configuration_settings[
'ssl_verification'])
if response.status_code >= 400:
logger.error(
"HTTP error {0} occurred when uploading file: {1} "
.format(str(response.status_code),
file_path))
print(response.text)
error = response.text if response.status_code == 500 else json.loads(
response.text)
logger.error(
"Error details: {}".format(error))
dict_object.update(
{"error": response.text})
errors.append(dict_object)
else:
dict_object.update({
"response":
response.text,
"output_type":
configuration_settings[
'output_options'].split(",")
})
output_results.append(dict_object)
else:
message = "Zen token is required to upload the files, filename {}".format(
file_name)
logger.error(message)
error.append({'error': message})
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
except SSLError as sslerror:
logger.error(
"SSL error was thrown due to certificate failure, set ssl_verification to false in configuration config.json file."
)
dict_object.update({"error": str(sslerror)})
errors.append(dict_object)
except Exception as ex:
dict_object.update({"error": str(ex)})
errors.append(dict_object)
logger.error(
"An error occurred when trying to upload file "
+ file_path)
logger.debug(ex, exc_info=True)
pass
endtime = dt.datetime.now()
seconds = (endtime - starttime).total_seconds()
result = {
"zen_token": token,
"starttime": str(starttime),
"endtime": str(endtime),
"no_of_files": count,
"output_results": output_results,
"no_output_results": len(output_results),
"total_upload_seconds": seconds,
"upload_errors": errors,
"no_of_upload_errors": len(errors)
}
json.dump(result,
open(os.path.join(os.getcwd(), "output.json"),
'w'),
indent=4)
logger.info("Upload status reported in output.json")
if count == 0:
logger.error("No files found in directory_path, " + dir_path)
return False
elif len(errors) > 0:
logger.error("Errors occurred during upload.")
return True
else:
logger.info("Done uploading {0} files".format(count))
return True
else:
logger.error("Zen token is required to upload the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
def downloadFiles():
pending_completion = True
configuration, configuration_settings = readJSON()
if (configuration):
loop = 0
failed_download = []
completed_download = []
completed_count = 0
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
while pending_completion and loop < 1000:
output_json = json.load(open(output_json_path, "r"))
loop += 1
logger.info("Loop " + str(loop))
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "data" in json.loads(
result["response"]
) and "analyzerId" in json.loads(
result["response"])["data"]:
if "download_success" not in result:
response = json.loads(result["response"])
path = result["path"]
filename = result["filename"]
analyzerId = response["data"]["analyzerId"]
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
if ("download_completed" not in result):
completed = checkCompleted(
latest_output_outputs, result)
if (completed):
result["download_completed"] = True
else:
logger.info(
"Checking status of analyzerId: "
+ analyzerId)
status, result_response = checkStatus(
configuration_settings,
analyzerId)
if (status):
if ("data" in result_response
and "statusDetails" in
result_response["data"]
):
status_result_response = result_response[
"data"][
"statusDetails"]
done_output = []
for output in status_result_response:
if (output["type"] in
latest_output_outputs
and output[
"status"]
== "Completed"
and
output["type"]
not in result):
logger.info(
"Downloading {0} of analyzerId: {1}"
.format(
output[
"type"],
analyzerId
))
response, reason = downloadFile(
configuration_settings,
analyzerId,
output["type"],
path,
filename.
rsplit(".")[0])
result[output[
"type"]] = response
if (not response):
result[
output[
"type"]
+
"_error"] = reason
done_output.append(
output["type"])
elif (output["type"] in
latest_output_outputs
and
output["status"]
== "Failed" and
output["type"]
not in result):
result[output[
"type"]] = False
result[
output["type"]
+
"_error"] = output[
"status"]
done_output.append(
output["type"])
elif output[
"type"] in result:
done_output.append(
output["type"])
if (len(
done_output
) == len(
latest_output_outputs
)):
completed_download.append(
True)
completed = checkCompleted(
latest_output_outputs,
result)
if (completed):
result[
"download_completed"] = True
else:
result[
"download_success"] = False
result[
"download_failure_reason"] = result_response
failed_download.append(True)
else:
logger.error(
"We could not find any information to download files from."
)
result["download_success"] = False
result[
"download_failure_reason"] = "No available data to download from"
failed_download.append(True)
except:
result["download_success"] = False
result[
"download_failure_reason"] = "No analyzerID available to download results"
logger.error(
"No analyzerID available to download results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
failed_download.append(True)
# new_output_json_result.append(result)
# print(len(new_output_json_result))
endtime = dt.datetime.now()
output_json["output_results"] = output_json_result
output_json["endtime"] = str(endtime)
json.dump(output_json,
open("output.json", 'w'),
indent=4)
completed_count = len(failed_download) + len(
completed_download)
if (completed_count >= len(output_json_result)):
pending_completion = False
else:
if (loop >= 1000):
pending_completion = False
logger.error(
"Reached maximum number of download retries.")
else:
time.sleep(5)
else:
pending_completion = False
logger.error("No results available to download.")
return True
else:
logger.info(
"Done downloading all output files to your output_directory_path"
)
logger.info("Download status reported in output.json")
if (loop < 999):
return True
else:
return False
else:
logger.error(
"output.json file does not exist. No results available to download."
)
return False
def downloadFiles(token):
pending_completion = True
configuration, configuration_settings = readJSON()
if (configuration):
token = checkTokenValid(token, configuration_settings)
starttime = dt.datetime.now()
if token:
loop = 0
failed_download = []
completed_download = []
compelted_check = []
completed_count = 0
output_json_path = os.path.join(os.getcwd(), "output.json")
if (os.path.exists(output_json_path)):
while pending_completion and loop < 1000:
output_json = json.load(open(output_json_path, "r"))
loop += 1
logger.info("Loop " + str(loop))
# print(output_json)
if ("output_results" in output_json
and len(output_json["output_results"]) > 0):
output_json_result = output_json["output_results"]
new_output_json_result = []
for outresult in output_json_result:
result = outresult
try:
if "response" in result and "result" in json.loads(
result["response"]
) and "data" in json.loads(
result["response"]
)["result"][0] and "analyzerId" in json.loads(
result["response"]
)["result"][0]["data"]:
if "download_success" not in result:
response = json.loads(
result["response"])
path = result["path"]
filename = result["filename"]
analyzerId = response["result"][0][
"data"]["analyzerId"]
output_outputs = result["output_type"]
latest_output_outputs = [
output.replace("\"", "").upper()
for output in output_outputs
]
if ("download_completed"
not in result):
completed = checkCompleted(
latest_output_outputs, result)
if (completed):
result[
"download_completed"] = True
else:
current_time = dt.datetime.now(
)
seconds = (
current_time -
starttime).total_seconds()
if seconds < 7000 * 5: # refresh zen token every 10 hours (7199 = 2 hours)
if token:
headers = {
'Authorization':
'Bearer {}'.format(
token)
}
logger.info(
"Checking status of analyzerId: {}, filename: {}"
.format(
analyzerId,
filename))
status, result_response = checkStatus(
configuration_settings,
token, analyzerId)
if (status):
if ("data" in
result_response
and
"statusDetails"
in
result_response[
"data"]
):
status_result_response = result_response[
"data"][
"statusDetails"]
done_output = []
for output in status_result_response:
# print(output)
logger.info(
"status: {}, type {}, analyzerId: {}, filename: {}"
.
format(
output
.
get(
"status"
),
output
.
get(
"type"
),
analyzerId,
filename
))
if (output[
"type"]
in
latest_output_outputs
and
output[
"status"]
==
"Completed"
and
output[
"type"]
not in
result
):
logger.info(
"Downloading {0} of analyzerId: {1}"
.
format(
output[
"type"],
analyzerId
))
response, reason = downloadFile(
configuration_settings,
token,
analyzerId,
output[
"type"],
path,
filename
.
rsplit(
"."
)
[0]
)
result[output[
"type"]] = response
if (not response
):
result[
output[
"type"]
+
"_error"] = reason
done_output.append(
output[
"type"]
)
elif (output[
"type"]
in
latest_output_outputs
and
output[
"status"]
==
"Failed"
and
output[
"type"]
not in
result
):
result[output[
"type"]] = False
result[
output[
"type"]
+
"_error"] = output[
"status"]
logger.info(
"Processing failed of analyzerId: {}, filename: {}"
.
format(
analyzerId,
filename
))
done_output.append(
output[
"type"]
)
elif output[
"type"] in result:
logger.info(
"Processing of analyzerId: {}, filename: {}"
.
format(
analyzerId,
filename
))
done_output.append(
output[
"type"]
)
if (len(
done_output
) == len(
latest_output_outputs
)):
completed_download.append(
True)
compelted_check.append(
filename
)
completed = checkCompleted(
latest_output_outputs,
result)
if (completed):
result[
"download_completed"] = True
else:
logger.error(
"Cannot get status of analyzerId: {}, filename: {}"
.format(
analyzerId,
filename))
result[
"download_success"] = False
result[
"download_failure_reason"] = result_response
failed_download.append(
True)
else:
message = "Zen token is required to check file status, filename {}".format(
filename)
logger.error(message)
failed_download.append(
True)
else:
token, checked_time = generateToken_pw_flow(
configuration_settings)
else:
logger.error(
"We could not find any information to download files from."
)
result["download_success"] = False
result[
"download_failure_reason"] = "No available data to download from"
failed_download.append(True)
except Exception as err:
result["download_success"] = False
result[
"download_failure_reason"] = "No analyzerID available to download results"
logger.error(
"No analyzerID available to download results. The file upload may have failed. File name: {0}"
.format(result["filename"]))
failed_download.append(True)
logger.error(err)
# new_output_json_result.append(result)
# print(len(new_output_json_result))
endtime = dt.datetime.now()
output_json["output_results"] = output_json_result
output_json["endtime"] = str(endtime)
json.dump(output_json,
open("output.json", 'w'),
indent=4)
completed_count = len(failed_download) + len(
completed_download)
logger.info("count, failed {}, completed {}".format(
len(failed_download), len(completed_download)))
if (completed_count >= len(output_json_result)):
print(compelted_check)
pending_completion = False
else:
if (loop >= 1000):
pending_completion = False
logger.error(
"Reached maximum number of download retries."
)
else:
time.sleep(10)
else:
pending_completion = False
logger.error("No results available to download.")
return True
logger.info(
"Done downloading all output files to your output_directory_path"
)
logger.info("Download status reported in output.json")
if (loop < 999):
return True
else:
return False
else:
logger.error(
"output.json file does not exist. No results available to download."
)
return False
else:
logger.error("Zen token is required to download the files")
return False
else:
logger.error(
"Check your configuration file (config.json) for correct format and valid parameters"
)
return False
