def document_id_beginning_is_letter(document_id):
"""
:param document_id:
:return:
"""
letter = document_id[0]
result = letter.isalpha()
if result is True:
return True
logger.warning('Document ID does not begin with a letter')
return False
def create_document_job(work_file, job_id):
"""
Creates a job for the server to provide to clients
:param work_file: The list of ids for the clients to retrieve
:param job_id: The id for the job
:return: A json in the form of a dictionary
"""
logger.warning('Creating document job...')
dictionary = {'job_id': job_id,
'type': 'doc',
'data': work_file,
'version': VERSION}
return json.dumps(dictionary)
def copy_file_safely(directory, filepath):
"""
Safely copies a file to a directory; if the file isn't there to be copied, it won't be copied.
:param directory: Directory to copy to
:param filepath: File to copy
"""
if Path(filepath).exists():
if Path(directory).exists():
shutil.copy(filepath, directory)
else:
logger.warning('File not copied, directory does not exist')
else:
logger.warning('File not copied, file does not exist')
def add_document_job_to_queue(redis_server, json_data):
"""
Creates a job for each work file and then adds each job to the "queue"
:param json_data: the json data containing all the work files
:return:
"""
logger.warning('Adding document job to the queue...')
for work_file in json_data["data"]:
random_id = \
''.join(random.choices(string.ascii_letters + string.digits, k=16))
job = create_document_job(work_file, random_id)
redis_server.add_to_queue(job)
logger.warning('Document job successfully added to queue')
def get_work():
"""
Endpoint the user will use to get work from the queue
client_id will be one of the parameters given for logging purposes
:return: Returns the json containing the job_id, the type of work to be done, the work that nees to be done, and
the version number
"""
logger.warning("Successful API Call: %s", 'get_work: get_work')
if len(request.args) != 1:
logger.error('Error - number of parameters incorrect')
return 'Parameter Missing', 400
client_id = request.args.get('client_id')
if client_id is None:
logger.warning("Exception: %s",
'get_work: BadParameterException, client id was none')
logger.error('Error - no client ID')
return 'Bad Parameter', 400
json_info = redis_server().get_work()
return json.dumps(json_info)
def check_workfile_length(json_data):
"""
Checks the file count and attachment count of each work file
:param json_data: the json containing the work files
:return: True if there are 1000 or less document ids and 1000
or less attachments per work file False if either
the ids or attachments are over 1000
"""
file_count = 0
attachment_count = 0
for work_file in json_data['data']:
for line in work_file:
file_count += 1
attachment_count += line["count"]
is_file_count_too_big = file_count > 1000
is_attachment_count_too_big = attachment_count > 1000
if is_file_count_too_big or is_attachment_count_too_big:
return False
logger.warning('Workfile length check completed')
return True
def call(url):
"""
Sends an API call to regulations.gov
Raises exceptions if it is not a valid API call
When a 300 status code is given, return a temporary exception so the user can retry the API call
When a 429 status code is given, the user is out of API calls and must wait an hour to make more
When 400 or 500 status codes are given there is a problem with the API connection
:param url: the url that will be used to make the API call
:return: returns the json format information of the documents
"""
logger.warning('Making API call...')
result = requests.get(url)
if 300 <= result.status_code < 400:
logger.warning('API call failed')
raise TemporaryException
if result.status_code == 429:
logger.warning('API call failed')
raise ApiCountZeroException
if 400 <= result.status_code < 600:
logger.warning('API call failed')
raise PermanentException
logger.warning('API call successfully made')
return result
def get_keys_from_progress_no_lock(self, job_id):
"""
Get the key of a job that is the "progress" queue
:param job_id: The id of the job you want to get the key for
:return: '' if the job does not exist, or the key if the job does exist
"""
key_list = self.r.hgetall('progress')
logger.warning('Variable Success: %s', 'get_keys_from_progress_no_lock: list of keys successfully received')
logger.warning('CLIENT_JOB_ID: %s', 'get_keys_from_progress_no_lock: ' + str(job_id))
for key in key_list:
logger.warning('CURRENT_KEY: %s', key)
logger.warning('Assign Variable: %s', 'get_keys_from_progress_no_lock: attempt to get the json using the key')
json_info = self.get_specific_job_from_progress_no_lock(key)
info = literal_eval(json_info)
if info["job_id"] == job_id:
return key.decode("utf-8")
return -1
def __init__(self):
logger.warning('API call failed...')