def loop_through_paras(indata):
''' looping through paras from Step 1 output (Step 3 input) '''
print(f'num paras =={len(indata["paragraphs"])}')
print(f'num gp =={len(indata["group_paragraph"])}')
no_guid = 0
no_id = 0
for para in indata['paragraphs']:
guid = para['guid']
pk_id = para['id']
subtitle = para['subtitle']
str_id = str(pk_id)
if utils.key_not_in_dictionary(indata['record_lookups']['paragraphs'],
guid):
no_guid += 1
print(f'{no_guid}. no guid key for subtitle=={subtitle}')
if utils.key_not_in_dictionary(indata['record_lookups']['paragraphs'],
str_id):
no_id += 1
print(f'{no_id}. no pk_id key for subtitle=={subtitle}')
else:
print(f'results for subtitle=={subtitle}:')
res = utils.find_dictionary_from_list_by_key_and_value(
indata['group_paragraph'], 'paragraph_id', pk_id)
print(f'group_para = {res}')
def incorrect_keys_based_on_owner_record(self):
'''
if the correct child record key and / or association record key exist
will return False (it is truly incorrect), otherwise True (not incorrect)
Will also be incorrect if neither a group id or a category id was passed in
'''
if self.group_id:
return (utils.key_not_in_dictionary(self.file_data, 'paragraphs')
and utils.key_not_in_dictionary(self.file_data,
'group_paragraph'))
if self.category_id:
return utils.key_not_in_dictionary(self.file_data, 'groups')
return True
def add_or_delete_associations(self):
'''
add_or_delete_associations starts the process of reading data from the add and delete
associations portion of the input data. This will be used when you want to associate a
paragraph with another group, for example. Usually adding a reference will be done in a
normal update, but if you just forgot to add a reference, this is an easy fix.
This is called only in Step 3 since it involves changing data in the database
Throughout much of this process we have constants that drive the input data, directing to
the correct process
'''
for input_key in crud.ASSOCIATION_KEYS:
if utils.key_not_in_dictionary(self.file_data, input_key):
continue
function, data_key = input_key.split('_', 1)
input_dictionaries = self.prepare_association_data(
self.file_data[input_key], data_key)
if function == 'delete':
self.delete_associations(data_key, input_key,
input_dictionaries)
json_helper.write_dictionary_to_file(
self.file_data[input_key],
prefix=scripts.PROD_PROCESS_IND,
directory_path=scripts.PROD_INPUT_JSON)
elif function == 'add':
self.add_associations(input_key, input_dictionaries)
def initiate_paragraph_associations(para, key_vars, association_list=None):
'''
add_ref_para_associations is a convenience method allowing the user to simply list the link text
associated with given para, instead of having to manually associate each link to its paragraph
for example, we may create and assign an association list as follows:
self.input_data["add_paragraph_reference"] = [{"paragraph_guid": "val", "reference_slug": "val"}]
:param para: one para
:type para: dict
:param key_vars: key to a foreign key's identifier field name or its value
:type key_vars: dictionary of strings
:param association_list: existing association records, works with None (default)
:type association_list: list, optional
:return: the dicionary with the keys and values necessary to create paragraph reference associations
:rtype: dict
'''
if utils.key_not_in_dictionary(para, key_vars['para_val_2_key']):
return
if not para[key_vars['para_val_2_key']]:
return
return add_to_associations(key_vars['key_1'],
para[key_vars['para_val_1_key']],
key_vars['key_2'],
para[key_vars['para_val_2_key']],
association_list)
def use_default_input_file(process_data, file_list):
''' True if we use default input (for_prod, updated within 2 days) '''
if utils.key_not_in_dictionary(process_data, 'for_prod'):
return False
if process_data['for_prod'] and len(file_list) == 0:
return True
return False
def get_context_data(self, **kwargs):
context = self._add_to_context(super().get_context_data(**kwargs))
if utils.key_not_in_dictionary(context, 'ordered'):
self.template_name = STANDALONE_TMPLT
return context
self.template_name = ORDERED_TMPLT if context[
'ordered'] else STANDALONE_TMPLT
return context
def update_record_loop(self):
'''
update_record_loop finds and updates the record, based on the keys in the input data.
It loops through the UPDATE_RECORD_KEYS to know which keys to look for and then calls the
find_and_update_wrapper method with the necessary arguments to do the actual find and update
CRUD.
'''
for key in crud.UPDATE_RECORD_KEYS:
if utils.key_not_in_dictionary(self.file_data, key):
continue
for record in self.file_data[key]:
self.find_and_update_wrapper(key, record)
def copy_directly_to_output(self):
'''
Copy_directly_to_output is a convenience feature to make it so the user can itemize all the
work needed in step 1. This prepatory process step retrieves data for updates and must be
run beforehand (to capture the existing data and relationships), but the input can also have
some add_* or delete_* keys, which are not prepared for or implemented until step 3.
For COPY_DIRECTLY_TO_OUTPUT data, you can prepare the input and it will carry over to the
next step by copying it directly to the manual json file.
'''
for key in crud.COPY_DIRECTLY_TO_OUTPUT:
if utils.key_not_in_dictionary(self.file_data, key):
continue
self.output_data[key] = self.file_data[key]
def find_or_create_references(self):
'''
find_or_create_references will find based on link_text which must be unique.
'''
if utils.key_not_in_dictionary(self.input_data, 'references'):
return
references = self.input_data['references']
for ref in references:
create_dict = {
'link_text': ref['link_text'],
'url': ref['url'],
'short_text': ref['short_text']
}
find_dict = {'link_text': ref['link_text']}
ref = self.find_or_create_record(Reference, find_dict, create_dict)
def preliminary_record_setup(self):
'''
preliminary_record_setup will loop through input data and set up the data in two ways:
1. Non-association records - need to add the production ids to the lookup table
2. Always adds association records to the create list, because until all the parent records
are created, we will not be able to find the ids necessary for associations
'''
for key in crud.UPDATE_RECORD_KEYS:
if utils.key_not_in_dictionary(self.file_data, key):
continue
self.prod_results[key] = {'created': [], 'updated': []}
for record in self.file_data[key]:
if key in crud.ASSOCIATION_RECORD_KEYS:
self.process_data[key]['create'].append(record)
continue
self.unique_field_lookup(record, key)
def create_links_from_references(self):
'''
create_links_from_references user the link text and url to create html links
'''
references = self.input_data['references']
for ref in references:
link_text = ref['link_text'].strip()
link = para_helpers.create_link(ref['url'], link_text)
self.reference_links[link_text] = link.strip()
if utils.key_not_in_dictionary(ref, 'slug'):
continue
slug = ref['slug']
self.para_link_lookup['ref'][slug] = {
'link_text': ref['short_text'],
'url': ref['url']
}
def expect_to_create_group(self):
'''
expect_to_create_group returns True if the user wants to create a new group
Its purpose is to fail early if a group for a given title does not exist, in order
to prevent a title change or typo making it so garbage groups and group para relationships
are created
:return: returns True if the user wants to create a new group
:rtype: bool
'''
if utils.key_not_in_dictionary(self.input_data['group'], 'short_name'):
return False
if len(self.input_data['group']['short_name']) > 2:
return True
return False
def retrieve_existing_data(self):
'''
retrieve_existing_data is necessary for updating records. It builds the query according to
the input critera, retrieves the records from the database, creates a dictionary with the
information and then writes the dictionary to the output directory as a JSON file.
After any manual updates (step 2), the file becomes input to the db_update_s3 process.
'''
for key in crud.VALID_RETRIEVAL_KEYS:
if utils.key_not_in_dictionary(self.file_data, key):
continue
query = self.build_sql(key)
if query is None:
continue
# print(f'Retrieval query == {query}')
raw_queryset = ParaDbMethods.class_based_rawsql_retrieval(
query, Paragraph)
self.add_existing_data_to_output(raw_queryset)
def link_text_list(self, para):
'''
link_text_list initiates the process of turning the list of references' link_text to
associations between a given paragraph and all of its references
:param para: one paragraph
:type para: dict
'''
if utils.key_not_in_dictionary(self.input_data, 'ref_link_paragraph'):
self.input_data['ref_link_paragraph'] = []
updated_para_ref = helpers.initiate_paragraph_associations(
para, crud.PARA_ID_REF_LINK_TEXT,
self.input_data['ref_link_paragraph'])
if updated_para_ref is not None:
self.input_data['ref_link_paragraph'] = updated_para_ref
if utils.key_in_dictionary(para, 'link_text_list'):
para.pop('link_text_list')
def decide_standalone(self, para):
'''
Decide_standalone says whether to make the standalone field in the paragraph record
True or False. If self.ordered is False (group_type is standalone), the paragraph must be a
standalone paragraph, even if it is used elsewhere as part of an ordered group.
If the group is ordered, but the paragraph is designed to also be a standalone paragraph then
the paragraph record will be standalone. In this case, it is up to the person creating the
input data to make sure the paragraph is standalone.
:param para: paragraph record from the input
:type para: dict
:return: True or False based on whether the paragraph stands alone
:rtype: Boolean
'''
if not self.ordered:
return True
if utils.key_not_in_dictionary(para, 'standalone'):
return False
return True if para['standalone'] in ('yes', 'true', 'True') else False
def record_lookups(self, top_key, pk_id, class_):
'''
record_lookups will make sure that there is a way to uniquely identify records
that may have a different primary key in production
:param top_key: top key to lookup table: plural form of the main four paragraph records
:type top_key: str
:param pk_id: key for lookup: primary key of the record we need to look up
:type pk_id: int
:param class_: models.Model class for the lookup
:type class_: models.Model
'''
if top_key == 'categories' and pk_id is None:
return
dict_to_check = self.output_data['record_lookups'][top_key]
if utils.key_not_in_dictionary(dict_to_check, pk_id):
rec = class_.objects.get(pk=pk_id)
if top_key == 'paragraphs':
self.for_prod_lookup(top_key, rec.id, rec.guid)
return
self.for_prod_lookup(top_key, rec.id, rec.slug)
def substitute_prod_foreign_key(self, record, record_key, top_level_key):
'''
substitute_prod_foreign_key finds the production id for the foreign keys in the association
records. For example, group_id and paragraph_id in the GroupParagraph record
Two errors could be thrown, theoretically, but it would mean a programming mistake in Step 1.
1. A Value Error if a dev id is not an integer
2. A Key Error if there is no prod id for the given category in the lookup table
:param record: record created manually when a record has foreign keys
:type record: dict
:return: group record with the production id (or, for testing test id) added
:rtype: dict
'''
str_dev_id = str(record[record_key])
unique_field = self.record_lookups[top_level_key][str_dev_id]
if utils.key_not_in_dictionary(
self.record_lookups[top_level_key][unique_field], 'prod_id'):
self.assign_existing_record_prod_id(top_level_key, str_dev_id)
record[record_key] = self.record_lookups[top_level_key][unique_field][
'prod_id']
return record
def associate_ref_para(self, para):
'''
associate_ref_para initiates the process of turning the list of references' slug or
link_text to associations between a given paragraph and all of its references
:param para: one paragraph
:type para: dict
'''
if utils.no_keys_from_list_in_dictionary(
('ref_slug_list', 'link_text_list'), para):
return para
if utils.key_not_in_dictionary(self.file_data,
'add_paragraph_reference'):
self.file_data['add_paragraph_reference'] = []
add_para_refs = helpers.initiate_paragraph_associations(
para, self.correct_ref_data(para.keys()),
self.file_data['add_paragraph_reference'])
if add_para_refs is not None:
self.file_data['add_paragraph_reference'] = add_para_refs
para = utils.pop_keys(('ref_slug_list', 'link_text_list'), para)
return para
def associate_paragraphs_with_references(self):
'''
associate_paragraphs_with_references deals with the association table for paragraphs to
references. It was a pain to create manually, so now it is created programmatically
(see self.link_text_list()). The association is made, as part of the create paragraph
process with 'link_text_list': []. OR as part of the update paragraph process
(ParaDbUpdateProcess) with either 'ref_slug_list': [] or 'link_text_list': [].
All variations link through the list of unique reference fields and associate the reference
with the paragraph that was just created or updated (in another class). This is easier in
terms of creating the data.
Note - the para reference association is made after the paragraph, so there is no need
for a fake paragraph id. It adds flexibility, however, so leaving for now
'''
if utils.key_not_in_dictionary(self.input_data, 'ref_link_paragraph'):
return
ref_link_paras = self.input_data['ref_link_paragraph']
for ref_para in ref_link_paras:
ref = Reference.objects.get(link_text=ref_para['link_text'])
para = Paragraph.objects.get(
pk=self.fake_to_real_para_id[ref_para['paragraph_id']])
para.references.add(ref)
def test_key_not_in_dictionary(key, expected):
result = utils.key_not_in_dictionary({'name': 'Nemo'}, key)
assert result == expected
