def filter_words_list():
"""
Apply filter words to the list, this is done post tagging in order
that the tagged objects have the correct lexical context
"""
while list_len(DbList.analysis_objects0) != 0:
ao = pop_object(AnalysisObject, DbList.analysis_objects0)
fao = filter_words(ao)
push_object(fao, DbList.filtered_objects)
def get_objects(list_name, model):
"""
Return a list of objects from redis table
:param list_name: name of redis list to search
:type list_name: str
:param model: model class to convert results to
:type model: object
:return: iter of current objects in redis list
:rtype: iter
"""
objs = []
for i in range(list_len(list_name)):
objs.append(get_index(model, list_name, i))
return iter(objs)
def get_test_data():
# flush the database to remove old data
if os.path.exists('analysis_objects0.pickle'):
os.remove('analysis_objects0.pickle')
flush_redis()
# call the main function of get_data to populate analysis objects
app.get_data.get_data_main()
# pull the analysis data from the database
aos = []
while list_len('analysis_objects0') != 0:
ao = pop_object(AnalysisObject, 'analysis_objects0')
aos.append(ao)
# write objects to pickle file
with open('analysis_objects0.pickle', 'wb') as fl:
pickle.dump(aos, fl)
def indexer(list_name):
"""
Loop through the list of AnalysisObjects and then through each info property
create a Dict of:
{'word': N}
- 'N': Number of occurrences of the word in all info lists
:param list_name: name of redis list
:type list_name: str
:return: Dictionary of words and occurrence count
:rtype: dict
"""
word_lookup = {}
for i in range(list_len(list_name)):
ao_info = get_index(AnalysisObject, list_name, i, 'info')
for j in ao_info:
if j[0] not in word_lookup.keys():
word_lookup[j[0]] = 1
else:
word_lookup[j[0]] += 1
return word_lookup
def data_setup():
if list_len('analysis_objects0') in [None, 0]:
tests.setup_data.main()