def test_updates_to_file_cause_lookup_rebuild(self):
spec = BaseTest.load_test_spec('enum_options_with_search')
workflow = self.create_workflow('enum_options_with_search')
file_model = session.query(FileModel).filter(
FileModel.name == "sponsors.xls").first()
LookupService.lookup(workflow, "sponsor", "sam", limit=10)
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(1, len(lookup_records))
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEqual(28, len(lookup_data))
# Update the workflow specification file.
file_path = os.path.join(app.root_path, '..', 'tests', 'data',
'enum_options_with_search',
'sponsors_modified.xls')
file = open(file_path, 'rb')
FileService.update_file(file_model, file.read(), CONTENT_TYPES['xls'])
file.close()
# restart the workflow, so it can pick up the changes.
WorkflowProcessor(workflow, soft_reset=True)
LookupService.lookup(workflow, "sponsor", "sam", limit=10)
lookup_records = session.query(LookupFileModel).all()
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEqual(4, len(lookup_data))
def test_lookup_fails_for_xls(self):
BaseTest.load_test_spec('enum_options_with_search')
# Using an old xls file should raise an error
file_model_xls = session.query(FileModel).filter(
FileModel.name == 'sponsors.xls').first()
file_data_xls = SpecFileService().get_spec_file_data(file_model_xls.id)
# file_data_model_xls = session.query(FileDataModel).filter(FileDataModel.file_model_id == file_model_xls.id).first()
with self.assertRaises(ApiError) as ae:
LookupService.build_lookup_table(file_model_xls.id, 'sponsors.xls',
file_data_xls.data,
'CUSTOMER_NUMBER',
'CUSTOMER_NAME')
self.assertIn('Error opening excel file', ae.exception.args[0])
# Using an xlsx file should work
file_model_xlsx = session.query(FileModel).filter(
FileModel.name == 'sponsors.xlsx').first()
file_data_xlsx = SpecFileService().get_spec_file_data(
file_model_xlsx.id)
# file_data_model_xlsx = session.query(FileDataModel).filter(FileDataModel.file_model_id == file_model_xlsx.id).first()
lookup_model = LookupService.build_lookup_table(
file_model_xlsx.id, 'sponsors.xlsx', file_data_xlsx.data,
'CUSTOMER_NUMBER', 'CUSTOMER_NAME')
self.assertEqual(28, len(lookup_model.dependencies))
self.assertIn('CUSTOMER_NAME',
lookup_model.dependencies[0].data.keys())
self.assertIn('CUSTOMER_NUMBER',
lookup_model.dependencies[0].data.keys())
def test_lookup_with_two_spreadsheets_with_the_same_field_name_in_different_forms(
self):
spec = BaseTest.load_test_spec('enum_options_competing_files')
workflow = self.create_workflow('enum_options_competing_files')
processor = WorkflowProcessor(workflow)
processor.do_engine_steps()
task = processor.get_ready_user_tasks()[0]
task.data = {"type": "animals"}
processor.complete_task(task)
processor.do_engine_steps()
task = processor.get_ready_user_tasks()[0]
results = LookupService.lookup(workflow,
task.task_spec.name,
"selectedItem",
"",
value="pigs",
limit=10)
self.assertEqual(
1, len(results),
"It is possible to find an item based on the id, rather than as a search"
)
self.assertIsNotNone(results[0])
results = LookupService.lookup(workflow,
task.task_spec.name,
"selectedItem",
"",
value="apples",
limit=10)
self.assertEqual(
0, len(results),
"We shouldn't find our fruits mixed in with our animals.")
processor = WorkflowProcessor.reset(workflow, clear_data=True)
processor.do_engine_steps()
task = processor.get_ready_user_tasks()[0]
task.data = {"type": "fruits"}
processor.complete_task(task)
processor.do_engine_steps()
task = processor.get_ready_user_tasks()[0]
results = LookupService.lookup(workflow,
task.task_spec.name,
"selectedItem",
"",
value="apples",
limit=10)
self.assertEqual(
1, len(results),
"It is possible to find an item based on the id, rather than as a search"
)
self.assertIsNotNone(results[0])
results = LookupService.lookup(workflow,
task.task_spec.name,
"selectedItem",
"",
value="pigs",
limit=10)
self.assertEqual(
0, len(results),
"We shouldn't find our animals mixed in with our fruits.")
def test_lookup_returns_good_error_on_bad_field(self):
spec = BaseTest.load_test_spec('enum_options_with_search')
workflow = self.create_workflow('enum_options_with_search')
file_model = session.query(FileModel).filter(
FileModel.name == "customer_list.xls").first()
file_data_model = session.query(FileDataModel).filter(
FileDataModel.file_model == file_model).first()
with self.assertRaises(ApiError):
LookupService.lookup(workflow,
"not_the_right_field",
"sam",
limit=10)
def test_updates_to_file_cause_lookup_rebuild(self):
spec = BaseTest.load_test_spec('enum_options_with_search')
workflow = self.create_workflow('enum_options_with_search')
file_model = session.query(FileModel).filter(
FileModel.name == "sponsors.xlsx").first()
LookupService.lookup(workflow,
"Task_Enum_Lookup",
"sponsor",
"sam",
limit=10)
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(1, len(lookup_records))
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEqual(28, len(lookup_data))
# Update the workflow specification file.
file_path = os.path.join(app.root_path, '..', 'tests', 'data',
'enum_options_with_search',
'sponsors_modified.xlsx')
file = open(file_path, 'rb')
if file_model.workflow_spec_id is not None:
workflow_spec_model = session.query(WorkflowSpecModel).filter(
WorkflowSpecModel.id == file_model.workflow_spec_id).first()
SpecFileService().update_spec_file_data(workflow_spec_model,
file_model.name,
file.read())
elif file_model.is_reference:
ReferenceFileService().update_reference_file(
file_model, file.read())
else:
FileService.update_file(file_model, file.read(),
CONTENT_TYPES['xlsx'])
file.close()
# restart the workflow, so it can pick up the changes.
processor = WorkflowProcessor.reset(workflow)
workflow = processor.workflow_model
LookupService.lookup(workflow,
"Task_Enum_Lookup",
"sponsor",
"sam",
limit=10)
lookup_records = session.query(LookupFileModel).all()
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEqual(4, len(lookup_data))
def lookup_label(self, workflow_model, task_name, field, value):
label_column = field.get_property(Task.FIELD_PROP_LABEL_COLUMN)
result = LookupService().lookup(workflow_model, task_name, field.id, '', value=value, limit=1)
if len(result) > 0:
return result[0][label_column]
else:
return self.UNKNOWN
def lookup_ldap(query=None, limit=10):
"""
perform a lookup against the LDAP server without needing a provided workflow.
"""
value = None
lookup_data = LookupService._run_ldap_query(query, value, limit)
return lookup_data
def lookup(workflow_id, field_id, query=None, value=None, limit=10):
"""
given a field in a task, attempts to find the lookup table or function associated
with that field and runs a full-text query against it to locate the values and
labels that would be returned to a type-ahead box.
Tries to be fast, but first runs will be very slow.
"""
workflow = session.query(WorkflowModel).filter(
WorkflowModel.id == workflow_id).first()
lookup_data = LookupService.lookup(workflow, field_id, query, value, limit)
return LookupDataSchema(many=True).dump(lookup_data)
def get_investigator_dictionary():
"""Returns a dictionary of document details keyed on the doc_code."""
file_id = session.query(FileModel.id). \
filter(FileModel.name == StudyService.INVESTIGATOR_LIST). \
filter(FileModel.is_reference == True). \
scalar()
lookup_model = LookupService.get_lookup_model_for_file_data(
file_id, StudyService.INVESTIGATOR_LIST, 'code', 'label')
doc_dict = {}
for lookup_data in lookup_model.dependencies:
doc_dict[lookup_data.value] = lookup_data.data
return doc_dict
def get_dictionary():
"""Returns a dictionary of document details keyed on the doc_code."""
file_id = session.query(FileModel.id). \
filter(FileModel.name == DocumentService.DOCUMENT_LIST). \
filter(FileModel.is_reference == True). \
scalar()
lookup_model = LookupService.get_lookup_model_for_file_data(
file_id, DocumentService.DOCUMENT_LIST, 'code', 'description')
doc_dict = {}
for lookup_data in lookup_model.dependencies:
doc_dict[lookup_data.value] = lookup_data.data
return doc_dict
def test_lookup_table_is_not_created_more_than_once(self):
spec = BaseTest.load_test_spec('enum_options_with_search')
workflow = self.create_workflow('enum_options_with_search')
LookupService.lookup(workflow,
"Task_Enum_Lookup",
"sponsor",
"sam",
limit=10)
LookupService.lookup(workflow,
"Task_Enum_Lookup",
"sponsor",
"something",
limit=10)
LookupService.lookup(workflow,
"Task_Enum_Lookup",
"sponsor",
"blah",
limit=10)
lookup_records = session.query(LookupFileModel).all()
self.assertIsNotNone(lookup_records)
self.assertEqual(1, len(lookup_records))
lookup_record = lookup_records[0]
lookup_data = session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_record).all()
self.assertEqual(28, len(lookup_data))
def test_find_by_id(self):
spec = BaseTest.load_test_spec('enum_options_from_file')
workflow = self.create_workflow('enum_options_from_file')
processor = WorkflowProcessor(workflow)
processor.do_engine_steps()
result = LookupService.lookup(workflow,
"TaskEnumLookup",
"AllTheNames",
None,
value="1000")
first_result = result[0]
self.assertEquals(1000, first_result['CUSTOMER_NUMBER'])
self.assertEquals('UVA - INTERNAL - GM USE ONLY',
first_result['CUSTOMER_NAME'])
def test_lookup_based_on_id(self):
spec = BaseTest.load_test_spec('enum_options_from_file')
workflow = self.create_workflow('enum_options_from_file')
processor = WorkflowProcessor(workflow)
processor.do_engine_steps()
results = LookupService.lookup(workflow,
"AllTheNames",
"",
value="1000",
limit=10)
self.assertEqual(
1, len(results),
"It is possible to find an item based on the id, rather than as a search"
)
self.assertIsNotNone(results[0].data)
self.assertIsInstance(results[0].data, dict)
def process_options(spiff_task, field):
# If this is an auto-complete field, do not populate options, a lookup will happen later.
if field.type == Task.FIELD_TYPE_AUTO_COMPLETE:
pass
elif field.has_property(Task.PROP_OPTIONS_FILE_NAME):
lookup_model = LookupService.get_lookup_model(spiff_task, field)
data = db.session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_model).all()
if not hasattr(field, 'options'):
field.options = []
for d in data:
field.options.append({
"id": d.value,
"name": d.label,
"data": d.data
})
elif field.has_property(Task.PROP_OPTIONS_DATA_NAME):
field.options = WorkflowService.get_options_from_task_data(
spiff_task, field)
return field
def test_some_full_text_queries(self):
spec = BaseTest.load_test_spec('enum_options_from_file')
workflow = self.create_workflow('enum_options_from_file')
processor = WorkflowProcessor(workflow)
processor.do_engine_steps()
results = LookupService.lookup(workflow, "AllTheNames", "", limit=10)
self.assertEqual(10, len(results),
"Blank queries return everything, to the limit")
results = LookupService.lookup(workflow,
"AllTheNames",
"medicines",
limit=10)
self.assertEqual(1, len(results),
"words in the middle of label are detected.")
self.assertEqual("The Medicines Company", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"UVA",
limit=10)
self.assertEqual(1, len(results), "Beginning of label is found.")
self.assertEqual("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"uva",
limit=10)
self.assertEqual(1, len(results), "case does not matter.")
self.assertEqual("UVA - INTERNAL - GM USE ONLY", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"medici",
limit=10)
self.assertEqual(1, len(results), "partial words are picked up.")
self.assertEqual("The Medicines Company", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"Genetics Savings",
limit=10)
self.assertEqual(1, len(results), "multiple terms are picked up..")
self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"Genetics Sav",
limit=10)
self.assertEqual(1, len(results),
"prefix queries still work with partial terms")
self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"Gen Sav",
limit=10)
self.assertEqual(
1, len(results),
"prefix queries still work with ALL the partial terms")
self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"Inc",
limit=10)
self.assertEqual(7, len(results),
"short terms get multiple correct results.")
self.assertEqual("Genetics Savings & Clone, Inc.", results[0].label)
results = LookupService.lookup(workflow,
"AllTheNames",
"reaction design",
limit=10)
self.assertEqual(5, len(results),
"all results come back for two terms.")
self.assertEqual("Reaction Design", results[0].label,
"Exact matches come first.")
results = LookupService.lookup(workflow,
"AllTheNames",
"1 Something",
limit=10)
self.assertEqual("1 Something", results[0].label,
"Exact matches are prefered")
results = LookupService.lookup(workflow,
"AllTheNames",
"1 (!-Something",
limit=10)
self.assertEqual("1 Something", results[0].label,
"special characters don't flake out")
results = LookupService.lookup(workflow,
"AllTheNames",
"1 Something",
limit=10)
self.assertEqual("1 Something", results[0].label,
"double spaces should not be an issue.")
def get_random_data_for_field(field, task):
has_ldap_lookup = field.has_property(Task.PROP_LDAP_LOOKUP)
has_file_lookup = field.has_property(Task.PROP_OPTIONS_FILE_NAME)
has_data_lookup = field.has_property(Task.PROP_OPTIONS_DATA_NAME)
has_lookup = has_ldap_lookup or has_file_lookup or has_data_lookup
if field.type == "enum" and not has_lookup:
# If it's a normal enum field with no lookup,
# return a random option.
if len(field.options) > 0:
random_choice = random.choice(field.options)
if isinstance(random_choice, dict):
choice = random.choice(field.options)
return {'value': choice['id'], 'label': choice['name']}
else:
# fixme: why it is sometimes an EnumFormFieldOption, and other times not?
# Assume it is an EnumFormFieldOption
return {
'value': random_choice.id,
'label': random_choice.name
}
else:
raise ApiError.from_task(
"invalid_enum", "You specified an enumeration field (%s),"
" with no options" % field.id, task)
elif field.type == "autocomplete" or field.type == "enum":
# If it has a lookup, get the lookup model from the spreadsheet or task data, then return a random option
# from the lookup model
lookup_model = LookupService.get_lookup_model(task, field)
if has_ldap_lookup: # All ldap records get the same person.
return {
"label": "dhf8r",
"value": "Dan Funk",
"data": {
"uid": "dhf8r",
"display_name": "Dan Funk",
"given_name": "Dan",
"email_address": "*****@*****.**",
"department": "Depertment of Psychocosmographictology",
"affiliation": "Rousabout",
"sponsor_type": "Staff"
}
}
elif lookup_model:
data = db.session.query(LookupDataModel).filter(
LookupDataModel.lookup_file_model == lookup_model).limit(
10).all()
options = [{
"value": d.value,
"label": d.label,
"data": d.data
} for d in data]
return random.choice(options)
else:
raise ApiError.from_task(
"unknown_lookup_option",
"The settings for this auto complete field "
"are incorrect: %s " % field.id, task)
elif field.type == "long":
return random.randint(1, 1000)
elif field.type == 'boolean':
return random.choice([True, False])
elif field.type == 'file':
# fixme: produce some something sensible for files.
return random.randint(1, 100)
# fixme: produce some something sensible for files.
elif field.type == 'files':
return random.randrange(1, 100)
else:
return WorkflowService._random_string()