def get_and_load_choices_first_step(self):
""" Load the first query able step and return it.
Returns:
The first query able step
"""
first_step = self.get_first_query_able_step()
if first_step is None:
return None
else:
full_projection = "dict_content." + first_step.full_xpath
if first_step.step.target == 'attribute':
full_projection += ".@" + first_step.step.value
elif first_step.step.target == 'name':
full_projection += ".@title"
# FIXME: #text was hardcoded, but elements don't have #text if the xml tag does not have any attributes
# else:
# full_projection += ".#text"
full_projection = possible_projection(full_projection)
dict_query = {'template': ObjectId(self.query.schema)}
cursor_xml_data = system_api.execute_query_with_projection(dict_query, full_projection)
if cursor_xml_data.count() == 0:
schema_name = Template.objects.get(pk=self.query.schema).display_name
raise EmptyChoicesFromQuery(schema=schema_name)
elif first_step.step.data_type == "data":
first_step.load_choices_and_xml_data(cursor_xml_data=cursor_xml_data)
else:
for file_xml_data in cursor_xml_data:
first_step.files.append(str(file_xml_data.id))
return first_step
def get_and_load_choices_next_step(self):
""" Get and load the next query able step.
Returns:
the next query able step
"""
current_step = self.get_current_step()
next_step = self.get_next_query_able_step()
if next_step is None:
return None
if current_step.step.data_type == 'data':
unique_ids = current_step.get_id_files_from_user_choices()
else:
unique_ids = [ObjectId(x) for x in list(set(current_step.files))]
if next_step.step.data_type == 'data':
full_projection = 'dict_content.' + next_step.full_xpath
if next_step.step.target == 'attribute':
full_projection += ".@" + next_step.step.value
elif next_step.step.target == 'name':
full_projection += ".@title"
# TODO: remove #text hardcoded
# else:
# full_projection += ".#text"
full_projection = possible_projection(full_projection)
dict_query = {
"template": ObjectId(self.query.schema),
"_id": {
"$in": unique_ids
}
}
cursor_xml_data = system_api.execute_query_with_projection(
dict_query, full_projection)
if cursor_xml_data.count() == 0:
schema_name = Template.objects.get(
id=self.query.schema).display_name
raise EmptyChoicesFromQuery(schema=schema_name)
next_step.dict_choices_id_file = dict()
next_step.load_choices_and_xml_data(
cursor_xml_data=cursor_xml_data)
next_step.update_choices_id_files_to_db()
else:
next_step.files = unique_ids
next_step.update_files_in_db()
return next_step
def init_data_result(self):
""" Initialize the variables to create the files. Create the files and save them into the database.
"""
translate_dt = maketrans("-:.T", ' ') # Translate object for element date time
translate_bounds = maketrans("-:", ' ') # Translate object for bounds date time
max_depth = self.number_of_step
# Create the query
ids_xml_data = self.get_ids_files_last_step()
nb_files_total = len(ids_xml_data)
first_step_xpath = self.list_steps[0].step.xpath.split("*")[0]
full_projection = possible_projection("dict_content." + first_step_xpath)
dict_query = {
"template": ObjectId(self.query.schema),
"_id": {
"$in": ids_xml_data
}
}
xml_data = [
data.to_mongo()['dict_content']
for data in system_api.execute_query_with_projection(dict_query, full_projection)
]
depth = 0
list_keys_xpath = []
dict_data = dict()
map_key = dict()
list_nb_file_treated = list()
list_nb_file_treated.append(1.0)
# Get the filtered elements form the database.
self.explore_elements(list(xml_data), list_keys_xpath, depth, max_depth, dict_data,
translate_dt, translate_bounds, map_key, list_nb_file_treated, nb_files_total)
self.update_message("Creating output files.")
list_step_dot_only = [x for x in self.list_steps if x.step.xpath == "."]
max_depth -= len(list_step_dot_only) + 1
# Create the output files
depth = 0
list_file_xml = list()
list_file_json = list()
list_file_csv = list()
list_headers = list()
list_file_json.append("{\r\n\t\"Data\": {\r\n\t\t\"Item\": [\r\n")
list_file_xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\r\n<Data>\r\n")
self.create_files(dict_data, map_key, list_headers, list_file_xml, list_file_json,
list_file_csv, depth, max_depth)
file_csv = ''.join(list_file_csv)
list_file_xml.append('</Data>')
file_xml = ''.join(list_file_xml)
file_json = ''.join(list_file_json)
file_json = file_json[:-6] + file_json[-6:].replace(',', '')
file_json += "\t\t]\r\n\t}\r\n}"
self.update(str_id_file_json=TempOutputFile().save_file(str_file=file_json, type_file="json"))
self.update(str_id_file_csv=TempOutputFile().save_file(str_file=file_csv, type_file="csv"))
self.update(str_id_file_xml=TempOutputFile().save_file(str_file=file_xml, type_file="xml"))
return