def _retrieve_data(self, request):
data = ""
if request.method == "GET":
if "data" in request.GET:
data = request.GET["data"]
elif request.method == "POST":
form = RefinementForm(request.POST)
if not form.is_valid():
raise ModuleError("Data not properly sent to server.")
if "data[]" in request.POST:
try:
category_id_list = request.POST.getlist("data[]")
for category_id in category_id_list:
category = category_api.get_by_id(category_id)
split_category_path = category.path.split(".")
data += "<{0}><{1}>{2}</{1}></{0}>".format(
split_category_path[-2],
split_category_path[-1],
category.value
if not category.value.endswith(CATEGORY_SUFFIX)
else category.value[:-len(CATEGORY_SUFFIX)],
)
except Exception as e:
raise ModuleError(
"Something went wrong during the processing of posted data: "
+ str(e))
return data
def _retrieve_data(self, request):
data = ''
if request.method == 'GET':
if 'data' in request.GET:
data = request.GET['data']
elif request.method == 'POST':
form = RefinementForm(request.POST)
if not form.is_valid():
raise ModuleError('Data not properly sent to server.')
if 'data[]' in request.POST:
try:
category_id_list = request.POST.getlist('data[]')
for category_id in category_id_list:
category = category_api.get_by_id(category_id)
if not category.value.endswith(CATEGORY_PREFIX):
split_category_path = category.path.split('.')
data += "<{0}><{1}>{2}</{1}></{0}>".format(
split_category_path[-2],
split_category_path[-1], category.value)
except Exception, e:
raise ModuleError(
'Something went wrong during the processing of posted data: '
+ e.message)
def _get(self, request):
"""Manage the GET requests
Args:
request:
Returns:
"""
module_id = request.GET["module_id"]
url = (
request.GET["url"]
if "url" in request.GET
else data_structure_element_api.get_by_id(module_id, request).options["url"]
)
template_data = {
"module_id": module_id,
"module": "",
"display": "",
"url": url,
}
try:
# retrieve module's data
self.data = self._retrieve_data(request)
# get module's rendering
template_data["module"] = self._render_module(request)
# get nodule's data rendering
template_data["display"] = self._render_data(request)
# get module element
module_element = data_structure_element_api.get_by_id(module_id, request)
# get its options
options = module_element.options
# update module element data
options["data"] = self.data
# set updated options
module_element.options = options
# save module element
data_structure_element_api.upsert(module_element, request)
except Exception as e:
raise ModuleError(
"Something went wrong during module initialization: " + str(e)
)
# Check that values are not None
for key, val in list(template_data.items()):
if val is None:
raise ModuleError(
"Variable "
+ key
+ " cannot be None. Module initialization cannot be completed."
)
# TODO Add additional checks
# Apply tags to the template
html_string = AbstractModule.render_template(self.template_name, template_data)
return HttpResponse(html_string, status=HTTP_200_OK)
def __init__(
self,
scripts=list(),
styles=list(),
label=None,
name=None,
options=None,
selected=list(),
):
"""Initialize the module
Args:
scripts:
styles:
label:
name:
options:
selected:
"""
scripts = ["core_parser_app/js/builtin/checkboxes.js"] + scripts
styles = ["core_parser_app/css/builtin/checkboxes.css"] + styles
AbstractModule.__init__(self, scripts=scripts, styles=styles)
if name is None:
raise ModuleError("The name can't be empty.")
self.selected = selected
self.options = options if options is not None else dict()
self.label = label
def __init__(
self,
scripts=list(),
styles=list(),
label=None,
default_value=None,
modclass=None,
disabled=False,
):
"""Initialize the module
Args:
scripts:
styles:
label:
default_value:
modclass:
disabled:
"""
scripts = ["core_parser_app/js/builtin/sync_input.js"] + scripts
AbstractModule.__init__(self, scripts=scripts, styles=styles)
if modclass is None:
raise ModuleError("'modclass' is required.")
self.modclass = modclass
self.label = label
self.default_value = default_value
self.disabled = disabled
def get_updated_keys(request):
"""Return current keys
updated_keys[key] = {'ids': [],
'tagIDs': []}
key = key name
ids = list of possible values for a key
tagIDs = HTML element that needs to be updated with the values (keyrefs)
Args:
request:
Returns:
"""
try:
# get root id
root_id = request.GET["root_id"]
# get root element from id
root_element = data_structure_element_api.get_by_id(root_id, request)
# get auto key manager from root
auto_key = auto_key_api.get_by_root(root_element)
# go through all existing keys
for key, module_ids in auto_key.keys.items():
# get list of current module ids
current_module_ids = _get_current_module_ids(module_ids, request)
# update list of module ids in auto key manager
auto_key.keys[key] = current_module_ids
# update auto key manager
auto_key_api.upsert(auto_key)
# get auto keyref manager from root
auto_keyref = auto_keyref_api.get_by_root(root_element)
# go through all existing keyrefs
for keyref, module_ids in auto_keyref.keyrefs.items():
# get list of current module ids
current_module_ids = _get_current_module_ids(module_ids, request)
# update list of module ids in auto keyref manager
auto_keyref.keyrefs[keyref] = current_module_ids
# update auto keyref manager
auto_keyref_api.upsert(auto_keyref)
# get the list of keyrefs to update
updated_keyrefs = []
for keyref, module_ids in auto_keyref.keyrefs.items():
updated_keyrefs.extend(module_ids)
except Exception as e:
raise ModuleError(
"An unexpected error occurred while getting updated list of keys: "
+ str(e))
return HttpResponse(json.dumps(updated_keyrefs),
content_type="application/javascript")
def __init__(self, key_gen_func):
"""Initializes the module"""
if key_gen_func is None:
raise ModuleError(
"A function for the generation of the keys should be provided (key_gen_func is None)."
)
self.key_gen_func = key_gen_func
AbstractSyncInputModule.__init__(self,
modclass="mod_auto_key",
disabled=True)
def discover_modules(urls):
"""
:return:
"""
logger.info("START discover modules.")
# Remove all existing modules
module_api.delete_all()
# Look for modules in project urls
try:
for url in urls:
if hasattr(url, "url_patterns"):
for url_pattern in url.url_patterns:
module_view_name = (url_pattern.lookup_str if hasattr(
url_pattern, "lookup_str") else "")
if module_view_name.startswith("core_module_"):
module_view = AbstractModule.get_view_from_view_path(
module_view_name)
if issubclass(module_view, AbstractModule):
# FIXME: do not use private field
module_object = Module(
url=url_pattern.pattern._regex,
name=url_pattern.name,
view=module_view_name,
multiple=module_view.is_managing_occurrences,
)
try:
module_api.upsert(module_object)
except NotUniqueError:
logger.error(
"The module %s is already present in the database."
"Please check the list of urls for duplicates."
% url_pattern.name)
except ValidationError:
# something went wrong, delete already added modules
module_api.delete_all()
error_msg = (
"A validation error occurred during the module discovery. "
"Please provide a name to all modules urls using the name argument."
)
logger.error("Discover modules failed with %s." % error_msg)
raise ModuleError(error_msg)
except Exception as e:
# something went wrong, delete already added modules
module_api.delete_all()
logger.error("Discover modules failed with %s." % str(e))
raise e
logger.info("FINISH discover modules.")
def post(self, request, *args, **kwargs):
"""Manage POST requests
Args:
request:
*args:
**kwargs:
Returns:
"""
template_data = {"display": "", "url": ""}
try:
if "module_id" not in request.POST:
return HttpResponseBadRequest(
{"error": 'No "module_id" parameter provided'}
)
module_element = data_structure_element_api.get_by_id(
request.POST["module_id"], request
)
# retrieve module's data
self.data = self._retrieve_data(request)
template_data["display"] = self._render_data(request)
options = module_element.options
# TODO: needs to be updated
if type(self.data) == dict:
options["data"] = self.data["data"]
options["attributes"] = self.data["attributes"]
else:
options["data"] = self.data
# TODO Implement this system instead
# options['content'] = self._get_content(request)
# options['attributes'] = self._get_attributes(request)
module_element.options = options
data_structure_element_api.upsert(module_element, request)
except Exception as e:
raise ModuleError("Something went wrong during module update: " + str(e))
html_code = AbstractModule.render_template(self.template_name, template_data)
response_dict = dict()
response_dict["html"] = html_code
if hasattr(self, "get_XpathAccessor"):
response_dict.update(self.get_XpathAccessor())
return HttpResponse(json.dumps(response_dict))
def _get(self, request):
""" Manage the GET requests
Args:
request:
Returns:
"""
module_id = request.GET['module_id']
url = request.GET['url'] if 'url' in request.GET else \
data_structure_element_api.get_by_id(module_id) .options['url']
template_data = {
'module_id': module_id,
'module': '',
'display': '',
'url': url
}
try:
# retrieve module's data
self.data = self._retrieve_data(request)
# get module's rendering
template_data['module'] = self._render_module(request)
# get nodule's data rendering
template_data['display'] = self._render_data(request)
# get module element
module_element = data_structure_element_api.get_by_id(module_id)
# get its options
options = module_element.options
# update module element data
options['data'] = self.data
# set updated options
module_element.options = options
# save module element
data_structure_element_api.upsert(module_element)
except Exception, e:
raise ModuleError(
'Something went wrong during module initialization: ' +
e.message)
def post(self, request, *args, **kwargs):
""" Manage POST requests
Args:
request:
*args:
**kwargs:
Returns:
"""
template_data = {'display': '', 'url': ''}
try:
if 'module_id' not in request.POST:
return HttpResponseBadRequest(
{'error': 'No "module_id" parameter provided'})
module_element = data_structure_element_api.get_by_id(
request.POST['module_id'])
# retrieve module's data
self.data = self._retrieve_data(request)
template_data['display'] = self._render_data(request)
options = module_element.options
# TODO: needs to be updated
if type(self.data) == dict:
options['data'] = self.data['data']
options['attributes'] = self.data['attributes']
else:
options['data'] = self.data
# TODO Implement this system instead
# options['content'] = self._get_content(request)
# options['attributes'] = self._get_attributes(request)
module_element.options = options
data_structure_element_api.upsert(module_element)
except Exception, e:
raise ModuleError('Something went wrong during module update: ' +
e.message)
reload_form_data[
refinement_form_field] = reload_categories_id_list
return AbstractModule.render_template(
'core_module_fancy_tree_registry_app/fancy_tree.html', {
'form':
RefinementForm(refinement=refinement,
field_id=field_id,
data=reload_form_data)
})
except Exception, e:
raise ModuleError(
"Something went wrong when rendering the module: " +
e.message)
else:
raise ModuleError(
"xml_xpath was not found in request GET parameters.")
def _retrieve_data(self, request):
data = ''
if request.method == 'GET':
if 'data' in request.GET:
data = request.GET['data']
elif request.method == 'POST':
form = RefinementForm(request.POST)
if not form.is_valid():
raise ModuleError('Data not properly sent to server.')
if 'data[]' in request.POST:
try:
category_id_list = request.POST.getlist('data[]')
class AbstractModule(View):
"""Abstract module class
"""
__metaclass__ = ABCMeta
# Is the module managing occurrences by its own? (False by default)
# NOTE: needs to be redefined in subclasses
is_managing_occurrences = False
def __init__(self, scripts=list(), styles=list()):
"""Initializes the module
:param scripts:
:param styles:
"""
# JS scripts
self.scripts = scripts
# CSS spreadsheets
self.styles = styles
# Skeleton of the modules
self.template_name = 'core_parser_app/module.html'
# initialize data
self.data = None
def get(self, request, *args, **kwargs):
""" Manage the GET requests
Args:
request:
*args:
**kwargs:
Returns:
"""
if 'resources' in request.GET:
return self._get_resources()
elif 'managing_occurrences' in request.GET:
return HttpResponse(json.dumps(self.is_managing_occurrences),
HTTP_200_OK)
else:
return self._get(request)
def _get(self, request):
""" Manage the GET requests
Args:
request:
Returns:
"""
module_id = request.GET['module_id']
url = request.GET['url'] if 'url' in request.GET else \
data_structure_element_api.get_by_id(module_id) .options['url']
template_data = {
'module_id': module_id,
'module': '',
'display': '',
'url': url
}
try:
# retrieve module's data
self.data = self._retrieve_data(request)
# get module's rendering
template_data['module'] = self._render_module(request)
# get nodule's data rendering
template_data['display'] = self._render_data(request)
# get module element
module_element = data_structure_element_api.get_by_id(module_id)
# get its options
options = module_element.options
# update module element data
options['data'] = self.data
# set updated options
module_element.options = options
# save module element
data_structure_element_api.upsert(module_element)
except Exception, e:
raise ModuleError(
'Something went wrong during module initialization: ' +
e.message)
# Check that values are not None
for key, val in template_data.items():
if val is None:
raise ModuleError(
'Variable ' + key +
' cannot be None. Module initialization cannot be completed.'
)
# TODO Add additional checks
# Apply tags to the template
html_string = AbstractModule.render_template(self.template_name,
template_data)
return HttpResponse(html_string, status=HTTP_200_OK)
def _render_module(self, request):
# get the xml path of the element on which the module is placed
xml_xpath = request.GET.get('xml_xpath', None)
# xml path is provided
if xml_xpath is not None:
try:
# create unique field id from xpath
field_id = re.sub('[/.:\[\]]', '', xml_xpath)
# split the xpath
split_xml_xpath = xml_xpath.split('/')
# get the last element of the xpath
xml_element = split_xml_xpath[-1]
# check if namespace is present
if ":" in xml_element:
# split element name
split_xml_element = xml_element.split(":")
# only keep element name if namespace is present
xml_element = split_xml_element[-1]
# get registry template
template = template_registry_api.get_current_registry_template(
)
# get all refinements for this template
refinements = refinement_api.get_all_filtered_by_template_hash(
template.hash)
# get the refinement for the xml element
refinement = refinements.get(xsd_name=xml_element)
# initialize reload data for the form
reload_form_data = {}
# data to reload were provided
if self.data != '':
# build filed for the refinement form for the current xml element
refinement_form_field = "{0}-{1}".format(
RefinementForm.prefix, field_id)
# get the categories for the current refinement
categories = category_api.get_all_filtered_by_refinement_id(
refinement.id)
# Initialize list of categories id
reload_categories_id_list = []
# load list of data to reload from XML
reload_data = XSDTree.fromstring("<root>" + self.data +
"</root>")
# Iterate xml elements
for reload_data_element in list(reload_data):
try:
if len(reload_data_element) > 0:
# The xml element to be reloaded is the child element
child = reload_data_element[0]
# get its value
selected_value = child.text
# find the corresponding category and add its id to the list
reload_categories_id_list.append(
categories.get(value=selected_value).id)
except Exception, e:
raise ModuleError(
"Something went wrong when reloading data from XML."
+ e.message)
# set data to reload in the form
reload_form_data[
refinement_form_field] = reload_categories_id_list
return AbstractModule.render_template(
'core_module_fancy_tree_registry_app/fancy_tree.html', {
'form':
RefinementForm(refinement=refinement,
field_id=field_id,
data=reload_form_data)
})
except Exception, e:
raise ModuleError(
"Something went wrong when rendering the module: " +
e.message)
def _retrieve_data(self, request):
"""
Args:
request:
Returns:
"""
data = ""
if request.method == "GET":
try:
# get module id
module_id = request.GET["module_id"]
# get module element form module id
module = data_structure_element_api.get_by_id(
module_id, request)
# get key id form module
key_id = module.options["params"]["key"]
# get XML document root element from module element
root_element = data_structure_element_api.get_root_element(
module, request)
try:
# get auto key manager from db
auto_key = auto_key_api.get_by_root(root_element)
except:
# create auto key manager if doesn't exist
auto_key = AutoKey(root=root_element)
auto_key_api.upsert(auto_key)
# if key id not already present in auto key manager
if key_id not in list(auto_key.keys.keys()):
# initialize key id entry
auto_key.keys[key_id] = []
# if module id not already present in auto key manager
if str(module_id) not in auto_key.keys[key_id]:
# add module id to auto key manager
auto_key.keys[key_id].append(str(module_id))
# update auto key
auto_key_api.upsert(auto_key)
# if data are present
if "data" in request.GET:
# set the key coming from data
data = request.GET["data"]
else:
# get the list of values for this key
values = []
module_ids = auto_key.keys[key_id]
for key_module_id in module_ids:
try:
key_module = data_structure_element_api.get_by_id(
key_module_id, request)
if key_module.options["data"] is not None:
values.append(key_module.options["data"])
except Exception as e:
logger.warning(
"_retrieve_data threw an exception: {0}".
format(str(e)))
# generate next key
data = str(self.key_gen_func(values))
except Exception as e:
raise ModuleError(
"An unexpected error occurred in AutoKeyModule: " + str(e))
return data
def _render_module(self, request):
# get the xml path of the element on which the module is placed
xml_xpath = request.GET.get("xml_xpath", None)
# xml path is provided
if xml_xpath is not None:
try:
# create unique field id from xpath
field_id = re.sub("[/.:\[\]]", "", xml_xpath)
# split the xpath
split_xml_xpath = xml_xpath.split("/")
# get the last element of the xpath
xml_element = split_xml_xpath[-1]
# check if namespace is present
if ":" in xml_element:
# split element name
split_xml_element = xml_element.split(":")
# only keep element name if namespace is present
xml_element = split_xml_element[-1]
# get registry template
template = template_registry_api.get_current_registry_template(
request=request)
# get all refinements for this template
refinements = refinement_api.get_all_filtered_by_template_hash(
template.hash)
# get the refinement for the xml element
refinement = refinements.get(xsd_name=xml_element)
# initialize reload data for the form
reload_form_data = {}
# data to reload were provided
if self.data != "":
# build filed for the refinement form for the current xml element
refinement_form_field = "{0}-{1}".format(
RefinementForm.prefix, field_id)
# get the categories for the current refinement
categories = category_api.get_all_filtered_by_refinement_id(
refinement.id)
# Initialize list of categories id
reload_categories_id_list = []
# load list of data to reload from XML
reload_data = XSDTree.fromstring("<root>" + self.data +
"</root>")
# Iterate xml elements
for reload_data_element in list(reload_data):
try:
if len(reload_data_element) > 0:
# The xml element to be reloaded is the child element
child = reload_data_element[0]
# get its value
selected_value = child.text
# find the corresponding category and add its id to the list
category = categories.get(value=selected_value)
# if the element is an unspecified element
if category.slug.startswith(UNSPECIFIED_LABEL):
# get the parent category
selected_value += CATEGORY_SUFFIX
category = categories.get(
value=selected_value)
reload_categories_id_list.append(category.id)
except Exception as e:
raise ModuleError(
"Something went wrong when reloading data from XML."
+ str(e))
# set data to reload in the form
reload_form_data[
refinement_form_field] = reload_categories_id_list
return AbstractModule.render_template(
"core_module_fancy_tree_registry_app/fancy_tree.html",
{
"form":
RefinementForm(
refinement=refinement,
field_id=field_id,
data=reload_form_data,
)
},
)
except Exception as e:
raise ModuleError(
"Something went wrong when rendering the module: " +
str(e))
else:
raise ModuleError(
"xml_xpath was not found in request GET parameters.")
def _retrieve_data(self, request):
"""Retrieve module's data
Args:
request:
Returns:
"""
data = ""
if request.method == "GET":
# look for existing values
try:
# get module id
module_id = request.GET["module_id"]
# get module element from module id
module = data_structure_element_api.get_by_id(
module_id, request)
# get keyref id in moduke
keyref_id = module.options["params"]["keyref"]
# get XML document root element
root_element = data_structure_element_api.get_root_element(
module, request)
try:
# get auto keyref manager by root
auto_keyref = auto_keyref_api.get_by_root(root_element)
except:
# if auto keyref manager does not exist, create it
auto_keyref = AutoKeyref(root=root_element)
auto_keyref_api.upsert(auto_keyref)
# if keyref id not already present
if keyref_id not in list(auto_keyref.keyrefs.keys()):
# initialize keyref entry
auto_keyref.keyrefs[keyref_id] = []
# if module id not already registered
if str(module_id) not in auto_keyref.keyrefs[keyref_id]:
# add module id to keyref manager
auto_keyref.keyrefs[keyref_id].append(str(module_id))
# update auto keyref
auto_keyref_api.upsert(auto_keyref)
# get key id from root element
key_id = root_element.options["keyrefs"][keyref_id]["refer"]
try:
# get auto key
auto_key = auto_key_api.get_by_root(root_element)
# get list of module ids for the given key id
modules_ids = auto_key.keys[key_id]
except:
# no auto key found, create an empty list of module ids
modules_ids = []
self.values = []
for key_module_id in modules_ids:
key_module = data_structure_element_api.get_by_id(
key_module_id, request)
if key_module.options["data"] is not None:
self.values.append(key_module.options["data"])
# get data from request
if "data" in request.GET:
data = request.GET["data"]
# get data from db
elif "data" in module.options and module.options[
"data"] is not None:
data = str(module.options["data"])
except Exception as e:
raise ModuleError(
"An unexpected error occurred in AutoKeyrefModule: " +
str(e))
elif request.method == "POST":
if "data" in request.POST:
data = request.POST["data"]
return data
def _retrieve_data(self, request):
"""Return module"s data
Args:
request:
Returns:
"""
data = ""
if request.method == "GET":
if "data" in request.GET and request.GET["data"] != "":
xml_table = XSDTree.fromstring("<table>" +
request.GET["data"] +
"</table>")
self.table_name = "name"
self.table = {"headers": [], "values": []}
headers = xml_table[0]
for header in headers.iter("column"):
self.table["headers"].append(header.text)
values = xml_table[1]
for row in values.iter("row"):
value_list = []
for data in row.iter("column"):
value_list.append(data.text)
self.table["values"].append(value_list)
data = ExcelUploaderModule.extract_xml_from_table(
self.table_name, self.table)
elif request.method == "POST":
form = ExcelUploaderForm(request.POST, request.FILES)
if not form.is_valid():
raise ModuleError(
"Data not properly sent to server. Please set 'file' in POST data."
)
try:
input_excel = request.FILES["file"]
book = open_workbook(file_contents=input_excel.read())
sheet = book.sheet_by_index(0)
self.table = {"headers": [], "values": []}
for row_index in range(sheet.nrows):
row_values = []
for col_index in range(sheet.ncols):
cell_text = str(sheet.cell(row_index, col_index).value)
if row_index == 0:
self.table["headers"].append(cell_text)
else:
row_values.append(cell_text)
if len(row_values) != 0:
self.table["values"].append(row_values)
self.table_name = str(input_excel)
except Exception as e:
logger.warning("_retrieve_data threw an exception: {0}".format(
str(e)))
data = ExcelUploaderModule.extract_xml_from_table(
self.table_name, self.table)
return data
def _retrieve_data(self, request):
""" Return module's data
Args:
request:
Returns:
"""
data = ''
if request.method == 'GET':
if 'data' in request.GET:
xml_table = XSDTree.fromstring("<table>" + request.GET['data'] + "</table>")
self.table_name = 'name'
self.table = {
'headers': [],
'values': []
}
headers = xml_table[0]
for header in headers.iter('column'):
self.table['headers'].append(header.text)
values = xml_table[1]
for row in values.iter('row'):
value_list = []
for data in row.iter('column'):
value_list.append(data.text)
self.table['values'].append(value_list)
data = ExcelUploaderModule.extract_xml_from_table(self.table_name, self.table)
elif request.method == 'POST':
form = ExcelUploaderForm(request.POST, request.FILES)
if not form.is_valid():
raise ModuleError('Data not properly sent to server. Please set "file" in POST data.')
try:
input_excel = request.FILES['file']
book = open_workbook(file_contents=input_excel.read())
sheet = book.sheet_by_index(0)
self.table = {
'headers': [],
'values': []
}
for row_index in range(sheet.nrows):
row_values = []
for col_index in range(sheet.ncols):
cell_text = sheet.cell(row_index, col_index).value
if isinstance(cell_text, unicode):
cell_text = unicode(cell_text)
else:
cell_text = str(cell_text)
if row_index == 0:
self.table['headers'].append(cell_text)
else:
row_values.append(cell_text)
if len(row_values) != 0:
self.table['values'].append(row_values)
self.table_name = unicode(input_excel)
except:
pass
data = ExcelUploaderModule.extract_xml_from_table(self.table_name, self.table)
return data
