def serialize(self, fields=None, exclude=None):
"""
serialize to a different form then used by the internal class structure
used to append additional values (like parent ontology properties) that
internal objects (like models.Nodes) don't support
"""
serializer = JSONSerializer()
serializer.geom_format = 'geojson'
obj = serializer.handle_model(self)
ordered_cards = self.get_ordered_cards()
ret = JSONSerializer().serializeToPython(obj)
graphs = []
graphids = []
for card in self.cards.all():
if card.graph_id not in graphids:
graphids.append(card.graph_id)
#we may want the full proxy model at some point, but for now just the root node color
# graphs.append(Graph.objects.get(pk=card.graph_id))
proxy_graph = Graph.objects.get(pk=card.graph_id)
color = proxy_graph.root.config['fillColor']
graph = card.graph
graph = JSONSerializer().serializeToPython(graph, exclude=['functions','disable_instance_creation','deploymentdate','deploymentfile'])
graph['color'] = color
graphs.append(graph)
ret['graphs'] = graphs
ret['cards'] = ordered_cards
try:
ret['bounds'] = json.loads(ret['bounds'])
except TypeError as e:
print 'Could not parse', ret['bounds'], e
return ret
def __init__(self):
#
serializer = JSONSerializer()
serializer.mimetype = 'application/json'
serializer.dumps = serializer.serialize
serializer.loads = JSONDeserializer().deserialize
self.es = Elasticsearch(hosts=settings.ELASTICSEARCH_HOSTS, serializer=serializer, **settings.ELASTICSEARCH_CONNECTION_OPTIONS)
self.logger = logging.getLogger(__name__)
def serialize(self, fields=None, exclude=None):
"""
serialize to a different form then used by the internal class structure
"""
exclude = [] if exclude == None else exclude
ret = JSONSerializer().handle_model(self, fields, exclude)
ret['cardinality'] = self.cardinality if 'cardinality' not in exclude else ret.pop('cardinality', None)
ret['cards'] = self.cards if 'cards' not in exclude else ret.pop('cards', None)
ret['nodes'] = list(self.nodegroup.node_set.all()) if 'nodes' not in exclude else ret.pop('nodes', None)
ret['visible'] = self.visible if 'visible' not in exclude else ret.pop('visible', None)
ret['active'] = self.active if 'active' not in exclude else ret.pop('active', None)
ret['is_editable'] = self.is_editable() if 'is_editable' not in exclude else ret.pop('is_editable', None)
ret['ontologyproperty'] = self.ontologyproperty if 'ontologyproperty' not in exclude else ret.pop('ontologyproperty', None)
ret['disabled'] = self.disabled if 'disabled' not in exclude else ret.pop('disabled', None)
if self.graph and self.graph.ontology and self.graph.isresource:
edge = self.get_edge_to_parent()
ret['ontologyproperty'] = edge.ontologyproperty
# provide a models.CardXNodeXWidget model for every node
# even if a widget hasn't been configured
ret['widgets'] = self.widgets
if 'widgets' not in exclude:
for node in ret['nodes']:
found = False
for widget in ret['widgets']:
if node.nodeid == widget.node_id:
found = True
if not found:
widget = models.DDataType.objects.get(pk=node.datatype).defaultwidget
if widget:
widget_model = models.CardXNodeXWidget()
widget_model.node_id = node.nodeid
widget_model.card_id = self.cardid
widget_model.widget_id = widget.pk
widget_model.config = JSONSerializer().serialize(widget.defaultconfig)
widget_model.label = node.name
ret['widgets'].append(widget_model)
else:
ret.pop('widgets', None)
return ret
def write_v4_jsonl(self, dest_path):
ct = 0
with open(self.source_file, "rb") as openv3:
lines = openv3.readlines()
with open(dest_path, "wb") as openv4:
for n, line in enumerate(lines):
v3_json = json.loads(line)
# set of checks that basically mimicks what happens
# in the load_v3_data method that is used for JSON.
if v3_json["entitytypeid"] != self.v3_graph_name:
continue
if len(v3_json["child_entities"]) == 0:
continue
resid = v3_json["entityid"]
if self.truncate:
if n - 2 == self.truncate:
break
if len(self.only) > 0:
if resid not in self.only:
continue
if resid in self.exclude:
continue
v4_json = self.process_one_resource(v3_json)
v4_line = JSONSerializer().serialize(v4_json)
openv4.write(v4_line + "\n")
ct += 1
# simple progress printing
if ct == 0:
print("")
if ct % 1000 == 0:
print(ct)
elif ct % 100 == 0:
print(".")
if ct == self.truncate:
break
if ct > 0:
print("\ntotal resources converted: {}".format(ct))
if ct == 0:
os.remove(dest_path)
return False
return dest_path
def index(self):
"""
Indexes all the nessesary items values of a resource to support search
"""
if unicode(self.graph_id) != unicode(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID):
se = SearchEngineFactory().create()
datatype_factory = DataTypeFactory()
node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list('nodeid', 'datatype')}
document, terms = self.get_documents_to_index(datatype_factory=datatype_factory, node_datatypes=node_datatypes)
document['root_ontology_class'] = self.get_root_ontology()
se.index_data('resource', self.graph_id, JSONSerializer().serializeToPython(document), id=self.pk)
for term in terms:
se.index_data('strings', 'term', term['_source'], id=term['_id'])
def EntityTypes(request, entitytypeid):
entityschema = []
if entitytypeid == '':
return HttpResponse(JSONSerializer().serialize({},
ensure_ascii=True,
indent=4))
else:
if request.GET.get('f') is None:
return render_to_response('graph.htm', {},
context_instance=RequestContext(request))
else:
entityschema = {
entitytypeid: Entity.get_mapping_schema(entitytypeid)
}
if request.GET.get('f') == 'json':
return HttpResponse(JSONSerializer().serialize(
entityschema, ensure_ascii=True, indent=4),
content_type='application/json')
if request.GET.get('f') == 'd3':
d3Schema = d3Obj()
d3Schema.name = entitytypeid
for assestAttr in entityschema[entitytypeid]:
d3ObjAssestAttr = d3Obj()
d3ObjAssestAttr.name = assestAttr
for step in entityschema[entitytypeid][assestAttr][
'steps']:
d3ObjStep = d3Obj()
d3ObjStep.name = step['entitytypedomain'] + ' ' + step[
'propertyid'] + ' ' + step['entitytyperange']
d3ObjAssestAttr.children.append(d3ObjStep)
d3Schema.children.append(d3ObjAssestAttr)
return HttpResponse(JSONSerializer().serialize(
d3Schema, ensure_ascii=True, indent=4))
def index(self):
"""
Indexes all the nessesary items values of a resource to support search
"""
if str(self.graph_id) != str(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID):
se = SearchEngineFactory().create()
datatype_factory = DataTypeFactory()
node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list("nodeid", "datatype")}
document, terms = self.get_documents_to_index(datatype_factory=datatype_factory, node_datatypes=node_datatypes)
document["root_ontology_class"] = self.get_root_ontology()
doc = JSONSerializer().serializeToPython(document)
se.index_data(index="resources", body=doc, id=self.pk)
for term in terms:
se.index_data("terms", body=term["_source"], id=term["_id"])
def test_node_delete(self):
"""
Test delete a node (HERITAGE_RESOURCE_PLACE) via node view
"""
self.client.login(username="******", password="******")
node = Node.objects.get(nodeid=str(self.appended_branch_1.root.pk))
url = reverse("delete_node", kwargs={"graphid": self.GRAPH_ID})
post_data = JSONSerializer().serialize({"nodeid": node.nodeid})
response = self.client.delete(url, post_data)
self.assertEqual(response.status_code, 200)
graph = Graph.objects.get(graphid=self.GRAPH_ID).serialize()
self.assertEqual(len(graph["nodes"]), 3)
self.assertEqual(len(graph["edges"]), 2)
def get(self, request, formid):
try:
form = models.Form.objects.get(formid=formid)
self.graph = Graph.objects.get(graphid=form.graph.pk)
icons = models.Icon.objects.order_by('name')
cards = models.CardModel.objects.filter(
nodegroup__parentnodegroup=None, graph=self.graph)
context = self.get_context_data(
main_script='views/graph/form-configuration',
graph_id=self.graph.pk,
icons=JSONSerializer().serialize(icons),
form=JSONSerializer().serialize(form),
forms=JSONSerializer().serialize(self.graph.form_set.all()),
cards=JSONSerializer().serialize(cards),
forms_x_cards=JSONSerializer().serialize(
models.FormXCard.objects.filter(
form=form).order_by('sortorder')),
)
context['nav']['title'] = self.graph.name
context['nav']['menu'] = True
context['nav']['help'] = (_('Configuring Menus'),
'help/menu-designer-help.htm')
return render(request, 'views/graph/form-configuration.htm',
context)
except (models.Form.DoesNotExist):
# assume the formid is a graph id
graph = Graph.objects.get(graphid=formid)
if graph.isresource == False:
return redirect('graph_settings', graphid=graph.graphid)
else:
return redirect('form_manager', graphid=graph.graphid)
def get(self, request, graphid):
self.graph = Graph.objects.get(graphid=graphid)
if self.graph.isresource:
context = self.get_context_data(
main_script='views/graph/function-manager',
functions=JSONSerializer().serialize(models.Function.objects.all()),
applied_functions=JSONSerializer().serialize(models.FunctionXGraph.objects.filter(graph=self.graph)),
function_templates=models.Function.objects.exclude(component__isnull=True),
)
context['graphs'] = JSONSerializer().serialize(
models.GraphModel.objects.all().exclude(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID),
exclude=['functions'])
context['nav']['title'] = self.graph.name
context['nav']['menu'] = True
context['nav']['help'] = {
'title': _('Managing Functions'),
'template': 'function-help',
}
return render(request, 'views/graph/function-manager.htm', context)
else:
return redirect('graph_designer', graphid=graphid)
def get(self, request):
map_layers = models.MapLayer.objects.all()
map_markers = models.MapMarker.objects.all()
map_sources = models.MapSource.objects.all()
resource_graphs = (models.GraphModel.objects.exclude(
pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude(
isresource=False).exclude(isactive=False))
geocoding_providers = models.Geocoder.objects.all()
search_components = models.SearchComponent.objects.all()
datatypes = models.DDataType.objects.all()
context = self.get_context_data(
map_layers=map_layers,
map_markers=map_markers,
map_sources=map_sources,
geocoding_providers=geocoding_providers,
search_components=search_components,
main_script="views/search",
resource_graphs=resource_graphs,
datatypes=datatypes,
user_is_reviewer=user_is_resource_reviewer(request.user),
)
graphs = JSONSerializer().serialize(
context["resource_graphs"],
exclude=[
"functions",
"author",
"deploymentdate",
"deploymentfile",
"version",
"subtitle",
"description",
"disable_instance_creation",
"ontology_id",
],
)
context["graphs"] = graphs
context["nav"]["title"] = _("Search")
context["nav"]["icon"] = "fa-search"
context["nav"]["search"] = False
context["nav"]["help"] = {
"title": _("Searching the Database"),
"template": "search-help",
}
context["celery_running"] = task_management.check_if_celery_available()
return render(request, "views/search.htm", context)
def get_documents_to_index(self,
fetchTiles=True,
datatype_factory=None,
node_datatypes=None):
"""
Gets all the documents nessesary to index a single resource
returns a tuple of a document and list of terms
Keyword Arguments:
fetchTiles -- instead of fetching the tiles from the database get them off the model itself
datatype_factory -- refernce to the DataTypeFactory instance
node_datatypes -- a dictionary of datatypes keyed to node ids
"""
document = JSONSerializer().serializeToPython(self)
document['tiles'] = models.TileModel.objects.filter(
resourceinstance=self) if fetchTiles else self.tiles
document['strings'] = []
document['dates'] = []
document['domains'] = []
document['geometries'] = []
document['points'] = []
document['numbers'] = []
terms = []
for tile in document['tiles']:
for nodeid, nodevalue in tile.data.iteritems():
datatype = node_datatypes[nodeid]
if nodevalue != '' and nodevalue != [] and nodevalue != {} and nodevalue is not None:
datatype_instance = datatype_factory.get_instance(datatype)
datatype_instance.append_to_document(document, nodevalue)
term = datatype_instance.get_search_term(nodevalue)
if term is not None:
terms.append({
'_id':
unicode(nodeid) + unicode(tile.tileid),
'_source': {
'value': term,
'nodeid': nodeid,
'nodegroupid': tile.nodegroup_id,
'tileid': tile.tileid,
'resourceinstanceid': tile.resourceinstance_id
}
})
return document, terms
def get(self, request, graphid):
self.graph = Graph.objects.get(graphid=graphid)
branch_graphs = Graph.objects.exclude(pk=graphid).exclude(isresource=True).exclude(isactive=False)
if self.graph.ontology is not None:
branch_graphs = branch_graphs.filter(ontology=self.graph.ontology)
context = self.get_context_data(
main_script='views/graph/card-manager',
branches=JSONSerializer().serialize(branch_graphs),
)
context['nav']['title'] = self.graph.name
context['nav']['menu'] = True
context['nav']['help'] = ('Managing Cards','help/card-manager-help.htm')
return render(request, 'views/graph/card-manager.htm', context)
def get_context_data(self, **kwargs):
datatype_factory = DataTypeFactory()
context = super(BaseManagerView, self).get_context_data(**kwargs)
context['graph_models'] = models.GraphModel.objects.all()
context['graphs'] = JSONSerializer().serialize(context['graph_models'])
if 'Resource Editor' in self.request.user.user_groups:
context['resource_instances'] = Resource.objects.all().order_by(
'-createdtime')[:100]
else:
context['resource_instances'] = []
context['nav'] = {
'icon': 'fa fa-chevron-circle-right',
'title': '',
'help': ('', ''),
'menu': False,
'search': True,
'res_edit': False,
'edit_history': True,
'login': True,
'print': False,
}
geom_datatypes = [
d.pk for d in models.DDataType.objects.filter(isgeometric=True)
]
geom_nodes = models.Node.objects.filter(graph__isresource=True,
graph__isactive=True,
datatype__in=geom_datatypes)
resource_layers = []
resource_sources = []
for node in geom_nodes:
# TODO: check user node level permissions here, if user does not
# have read permissions on this node, then do not create map layer
# or source
datatype = datatype_factory.get_instance(node.datatype)
map_source = datatype.get_map_source(node)
if map_source is not None:
resource_sources.append(map_source)
map_layer = datatype.get_map_layer(node)
if map_layer is not None:
resource_layers.append(map_layer)
context['geom_nodes'] = geom_nodes
context['resource_map_layers'] = resource_layers
context['resource_map_sources'] = resource_sources
context['iiif_manifests'] = models.IIIFManifest.objects.all()
return context
def get(self, request, graphid, nodeid=None):
if self.action == 'export_graph':
graph = get_graphs_for_export([graphid])
graph['metadata'] = system_metadata()
f = JSONSerializer().serialize(graph, indent=4)
graph_name = JSONDeserializer().deserialize(f)['graph'][0]['name']
response = HttpResponse(f, content_type='json/plain')
response[
'Content-Disposition'] = 'attachment; filename="%s.json"' % (
graph_name)
return response
elif self.action == 'export_mapping_file':
files_for_export = create_mapping_configuration_file(graphid)
file_name = Graph.objects.get(graphid=graphid).name
buffer = StringIO()
with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as zip:
for f in files_for_export:
f['outputfile'].seek(0)
zip.writestr(f['name'], f['outputfile'].read())
zip.close()
buffer.flush()
zip_stream = buffer.getvalue()
buffer.close()
response = HttpResponse()
response[
'Content-Disposition'] = 'attachment; filename=' + file_name + '.zip'
response['Content-length'] = str(len(zip_stream))
response['Content-Type'] = 'application/zip'
response.write(zip_stream)
return response
else:
graph = Graph.objects.get(graphid=graphid)
if self.action == 'get_related_nodes':
ret = graph.get_valid_ontology_classes(nodeid=nodeid)
elif self.action == 'get_valid_domain_nodes':
ret = graph.get_valid_domain_ontology_classes(nodeid=nodeid)
return JSONResponse(ret)
return HttpResponseNotFound()
def get_context_data(self, **kwargs):
context = super(BaseManagerView, self).get_context_data(**kwargs)
context[
"system_settings_graphid"] = settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID
context["graph_models"] = []
context["graphs"] = "[]"
context["plugins"] = []
for plugin in models.Plugin.objects.all().order_by("sortorder"):
if self.request.user.has_perm("view_plugin", plugin):
context["plugins"].append(plugin)
createable = get_createable_resource_types(self.request.user)
createable.sort(key=lambda x: x.name.lower())
context["createable_resources"] = JSONSerializer().serialize(
createable,
exclude=[
"functions",
"ontology",
"isactive",
"isresource",
"version",
"deploymentdate",
"deploymentfile",
"author",
],
)
context["notifications"] = models.UserXNotification.objects.filter(
recipient=self.request.user, isread=False)
context["nav"] = {
"icon": "fa fa-chevron-circle-right",
"title": "",
"help": {
# title:'',template:'' (leave this commented out)
},
"menu": False,
"search": True,
"notifs": True,
"res_edit": False,
"login": True,
"print": False,
}
context["user_is_reviewer"] = user_is_resource_reviewer(
self.request.user)
context["app_name"] = settings.APP_NAME
context["show_language_swtich"] = settings.SHOW_LANGUAGE_SWITCH
return context
def index(self, scheme=None):
if self.category == 'label':
se = SearchEngineFactory().create()
data = JSONSerializer().serializeToPython(self)
if scheme == None:
scheme = self.get_scheme_id()
if scheme == None:
raise Exception('Index of label failed. Index type (scheme id) could not be derived from the label.')
se.create_mapping('concept_labels', scheme.id, fieldname='conceptid', fieldtype='string', fieldindex='not_analyzed')
se.index_data('concept_labels', scheme.id, data, 'id')
#Looks up whether the label is actually a dropdown label or an entity label and, if so, excludes them from the term search index.
entity_or_dropdown= archesmodels.ConceptRelations.objects.filter(Q(relationtype ='hasCollection') | Q(relationtype ='hasEntity'),conceptidto = scheme.id)
is_entity_or_dropdown = False if entity_or_dropdown.count() == 0 else True
# don't create terms for entity type concepts
if not(scheme.id == '00000000-0000-0000-0000-000000000003' or scheme.id == '00000000-0000-0000-0000-000000000004') and is_entity_or_dropdown ==False:
se.index_term(self.value, self.id, scheme.id, {'conceptid': self.conceptid})
def index_concepts_by_entitytypeid(self, entitytypeid):
entitytype = archesmodels.EntityTypes.objects.get(pk=entitytypeid)
conceptid = entitytype.conceptid_id
concept_graph = Concept().get(id=conceptid,
include_subconcepts=True,
exclude=['note'])
if len(concept_graph.subconcepts) > 0:
data = JSONSerializer().serializeToPython(concept_graph,
ensure_ascii=True,
indent=4)
self.index(data,
'concept',
entitytypeid,
'id',
processdoc=None,
getid=None,
bulk=False)
def map_info(request):
return {
'map_info': {
'x': settings.DEFAULT_MAP_X,
'y': settings.DEFAULT_MAP_Y,
'zoom': settings.DEFAULT_MAP_ZOOM,
'map_min_zoom': settings.MAP_MIN_ZOOM,
'map_max_zoom': settings.MAP_MAX_ZOOM,
'mapbox_api_key': settings.MAPBOX_API_KEY,
'hex_bin_size': settings.HEX_BIN_SIZE,
'mapbox_sprites': settings.MAPBOX_SPRITES,
'mapbox_glyphs': settings.MAPBOX_GLYPHS,
'hex_bin_bounds':
JSONSerializer().serialize(settings.HEX_BIN_BOUNDS),
'geocoder_default': settings.DEFAULT_SEARCH_GEOCODER
}
}
def get(self, request, resourceid=None):
resource_instance = models.ResourceInstance.objects.get(pk=resourceid)
tiles = models.TileModel.objects.filter(
resourceinstance=resource_instance)
try:
report = models.Report.objects.get(graph=resource_instance.graph,
active=True)
except models.Report.DoesNotExist:
report = None
graph = Graph.objects.get(graphid=resource_instance.graph.pk)
resource_graphs = Graph.objects.exclude(
pk=resource_instance.graph.pk).exclude(
pk='22000000-0000-0000-0000-000000000002').exclude(
isresource=False).exclude(isactive=False)
forms = resource_instance.graph.form_set.filter(visible=True)
forms_x_cards = models.FormXCard.objects.filter(
form__in=forms).order_by('sortorder')
cards = Card.objects.filter(nodegroup__parentnodegroup=None,
graph=resource_instance.graph)
datatypes = models.DDataType.objects.all()
widgets = models.Widget.objects.all()
map_layers = models.MapLayers.objects.all()
map_sources = models.MapSources.objects.all()
templates = models.ReportTemplate.objects.all()
context = self.get_context_data(
main_script='views/resource/report',
report=JSONSerializer().serialize(report),
report_templates=templates,
templates_json=JSONSerializer().serialize(templates),
forms=JSONSerializer().serialize(forms),
tiles=JSONSerializer().serialize(tiles),
forms_x_cards=JSONSerializer().serialize(forms_x_cards),
cards=JSONSerializer().serialize(cards),
datatypes_json=JSONSerializer().serialize(datatypes),
widgets=widgets,
map_layers=map_layers,
map_sources=map_sources,
resource_graphs=resource_graphs,
graph_id=resource_instance.graph.pk,
graph_name=resource_instance.graph.name,
graph_json=JSONSerializer().serialize(graph))
return render(request, 'views/resource/report.htm', context)
def get_instance_permissions(self, resource_instance):
permission_order = [
"view_resourceinstance", "change_resourceinstance",
"delete_resourceinstance", "no_access_to_resourceinstance"
]
perms = json.loads(JSONSerializer().serialize({
p.codename: p
for p in get_perms_for_model(resource_instance)
if p.codename != "add_resourceinstance"
}))
ordered_perms = []
for p in permission_order:
ordered_perms.append(perms[p])
identities = [{
"name":
user.username,
"id":
user.id,
"type":
"user",
"default_permissions":
self.get_perms(user, "user", resource_instance, ordered_perms),
"is_editor_or_reviewer":
bool(
user_is_resource_editor(user)
or user_is_resource_reviewer(user)),
} for user in User.objects.all()]
identities += [{
"name":
group.name,
"id":
group.id,
"type":
"group",
"default_permissions":
self.get_perms(group, "group", resource_instance, ordered_perms),
} for group in Group.objects.all()]
result = {"identities": identities}
result["permissions"] = ordered_perms
result["limitedaccess"] = (
len(get_users_with_perms(resource_instance)) +
len(get_groups_with_perms(resource_instance))) > 1
instance_creator = get_instance_creator(resource_instance)
result["creatorid"] = instance_creator["creatorid"]
return result
def _buffer(geojson, width=0, unit="ft"):
geojson = JSONSerializer().serialize(geojson)
geom = GEOSGeometry(geojson, srid=4326)
try:
width = float(width)
except Exception:
width = 0
if width > 0:
if unit == "ft":
width = width / 3.28084
geom.transform(settings.ANALYSIS_COORDINATE_SYSTEM_SRID)
geom = geom.buffer(width)
geom.transform(4326)
return geom
def post(self, request):
username = request.POST.get("username", None)
password = request.POST.get("password", None)
user = authenticate(username=username, password=password)
if user:
if hasattr(user, "userprofile") is not True:
models.UserProfile.objects.create(user=user)
userDict = JSONSerializer().serializeToPython(user)
userDict["password"] = None
userDict["is_reviewer"] = user_is_resource_reviewer(user)
userDict["viewable_nodegroups"] = user.userprofile.viewable_nodegroups
userDict["editable_nodegroups"] = user.userprofile.editable_nodegroups
userDict["deletable_nodegroups"] = user.userprofile.deletable_nodegroups
response = JSONResponse(userDict)
else:
response = Http401Response()
return response
def home_page(request):
lang = request.GET.get('lang', settings.LANGUAGE_CODE)
min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val'))
return render_to_response('search.htm', {
'main_script':
'search',
'active_page':
'Search',
'min_date':
min_max_dates['val__min'].year
if min_max_dates['val__min'] != None else 0,
'max_date':
min_max_dates['val__max'].year
if min_max_dates['val__min'] != None else 1,
'timefilterdata':
JSONSerializer().serialize(Concept.get_time_filter_data()),
},
context_instance=RequestContext(request))
def test_rules_for_appending(self):
"""
test the rules that control the appending of branches to graphs
"""
graph = Graph.objects.get(node=self.rootNode)
graph.isresource = True
self.assertIsNotNone(graph.append_branch('http://www.ics.forth.gr/isl/CRMdig/L54_is_same-as', graphid=self.NODE_NODETYPE_GRAPHID))
graph = Graph.new()
graph.root.datatype = 'string'
graph.update_node(JSONSerializer().serializeToPython(graph.root))
# create card collector graph to use for appending on to other graphs
collector_graph = Graph.new()
collector_graph.append_branch('http://www.ics.forth.gr/isl/CRMdig/L54_is_same-as', graphid=self.NODE_NODETYPE_GRAPHID)
collector_graph.save()
def _buffer(geojson, width=0, unit='ft'):
geojson = JSONSerializer().serialize(geojson)
geom = GEOSGeometry(geojson, srid=4326)
try:
width = float(width)
except:
width = 0
if width > 0:
if unit == 'ft':
width = width / 3.28084
geom.transform(3857)
geom = geom.buffer(width)
geom.transform(4326)
return geom
def test_delete_mobile_survey(self):
"""
Test that deletion of a mobile survey deletes its couchdb
"""
self.client.login(username='******', password='******')
url = reverse('mobile_survey_manager')
post_data = {"id": str(self.mobile_survey.id)}
post_data = JSONSerializer().serialize(post_data)
content_type = 'application/x-www-form-urlencoded'
response = self.client.delete(url, post_data, content_type)
response_json = json.loads(response.content)
couch = couchdb.Server(settings.COUCHDB_URL)
self.assertFalse(
MobileSurvey.objects.filter(pk=self.mobile_survey.id).exists())
self.assertTrue('project_' + str(self.mobile_survey.id) not in couch)
def get(self, request):
if self.action == 'tile_history':
start = request.GET.get('start')
end = request.GET.get('end')
edits = EditLog.objects.filter(provisional_userid=request.user.id).filter(timestamp__range=[start, end]).order_by('tileinstanceid', 'timestamp')
resourceinstanceids = [e['resourceinstanceid'] for e in edits.values('resourceinstanceid')]
deleted_resource_edits = EditLog.objects.filter(resourceinstanceid__in=resourceinstanceids).filter(edittype='delete')
deleted_resource_instances = [e['resourceinstanceid'] for e in deleted_resource_edits.values('resourceinstanceid')]
summary = {}
for edit in edits:
if edit.tileinstanceid not in summary:
summary[edit.tileinstanceid] = {'pending': False, 'tileid': edit.tileinstanceid}
summary[edit.tileinstanceid]['lasttimestamp'] = edit.timestamp
summary[edit.tileinstanceid]['lastedittype'] = edit.provisional_edittype
summary[edit.tileinstanceid]['reviewer'] = ''
summary[edit.tileinstanceid]['resourceinstanceid'] = edit.resourceinstanceid
summary[edit.tileinstanceid]['resourcedisplayname'] = edit.resourcedisplayname
summary[edit.tileinstanceid]['resourcemodelid'] = edit.resourceclassid
summary[edit.tileinstanceid]['nodegroupid'] = edit.nodegroupid
summary[edit.tileinstanceid]['resource_deleted'] = True if edit.resourceinstanceid in deleted_resource_instances else False
if edit.provisional_edittype in ['accept edit', 'delete edit']:
summary[edit.tileinstanceid]['reviewer'] = edit.user_username
chronological_summary = []
resource_models = models.GraphModel.objects.filter(isresource=True).exclude(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).values('iconclass','color','graphid','name')
cards = models.CardModel.objects.all().values('name', 'nodegroup_id')
card_lookup = {str(card['nodegroup_id']): card for card in cards}
resource_model_lookup = {str(graph['graphid']): graph for graph in resource_models}
for k, v in summary.iteritems():
if v['lastedittype'] not in ['accept edit', 'delete edit']:
if models.TileModel.objects.filter(pk=k).exists():
tile = models.TileModel.objects.get(pk=k)
if tile.provisionaledits is not None and str(request.user.id) in tile.provisionaledits:
v['pending'] = True
v['resourcemodel'] = resource_model_lookup[v['resourcemodelid']]
v['card'] = card_lookup[v['nodegroupid']]
if 'graphid' in v['resourcemodel']:
v['resourcemodel'].pop('graphid')
if 'nodegroup_id' in v['card']:
v['card'].pop('nodegroup_id')
chronological_summary.append(v)
return JSONResponse(JSONSerializer().serialize(sorted(chronological_summary, key=lambda k: k['lasttimestamp'], reverse=True)))
def test_node_delete(self):
"""
Test delete a node (HERITAGE_RESOURCE_PLACE) via node view
"""
self.client.login(username='******', password='******')
graphid = Node.objects.get(nodeid=self.ROOT_ID).graph.pk
node = Node.objects.get(nodeid=self.HERITAGE_RESOURCE_PLACE_ID)
url = reverse('delete_node', kwargs={'graphid': graphid})
post_data = JSONSerializer().serialize({'nodeid': node.nodeid})
response = self.client.delete(url, post_data)
self.assertEqual(response.status_code, 200)
new_count = self.NODE_COUNT - self.PLACE_BRANCH_COUNT
root = Node.objects.get(nodeid=self.ROOT_ID)
nodes, edges = root.get_child_nodes_and_edges()
self.assertEqual(len(nodes), new_count)
self.assertEqual(len(edges), new_count)
def load_tiles_into_couch(self, instances, nodegroup):
"""
Takes a mobile survey object, a couch database instance, and a dictionary
of resource instances to identify eligible tiles and load them into the
database instance
"""
db = self.couch.create_db("project_" + str(self.id))
tiles = models.TileModel.objects.filter(nodegroup=nodegroup)
tiles_serialized = json.loads(JSONSerializer().serialize(tiles))
for tile in tiles_serialized:
if str(tile["resourceinstance_id"]) in instances:
try:
tile["type"] = "tile"
self.couch.update_doc(db, tile, tile["tileid"])
except Exception as e:
print(e, tile)
nodegroups = models.NodeGroup.objects.filter(parentnodegroup=nodegroup)
for nodegroup in nodegroups:
self.load_tiles_into_couch(instances, nodegroup)
def serialize(self, fields=None, exclude=None):
"""
serialize to a different form then used by the internal class structure
used to append additional values (like parent ontology properties) that
internal objects (like models.Nodes) don't support
"""
exclude = [] if exclude == None else exclude
ret = JSONSerializer().handle_model(self, fields, exclude)
ret['root'] = self.root
if 'relatable_resource_model_ids' not in exclude:
ret['relatable_resource_model_ids'] = [str(relatable_node.graph_id) for relatable_node in self.root.get_relatable_resources()]
else:
ret.pop('relatable_resource_model_ids', None)
ret['cards'] = self.get_cards() if 'cards' not in exclude else ret.pop('cards', None)
ret['nodegroups'] = self.get_nodegroups() if 'nodegroups' not in exclude else ret.pop('nodegroups', None)
ret['domain_connections'] = self.get_valid_domain_ontology_classes() if 'domain_connections' not in exclude else ret.pop('domain_connections', None)
ret['is_editable'] = self.is_editable() if 'is_editable' not in exclude else ret.pop('is_editable', None)
ret['functions'] = models.FunctionXGraph.objects.filter(graph_id=self.graphid) if 'functions' not in exclude else ret.pop('functions', None)
parentproperties = {
self.root.nodeid: ''
}
for edge_id, edge in self.edges.iteritems():
parentproperties[edge.rangenode_id] = edge.ontologyproperty
ret['edges'] = [edge for key, edge in self.edges.iteritems()] if 'edges' not in exclude else ret.pop('edges', None)
if 'nodes' not in exclude:
ret['nodes'] = []
for key, node in self.nodes.iteritems():
nodeobj = JSONSerializer().serializeToPython(node)
nodeobj['parentproperty'] = parentproperties[node.nodeid]
ret['nodes'].append(nodeobj)
else:
ret.pop('nodes', None)
res = JSONSerializer().serializeToPython(ret)
return res
def index(self):
"""
Indexes all the nessesary items values of a resource to support search
"""
if str(self.graph_id) != str(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID):
datatype_factory = DataTypeFactory()
node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list("nodeid", "datatype")}
document, terms = self.get_documents_to_index(datatype_factory=datatype_factory, node_datatypes=node_datatypes)
document["root_ontology_class"] = self.get_root_ontology()
doc = JSONSerializer().serializeToPython(document)
se.index_data(index=RESOURCES_INDEX, body=doc, id=self.pk)
for term in terms:
se.index_data("terms", body=term["_source"], id=term["_id"])
for index in settings.ELASTICSEARCH_CUSTOM_INDEXES:
es_index = import_class_from_string(index["module"])(index["name"])
doc, doc_id = es_index.get_documents_to_index(self, document["tiles"])
es_index.index_document(document=doc, id=doc_id)
def post(self, request):
username = request.POST.get("username", None)
password = request.POST.get("password", None)
user = authenticate(username=username, password=password)
if settings.MOBILE_OAUTH_CLIENT_ID == "":
message = _("Make sure to set your MOBILE_OAUTH_CLIENT_ID in settings.py")
response = HttpResponse(message, status=500)
logger.warning(message)
else:
if user:
if hasattr(user, "userprofile") is not True:
models.UserProfile.objects.create(user=user)
is_reviewer = user_is_resource_reviewer(user)
user = JSONSerializer().serializeToPython(user)
user["password"] = None
user["is_reviewer"] = is_reviewer
response = JSONResponse({"user": user, "clientid": settings.MOBILE_OAUTH_CLIENT_ID})
else:
response = Http401Response()
return response
def get(self, request):
map_layers = models.MapLayer.objects.all()
map_markers = models.MapMarker.objects.all()
map_sources = models.MapSource.objects.all()
resource_graphs = models.GraphModel.objects.exclude(pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude(isresource=False).exclude(isactive=False)
geocoding_providers = models.Geocoder.objects.all()
search_components = models.SearchComponent.objects.all()
datatypes = models.DDataType.objects.all()
context = self.get_context_data(
map_layers=map_layers,
map_markers=map_markers,
map_sources=map_sources,
geocoding_providers=geocoding_providers,
search_components=search_components,
main_script='views/search',
resource_graphs=resource_graphs,
datatypes=datatypes
)
graphs = JSONSerializer().serialize(
context['resource_graphs'],
exclude=['functions',
'author',
'deploymentdate',
'deploymentfile',
'version',
'subtitle',
'description',
'disable_instance_creation',
'ontology_id'])
context['graphs'] = graphs
context['nav']['title'] = _('Search')
context['nav']['icon'] = 'fa-search'
context['nav']['search'] = False
context['nav']['help'] = {
'title': _('Searching the Database'),
'template': 'search-help',
}
return render(request, 'views/search.htm', context)
def get(self, request):
def get_last_login(date):
result = _("Not yet logged in")
try:
if date is not None:
result = datetime.strftime(date, '%Y-%m-%d %H:%M')
except TypeError as e:
print e
return result
identities = []
for group in Group.objects.all():
users = group.user_set.all()
if len(users) > 0:
groupUsers = [{'id': user.id, 'first_name': user.first_name, 'last_name': user.last_name, 'email': user.email, 'last_login': get_last_login(user.last_login), 'username': user.username, 'groups': [g.id for g in user.groups.all()], 'group_names': ', '.join([g.name for g in user.groups.all()]) } for user in users]
identities.append({'name': group.name, 'type': 'group', 'id': group.pk, 'users': groupUsers, 'default_permissions': group.permissions.all()})
for user in User.objects.filter():
groups = []
group_ids = []
default_perms = []
for group in user.groups.all():
groups.append(group.name)
group_ids.append(group.id)
default_perms = default_perms + list(group.permissions.all())
identities.append({'name': user.email or user.username, 'groups': ', '.join(groups), 'type': 'user', 'id': user.pk, 'default_permissions': set(default_perms), 'is_superuser':user.is_superuser, 'group_ids': group_ids, 'first_name': user.first_name, 'last_name': user.last_name, 'email': user.email})
map_layers = models.MapLayer.objects.all()
map_markers = models.MapMarker.objects.all()
map_sources = models.MapSource.objects.all()
geocoding_providers = models.Geocoder.objects.all()
mobile_survey_models = models.MobileSurveyModel.objects.order_by('name')
mobile_surveys, resources = self.get_survey_resources(mobile_survey_models)
for mobile_survey in mobile_surveys:
try:
mobile_survey['datadownloadconfig'] = json.loads(mobile_survey['datadownloadconfig'])
except TypeError:
pass
multipart = mobile_survey['bounds']
singlepart = GeoUtils().convert_multipart_to_singlepart(multipart)
mobile_survey['bounds'] = singlepart
serializer = JSONSerializer()
context = self.get_context_data(
map_layers=map_layers,
map_markers=map_markers,
map_sources=map_sources,
geocoding_providers=geocoding_providers,
mobile_surveys=serializer.serialize(mobile_surveys, sort_keys=False),
identities=serializer.serialize(identities, sort_keys=False),
resources=serializer.serialize(resources, sort_keys=False),
resource_download_limit=settings.MOBILE_DOWNLOAD_RESOURCE_LIMIT,
main_script='views/mobile-survey-manager',
)
context['nav']['title'] = _('Mobile Survey Manager')
context['nav']['icon'] = 'fa-server'
context['nav']['help'] = (_('Mobile Survey Manager'),'help/base-help.htm')
context['help'] = 'mobile-survey-manager-help'
return render(request, 'views/mobile-survey-manager.htm', context)