def serialize(self, fields=None, exclude=None): """ serialize to a different form then used by the internal class structure used to append additional values (like parent ontology properties) that internal objects (like models.Nodes) don't support """ serializer = JSONSerializer() serializer.geom_format = 'geojson' obj = serializer.handle_model(self) ordered_cards = self.get_ordered_cards() ret = JSONSerializer().serializeToPython(obj) graphs = [] graphids = [] for card in self.cards.all(): if card.graph_id not in graphids: graphids.append(card.graph_id) #we may want the full proxy model at some point, but for now just the root node color # graphs.append(Graph.objects.get(pk=card.graph_id)) proxy_graph = Graph.objects.get(pk=card.graph_id) color = proxy_graph.root.config['fillColor'] graph = card.graph graph = JSONSerializer().serializeToPython(graph, exclude=['functions','disable_instance_creation','deploymentdate','deploymentfile']) graph['color'] = color graphs.append(graph) ret['graphs'] = graphs ret['cards'] = ordered_cards try: ret['bounds'] = json.loads(ret['bounds']) except TypeError as e: print 'Could not parse', ret['bounds'], e return ret
def __init__(self): # serializer = JSONSerializer() serializer.mimetype = 'application/json' serializer.dumps = serializer.serialize serializer.loads = JSONDeserializer().deserialize self.es = Elasticsearch(hosts=settings.ELASTICSEARCH_HOSTS, serializer=serializer, **settings.ELASTICSEARCH_CONNECTION_OPTIONS) self.logger = logging.getLogger(__name__)
def serialize(self, fields=None, exclude=None): """ serialize to a different form then used by the internal class structure """ exclude = [] if exclude == None else exclude ret = JSONSerializer().handle_model(self, fields, exclude) ret['cardinality'] = self.cardinality if 'cardinality' not in exclude else ret.pop('cardinality', None) ret['cards'] = self.cards if 'cards' not in exclude else ret.pop('cards', None) ret['nodes'] = list(self.nodegroup.node_set.all()) if 'nodes' not in exclude else ret.pop('nodes', None) ret['visible'] = self.visible if 'visible' not in exclude else ret.pop('visible', None) ret['active'] = self.active if 'active' not in exclude else ret.pop('active', None) ret['is_editable'] = self.is_editable() if 'is_editable' not in exclude else ret.pop('is_editable', None) ret['ontologyproperty'] = self.ontologyproperty if 'ontologyproperty' not in exclude else ret.pop('ontologyproperty', None) ret['disabled'] = self.disabled if 'disabled' not in exclude else ret.pop('disabled', None) if self.graph and self.graph.ontology and self.graph.isresource: edge = self.get_edge_to_parent() ret['ontologyproperty'] = edge.ontologyproperty # provide a models.CardXNodeXWidget model for every node # even if a widget hasn't been configured ret['widgets'] = self.widgets if 'widgets' not in exclude: for node in ret['nodes']: found = False for widget in ret['widgets']: if node.nodeid == widget.node_id: found = True if not found: widget = models.DDataType.objects.get(pk=node.datatype).defaultwidget if widget: widget_model = models.CardXNodeXWidget() widget_model.node_id = node.nodeid widget_model.card_id = self.cardid widget_model.widget_id = widget.pk widget_model.config = JSONSerializer().serialize(widget.defaultconfig) widget_model.label = node.name ret['widgets'].append(widget_model) else: ret.pop('widgets', None) return ret
def write_v4_jsonl(self, dest_path): ct = 0 with open(self.source_file, "rb") as openv3: lines = openv3.readlines() with open(dest_path, "wb") as openv4: for n, line in enumerate(lines): v3_json = json.loads(line) # set of checks that basically mimicks what happens # in the load_v3_data method that is used for JSON. if v3_json["entitytypeid"] != self.v3_graph_name: continue if len(v3_json["child_entities"]) == 0: continue resid = v3_json["entityid"] if self.truncate: if n - 2 == self.truncate: break if len(self.only) > 0: if resid not in self.only: continue if resid in self.exclude: continue v4_json = self.process_one_resource(v3_json) v4_line = JSONSerializer().serialize(v4_json) openv4.write(v4_line + "\n") ct += 1 # simple progress printing if ct == 0: print("") if ct % 1000 == 0: print(ct) elif ct % 100 == 0: print(".") if ct == self.truncate: break if ct > 0: print("\ntotal resources converted: {}".format(ct)) if ct == 0: os.remove(dest_path) return False return dest_path
def index(self): """ Indexes all the nessesary items values of a resource to support search """ if unicode(self.graph_id) != unicode(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID): se = SearchEngineFactory().create() datatype_factory = DataTypeFactory() node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list('nodeid', 'datatype')} document, terms = self.get_documents_to_index(datatype_factory=datatype_factory, node_datatypes=node_datatypes) document['root_ontology_class'] = self.get_root_ontology() se.index_data('resource', self.graph_id, JSONSerializer().serializeToPython(document), id=self.pk) for term in terms: se.index_data('strings', 'term', term['_source'], id=term['_id'])
def EntityTypes(request, entitytypeid): entityschema = [] if entitytypeid == '': return HttpResponse(JSONSerializer().serialize({}, ensure_ascii=True, indent=4)) else: if request.GET.get('f') is None: return render_to_response('graph.htm', {}, context_instance=RequestContext(request)) else: entityschema = { entitytypeid: Entity.get_mapping_schema(entitytypeid) } if request.GET.get('f') == 'json': return HttpResponse(JSONSerializer().serialize( entityschema, ensure_ascii=True, indent=4), content_type='application/json') if request.GET.get('f') == 'd3': d3Schema = d3Obj() d3Schema.name = entitytypeid for assestAttr in entityschema[entitytypeid]: d3ObjAssestAttr = d3Obj() d3ObjAssestAttr.name = assestAttr for step in entityschema[entitytypeid][assestAttr][ 'steps']: d3ObjStep = d3Obj() d3ObjStep.name = step['entitytypedomain'] + ' ' + step[ 'propertyid'] + ' ' + step['entitytyperange'] d3ObjAssestAttr.children.append(d3ObjStep) d3Schema.children.append(d3ObjAssestAttr) return HttpResponse(JSONSerializer().serialize( d3Schema, ensure_ascii=True, indent=4))
def index(self): """ Indexes all the nessesary items values of a resource to support search """ if str(self.graph_id) != str(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID): se = SearchEngineFactory().create() datatype_factory = DataTypeFactory() node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list("nodeid", "datatype")} document, terms = self.get_documents_to_index(datatype_factory=datatype_factory, node_datatypes=node_datatypes) document["root_ontology_class"] = self.get_root_ontology() doc = JSONSerializer().serializeToPython(document) se.index_data(index="resources", body=doc, id=self.pk) for term in terms: se.index_data("terms", body=term["_source"], id=term["_id"])
def test_node_delete(self): """ Test delete a node (HERITAGE_RESOURCE_PLACE) via node view """ self.client.login(username="******", password="******") node = Node.objects.get(nodeid=str(self.appended_branch_1.root.pk)) url = reverse("delete_node", kwargs={"graphid": self.GRAPH_ID}) post_data = JSONSerializer().serialize({"nodeid": node.nodeid}) response = self.client.delete(url, post_data) self.assertEqual(response.status_code, 200) graph = Graph.objects.get(graphid=self.GRAPH_ID).serialize() self.assertEqual(len(graph["nodes"]), 3) self.assertEqual(len(graph["edges"]), 2)
def get(self, request, formid): try: form = models.Form.objects.get(formid=formid) self.graph = Graph.objects.get(graphid=form.graph.pk) icons = models.Icon.objects.order_by('name') cards = models.CardModel.objects.filter( nodegroup__parentnodegroup=None, graph=self.graph) context = self.get_context_data( main_script='views/graph/form-configuration', graph_id=self.graph.pk, icons=JSONSerializer().serialize(icons), form=JSONSerializer().serialize(form), forms=JSONSerializer().serialize(self.graph.form_set.all()), cards=JSONSerializer().serialize(cards), forms_x_cards=JSONSerializer().serialize( models.FormXCard.objects.filter( form=form).order_by('sortorder')), ) context['nav']['title'] = self.graph.name context['nav']['menu'] = True context['nav']['help'] = (_('Configuring Menus'), 'help/menu-designer-help.htm') return render(request, 'views/graph/form-configuration.htm', context) except (models.Form.DoesNotExist): # assume the formid is a graph id graph = Graph.objects.get(graphid=formid) if graph.isresource == False: return redirect('graph_settings', graphid=graph.graphid) else: return redirect('form_manager', graphid=graph.graphid)
def get(self, request, graphid): self.graph = Graph.objects.get(graphid=graphid) if self.graph.isresource: context = self.get_context_data( main_script='views/graph/function-manager', functions=JSONSerializer().serialize(models.Function.objects.all()), applied_functions=JSONSerializer().serialize(models.FunctionXGraph.objects.filter(graph=self.graph)), function_templates=models.Function.objects.exclude(component__isnull=True), ) context['graphs'] = JSONSerializer().serialize( models.GraphModel.objects.all().exclude(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID), exclude=['functions']) context['nav']['title'] = self.graph.name context['nav']['menu'] = True context['nav']['help'] = { 'title': _('Managing Functions'), 'template': 'function-help', } return render(request, 'views/graph/function-manager.htm', context) else: return redirect('graph_designer', graphid=graphid)
def get(self, request): map_layers = models.MapLayer.objects.all() map_markers = models.MapMarker.objects.all() map_sources = models.MapSource.objects.all() resource_graphs = (models.GraphModel.objects.exclude( pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude( isresource=False).exclude(isactive=False)) geocoding_providers = models.Geocoder.objects.all() search_components = models.SearchComponent.objects.all() datatypes = models.DDataType.objects.all() context = self.get_context_data( map_layers=map_layers, map_markers=map_markers, map_sources=map_sources, geocoding_providers=geocoding_providers, search_components=search_components, main_script="views/search", resource_graphs=resource_graphs, datatypes=datatypes, user_is_reviewer=user_is_resource_reviewer(request.user), ) graphs = JSONSerializer().serialize( context["resource_graphs"], exclude=[ "functions", "author", "deploymentdate", "deploymentfile", "version", "subtitle", "description", "disable_instance_creation", "ontology_id", ], ) context["graphs"] = graphs context["nav"]["title"] = _("Search") context["nav"]["icon"] = "fa-search" context["nav"]["search"] = False context["nav"]["help"] = { "title": _("Searching the Database"), "template": "search-help", } context["celery_running"] = task_management.check_if_celery_available() return render(request, "views/search.htm", context)
def get_documents_to_index(self, fetchTiles=True, datatype_factory=None, node_datatypes=None): """ Gets all the documents nessesary to index a single resource returns a tuple of a document and list of terms Keyword Arguments: fetchTiles -- instead of fetching the tiles from the database get them off the model itself datatype_factory -- refernce to the DataTypeFactory instance node_datatypes -- a dictionary of datatypes keyed to node ids """ document = JSONSerializer().serializeToPython(self) document['tiles'] = models.TileModel.objects.filter( resourceinstance=self) if fetchTiles else self.tiles document['strings'] = [] document['dates'] = [] document['domains'] = [] document['geometries'] = [] document['points'] = [] document['numbers'] = [] terms = [] for tile in document['tiles']: for nodeid, nodevalue in tile.data.iteritems(): datatype = node_datatypes[nodeid] if nodevalue != '' and nodevalue != [] and nodevalue != {} and nodevalue is not None: datatype_instance = datatype_factory.get_instance(datatype) datatype_instance.append_to_document(document, nodevalue) term = datatype_instance.get_search_term(nodevalue) if term is not None: terms.append({ '_id': unicode(nodeid) + unicode(tile.tileid), '_source': { 'value': term, 'nodeid': nodeid, 'nodegroupid': tile.nodegroup_id, 'tileid': tile.tileid, 'resourceinstanceid': tile.resourceinstance_id } }) return document, terms
def get(self, request, graphid): self.graph = Graph.objects.get(graphid=graphid) branch_graphs = Graph.objects.exclude(pk=graphid).exclude(isresource=True).exclude(isactive=False) if self.graph.ontology is not None: branch_graphs = branch_graphs.filter(ontology=self.graph.ontology) context = self.get_context_data( main_script='views/graph/card-manager', branches=JSONSerializer().serialize(branch_graphs), ) context['nav']['title'] = self.graph.name context['nav']['menu'] = True context['nav']['help'] = ('Managing Cards','help/card-manager-help.htm') return render(request, 'views/graph/card-manager.htm', context)
def get_context_data(self, **kwargs): datatype_factory = DataTypeFactory() context = super(BaseManagerView, self).get_context_data(**kwargs) context['graph_models'] = models.GraphModel.objects.all() context['graphs'] = JSONSerializer().serialize(context['graph_models']) if 'Resource Editor' in self.request.user.user_groups: context['resource_instances'] = Resource.objects.all().order_by( '-createdtime')[:100] else: context['resource_instances'] = [] context['nav'] = { 'icon': 'fa fa-chevron-circle-right', 'title': '', 'help': ('', ''), 'menu': False, 'search': True, 'res_edit': False, 'edit_history': True, 'login': True, 'print': False, } geom_datatypes = [ d.pk for d in models.DDataType.objects.filter(isgeometric=True) ] geom_nodes = models.Node.objects.filter(graph__isresource=True, graph__isactive=True, datatype__in=geom_datatypes) resource_layers = [] resource_sources = [] for node in geom_nodes: # TODO: check user node level permissions here, if user does not # have read permissions on this node, then do not create map layer # or source datatype = datatype_factory.get_instance(node.datatype) map_source = datatype.get_map_source(node) if map_source is not None: resource_sources.append(map_source) map_layer = datatype.get_map_layer(node) if map_layer is not None: resource_layers.append(map_layer) context['geom_nodes'] = geom_nodes context['resource_map_layers'] = resource_layers context['resource_map_sources'] = resource_sources context['iiif_manifests'] = models.IIIFManifest.objects.all() return context
def get(self, request, graphid, nodeid=None): if self.action == 'export_graph': graph = get_graphs_for_export([graphid]) graph['metadata'] = system_metadata() f = JSONSerializer().serialize(graph, indent=4) graph_name = JSONDeserializer().deserialize(f)['graph'][0]['name'] response = HttpResponse(f, content_type='json/plain') response[ 'Content-Disposition'] = 'attachment; filename="%s.json"' % ( graph_name) return response elif self.action == 'export_mapping_file': files_for_export = create_mapping_configuration_file(graphid) file_name = Graph.objects.get(graphid=graphid).name buffer = StringIO() with zipfile.ZipFile(buffer, 'w', zipfile.ZIP_DEFLATED) as zip: for f in files_for_export: f['outputfile'].seek(0) zip.writestr(f['name'], f['outputfile'].read()) zip.close() buffer.flush() zip_stream = buffer.getvalue() buffer.close() response = HttpResponse() response[ 'Content-Disposition'] = 'attachment; filename=' + file_name + '.zip' response['Content-length'] = str(len(zip_stream)) response['Content-Type'] = 'application/zip' response.write(zip_stream) return response else: graph = Graph.objects.get(graphid=graphid) if self.action == 'get_related_nodes': ret = graph.get_valid_ontology_classes(nodeid=nodeid) elif self.action == 'get_valid_domain_nodes': ret = graph.get_valid_domain_ontology_classes(nodeid=nodeid) return JSONResponse(ret) return HttpResponseNotFound()
def get_context_data(self, **kwargs): context = super(BaseManagerView, self).get_context_data(**kwargs) context[ "system_settings_graphid"] = settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID context["graph_models"] = [] context["graphs"] = "[]" context["plugins"] = [] for plugin in models.Plugin.objects.all().order_by("sortorder"): if self.request.user.has_perm("view_plugin", plugin): context["plugins"].append(plugin) createable = get_createable_resource_types(self.request.user) createable.sort(key=lambda x: x.name.lower()) context["createable_resources"] = JSONSerializer().serialize( createable, exclude=[ "functions", "ontology", "isactive", "isresource", "version", "deploymentdate", "deploymentfile", "author", ], ) context["notifications"] = models.UserXNotification.objects.filter( recipient=self.request.user, isread=False) context["nav"] = { "icon": "fa fa-chevron-circle-right", "title": "", "help": { # title:'',template:'' (leave this commented out) }, "menu": False, "search": True, "notifs": True, "res_edit": False, "login": True, "print": False, } context["user_is_reviewer"] = user_is_resource_reviewer( self.request.user) context["app_name"] = settings.APP_NAME context["show_language_swtich"] = settings.SHOW_LANGUAGE_SWITCH return context
def index(self, scheme=None): if self.category == 'label': se = SearchEngineFactory().create() data = JSONSerializer().serializeToPython(self) if scheme == None: scheme = self.get_scheme_id() if scheme == None: raise Exception('Index of label failed. Index type (scheme id) could not be derived from the label.') se.create_mapping('concept_labels', scheme.id, fieldname='conceptid', fieldtype='string', fieldindex='not_analyzed') se.index_data('concept_labels', scheme.id, data, 'id') #Looks up whether the label is actually a dropdown label or an entity label and, if so, excludes them from the term search index. entity_or_dropdown= archesmodels.ConceptRelations.objects.filter(Q(relationtype ='hasCollection') | Q(relationtype ='hasEntity'),conceptidto = scheme.id) is_entity_or_dropdown = False if entity_or_dropdown.count() == 0 else True # don't create terms for entity type concepts if not(scheme.id == '00000000-0000-0000-0000-000000000003' or scheme.id == '00000000-0000-0000-0000-000000000004') and is_entity_or_dropdown ==False: se.index_term(self.value, self.id, scheme.id, {'conceptid': self.conceptid})
def index_concepts_by_entitytypeid(self, entitytypeid): entitytype = archesmodels.EntityTypes.objects.get(pk=entitytypeid) conceptid = entitytype.conceptid_id concept_graph = Concept().get(id=conceptid, include_subconcepts=True, exclude=['note']) if len(concept_graph.subconcepts) > 0: data = JSONSerializer().serializeToPython(concept_graph, ensure_ascii=True, indent=4) self.index(data, 'concept', entitytypeid, 'id', processdoc=None, getid=None, bulk=False)
def map_info(request): return { 'map_info': { 'x': settings.DEFAULT_MAP_X, 'y': settings.DEFAULT_MAP_Y, 'zoom': settings.DEFAULT_MAP_ZOOM, 'map_min_zoom': settings.MAP_MIN_ZOOM, 'map_max_zoom': settings.MAP_MAX_ZOOM, 'mapbox_api_key': settings.MAPBOX_API_KEY, 'hex_bin_size': settings.HEX_BIN_SIZE, 'mapbox_sprites': settings.MAPBOX_SPRITES, 'mapbox_glyphs': settings.MAPBOX_GLYPHS, 'hex_bin_bounds': JSONSerializer().serialize(settings.HEX_BIN_BOUNDS), 'geocoder_default': settings.DEFAULT_SEARCH_GEOCODER } }
def get(self, request, resourceid=None): resource_instance = models.ResourceInstance.objects.get(pk=resourceid) tiles = models.TileModel.objects.filter( resourceinstance=resource_instance) try: report = models.Report.objects.get(graph=resource_instance.graph, active=True) except models.Report.DoesNotExist: report = None graph = Graph.objects.get(graphid=resource_instance.graph.pk) resource_graphs = Graph.objects.exclude( pk=resource_instance.graph.pk).exclude( pk='22000000-0000-0000-0000-000000000002').exclude( isresource=False).exclude(isactive=False) forms = resource_instance.graph.form_set.filter(visible=True) forms_x_cards = models.FormXCard.objects.filter( form__in=forms).order_by('sortorder') cards = Card.objects.filter(nodegroup__parentnodegroup=None, graph=resource_instance.graph) datatypes = models.DDataType.objects.all() widgets = models.Widget.objects.all() map_layers = models.MapLayers.objects.all() map_sources = models.MapSources.objects.all() templates = models.ReportTemplate.objects.all() context = self.get_context_data( main_script='views/resource/report', report=JSONSerializer().serialize(report), report_templates=templates, templates_json=JSONSerializer().serialize(templates), forms=JSONSerializer().serialize(forms), tiles=JSONSerializer().serialize(tiles), forms_x_cards=JSONSerializer().serialize(forms_x_cards), cards=JSONSerializer().serialize(cards), datatypes_json=JSONSerializer().serialize(datatypes), widgets=widgets, map_layers=map_layers, map_sources=map_sources, resource_graphs=resource_graphs, graph_id=resource_instance.graph.pk, graph_name=resource_instance.graph.name, graph_json=JSONSerializer().serialize(graph)) return render(request, 'views/resource/report.htm', context)
def get_instance_permissions(self, resource_instance): permission_order = [ "view_resourceinstance", "change_resourceinstance", "delete_resourceinstance", "no_access_to_resourceinstance" ] perms = json.loads(JSONSerializer().serialize({ p.codename: p for p in get_perms_for_model(resource_instance) if p.codename != "add_resourceinstance" })) ordered_perms = [] for p in permission_order: ordered_perms.append(perms[p]) identities = [{ "name": user.username, "id": user.id, "type": "user", "default_permissions": self.get_perms(user, "user", resource_instance, ordered_perms), "is_editor_or_reviewer": bool( user_is_resource_editor(user) or user_is_resource_reviewer(user)), } for user in User.objects.all()] identities += [{ "name": group.name, "id": group.id, "type": "group", "default_permissions": self.get_perms(group, "group", resource_instance, ordered_perms), } for group in Group.objects.all()] result = {"identities": identities} result["permissions"] = ordered_perms result["limitedaccess"] = ( len(get_users_with_perms(resource_instance)) + len(get_groups_with_perms(resource_instance))) > 1 instance_creator = get_instance_creator(resource_instance) result["creatorid"] = instance_creator["creatorid"] return result
def _buffer(geojson, width=0, unit="ft"): geojson = JSONSerializer().serialize(geojson) geom = GEOSGeometry(geojson, srid=4326) try: width = float(width) except Exception: width = 0 if width > 0: if unit == "ft": width = width / 3.28084 geom.transform(settings.ANALYSIS_COORDINATE_SYSTEM_SRID) geom = geom.buffer(width) geom.transform(4326) return geom
def post(self, request): username = request.POST.get("username", None) password = request.POST.get("password", None) user = authenticate(username=username, password=password) if user: if hasattr(user, "userprofile") is not True: models.UserProfile.objects.create(user=user) userDict = JSONSerializer().serializeToPython(user) userDict["password"] = None userDict["is_reviewer"] = user_is_resource_reviewer(user) userDict["viewable_nodegroups"] = user.userprofile.viewable_nodegroups userDict["editable_nodegroups"] = user.userprofile.editable_nodegroups userDict["deletable_nodegroups"] = user.userprofile.deletable_nodegroups response = JSONResponse(userDict) else: response = Http401Response() return response
def home_page(request): lang = request.GET.get('lang', settings.LANGUAGE_CODE) min_max_dates = models.Dates.objects.aggregate(Min('val'), Max('val')) return render_to_response('search.htm', { 'main_script': 'search', 'active_page': 'Search', 'min_date': min_max_dates['val__min'].year if min_max_dates['val__min'] != None else 0, 'max_date': min_max_dates['val__max'].year if min_max_dates['val__min'] != None else 1, 'timefilterdata': JSONSerializer().serialize(Concept.get_time_filter_data()), }, context_instance=RequestContext(request))
def test_rules_for_appending(self): """ test the rules that control the appending of branches to graphs """ graph = Graph.objects.get(node=self.rootNode) graph.isresource = True self.assertIsNotNone(graph.append_branch('http://www.ics.forth.gr/isl/CRMdig/L54_is_same-as', graphid=self.NODE_NODETYPE_GRAPHID)) graph = Graph.new() graph.root.datatype = 'string' graph.update_node(JSONSerializer().serializeToPython(graph.root)) # create card collector graph to use for appending on to other graphs collector_graph = Graph.new() collector_graph.append_branch('http://www.ics.forth.gr/isl/CRMdig/L54_is_same-as', graphid=self.NODE_NODETYPE_GRAPHID) collector_graph.save()
def _buffer(geojson, width=0, unit='ft'): geojson = JSONSerializer().serialize(geojson) geom = GEOSGeometry(geojson, srid=4326) try: width = float(width) except: width = 0 if width > 0: if unit == 'ft': width = width / 3.28084 geom.transform(3857) geom = geom.buffer(width) geom.transform(4326) return geom
def test_delete_mobile_survey(self): """ Test that deletion of a mobile survey deletes its couchdb """ self.client.login(username='******', password='******') url = reverse('mobile_survey_manager') post_data = {"id": str(self.mobile_survey.id)} post_data = JSONSerializer().serialize(post_data) content_type = 'application/x-www-form-urlencoded' response = self.client.delete(url, post_data, content_type) response_json = json.loads(response.content) couch = couchdb.Server(settings.COUCHDB_URL) self.assertFalse( MobileSurvey.objects.filter(pk=self.mobile_survey.id).exists()) self.assertTrue('project_' + str(self.mobile_survey.id) not in couch)
def get(self, request): if self.action == 'tile_history': start = request.GET.get('start') end = request.GET.get('end') edits = EditLog.objects.filter(provisional_userid=request.user.id).filter(timestamp__range=[start, end]).order_by('tileinstanceid', 'timestamp') resourceinstanceids = [e['resourceinstanceid'] for e in edits.values('resourceinstanceid')] deleted_resource_edits = EditLog.objects.filter(resourceinstanceid__in=resourceinstanceids).filter(edittype='delete') deleted_resource_instances = [e['resourceinstanceid'] for e in deleted_resource_edits.values('resourceinstanceid')] summary = {} for edit in edits: if edit.tileinstanceid not in summary: summary[edit.tileinstanceid] = {'pending': False, 'tileid': edit.tileinstanceid} summary[edit.tileinstanceid]['lasttimestamp'] = edit.timestamp summary[edit.tileinstanceid]['lastedittype'] = edit.provisional_edittype summary[edit.tileinstanceid]['reviewer'] = '' summary[edit.tileinstanceid]['resourceinstanceid'] = edit.resourceinstanceid summary[edit.tileinstanceid]['resourcedisplayname'] = edit.resourcedisplayname summary[edit.tileinstanceid]['resourcemodelid'] = edit.resourceclassid summary[edit.tileinstanceid]['nodegroupid'] = edit.nodegroupid summary[edit.tileinstanceid]['resource_deleted'] = True if edit.resourceinstanceid in deleted_resource_instances else False if edit.provisional_edittype in ['accept edit', 'delete edit']: summary[edit.tileinstanceid]['reviewer'] = edit.user_username chronological_summary = [] resource_models = models.GraphModel.objects.filter(isresource=True).exclude(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).values('iconclass','color','graphid','name') cards = models.CardModel.objects.all().values('name', 'nodegroup_id') card_lookup = {str(card['nodegroup_id']): card for card in cards} resource_model_lookup = {str(graph['graphid']): graph for graph in resource_models} for k, v in summary.iteritems(): if v['lastedittype'] not in ['accept edit', 'delete edit']: if models.TileModel.objects.filter(pk=k).exists(): tile = models.TileModel.objects.get(pk=k) if tile.provisionaledits is not None and str(request.user.id) in tile.provisionaledits: v['pending'] = True v['resourcemodel'] = resource_model_lookup[v['resourcemodelid']] v['card'] = card_lookup[v['nodegroupid']] if 'graphid' in v['resourcemodel']: v['resourcemodel'].pop('graphid') if 'nodegroup_id' in v['card']: v['card'].pop('nodegroup_id') chronological_summary.append(v) return JSONResponse(JSONSerializer().serialize(sorted(chronological_summary, key=lambda k: k['lasttimestamp'], reverse=True)))
def test_node_delete(self): """ Test delete a node (HERITAGE_RESOURCE_PLACE) via node view """ self.client.login(username='******', password='******') graphid = Node.objects.get(nodeid=self.ROOT_ID).graph.pk node = Node.objects.get(nodeid=self.HERITAGE_RESOURCE_PLACE_ID) url = reverse('delete_node', kwargs={'graphid': graphid}) post_data = JSONSerializer().serialize({'nodeid': node.nodeid}) response = self.client.delete(url, post_data) self.assertEqual(response.status_code, 200) new_count = self.NODE_COUNT - self.PLACE_BRANCH_COUNT root = Node.objects.get(nodeid=self.ROOT_ID) nodes, edges = root.get_child_nodes_and_edges() self.assertEqual(len(nodes), new_count) self.assertEqual(len(edges), new_count)
def load_tiles_into_couch(self, instances, nodegroup): """ Takes a mobile survey object, a couch database instance, and a dictionary of resource instances to identify eligible tiles and load them into the database instance """ db = self.couch.create_db("project_" + str(self.id)) tiles = models.TileModel.objects.filter(nodegroup=nodegroup) tiles_serialized = json.loads(JSONSerializer().serialize(tiles)) for tile in tiles_serialized: if str(tile["resourceinstance_id"]) in instances: try: tile["type"] = "tile" self.couch.update_doc(db, tile, tile["tileid"]) except Exception as e: print(e, tile) nodegroups = models.NodeGroup.objects.filter(parentnodegroup=nodegroup) for nodegroup in nodegroups: self.load_tiles_into_couch(instances, nodegroup)
def serialize(self, fields=None, exclude=None): """ serialize to a different form then used by the internal class structure used to append additional values (like parent ontology properties) that internal objects (like models.Nodes) don't support """ exclude = [] if exclude == None else exclude ret = JSONSerializer().handle_model(self, fields, exclude) ret['root'] = self.root if 'relatable_resource_model_ids' not in exclude: ret['relatable_resource_model_ids'] = [str(relatable_node.graph_id) for relatable_node in self.root.get_relatable_resources()] else: ret.pop('relatable_resource_model_ids', None) ret['cards'] = self.get_cards() if 'cards' not in exclude else ret.pop('cards', None) ret['nodegroups'] = self.get_nodegroups() if 'nodegroups' not in exclude else ret.pop('nodegroups', None) ret['domain_connections'] = self.get_valid_domain_ontology_classes() if 'domain_connections' not in exclude else ret.pop('domain_connections', None) ret['is_editable'] = self.is_editable() if 'is_editable' not in exclude else ret.pop('is_editable', None) ret['functions'] = models.FunctionXGraph.objects.filter(graph_id=self.graphid) if 'functions' not in exclude else ret.pop('functions', None) parentproperties = { self.root.nodeid: '' } for edge_id, edge in self.edges.iteritems(): parentproperties[edge.rangenode_id] = edge.ontologyproperty ret['edges'] = [edge for key, edge in self.edges.iteritems()] if 'edges' not in exclude else ret.pop('edges', None) if 'nodes' not in exclude: ret['nodes'] = [] for key, node in self.nodes.iteritems(): nodeobj = JSONSerializer().serializeToPython(node) nodeobj['parentproperty'] = parentproperties[node.nodeid] ret['nodes'].append(nodeobj) else: ret.pop('nodes', None) res = JSONSerializer().serializeToPython(ret) return res
def index(self): """ Indexes all the nessesary items values of a resource to support search """ if str(self.graph_id) != str(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID): datatype_factory = DataTypeFactory() node_datatypes = {str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list("nodeid", "datatype")} document, terms = self.get_documents_to_index(datatype_factory=datatype_factory, node_datatypes=node_datatypes) document["root_ontology_class"] = self.get_root_ontology() doc = JSONSerializer().serializeToPython(document) se.index_data(index=RESOURCES_INDEX, body=doc, id=self.pk) for term in terms: se.index_data("terms", body=term["_source"], id=term["_id"]) for index in settings.ELASTICSEARCH_CUSTOM_INDEXES: es_index = import_class_from_string(index["module"])(index["name"]) doc, doc_id = es_index.get_documents_to_index(self, document["tiles"]) es_index.index_document(document=doc, id=doc_id)
def post(self, request): username = request.POST.get("username", None) password = request.POST.get("password", None) user = authenticate(username=username, password=password) if settings.MOBILE_OAUTH_CLIENT_ID == "": message = _("Make sure to set your MOBILE_OAUTH_CLIENT_ID in settings.py") response = HttpResponse(message, status=500) logger.warning(message) else: if user: if hasattr(user, "userprofile") is not True: models.UserProfile.objects.create(user=user) is_reviewer = user_is_resource_reviewer(user) user = JSONSerializer().serializeToPython(user) user["password"] = None user["is_reviewer"] = is_reviewer response = JSONResponse({"user": user, "clientid": settings.MOBILE_OAUTH_CLIENT_ID}) else: response = Http401Response() return response
def get(self, request): map_layers = models.MapLayer.objects.all() map_markers = models.MapMarker.objects.all() map_sources = models.MapSource.objects.all() resource_graphs = models.GraphModel.objects.exclude(pk=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID).exclude(isresource=False).exclude(isactive=False) geocoding_providers = models.Geocoder.objects.all() search_components = models.SearchComponent.objects.all() datatypes = models.DDataType.objects.all() context = self.get_context_data( map_layers=map_layers, map_markers=map_markers, map_sources=map_sources, geocoding_providers=geocoding_providers, search_components=search_components, main_script='views/search', resource_graphs=resource_graphs, datatypes=datatypes ) graphs = JSONSerializer().serialize( context['resource_graphs'], exclude=['functions', 'author', 'deploymentdate', 'deploymentfile', 'version', 'subtitle', 'description', 'disable_instance_creation', 'ontology_id']) context['graphs'] = graphs context['nav']['title'] = _('Search') context['nav']['icon'] = 'fa-search' context['nav']['search'] = False context['nav']['help'] = { 'title': _('Searching the Database'), 'template': 'search-help', } return render(request, 'views/search.htm', context)
def get(self, request): def get_last_login(date): result = _("Not yet logged in") try: if date is not None: result = datetime.strftime(date, '%Y-%m-%d %H:%M') except TypeError as e: print e return result identities = [] for group in Group.objects.all(): users = group.user_set.all() if len(users) > 0: groupUsers = [{'id': user.id, 'first_name': user.first_name, 'last_name': user.last_name, 'email': user.email, 'last_login': get_last_login(user.last_login), 'username': user.username, 'groups': [g.id for g in user.groups.all()], 'group_names': ', '.join([g.name for g in user.groups.all()]) } for user in users] identities.append({'name': group.name, 'type': 'group', 'id': group.pk, 'users': groupUsers, 'default_permissions': group.permissions.all()}) for user in User.objects.filter(): groups = [] group_ids = [] default_perms = [] for group in user.groups.all(): groups.append(group.name) group_ids.append(group.id) default_perms = default_perms + list(group.permissions.all()) identities.append({'name': user.email or user.username, 'groups': ', '.join(groups), 'type': 'user', 'id': user.pk, 'default_permissions': set(default_perms), 'is_superuser':user.is_superuser, 'group_ids': group_ids, 'first_name': user.first_name, 'last_name': user.last_name, 'email': user.email}) map_layers = models.MapLayer.objects.all() map_markers = models.MapMarker.objects.all() map_sources = models.MapSource.objects.all() geocoding_providers = models.Geocoder.objects.all() mobile_survey_models = models.MobileSurveyModel.objects.order_by('name') mobile_surveys, resources = self.get_survey_resources(mobile_survey_models) for mobile_survey in mobile_surveys: try: mobile_survey['datadownloadconfig'] = json.loads(mobile_survey['datadownloadconfig']) except TypeError: pass multipart = mobile_survey['bounds'] singlepart = GeoUtils().convert_multipart_to_singlepart(multipart) mobile_survey['bounds'] = singlepart serializer = JSONSerializer() context = self.get_context_data( map_layers=map_layers, map_markers=map_markers, map_sources=map_sources, geocoding_providers=geocoding_providers, mobile_surveys=serializer.serialize(mobile_surveys, sort_keys=False), identities=serializer.serialize(identities, sort_keys=False), resources=serializer.serialize(resources, sort_keys=False), resource_download_limit=settings.MOBILE_DOWNLOAD_RESOURCE_LIMIT, main_script='views/mobile-survey-manager', ) context['nav']['title'] = _('Mobile Survey Manager') context['nav']['icon'] = 'fa-server' context['nav']['help'] = (_('Mobile Survey Manager'),'help/base-help.htm') context['help'] = 'mobile-survey-manager-help' return render(request, 'views/mobile-survey-manager.htm', context)