def clean_resource_cache(tile):
# get the tile model's bounds
datatype_factory = DataTypeFactory()
nodegroup = models.NodeGroup.objects.get(pk=tile.nodegroup_id)
for node in nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
if datatype.should_cache(node) and datatype.should_manage_cache(node):
bounds = datatype.get_bounds(tile, node)
if bounds is not None:
zooms = range(20)
config = TileStache.parseConfig(
get_tileserver_config(node.nodeid))
layer = config.layers[str(node.nodeid)]
mimetype, format = layer.getTypeByExtension('pbf')
lon1, lat1, lon2, lat2 = bounds
south, west = min(lat1, lat2), min(lon1, lon2)
north, east = max(lat1, lat2), max(lon1, lon2)
northwest = Location(north, west)
southeast = Location(south, east)
ul = layer.projection.locationCoordinate(northwest)
lr = layer.projection.locationCoordinate(southeast)
padding = 0
coordinates = generateCoordinates(ul, lr, zooms, padding)
for (offset, count, coord) in coordinates:
config.cache.remove(layer, coord, format)
for key, tile_list in tile.tiles.iteritems():
for child_tile in tile_list:
clean_resource_cache(child_tile)
def get(self, request, resourceid=None, include_display_values=True):
datatype_factory = DataTypeFactory()
nodeid = request.GET.get("nodeid", None)
search_term = request.GET.get("term", None)
permitted_tiles = []
perm = "read_nodegroup"
tiles = models.TileModel.objects.filter(resourceinstance_id=resourceid)
if nodeid is not None:
node = models.Node.objects.get(pk=nodeid)
tiles = tiles.filter(nodegroup=node.nodegroup)
for tile in tiles:
if request.user.has_perm(perm, tile.nodegroup):
tile = Tile.objects.get(pk=tile.tileid)
tile.filter_by_perm(request.user, perm)
tile_dict = model_to_dict(tile)
if include_display_values:
tile_dict["display_values"] = []
for node in models.Node.objects.filter(nodegroup=tile.nodegroup):
if str(node.nodeid) in tile.data:
datatype = datatype_factory.get_instance(node.datatype)
display_value = datatype.get_display_value(tile, node)
if search_term is not None and search_term in display_value:
tile_dict["display_values"].append({"value": display_value, "label": node.name, "nodeid": node.nodeid})
elif search_term is None:
tile_dict["display_values"].append({"value": display_value, "label": node.name, "nodeid": node.nodeid})
if search_term is None:
permitted_tiles.append(tile_dict)
elif len(tile_dict["display_values"]) > 0:
permitted_tiles.append(tile_dict)
return JSONResponse({"tiles": permitted_tiles})
def append_dsl(self, search_results_object, permitted_nodegroups, include_provisional):
querysting_params = self.request.GET.get(details["componentname"], "")
advanced_filters = JSONDeserializer().deserialize(querysting_params)
datatype_factory = DataTypeFactory()
search_query = Bool()
advanced_query = Bool()
grouped_query = Bool()
grouped_queries = [grouped_query]
for index, advanced_filter in enumerate(advanced_filters):
tile_query = Bool()
null_query = Bool()
for key, val in advanced_filter.items():
if key != "op":
node = models.Node.objects.get(pk=key)
if self.request.user.has_perm("read_nodegroup", node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
if ("op" in val and (val["op"] == "null" or val["op"] == "not_null")) or (
"val" in val and (val["val"] == "null" or val["val"] == "not_null")
):
# don't use a nested query with the null/not null search
datatype.append_search_filters(val, node, null_query, self.request)
else:
datatype.append_search_filters(val, node, tile_query, self.request)
nested_query = Nested(path="tiles", query=tile_query)
if advanced_filter["op"] == "or" and index != 0:
grouped_query = Bool()
grouped_queries.append(grouped_query)
grouped_query.must(nested_query)
grouped_query.must(null_query)
for grouped_query in grouped_queries:
advanced_query.should(grouped_query)
search_query.must(advanced_query)
search_results_object["query"].add_query(search_query)
def get(self, request, resourceid=None, include_display_values=True):
datatype_factory = DataTypeFactory()
nodeid = request.GET.get('nodeid', None)
permitted_tiles = []
perm = 'read_nodegroup'
tiles = models.TileModel.objects.filter(resourceinstance_id=resourceid)
if nodeid is not None:
node = models.Node.objects.get(pk=nodeid)
tiles = tiles.filter(nodegroup=node.nodegroup)
for tile in tiles:
if request.user.has_perm(perm, tile.nodegroup):
tile = Tile.objects.get(pk=tile.tileid)
tile.filter_by_perm(request.user, perm)
tile_dict = model_to_dict(tile)
if include_display_values:
tile_dict['display_values'] = []
for node in models.Node.objects.filter(
nodegroup=tile.nodegroup):
if str(node.nodeid) in tile.data:
datatype = datatype_factory.get_instance(
node.datatype)
tile_dict['display_values'].append({
'value':
datatype.get_display_value(tile, node),
'label':
node.name,
'nodeid':
node.nodeid
})
permitted_tiles.append(tile_dict)
return JSONResponse({'tiles': permitted_tiles})
def get_primary_descriptor_from_nodes(self, resource, config):
try:
if 'nodegroup_id' in config and config[
'nodegroup_id'] != '' and config[
'nodegroup_id'] is not None:
tiles = models.TileModel.objects.filter(
nodegroup_id=uuid.UUID(config['nodegroup_id']),
sortorder=0).filter(
resourceinstance_id=resource.resourceinstanceid)
if len(tiles) == 0:
tiles = models.TileModel.objects.filter(
nodegroup_id=uuid.UUID(config['nodegroup_id'])).filter(
resourceinstance_id=resource.resourceinstanceid)
for tile in tiles:
for node in models.Node.objects.filter(
nodegroup_id=uuid.UUID(config['nodegroup_id'])):
if len(tile.data.keys()) > 0:
data = tile.data
elif tile.provisionaledits is not None and len(
tile.provisionaledits.keys()) == 1:
userid = tile.provisionaledits.keys()[0]
data = tile.provisionaledits[userid]['value']
if str(node.nodeid) in data:
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(
node.datatype)
value = datatype.get_display_value(tile, node)
config['string_template'] = config[
'string_template'].replace(
'<%s>' % node.name, value)
except ValueError as e:
print(e,
'invalid nodegroupid participating in descriptor function.')
return config['string_template']
def save(self, *args, **kwargs):
request = kwargs.pop('request', None)
index = kwargs.pop('index', True)
self.__preSave(request)
if self.data != {}:
old_model = models.TileModel.objects.filter(pk=self.tileid)
old_data = old_model[0].data if len(old_model) > 0 else None
edit_type = 'tile create' if (old_data == None) else 'tile edit'
try:
user = request.user
except:
user = {}
self.save_edit(user=user, edit_type=edit_type, old_value=old_data, new_value=self.data)
for nodeid, value in self.data.iteritems():
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(models.Node.objects.get(nodeid=nodeid).datatype)
datatype.convert_value(self, nodeid)
super(Tile, self).save(*args, **kwargs)
if index and unicode(self.resourceinstance.graph_id) != unicode(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID):
self.index()
for tiles in self.tiles.itervalues():
for tile in tiles:
tile.resourceinstance = self.resourceinstance
tile.parenttile = self
tile.save(*args, request=request, index=index, **kwargs)
def get(self, request, resourceid=None, include_display_values=True):
datatype_factory = DataTypeFactory()
nodeid = request.GET.get('nodeid', None)
permitted_tiles = []
perm = 'read_nodegroup'
tiles = models.TileModel.objects.filter(resourceinstance_id=resourceid)
if nodeid is not None:
node = models.Node.objects.get(pk=nodeid)
tiles = tiles.filter(nodegroup=node.nodegroup)
for tile in tiles:
if request.user.has_perm(perm, tile.nodegroup):
tile = Tile.objects.get(pk=tile.tileid)
tile.filter_by_perm(request.user, perm)
tile_dict = model_to_dict(tile)
if include_display_values:
tile_dict['display_values'] = []
for node in models.Node.objects.filter(nodegroup=tile.nodegroup):
if str(node.nodeid) in tile.data:
datatype = datatype_factory.get_instance(node.datatype)
tile_dict['display_values'].append({
'value': datatype.get_display_value(tile, node),
'label': node.name,
'nodeid': node.nodeid
})
permitted_tiles.append(tile_dict)
return JSONResponse({'tiles': permitted_tiles})
def get_primary_descriptor_from_nodes(self, resource, config):
datatype_factory = None
try:
if "nodegroup_id" in config and config["nodegroup_id"] != "" and config["nodegroup_id"] is not None:
tiles = models.TileModel.objects.filter(nodegroup_id=uuid.UUID(config["nodegroup_id"]), sortorder=0).filter(
resourceinstance_id=resource.resourceinstanceid
)
if len(tiles) == 0:
tiles = models.TileModel.objects.filter(nodegroup_id=uuid.UUID(config["nodegroup_id"])).filter(
resourceinstance_id=resource.resourceinstanceid
)
for tile in tiles:
for node in models.Node.objects.filter(nodegroup_id=uuid.UUID(config["nodegroup_id"])):
data = {}
if len(list(tile.data.keys())) > 0:
data = tile.data
elif tile.provisionaledits is not None and len(list(tile.provisionaledits.keys())) == 1:
userid = list(tile.provisionaledits.keys())[0]
data = tile.provisionaledits[userid]["value"]
if str(node.nodeid) in data:
if not datatype_factory:
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(node.datatype)
value = datatype.get_display_value(tile, node)
if value is None:
value = ""
config["string_template"] = config["string_template"].replace("<%s>" % node.name, str(value))
except ValueError as e:
print(e, "invalid nodegroupid participating in descriptor function.")
if config["string_template"].strip() == "":
config["string_template"] = _("Undefined")
return config["string_template"]
def clean_resource_cache(tile):
# get the tile model's bounds
datatype_factory = DataTypeFactory()
nodegroup = models.NodeGroup.objects.get(pk=tile.nodegroup_id)
for node in nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
if datatype.should_cache(node) and datatype.should_manage_cache(node):
bounds = datatype.get_bounds(tile, node)
if bounds is not None:
zooms = range(20)
config = TileStache.parseConfig(
get_tileserver_config(node.nodeid))
layer = config.layers[str(node.nodeid)]
mimetype, format = layer.getTypeByExtension('pbf')
lon1, lat1, lon2, lat2 = bounds
south, west = min(lat1, lat2), min(lon1, lon2)
north, east = max(lat1, lat2), max(lon1, lon2)
northwest = Location(north, west)
southeast = Location(south, east)
ul = layer.projection.locationCoordinate(northwest)
lr = layer.projection.locationCoordinate(southeast)
padding = 0
coordinates = generateCoordinates(ul, lr, zooms, padding)
for (offset, count, coord) in coordinates:
config.cache.remove(layer, coord, format)
for key, tile_list in tile.tiles.iteritems():
for child_tile in tile_list:
clean_resource_cache(child_tile)
def __init__(self):
self.tiles = {}
self.errors = {}
self.resources = []
self.use_ids = False
self.datatype_factory = DataTypeFactory()
self.resource_model_root_classes = set()
self.non_unique_classes = set()
self.graph_id_lookup = {}
self.logger = logging.getLogger(__name__)
for graph in models.GraphModel.objects.filter(isresource=True):
node = models.Node.objects.get(graph_id=graph.pk, istopnode=True)
self.graph_id_lookup[node.ontologyclass] = graph.pk
if node.ontologyclass in self.resource_model_root_classes:
# make a note of non-unique root classes
self.non_unique_classes.add(node.ontologyclass)
else:
self.resource_model_root_classes.add(node.ontologyclass)
self.resource_model_root_classes = self.resource_model_root_classes - self.non_unique_classes
self.ontologyproperties = models.Edge.objects.values_list(
'ontologyproperty', flat=True).distinct()
self.logger.info("Initialized JsonLdReader")
self.logger.debug("Found {0} Non-unique root classes".format(
len(self.non_unique_classes)))
self.logger.debug("Found {0} Resource Model Root classes".format(
len(self.resource_model_root_classes)))
self.logger.debug("Resource Model Root classes: {0}".format("\n".join(
list(map(str, self.resource_model_root_classes)))))
def check_for_constraint_violation(self, request):
card = models.CardModel.objects.get(nodegroup=self.nodegroup)
constraints = models.ConstraintModel.objects.filter(card=card)
if constraints.count() > 0:
for constraint in constraints:
if constraint.uniquetoallinstances is True:
tiles = models.TileModel.objects.filter(nodegroup=self.nodegroup)
else:
tiles = models.TileModel.objects.filter(
Q(resourceinstance_id=self.resourceinstance.resourceinstanceid) & Q(nodegroup=self.nodegroup)
)
nodes = [node for node in constraint.nodes.all()]
for tile in tiles:
if str(self.tileid) != str(tile.tileid):
match = False
duplicate_values = []
for node in nodes:
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(node.datatype)
nodeid = str(node.nodeid)
if datatype.values_match(tile.data[nodeid], self.data[nodeid]):
match = True
duplicate_values.append(datatype.get_display_value(tile, node))
else:
match = False
break
if match is True:
message = _(
"This card violates a unique constraint. \
The following value is already saved: "
)
raise TileValidationError(message + (", ").join(duplicate_values))
def check_for_missing_nodes(self, request):
missing_nodes = []
for nodeid, value in self.data.items():
try:
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
datatype.clean(self, nodeid)
if request is not None:
if self.data[nodeid] is None and node.isrequired is True:
if len(node.cardxnodexwidget_set.all()) > 0:
missing_nodes.append(
node.cardxnodexwidget_set.all()[0].label)
else:
missing_nodes.append(node.name)
except Exception as e:
warning = _(
f"Error checking for missing node. Nodeid: {nodeid} with value: {value}, not in nodes. \
You may have a node in your business data that no longer exists in any graphs."
)
logger.warning(warning)
if missing_nodes != []:
message = _("This card requires values for the following: ")
message += (", ").join(missing_nodes)
raise TileValidationError(message)
def save(self, *args, **kwargs):
request = kwargs.pop('request', None)
index = kwargs.pop('index', True)
self.__preSave(request)
if self.data != {}:
old_model = models.TileModel.objects.filter(pk=self.tileid)
old_data = old_model[0].data if len(old_model) > 0 else None
edit_type = 'tile create' if (old_data == None) else 'tile edit'
try:
user = request.user
except:
user = {}
self.save_edit(user=user,
edit_type=edit_type,
old_value=old_data,
new_value=self.data)
for nodeid, value in self.data.iteritems():
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(
models.Node.objects.get(nodeid=nodeid).datatype)
datatype.convert_value(self, nodeid)
super(Tile, self).save(*args, **kwargs)
if index and unicode(self.resourceinstance.graph_id) != unicode(
settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID):
self.index()
for tiles in self.tiles.itervalues():
for tile in tiles:
tile.resourceinstance = self.resourceinstance
tile.parenttile = self
tile.save(*args, request=request, index=index, **kwargs)
def append_dsl(self, search_results_object, permitted_nodegroups,
include_provisional):
querysting_params = self.request.GET.get(details['componentname'], '')
advanced_filters = JSONDeserializer().deserialize(querysting_params)
datatype_factory = DataTypeFactory()
search_query = Bool()
advanced_query = Bool()
grouped_query = Bool()
grouped_queries = [grouped_query]
for index, advanced_filter in enumerate(advanced_filters):
tile_query = Bool()
for key, val in advanced_filter.iteritems():
if key != 'op':
node = models.Node.objects.get(pk=key)
if self.request.user.has_perm('read_nodegroup',
node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
datatype.append_search_filters(val, node, tile_query,
self.request)
nested_query = Nested(path='tiles', query=tile_query)
if advanced_filter['op'] == 'or' and index != 0:
grouped_query = Bool()
grouped_queries.append(grouped_query)
grouped_query.must(nested_query)
for grouped_query in grouped_queries:
advanced_query.should(grouped_query)
search_query.must(advanced_query)
search_results_object['query'].add_query(search_query)
def get_context_data(self, **kwargs):
context = super(MapBaseManagerView, self).get_context_data(**kwargs)
datatype_factory = DataTypeFactory()
geom_datatypes = [
d.pk for d in models.DDataType.objects.filter(isgeometric=True)
]
geom_nodes = models.Node.objects.filter(
graph__isresource=True,
graph__isactive=True,
datatype__in=geom_datatypes).exclude(
graph__graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID)
resource_layers = []
resource_sources = []
for node in geom_nodes:
if self.request.user.has_perm("read_nodegroup", node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
map_source = datatype.get_map_source(node)
if map_source is not None:
resource_sources.append(map_source)
map_layer = datatype.get_map_layer(node)
if map_layer is not None:
resource_layers.append(map_layer)
context["geom_nodes"] = geom_nodes
context["resource_map_layers"] = resource_layers
context["resource_map_sources"] = resource_sources
return context
def datatype_post_save_actions(self, request=None):
for nodeid, value in self.data.items():
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
if request is not None:
datatype.handle_request(self, request, node)
def save(self, *args, **kwargs):
request = kwargs.pop('request', None)
index = kwargs.pop('index', True)
self.__preSave(request)
missing_nodes = []
if self.data != {}:
old_model = models.TileModel.objects.filter(pk=self.tileid)
old_data = old_model[0].data if len(old_model) > 0 else None
edit_type = 'tile create' if (old_data == None) else 'tile edit'
try:
user = request.user
except:
user = {}
self.save_edit(user=user, edit_type=edit_type, old_value=old_data, new_value=self.data)
for nodeid, value in self.data.iteritems():
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
datatype.convert_value(self, nodeid)
if self.data[nodeid] == None and node.isrequired == True:
missing_nodes.append(node.name)
if missing_nodes != []:
message = _('This card requires values for the following:')
raise ValidationError(message, (', ').join(missing_nodes))
super(Tile, self).save(*args, **kwargs)
if index and unicode(self.resourceinstance.graph_id) != unicode(settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID):
self.index()
for tiles in self.tiles.itervalues():
for tile in tiles:
tile.resourceinstance = self.resourceinstance
tile.parenttile = self
tile.save(*args, request=request, index=index, **kwargs)
def parse_and_validate_resource(self, request, node_id):
# currently not working
cell_value = json.loads(request.POST.get('cell_value'))
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(pk=node_id)
datatype = datatype_factory.get_instance(node.datatype)
errors = []
# GET RID OF TRY AFTER DOMAIN VALUE REFACTOR!
try:
validation_errors = datatype.validate(cell_value, node=node)
if validation_errors:
errors.append({
'errors': validation_errors,
'node_id': node_id,
'cell_value': cell_value
})
except Exception as e:
print(str(e))
return JSONResponse({'errors': errors})
def get_primary_descriptor_from_nodes(self, resource, config):
try:
for tile in models.TileModel.objects.filter(
nodegroup_id=uuid.UUID(config['nodegroup_id']),
sortorder=0).filter(
resourceinstance_id=resource.resourceinstanceid):
for node in models.Node.objects.filter(
nodegroup_id=uuid.UUID(config['nodegroup_id'])):
if str(node.nodeid) in tile.data:
value = tile.data[str(node.nodeid)]
if value:
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(
node.datatype)
display_value = datatype.get_display_value(
tile, node)
if display_value is not None:
value = display_value.value
config['string_template'] = config[
'string_template'].replace(
'<%s>' % node.name, value)
except ValueError as e:
print e, 'invalid nodegroupid'
return config['string_template']
def get(self, request):
se = SearchEngineFactory().create()
datatype_factory = DataTypeFactory()
datatypes = models.DDataType.objects.all()
widgets = models.Widget.objects.all()
map_layers = models.MapLayer.objects.all()
map_markers = models.MapMarker.objects.all()
map_sources = models.MapSource.objects.all()
icons = models.Icon.objects.order_by('name')
context = self.get_context_data(
icons=JSONSerializer().serialize(icons),
datatypes=datatypes,
widgets=widgets,
map_layers=map_layers,
map_markers=map_markers,
map_sources=map_sources,
datatypes_json=JSONSerializer().serialize(datatypes),
main_script='views/map-layer-manager',
)
def get_resource_bounds(node):
query = Query(se, start=0, limit=0)
search_query = Bool()
query.add_query(search_query)
query.add_aggregation(GeoBoundsAgg(field='points.point', name='bounds'))
results = query.search(index='resource', doc_type=[str(node.graph_id)])
bounds = results['aggregations']['bounds']['bounds'] if 'bounds' in results['aggregations']['bounds'] else None
return bounds
context['geom_nodes_json'] = JSONSerializer().serialize(context['geom_nodes'])
resource_layers = []
resource_sources = []
permissions = {}
for node in context['geom_nodes']:
datatype = datatype_factory.get_instance(node.datatype)
map_layer = datatype.get_map_layer(node=node, preview=True)
if map_layer is not None:
count = models.TileModel.objects.filter(data__has_key=str(node.nodeid)).count()
if count > 0:
map_layer['bounds'] = get_resource_bounds(node)
else:
map_layer['bounds'] = None
resource_layers.append(map_layer)
map_source = datatype.get_map_source(node=node, preview=True)
if map_source is not None:
resource_sources.append(map_source)
permissions[str(node.pk)] = {
"users": sorted([user.email or user.username for user in get_users_for_object('read_nodegroup', node.nodegroup)]),
"groups": sorted([group.name for group in get_groups_for_object('read_nodegroup', node.nodegroup)])
}
context['resource_map_layers_json'] = JSONSerializer().serialize(resource_layers)
context['resource_map_sources_json'] = JSONSerializer().serialize(resource_sources)
context['node_permissions'] = JSONSerializer().serialize(permissions)
context['nav']['title'] = _('Map Layer Manager')
context['nav']['icon'] = 'fa-server'
context['nav']['help'] = (_('Map Layer Manager'),'help/base-help.htm')
context['help'] = 'map-manager-help'
return render(request, 'views/map-layer-manager.htm', context)
def seed_resource_cache():
datatype_factory = DataTypeFactory()
zooms = range(settings.CACHE_SEED_MAX_ZOOM + 1)
extension = 'pbf'
lat1, lon1, lat2, lon2 = GeoUtils().get_bounds_from_geojson(settings.CACHE_SEED_BOUNDS)
south, west = min(lat1, lat2), min(lon1, lon2)
north, east = max(lat1, lat2), max(lon1, lon2)
northwest = Location(north, west)
southeast = Location(south, east)
padding = 0
datatypes = [
d.pk for d in models.DDataType.objects.filter(isgeometric=True)]
nodes = models.Node.objects.filter(
graph__isresource=True, datatype__in=datatypes)
for node in nodes:
datatype = datatype_factory.get_instance(node.datatype)
count = models.TileModel.objects.filter(
data__has_key=str(node.nodeid)).count()
if datatype.should_cache(node) and count > 0:
config = TileStache.parseConfig(get_tileserver_config(node.nodeid))
layer = config.layers[str(node.nodeid)]
ul = layer.projection.locationCoordinate(northwest)
lr = layer.projection.locationCoordinate(southeast)
coordinates = generateCoordinates(ul, lr, zooms, padding)
for (offset, count, coord) in coordinates:
path = '%s/%d/%d/%d.%s' % (layer.name(), coord.zoom,
coord.column, coord.row, extension)
progress = {"tile": path,
"offset": offset + 1,
"total": count}
attempts = 3
rendered = False
while not rendered:
print '%(offset)d of %(total)d...' % progress,
try:
mimetype, content = TileStache.getTile(
layer, coord, extension, True)
except:
attempts -= 1
print 'Failed %s, will try %s more.' % (progress['tile'], ['no', 'once', 'twice'][attempts])
if attempts == 0:
print 'Failed %(zoom)d/%(column)d/%(row)d, trying next tile.\n' % coord.__dict__
break
else:
rendered = True
progress['size'] = '%dKB' % (len(content) / 1024)
print '%(tile)s (%(size)s)' % progress
def after_update_all(self):
nodegroup = models.NodeGroup.objects.get(pk=self.nodegroup_id)
datatype_factory = DataTypeFactory()
for node in nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
datatype.after_update_all()
for tile in self.tiles:
tile.after_update_all()
def validate(self, errors=None):
for nodeid, value in self.data.iteritems():
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
error = datatype.validate(value)
if errors != None:
errors += error
return errors
def __init__(self, **kwargs):
super(CsvWriter, self).__init__(**kwargs)
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {str(nodeid): datatype for nodeid, datatype in Node.objects.values_list('nodeid', 'datatype').filter(~Q(datatype='semantic'), graph__isresource=True)}
self.single_file = kwargs.pop('single_file', False)
self.resource_export_configs = self.read_export_configs(kwargs.pop('configs', None))
if len(self.resource_export_configs) == 0:
raise MissingConfigException()
def after_update_all(self):
nodegroup = models.NodeGroup.objects.get(pk=self.nodegroup_id)
datatype_factory = DataTypeFactory()
for node in nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
datatype.after_update_all()
for key, tile_list in self.tiles.iteritems():
for child_tile in tile_list:
child_tile.after_update_all()
def save(self, tile, request):
if request:
datatype_factory = DataTypeFactory()
for node in tile.nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
previously_saved_tile = Tile.objects.filter(pk=tile.tileid)
datatype.manage_files(previously_saved_tile, tile, request, node)
return tile
def __init__(self):
super(CsvWriter, self).__init__()
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {
str(nodeid): datatype
for nodeid, datatype in Node.objects.values_list(
'nodeid', 'datatype').filter(~Q(datatype='semantic'),
graph__isresource=True)
}
def after_update_all(self):
nodegroup = models.NodeGroup.objects.get(pk=self.nodegroup_id)
datatype_factory = DataTypeFactory()
for node in nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
datatype.after_update_all()
for key, tile_list in self.tiles.iteritems():
for child_tile in tile_list:
child_tile.after_update_all()
def validate(self, value, row_number=None, source=''):
errors = []
## iterate list of values and use the concept validation on each one
validate_concept = DataTypeFactory().get_instance('concept')
for v in value:
val = v.strip()
errors += validate_concept.validate(val, row_number)
return errors
def validate(self, value, row_number=None, source=''):
errors = []
## iterate list of values and use the concept validation on each one
validate_concept = DataTypeFactory().get_instance('concept')
for v in value:
val = v.strip()
errors += validate_concept.validate(val, row_number)
return errors
def validate(self, value, row_number=None, source="", node=None, nodeid=None):
errors = []
# iterate list of values and use the concept validation on each one
if value is not None:
validate_concept = DataTypeFactory().get_instance("concept")
for v in value:
val = v.strip()
errors += validate_concept.validate(val, row_number)
return errors
def save(self, tile, request):
if request:
datatype_factory = DataTypeFactory()
for node in tile.nodegroup.node_set.all():
datatype = datatype_factory.get_instance(node.datatype)
previously_saved_tile = Tile.objects.filter(pk=tile.tileid)
datatype.manage_files(previously_saved_tile, tile, request,
node)
return tile
def validate(self, errors=None):
for nodeid, value in self.data.items():
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
error = datatype.validate(value, node=node)
for error_instance in error:
if error_instance["type"] == "ERROR":
raise TileValidationError(_("{0}".format(error_instance["message"])))
if errors is not None:
errors += error
return errors
def __init__(self, *args, **kwargs):
"""
Init a Tile from a dictionary representation of from a model method call
init this object by using Django query syntax, eg:
.. code-block:: python
Tile.objects.get(pk=some_tile_id)
# or
Tile.objects.filter(name=some_value_to_filter_by)
OR, init this object with a dictionary, eg:
.. code-block:: python
Tile({
name:'some name',
tileid: '12341234-1234-1234-1324-1234123433433',
...
})
Arguments:
args -- a dictionary of properties repsenting a Tile object
kwargs -- unused
"""
super(Tile, self).__init__(*args, **kwargs)
# from models.TileModel
# self.tileid
# self.resourceinstance
# self.parenttile
# self.data
# self.nodegroup
# self.sortorder
# end from models.TileModel
self.tiles = []
self.datatype_factory = DataTypeFactory()
if args:
if isinstance(args[0], dict):
for key, value in args[0].items():
if not (key == "tiles"):
setattr(self, key, value)
if self.tileid is None or self.tileid == "":
self.tileid = uuid.uuid4()
if "tiles" in args[0]:
for tile_obj in args[0]["tiles"]:
tile = Tile(tile_obj)
tile.parenttile = self
self.tiles.append(tile)
def validate(self, errors=None):
for nodeid, value in self.data.iteritems():
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
error = datatype.validate(value)
for error_instance in error:
if error_instance['type'] == 'ERROR':
print(str(error_instance)+" rejected tile with pk: "+ str(self.pk))
raise TileValidationError(_("Your tile: {0} ".format(error_instance["message"])))
if errors != None:
errors += error
return errors
def datatype_post_save_actions(self, request=None):
userid = None
if request is not None:
userid = str(request.user.id)
if hasattr(request.user, "userprofile") is not True:
models.UserProfile.objects.create(user=request.user)
tile_data = self.get_tile_data(userid)
for nodeid, value in list(tile_data.items()):
datatype_factory = DataTypeFactory()
node = models.Node.objects.get(nodeid=nodeid)
datatype = datatype_factory.get_instance(node.datatype)
if request is not None:
datatype.handle_request(self, request, node)
def import_business_data(self,
file_format=None,
business_data=None,
mapping=None,
overwrite='append',
bulk=False,
create_concepts=False,
create_collections=False):
reader = None
start = time()
cursor = connection.cursor()
try:
if file_format == None:
file_format = self.file_format
if business_data == None:
business_data = self.business_data
if mapping == None:
mapping = self.mapping
if file_format == 'json':
reader = ArchesFileReader()
reader.import_business_data(business_data, mapping)
elif file_format == 'csv' or file_format == 'shp' or file_format == 'zip':
if mapping != None:
reader = CsvReader()
reader.import_business_data(
business_data=business_data,
mapping=mapping,
overwrite=overwrite,
bulk=bulk,
create_concepts=create_concepts,
create_collections=create_collections)
else:
print '*' * 80
print 'ERROR: No mapping file detected. Please indicate one with the \'-c\' paramater or place one in the same directory as your business data.'
print '*' * 80
sys.exit()
elapsed = (time() - start)
print 'Time to import_business_data = {0}'.format(
datetime.timedelta(seconds=elapsed))
reader.report_errors()
finally:
datatype_factory = DataTypeFactory()
datatypes = DDataType.objects.all()
for datatype in datatypes:
datatype_instance = datatype_factory.get_instance(
datatype.datatype)
datatype_instance.after_update_all()
def import_business_data(self, file_format=None, business_data=None, mapping=None,
overwrite='append', bulk=False, create_concepts=False,
create_collections=False, use_multiprocessing=False):
reader = None
start = time()
cursor = connection.cursor()
try:
if file_format == None:
file_format = self.file_format
if business_data == None:
business_data = self.business_data
if mapping == None:
mapping = self.mapping
if file_format == 'json':
reader = ArchesFileReader()
reader.import_business_data(business_data, mapping)
elif file_format == 'jsonl':
with open(self.file[0], 'rU') as openf:
lines = openf.readlines()
if use_multiprocessing is True:
pool = Pool(cpu_count())
pool.map(import_one_resource, lines)
connections.close_all()
reader = ArchesFileReader()
else:
reader = ArchesFileReader()
for line in lines:
archesresource = JSONDeserializer().deserialize(line)
reader.import_business_data({"resources": [archesresource]})
elif file_format == 'csv' or file_format == 'shp' or file_format == 'zip':
if mapping != None:
reader = CsvReader()
reader.import_business_data(business_data=business_data, mapping=mapping, overwrite=overwrite, bulk=bulk, create_concepts=create_concepts, create_collections=create_collections)
else:
print('*'*80)
print('ERROR: No mapping file detected. Please indicate one with the \'-c\' paramater or place one in the same directory as your business data.')
print('*'*80)
sys.exit()
elapsed = (time() - start)
print('Time to import_business_data = {0}'.format(datetime.timedelta(seconds=elapsed)))
reader.report_errors()
finally:
datatype_factory = DataTypeFactory()
datatypes = DDataType.objects.all()
for datatype in datatypes:
datatype_instance = datatype_factory.get_instance(datatype.datatype)
datatype_instance.after_update_all()
def get_primary_descriptor_from_nodes(self, resource, config):
try:
for tile in models.TileModel.objects.filter(nodegroup_id=uuid.UUID(config['nodegroup_id']), sortorder=0).filter(resourceinstance_id=resource.resourceinstanceid):
for node in models.Node.objects.filter(nodegroup_id=uuid.UUID(config['nodegroup_id'])):
if str(node.nodeid) in tile.data:
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(node.datatype)
value = datatype.get_display_value(tile, node)
config['string_template'] = config['string_template'].replace('<%s>' % node.name, value)
except ValueError as e:
print e, 'invalid nodegroupid'
return config['string_template']
def save(self, *args, **kwargs):
request = kwargs.pop('request', None)
index = kwargs.pop('index', True)
self.__preSave(request)
if self.data != {}:
for nodeid, value in self.data.iteritems():
datatype_factory = DataTypeFactory()
datatype = datatype_factory.get_instance(models.Node.objects.get(nodeid=nodeid).datatype)
datatype.convert_value(self, nodeid)
super(Tile, self).save(*args, **kwargs)
if index:
self.index()
for tiles in self.tiles.itervalues():
for tile in tiles:
tile.resourceinstance = self.resourceinstance
tile.parenttile = self
tile.save(*args, request=request, index=index, **kwargs)
def __init__(self, **kwargs):
super(CsvWriter, self).__init__(**kwargs)
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {str(nodeid): datatype for nodeid, datatype in Node.objects.values_list('nodeid', 'datatype').filter(~Q(datatype='semantic'), graph__isresource=True)}
self.single_file = kwargs.pop('single_file', False)
self.resource_export_configs = self.read_export_configs(kwargs.pop('configs', None))
if len(self.resource_export_configs) == 0:
raise MissingConfigException()
def get(self, request):
datatype_factory = DataTypeFactory()
datatypes = models.DDataType.objects.all()
widgets = models.Widget.objects.all()
map_layers = models.MapLayer.objects.all()
map_sources = models.MapSource.objects.all()
icons = models.Icon.objects.order_by('name')
context = self.get_context_data(
icons=JSONSerializer().serialize(icons),
datatypes=datatypes,
widgets=widgets,
map_layers=map_layers,
map_sources=map_sources,
datatypes_json=JSONSerializer().serialize(datatypes),
main_script='views/map-layer-manager',
)
context['geom_nodes_json'] = JSONSerializer().serialize(context['geom_nodes'])
resource_layers = []
resource_sources = []
permissions = {}
for node in context['geom_nodes']:
datatype = datatype_factory.get_instance(node.datatype)
map_layer = datatype.get_map_layer(node=node, preview=True)
if map_layer is not None:
resource_layers.append(map_layer)
map_source = datatype.get_map_source(node=node, preview=True)
if map_source is not None:
resource_sources.append(map_source)
card = Card.objects.get(nodegroup_id=node.nodegroup_id)
permissions[str(node.pk)] = {
"users": card.users,
"groups": card.groups,
}
context['resource_map_layers_json'] = JSONSerializer().serialize(resource_layers)
context['resource_map_sources_json'] = JSONSerializer().serialize(resource_sources)
context['node_permissions'] = JSONSerializer().serialize(permissions)
context['nav']['title'] = _('Map Layer Manager')
context['nav']['icon'] = 'fa-server'
context['nav']['help'] = (_('Map Layer Manager'),'help/map-manager-help.htm')
return render(request, 'views/map-layer-manager.htm', context)
def get_context_data(self, **kwargs):
datatype_factory = DataTypeFactory()
context = super(BaseManagerView, self).get_context_data(**kwargs)
context['graph_models'] = models.GraphModel.objects.all()
context['graphs'] = JSONSerializer().serialize(context['graph_models'])
if 'Resource Editor' in self.request.user.user_groups:
context['resource_instances'] = Resource.objects.all().order_by('-createdtime')[:100]
else:
context['resource_instances'] = []
context['nav'] = {
'icon':'fa fa-chevron-circle-right',
'title':'',
'help':('',''),
'menu':False,
'search':True,
'res_edit':False,
'edit_history':True,
'login':True,
'print':False,
}
geom_datatypes = [d.pk for d in models.DDataType.objects.filter(isgeometric=True)]
geom_nodes = models.Node.objects.filter(graph__isresource=True, graph__isactive=True, datatype__in=geom_datatypes)
resource_layers = []
resource_sources = []
for node in geom_nodes:
# TODO: check user node level permissions here, if user does not
# have read permissions on this node, then do not create map layer
# or source
datatype = datatype_factory.get_instance(node.datatype)
map_source = datatype.get_map_source(node)
if map_source is not None:
resource_sources.append(map_source)
map_layer = datatype.get_map_layer(node)
if map_layer is not None:
resource_layers.append(map_layer)
context['geom_nodes'] = geom_nodes
context['resource_map_layers'] = resource_layers
context['resource_map_sources'] = resource_sources
context['iiif_manifests'] = models.IIIFManifest.objects.all()
return context
def get_tileserver_config(layer_id):
database = settings.DATABASES['default']
datatype_factory = DataTypeFactory()
try:
node = models.Node.objects.get(pk=layer_id)
# TODO: check user node permissions here, if user
# does not have read access to this node, fire an exception
datatype = datatype_factory.get_instance(node.datatype)
layer_config = datatype.get_layer_config(node)
except Exception:
layer_model = models.TileserverLayer.objects.get(name=layer_id)
layer_config = layer_model.config
config_dict = {
"cache": settings.TILE_CACHE_CONFIG,
"layers": {}
}
config_dict["layers"][str(layer_id)] = layer_config
return config_dict
def get_context_data(self, **kwargs):
datatype_factory = DataTypeFactory()
context = super(BaseManagerView, self).get_context_data(**kwargs)
context['system_settings_graphid'] = settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID
context['graph_models'] = models.GraphModel.objects.all().exclude(graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID)
context['graphs'] = JSONSerializer().serialize(context['graph_models'])
context['createable_resources'] = JSONSerializer().serialize(get_createable_resource_types(self.request.user))
context['nav'] = {
'icon':'fa fa-chevron-circle-right',
'title':'',
'help':('',''),
'menu':False,
'search':True,
'res_edit':False,
'login':True,
'print':False,
}
context['use_semantic_relationships'] = settings.USE_SEMANTIC_RESOURCE_RELATIONSHIPS
geom_datatypes = [d.pk for d in models.DDataType.objects.filter(isgeometric=True)]
geom_nodes = models.Node.objects.filter(graph__isresource=True, graph__isactive=True, datatype__in=geom_datatypes).exclude(graph__graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID)
resource_layers = []
resource_sources = []
for node in geom_nodes:
if self.request.user.has_perm('read_nodegroup', node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
map_source = datatype.get_map_source(node)
if map_source is not None:
resource_sources.append(map_source)
map_layer = datatype.get_map_layer(node)
if map_layer is not None:
resource_layers.append(map_layer)
context['app_name'] = settings.APP_NAME
context['geom_nodes'] = geom_nodes
context['resource_map_layers'] = resource_layers
context['resource_map_sources'] = resource_sources
context['iiif_manifests'] = models.IIIFManifest.objects.all()
return context
def get_context_data(self, **kwargs):
context = super(MapBaseManagerView, self).get_context_data(**kwargs)
datatype_factory = DataTypeFactory()
geom_datatypes = [d.pk for d in models.DDataType.objects.filter(isgeometric=True)]
geom_nodes = models.Node.objects.filter(graph__isresource=True, graph__isactive=True, datatype__in=geom_datatypes).exclude(graph__graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID)
resource_layers = []
resource_sources = []
for node in geom_nodes:
if self.request.user.has_perm('read_nodegroup', node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
map_source = datatype.get_map_source(node)
if map_source is not None:
resource_sources.append(map_source)
map_layer = datatype.get_map_layer(node)
if map_layer is not None:
resource_layers.append(map_layer)
context['geom_nodes'] = geom_nodes
context['resource_map_layers'] = resource_layers
context['resource_map_sources'] = resource_sources
return context
def get_tileserver_config(layer_id, request=None):
database = settings.DATABASES['default']
datatype_factory = DataTypeFactory()
try:
node = models.Node.objects.get(pk=layer_id)
datatype = datatype_factory.get_instance(node.datatype)
if request == None or request.user.has_perm('read_nodegroup', node.nodegroup):
layer_config = datatype.get_layer_config(node)
else:
layer_config = datatype.get_layer_config(None)
except Exception:
layer_model = models.TileserverLayer.objects.get(name=layer_id)
layer_config = layer_model.config
tile_cache_config = settings.TILE_CACHE_CONFIG
config_dict = {
"cache": tile_cache_config,
"layers": {}
}
config_dict["layers"][str(layer_id)] = layer_config
return config_dict
def import_business_data(self, file_format=None, business_data=None, mapping=None, overwrite='append', bulk=False):
reader = None
start = time()
cursor = connection.cursor()
try:
if file_format == None:
file_format = self.file_format
if business_data == None:
business_data = self.business_data
if mapping == None:
mapping = self.mapping
if file_format == 'json':
reader = ArchesFileReader()
reader.import_business_data(business_data, mapping)
elif file_format == 'csv':
if mapping != None:
reader = CsvReader()
reader.import_business_data(business_data=business_data, mapping=mapping, overwrite=overwrite, bulk=bulk)
else:
print '*'*80
print 'ERROR: No mapping file detected. Please indicate one with the \'-c\' paramater or place one in the same directory as your business data.'
print '*'*80
sys.exit()
elapsed = (time() - start)
print 'Time to import_business_data = {0}'.format(datetime.timedelta(seconds=elapsed))
reader.report_errors()
finally:
datatype_factory = DataTypeFactory()
datatypes = DDataType.objects.all()
for datatype in datatypes:
datatype_instance = datatype_factory.get_instance(datatype.datatype)
datatype_instance.after_update_all()
class CsvWriter(Writer):
def __init__(self):
super(CsvWriter, self).__init__()
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {str(nodeid): datatype for nodeid, datatype in Node.objects.values_list('nodeid', 'datatype').filter(~Q(datatype='semantic'), graph__isresource=True)}
def transform_value_for_export(self, datatype, value, concept_export_value_type):
datatype_instance = self.datatype_factory.get_instance(datatype)
value = datatype_instance.transform_export_values(value, concept_export_value_type=concept_export_value_type)
return value
def write_resources(self, resources, resource_export_configs=None):
csv_records = []
other_group_records = []
mapping = {}
concept_export_value_lookup = {}
for resource_export_config in resource_export_configs:
for node in resource_export_config['nodes']:
if node['file_field_name'] != '':
mapping[node['arches_nodeid']] = node['file_field_name']
if 'concept_export_value' in node:
concept_export_value_lookup[node['arches_nodeid']] = node['concept_export_value']
csv_header = ['ResourceID'] + mapping.values()
csvs_for_export = []
for resource in resources:
csv_record = {}
other_group_record = {}
resourceid = resource['_source']['resourceinstanceid']
resource_graphid = resource['_source']['graph_id']
legacyid = resource['_source']['legacyid']
csv_record['ResourceID'] = resourceid
other_group_record['ResourceID'] = resourceid
for tile in resource['_source']['tiles']:
if tile['data'] != {}:
for k in tile['data'].keys():
if tile['data'][k] != '' and k in mapping:
if mapping[k] not in csv_record:
concept_export_value_type = None
if k in concept_export_value_lookup:
concept_export_value_type = concept_export_value_lookup[k]
value = self.transform_value_for_export(self.node_datatypes[k], tile['data'][k], concept_export_value_type)
csv_record[mapping[k]] = value
del tile['data'][k]
else:
value = self.transform_value_for_export(self.node_datatypes[k], tile['data'][k], concept_export_value_type)
other_group_record[mapping[k]] = value
else:
del tile['data'][k]
csv_records.append(csv_record)
if other_group_record != {}:
other_group_records.append(other_group_record)
csv_name_prefix = resource_export_configs[0]['resource_model_name']
iso_date = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
csv_name = os.path.join('{0}_{1}.{2}'.format(csv_name_prefix, iso_date, 'csv'))
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csvs_for_export.append({'name':csv_name, 'outputfile': dest})
for csv_record in csv_records:
csvwriter.writerow({k:str(v) for k,v in csv_record.items()})
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csvs_for_export.append({'name':csv_name + '_groups', 'outputfile': dest})
for csv_record in other_group_records:
csvwriter.writerow({k:str(v) for k,v in csv_record.items()})
return csvs_for_export
class CsvWriter(Writer):
def __init__(self, **kwargs):
super(CsvWriter, self).__init__(**kwargs)
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {str(nodeid): datatype for nodeid, datatype in Node.objects.values_list('nodeid', 'datatype').filter(~Q(datatype='semantic'), graph__isresource=True)}
self.single_file = kwargs.pop('single_file', False)
self.resource_export_configs = self.read_export_configs(kwargs.pop('configs', None))
if len(self.resource_export_configs) == 0:
raise MissingConfigException()
def read_export_configs(self, configs):
'''
Reads the export configuration file or object and adds an array for records to store property data
'''
if configs:
resource_export_configs = json.load(open(configs, 'r'))
configs = [resource_export_configs]
else:
configs = []
for val in GraphXMapping.objects.values('mapping'):
configs.append(val['mapping'])
return configs
def transform_value_for_export(self, datatype, value, concept_export_value_type, node):
datatype_instance = self.datatype_factory.get_instance(datatype)
value = datatype_instance.transform_export_values(value, concept_export_value_type=concept_export_value_type, node=node)
return value
def write_resources(self, graph_id=None, resourceinstanceids=None):
graph_id = self.resource_export_configs[0]['resource_model_id']
super(CsvWriter, self).write_resources(graph_id=graph_id, resourceinstanceids=resourceinstanceids)
csv_records = []
other_group_records = []
mapping = {}
concept_export_value_lookup = {}
for resource_export_config in self.resource_export_configs:
for node in resource_export_config['nodes']:
if node['file_field_name'] != '' and node['export'] == True:
mapping[node['arches_nodeid']] = node['file_field_name']
if 'concept_export_value' in node:
concept_export_value_lookup[node['arches_nodeid']] = node['concept_export_value']
csv_header = ['ResourceID'] + mapping.values()
csvs_for_export = []
for resourceinstanceid, tiles in self.resourceinstances.iteritems():
csv_record = {}
csv_record['ResourceID'] = resourceinstanceid
csv_record['populated_node_groups'] = []
tiles = sorted(tiles, key=lambda k: k.parenttile_id)
for tile in tiles:
other_group_record = {}
other_group_record['ResourceID'] = resourceinstanceid
if tile.data != {}:
for k in tile.data.keys():
if tile.data[k] != '' and k in mapping and tile.data[k] != None:
if mapping[k] not in csv_record and tile.nodegroup_id not in csv_record['populated_node_groups']:
concept_export_value_type = None
if k in concept_export_value_lookup:
concept_export_value_type = concept_export_value_lookup[k]
if tile.data[k] != None:
value = self.transform_value_for_export(self.node_datatypes[k], tile.data[k], concept_export_value_type, k)
csv_record[mapping[k]] = value
del tile.data[k]
else:
concept_export_value_type = None
if k in concept_export_value_lookup:
concept_export_value_type = concept_export_value_lookup[k]
value = self.transform_value_for_export(self.node_datatypes[k], tile.data[k], concept_export_value_type, k)
other_group_record[mapping[k]] = value
else:
del tile.data[k]
csv_record['populated_node_groups'].append(tile.nodegroup_id)
if other_group_record != {'ResourceID': resourceinstanceid}:
other_group_records.append(other_group_record)
if csv_record != {'ResourceID': resourceinstanceid}:
csv_records.append(csv_record)
iso_date = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
file_name = '{0}_{1}'.format(self.file_prefix, iso_date)
csv_name = os.path.join('{0}.{1}'.format(file_name, 'csv'))
if self.single_file != True:
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csvs_for_export.append({'name':csv_name, 'outputfile': dest})
for csv_record in csv_records:
if 'populated_node_groups' in csv_record:
del csv_record['populated_node_groups']
csvwriter.writerow({k:str(v) for k,v in csv_record.items()})
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csvs_for_export.append({'name':csv_name.split('.')[0] + '_groups.' + csv_name.split('.')[1], 'outputfile': dest})
for csv_record in other_group_records:
if 'populated_node_groups' in csv_record:
del csv_record['populated_node_groups']
csvwriter.writerow({k:str(v) for k,v in csv_record.items()})
elif self.single_file == True:
all_records = csv_records + other_group_records
all_records = sorted(all_records, key=lambda k: k['ResourceID'])
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csvs_for_export.append({'name':csv_name, 'outputfile': dest})
for csv_record in all_records:
if 'populated_node_groups' in csv_record:
del csv_record['populated_node_groups']
csvwriter.writerow({k:str(v) for k,v in csv_record.items()})
if self.graph_id != None:
csvs_for_export = csvs_for_export + self.write_resource_relations(file_name=file_name)
return csvs_for_export
def write_resource_relations(self, file_name):
resourceids = self.resourceinstances.keys()
relations_file = []
if self.graph_id != settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID:
dest = StringIO()
csv_header = ['resourcexid','resourceinstanceidfrom','resourceinstanceidto','relationshiptype','datestarted','dateended','notes']
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csv_name = os.path.join('{0}.{1}'.format(file_name, 'relations'))
relations_file.append({'name':csv_name, 'outputfile': dest})
relations = ResourceXResource.objects.filter(Q(resourceinstanceidfrom__in=resourceids)|Q(resourceinstanceidto__in=resourceids)).values(*csv_header)
for relation in relations:
relation['datestarted'] = relation['datestarted'] if relation['datestarted'] != None else ''
relation['dateended'] = relation['dateended'] if relation['dateended'] != None else ''
relation['notes'] = relation['notes'] if relation['notes'] != None else ''
csvwriter.writerow({k:str(v) for k,v in relation.items()})
return relations_file
def __init__(self, **kwargs):
super(TileCsvWriter, self).__init__(**kwargs)
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {str(nodeid): datatype for nodeid, datatype in Node.objects.values_list('nodeid', 'datatype').filter(~Q(datatype='semantic'), graph__isresource=True)}
class TileCsvWriter(Writer):
def __init__(self, **kwargs):
super(TileCsvWriter, self).__init__(**kwargs)
self.datatype_factory = DataTypeFactory()
self.node_datatypes = {str(nodeid): datatype for nodeid, datatype in Node.objects.values_list('nodeid', 'datatype').filter(~Q(datatype='semantic'), graph__isresource=True)}
def transform_value_for_export(self, datatype, value, concept_export_value_type, node):
datatype_instance = self.datatype_factory.get_instance(datatype)
value = datatype_instance.transform_export_values(value, concept_export_value_type=concept_export_value_type, node=node)
return value
def write_resources(self, graph_id=None, resourceinstanceids=None):
super(TileCsvWriter, self).write_resources(graph_id=graph_id, resourceinstanceids=resourceinstanceids)
csv_records = []
other_group_records = []
concept_export_value_lookup = {}
csv_header = ['ResourceID']
mapping = {}
nodes = Node.objects.filter(graph_id=graph_id)
for node in nodes:
mapping[str(node.nodeid)] = node.name
csv_header = ['ResourceID', 'ResourceLegacyID', 'ResourceModelID', 'TileID', 'ParentTileID', 'NodeGroupID' ] + mapping.values()
csvs_for_export = []
for resourceinstanceid, tiles in self.resourceinstances.iteritems():
tiles = sorted(tiles, key=lambda k: k.parenttile_id)
for tile in tiles:
csv_record = {}
csv_record['ResourceID'] = resourceinstanceid
csv_record['ResourceModelID'] = graph_id
csv_record['TileID'] = tile.tileid
csv_record['ParentTileID'] = str(tile.parenttile_id)
csv_record['NodeGroupID'] = str(tile.nodegroup_id)
for k in tile.data.keys():
resource_instance = ResourceInstance.objects.get(resourceinstanceid=resourceinstanceid)
csv_record['ResourceLegacyID'] = str(resource_instance.legacyid) if resource_instance.legacyid is not None else str(resource_instance.resourceinstanceid)
if tile.data[k] != '' and tile.data[k] != None:
concept_export_value_type = 'label'
if k in concept_export_value_lookup:
concept_export_value_type = concept_export_value_lookup[k]
if tile.data[k] != None:
value = self.transform_value_for_export(self.node_datatypes[k], tile.data[k], concept_export_value_type, k)
csv_record[mapping[k]] = value
del tile.data[k]
else:
del tile.data[k]
if csv_record != {'ResourceID': resourceinstanceid}:
csv_records.append(csv_record)
iso_date = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
file_name = '{0}_{1}'.format(self.file_prefix, iso_date)
csv_name = os.path.join('{0}.{1}'.format(file_name, 'csv'))
all_records = sorted(csv_records, key=lambda k: k['ResourceID'])
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=csv_header)
csvwriter.writeheader()
csvs_for_export.append({'name':csv_name, 'outputfile': dest})
for csv_record in all_records:
if 'populated_node_groups' in csv_record:
del csv_record['populated_node_groups']
csvwriter.writerow({k:str(v) for k,v in csv_record.items()})
if self.graph_id != None:
csvs_for_export = csvs_for_export
return csvs_for_export
def build_search_results_dsl(request):
term_filter = request.GET.get('termFilter', '')
spatial_filter = JSONDeserializer().deserialize(request.GET.get('mapFilter', '{}'))
export = request.GET.get('export', None)
page = 1 if request.GET.get('page') == '' else int(request.GET.get('page', 1))
temporal_filter = JSONDeserializer().deserialize(request.GET.get('temporalFilter', '{}'))
advanced_filters = JSONDeserializer().deserialize(request.GET.get('advanced', '[]'))
search_buffer = None
se = SearchEngineFactory().create()
if export != None:
limit = settings.SEARCH_EXPORT_ITEMS_PER_PAGE
else:
limit = settings.SEARCH_ITEMS_PER_PAGE
query = Query(se, start=limit*int(page-1), limit=limit)
nested_agg = NestedAgg(path='points', name='geo_aggs')
nested_agg.add_aggregation(GeoHashGridAgg(field='points.point', name='grid', precision=settings.HEX_BIN_PRECISION))
nested_agg.add_aggregation(GeoBoundsAgg(field='points.point', name='bounds'))
query.add_aggregation(nested_agg)
search_query = Bool()
permitted_nodegroups = get_permitted_nodegroups(request.user)
if term_filter != '':
for term in JSONDeserializer().deserialize(term_filter):
term_query = Bool()
if term['type'] == 'term' or term['type'] == 'string':
string_filter = Bool()
if term['type'] == 'term':
string_filter.must(Match(field='strings.string', query=term['value'], type='phrase'))
elif term['type'] == 'string':
string_filter.should(Match(field='strings.string', query=term['value'], type='phrase_prefix'))
string_filter.should(Match(field='strings.string.folded', query=term['value'], type='phrase_prefix'))
string_filter.filter(Terms(field='strings.nodegroup_id', terms=permitted_nodegroups))
nested_string_filter = Nested(path='strings', query=string_filter)
if term['inverted']:
search_query.must_not(nested_string_filter)
else:
search_query.must(nested_string_filter)
# need to set min_score because the query returns results with score 0 and those have to be removed, which I don't think it should be doing
query.min_score('0.01')
elif term['type'] == 'concept':
concept_ids = _get_child_concepts(term['value'])
conceptid_filter = Bool()
conceptid_filter.filter(Terms(field='domains.conceptid', terms=concept_ids))
conceptid_filter.filter(Terms(field='domains.nodegroup_id', terms=permitted_nodegroups))
nested_conceptid_filter = Nested(path='domains', query=conceptid_filter)
if term['inverted']:
search_query.must_not(nested_conceptid_filter)
else:
search_query.filter(nested_conceptid_filter)
if 'features' in spatial_filter:
if len(spatial_filter['features']) > 0:
feature_geom = spatial_filter['features'][0]['geometry']
feature_properties = spatial_filter['features'][0]['properties']
buffer = {'width':0,'unit':'ft'}
if 'buffer' in feature_properties:
buffer = feature_properties['buffer']
search_buffer = _buffer(feature_geom, buffer['width'], buffer['unit'])
feature_geom = JSONDeserializer().deserialize(search_buffer.json)
geoshape = GeoShape(field='geometries.geom.features.geometry', type=feature_geom['type'], coordinates=feature_geom['coordinates'] )
invert_spatial_search = False
if 'inverted' in feature_properties:
invert_spatial_search = feature_properties['inverted']
spatial_query = Bool()
if invert_spatial_search == True:
spatial_query.must_not(geoshape)
else:
spatial_query.filter(geoshape)
# get the nodegroup_ids that the user has permission to search
spatial_query.filter(Terms(field='geometries.nodegroup_id', terms=permitted_nodegroups))
search_query.filter(Nested(path='geometries', query=spatial_query))
if 'fromDate' in temporal_filter and 'toDate' in temporal_filter:
now = str(datetime.utcnow())
start_date = SortableDate(temporal_filter['fromDate'])
end_date = SortableDate(temporal_filter['toDate'])
date_nodeid = str(temporal_filter['dateNodeId']) if 'dateNodeId' in temporal_filter and temporal_filter['dateNodeId'] != '' else None
query_inverted = False if 'inverted' not in temporal_filter else temporal_filter['inverted']
temporal_query = Bool()
if query_inverted:
# inverted date searches need to use an OR clause and are generally more complicated to structure (can't use ES must_not)
# eg: less than START_DATE OR greater than END_DATE
inverted_date_query = Bool()
inverted_date_ranges_query = Bool()
if start_date.is_valid():
inverted_date_query.should(Range(field='dates.date', lt=start_date.as_float()))
inverted_date_ranges_query.should(Range(field='date_ranges.date_range', lt=start_date.as_float()))
if end_date.is_valid():
inverted_date_query.should(Range(field='dates.date', gt=end_date.as_float()))
inverted_date_ranges_query.should(Range(field='date_ranges.date_range', gt=end_date.as_float()))
date_query = Bool()
date_query.filter(inverted_date_query)
date_query.filter(Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(inverted_date_ranges_query)
date_ranges_query.filter(Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups))
temporal_query.should(Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
else:
date_query = Bool()
date_query.filter(Range(field='dates.date', gte=start_date.as_float(), lte=end_date.as_float()))
date_query.filter(Terms(field='dates.nodegroup_id', terms=permitted_nodegroups))
if date_nodeid:
date_query.filter(Term(field='dates.nodeid', term=date_nodeid))
else:
date_ranges_query = Bool()
date_ranges_query.filter(Range(field='date_ranges.date_range', gte=start_date.as_float(), lte=end_date.as_float(), relation='intersects'))
date_ranges_query.filter(Terms(field='date_ranges.nodegroup_id', terms=permitted_nodegroups))
temporal_query.should(Nested(path='date_ranges', query=date_ranges_query))
temporal_query.should(Nested(path='dates', query=date_query))
search_query.filter(temporal_query)
#print search_query.dsl
datatype_factory = DataTypeFactory()
if len(advanced_filters) > 0:
advanced_query = Bool()
grouped_query = Bool()
grouped_queries = [grouped_query]
for index, advanced_filter in enumerate(advanced_filters):
tile_query = Bool()
for key, val in advanced_filter.iteritems():
if key != 'op':
node = models.Node.objects.get(pk=key)
if request.user.has_perm('read_nodegroup', node.nodegroup):
datatype = datatype_factory.get_instance(node.datatype)
datatype.append_search_filters(val, node, tile_query, request)
nested_query = Nested(path='tiles', query=tile_query)
if advanced_filter['op'] == 'or' and index != 0:
grouped_query = Bool()
grouped_queries.append(grouped_query)
grouped_query.must(nested_query)
for grouped_query in grouped_queries:
advanced_query.should(grouped_query)
search_query.must(advanced_query)
query.add_query(search_query)
if search_buffer != None:
search_buffer = search_buffer.geojson
return {'query': query, 'search_buffer':search_buffer}
