def collect_resource_instances_for_couch(self):
"""
Uses the data definition configs of a mobile survey object to search for
resource instances relevant to a mobile survey. Takes a user object which
is required for search.
"""
query = self.datadownloadconfig['custom']
resource_types = self.datadownloadconfig['resources']
instances = {}
if query in ('', None) and len(resource_types) == 0:
print "No resources or data query defined"
else:
request = HttpRequest()
request.user = self.lasteditedby
request.GET['mobiledownload'] = True
if query in ('', None):
if len(self.bounds.coords) == 0:
default_bounds = settings.DEFAULT_BOUNDS
default_bounds['features'][0]['properties']['inverted'] = False
request.GET['mapFilter'] = json.dumps(default_bounds)
else:
request.GET['mapFilter'] = json.dumps({u'type': u'FeatureCollection', 'features':[{'geometry': json.loads(self.bounds.json)}]})
request.GET['typeFilter'] = json.dumps([{'graphid': resourceid, 'inverted': False } for resourceid in self.datadownloadconfig['resources']])
else:
parsed = urlparse.urlparse(query)
urlparams = urlparse.parse_qs(parsed.query)
for k, v in urlparams.iteritems():
request.GET[k] = v[0]
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(search_res_json.content)
try:
instances = {hit['_source']['resourceinstanceid']: hit['_source'] for hit in search_res['results']['hits']['hits']}
except KeyError:
print 'no instances found in', search_res
return instances
def export(self):
from arches.app.views.search import search_results
search_res_json = search_results(self.search_request)
results = JSONDeserializer().deserialize(search_res_json.content)
instances = results["results"]["hits"]["hits"]
output = {}
ret = []
for resource_instance in instances:
resource_obj = self.flatten_tiles(
resource_instance["_source"]["tiles"],
self.datatype_factory,
compact=self.compact)
try:
output[resource_instance["_source"]
["graph_id"]]["output"].append(resource_obj)
except KeyError as e:
output[resource_instance["_source"]["graph_id"]] = {
"output": []
}
output[resource_instance["_source"]
["graph_id"]]["output"].append(resource_obj)
for graph_id, resources in output.items():
graph = models.GraphModel.objects.get(pk=graph_id)
headers = list(
graph.node_set.filter(exportable=True).values_list("fieldname",
flat=True))
ret.append(
self.to_csv(resources["output"],
headers=headers,
name=graph.name))
# output[graph_id]['csv'] = self.to_csv(resources['output'])
return ret
def __resourceinstance_from_eamenaid_es(self, eamenaid, graphid):
try:
rm = GraphModel.objects.get(graphid=graphid)
except:
return None
request = HttpRequest()
request.user = User.objects.get(username='******')
request.path = '/'
request.GET = {"paging-filter":"1","tiles":"true","format":"tilecsv","precision":"6","total":"1","term-filter":"[{\"inverted\":false,\"type\":\"string\",\"context\":\"\",\"context_label\":\"\",\"id\":\"" + eamenaid + "\",\"text\":\"" + eamenaid + "\",\"value\":\"" + eamenaid + "\"}]","resource-type-filter":"[{\"graphid\":\"" + str(rm.graphid) + "\",\"name\":\"" + str(rm.name) + "\",\"inverted\":false}]"}
with warnings.catch_warnings():
warnings.simplefilter("ignore")
response = search.search_results(request)
results = json.loads(response.content)
ret = None
for hit in results['results']['hits']['hits']:
hitid = str(hit['_id'])
hiteamenaid = str(hit['_source']['displayname'])
if hiteamenaid == eamenaid:
try:
ret = ResourceInstance.objects.get(resourceinstanceid=hitid, graph_id=rm.graphid)
except:
ret = None
if not(ret is None):
return ret
return ret
def append_to_instances(self, request, instances, resource_type_id):
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(search_res_json.content)
for hit in search_res['results']['hits']['hits']:
if hit['_type'] == resource_type_id and len(
instances.keys()) < int(self.datadownloadconfig['count']):
instances[hit['_source']
['resourceinstanceid']] = hit['_source']
def export(self, format):
ret = []
search_res_json = SearchView.search_results(self.search_request)
if search_res_json.status_code == 500:
return ret
results = JSONDeserializer().deserialize(search_res_json.content)
instances = results["results"]["hits"]["hits"]
output = {}
for resource_instance in instances:
use_fieldname = self.format in ("shp",)
resource_obj = self.flatten_tiles(
resource_instance["_source"]["tiles"], self.datatype_factory, compact=self.compact, use_fieldname=use_fieldname
)
has_geom = resource_obj.pop("has_geometry")
skip_resource = self.format in ("shp",) and has_geom is False
if skip_resource is False:
try:
output[resource_instance["_source"]["graph_id"]]["output"].append(resource_obj)
except KeyError as e:
output[resource_instance["_source"]["graph_id"]] = {"output": []}
output[resource_instance["_source"]["graph_id"]]["output"].append(resource_obj)
for graph_id, resources in output.items():
graph = models.GraphModel.objects.get(pk=graph_id)
if format == "geojson":
headers = list(graph.node_set.filter(exportable=True).values_list("name", flat=True))
ret = self.to_geojson(resources["output"], headers=headers, name=graph.name)
return ret, ""
if format == "tilecsv":
headers = list(graph.node_set.filter(exportable=True).values_list("name", flat=True))
headers.append("resourceid")
ret.append(self.to_csv(resources["output"], headers=headers, name=graph.name))
if format == "shp":
headers = graph.node_set.filter(exportable=True).values("fieldname", "datatype", "name")[::1]
missing_field_names = []
for header in headers:
if not header["fieldname"]:
missing_field_names.append(header["name"])
header.pop("name")
if len(missing_field_names) > 0:
message = _("Shapefile are fieldnames required for the following nodes: {0}".format(", ".join(missing_field_names)))
logger.error(message)
raise (Exception(message))
headers = graph.node_set.filter(exportable=True).values("fieldname", "datatype")[::1]
headers.append({"fieldname": "resourceid", "datatype": "str"})
ret += self.to_shp(resources["output"], headers=headers, name=graph.name)
full_path = self.search_request.get_full_path()
search_request_path = self.search_request.path if full_path is None else full_path
search_export_info = models.SearchExportHistory(
user=self.search_request.user, numberofinstances=len(instances), url=search_request_path
)
search_export_info.save()
return ret, search_export_info
def append_to_instances(self, request, instances, resource_type_id):
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(search_res_json.content)
try:
for hit in search_res["results"]["hits"]["hits"]:
if hit["_source"]["graph_id"] == resource_type_id and len(list(instances)) < int(self.datadownloadconfig["count"]):
instances[hit["_source"]["resourceinstanceid"]] = hit["_source"]
except Exception as e:
print(e)
def append_to_instances(self, request, instances, resource_type_id):
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(search_res_json.content)
try:
for hit in search_res['results']['hits']['hits']:
if hit['_source']['graph_id'] == resource_type_id and len(
list(instances)) < int(
self.datadownloadconfig['count']):
instances[hit['_source']
['resourceinstanceid']] = hit['_source']
except Exception as e:
print(e)
def get(self, request):
# via an id for a set, returns list of phys things and stuff necessary
set_resourceid = (None if request.GET.get("resourceid") == ""
or request.GET.get("resourceid") is None else
(request.GET.get("resourceid")))
nodegroupid = request.GET.get("nodegroupid")
nodeid = request.GET.get("nodeid")
related = []
results = []
tiles = models.TileModel.objects.filter(
resourceinstance_id=set_resourceid).filter(
nodegroup_id=nodegroupid)
if len(tiles) > 0:
tile = tiles[0]
related = tile.data[nodeid]
for related_resource_item in related:
search_request = HttpRequest()
search_request.user = request.user
search_request.GET["id"] = related_resource_item["resourceId"]
result = json.loads(search_results(search_request).content)
results.append(result["results"]["hits"]["hits"][0])
return JSONResponse({"items": results})
def collect_resource_instances_for_couch(self, mobile_survey, user):
"""
Uses the data definition configs of a mobile survey object to search for
resource instances relevant to a mobile survey. Takes a user object which
is required for search.
"""
query = mobile_survey.datadownloadconfig['custom']
resource_types = mobile_survey.datadownloadconfig['resources']
instances = {}
if query in ('', None) and len(resource_types) == 0:
print "No resources or data query defined"
else:
request = HttpRequest()
request.user = user
request.GET['mobiledownload'] = True
if query in ('', None):
if len(mobile_survey.bounds.coords) == 0:
default_bounds = settings.DEFAULT_BOUNDS
default_bounds['features'][0]['properties']['inverted'] = False
request.GET['mapFilter'] = json.dumps(default_bounds)
else:
request.GET['mapFilter'] = json.dumps({u'type': u'FeatureCollection', 'features':[{'geometry': json.loads(mobile_survey.bounds.json)}]})
request.GET['typeFilter'] = json.dumps([{'graphid': resourceid, 'inverted': False } for resourceid in mobile_survey.datadownloadconfig['resources']])
else:
parsed = urlparse.urlparse(query)
urlparams = urlparse.parse_qs(parsed.query)
for k, v in urlparams.iteritems():
request.GET[k] = v[0]
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(search_res_json.content)
try:
instances = {hit['_source']['resourceinstanceid']: hit['_source'] for hit in search_res['results']['hits']['hits']}
except KeyError:
print 'no instances found in', search_res
return instances
def collect_resource_instances_for_couch(self):
"""
Uses the data definition configs of a mobile survey object to search for
resource instances relevant to a mobile survey. Takes a user object which
is required for search.
"""
query = self.datadownloadconfig["custom"]
resource_types = self.datadownloadconfig["resources"]
all_instances = {}
if query in ("", None) and len(resource_types) == 0:
logger.info("No resources or data query defined")
else:
request = HttpRequest()
request.user = self.lasteditedby
request.GET["mobiledownload"] = True
request.GET["resourcecount"] = self.datadownloadconfig["count"]
if query in ("", None):
if len(self.bounds.coords) == 0:
default_bounds = settings.DEFAULT_BOUNDS
default_bounds["features"][0]["properties"][
"inverted"] = False
map_filter = json.dumps(default_bounds)
else:
map_filter = json.dumps({
"type":
"FeatureCollection",
"features": [{
"geometry": json.loads(self.bounds.json)
}]
})
try:
for res_type in resource_types:
instances = {}
request.GET["resource-type-filter"] = json.dumps([{
"graphid":
res_type,
"inverted":
False
}])
request.GET["map-filter"] = map_filter
request.GET["paging-filter"] = "1"
request.GET["resourcecount"] = self.datadownloadconfig[
"count"]
self.append_to_instances(request, instances, res_type)
if len(list(instances.keys())) < int(
self.datadownloadconfig["count"]):
request.GET["map-filter"] = "{}"
request.GET["resourcecount"] = int(
self.datadownloadconfig["count"]) - len(
list(instances.keys()))
self.append_to_instances(request, instances,
res_type)
for key, value in instances.items():
all_instances[key] = value
except Exception as e:
logger.exception(e)
else:
try:
instances = {}
parsed = urllib.parse.urlparse(query)
urlparams = urllib.parse.parse_qs(parsed.query)
for k, v in urlparams.items():
request.GET[k] = v[0]
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(
search_res_json.content)
for hit in search_res["results"]["hits"]["hits"]:
instances[hit["_source"]
["resourceinstanceid"]] = hit["_source"]
for key, value in instances.items():
all_instances[key] = value
except KeyError:
print("no instances found in", search_res)
return all_instances
def collect_resource_instances_for_couch(self):
"""
Uses the data definition configs of a mobile survey object to search for
resource instances relevant to a mobile survey. Takes a user object which
is required for search.
"""
query = self.datadownloadconfig['custom']
resource_types = self.datadownloadconfig['resources']
all_instances = {}
if query in ('', None) and len(resource_types) == 0:
logger.info("No resources or data query defined")
else:
request = HttpRequest()
request.user = self.lasteditedby
request.GET['mobiledownload'] = True
request.GET['resourcecount'] = self.datadownloadconfig['count']
if query in ('', None):
if len(self.bounds.coords) == 0:
default_bounds = settings.DEFAULT_BOUNDS
default_bounds['features'][0]['properties'][
'inverted'] = False
map_filter = json.dumps(default_bounds)
else:
map_filter = json.dumps({
'type':
'FeatureCollection',
'features': [{
'geometry': json.loads(self.bounds.json)
}]
})
try:
for res_type in resource_types:
instances = {}
request.GET['resource-type-filter'] = json.dumps([{
'graphid':
res_type,
'inverted':
False
}])
request.GET['map-filter'] = map_filter
request.GET['paging-filter'] = '1'
request.GET['resourcecount'] = self.datadownloadconfig[
'count']
self.append_to_instances(request, instances, res_type)
if len(list(instances.keys())) < int(
self.datadownloadconfig['count']):
request.GET['map-filter'] = '{}'
request.GET['resourcecount'] = int(
self.datadownloadconfig['count']) - len(
list(instances.keys()))
self.append_to_instances(request, instances,
res_type)
for key, value in instances.items():
all_instances[key] = value
except Exception as e:
logger.exception(e)
else:
try:
instances = {}
parsed = urllib.parse.urlparse(query)
urlparams = urllib.parse.parse_qs(parsed.query)
for k, v in urlparams.items():
request.GET[k] = v[0]
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(
search_res_json.content)
for hit in search_res['results']['hits']['hits']:
instances[hit['_source']
['resourceinstanceid']] = hit['_source']
for key, value in instances.items():
all_instances[key] = value
except KeyError:
print('no instances found in', search_res)
return all_instances
def collect_resource_instances_for_couch(self):
"""
Uses the data definition configs of a mobile survey object to search for
resource instances relevant to a mobile survey. Takes a user object which
is required for search.
"""
query = self.datadownloadconfig["custom"]
resource_types = self.datadownloadconfig["resources"]
all_instances = {}
if query in ("", None) and len(resource_types) == 0:
logger.info("No resources or data query defined")
else:
resources_in_couch = set()
resources_in_couch_by_type = {}
for res_type in resource_types:
resources_in_couch_by_type[res_type] = []
db = self.couch.create_db("project_" + str(self.id))
couch_query = {"selector": {"type": "resource"}, "fields": ["_id", "graph_id"]}
for doc in db.find(couch_query):
resources_in_couch.add(doc["_id"])
resources_in_couch_by_type[doc["graph_id"]].append(doc["_id"])
if self.datadownloadconfig["download"]:
request = HttpRequest()
request.user = self.lasteditedby
request.GET["mobiledownload"] = True
if query in ("", None):
if len(self.bounds.coords) == 0:
default_bounds = settings.DEFAULT_BOUNDS
default_bounds["features"][0]["properties"]["inverted"] = False
map_filter = json.dumps(default_bounds)
else:
map_filter = json.dumps({"type": "FeatureCollection", "features": [{"geometry": json.loads(self.bounds.json)}]})
try:
for res_type in resource_types:
instances = {}
request.GET["resource-type-filter"] = json.dumps([{"graphid": res_type, "inverted": False}])
request.GET["map-filter"] = map_filter
request.GET["paging-filter"] = "1"
request.GET["resourcecount"] = int(self.datadownloadconfig["count"]) - len(resources_in_couch_by_type[res_type])
self.append_to_instances(request, instances, res_type)
if len(list(instances.keys())) < request.GET["resourcecount"]:
request.GET["map-filter"] = "{}"
request.GET["resourcecount"] = request.GET["resourcecount"] - len(list(instances.keys()))
self.append_to_instances(request, instances, res_type)
for key, value in instances.items():
all_instances[key] = value
except Exception as e:
logger.exception(e)
else:
try:
request.GET["resourcecount"] = int(self.datadownloadconfig["count"]) - len(resources_in_couch)
parsed = urllib.parse.urlparse(query)
urlparams = urllib.parse.parse_qs(parsed.query)
for k, v in urlparams.items():
request.GET[k] = v[0]
search_res_json = search.search_results(request)
search_res = JSONDeserializer().deserialize(search_res_json.content)
for hit in search_res["results"]["hits"]["hits"]:
all_instances[hit["_source"]["resourceinstanceid"]] = hit["_source"]
except KeyError:
print("no instances found in", search_res)
# this effectively makes sure that resources in couch always get updated
# even if they weren't included in the search results above (assuming self.datadownloadconfig["download"] == True)
# if self.datadownloadconfig["download"] == False then this will always update the resources in couch
ids = list(resources_in_couch - set(all_instances.keys()))
if len(ids) > 0:
se = SearchEngineFactory().create()
query = Query(se, start=0, limit=settings.SEARCH_RESULT_LIMIT)
ids_query = Terms(field="_id", terms=ids)
query.add_query(ids_query)
results = query.search(index="resources")
if results is not None:
for result in results["hits"]["hits"]:
all_instances[result["_id"]] = result["_source"]
return all_instances
def export(self, format, report_link):
ret = []
search_res_json = SearchView.search_results(self.search_request)
if search_res_json.status_code == 500:
return ret
results = JSONDeserializer().deserialize(search_res_json.content)
instances = results["results"]["hits"]["hits"]
output = {}
for resource_instance in instances:
use_fieldname = self.format in ("shp",)
resource_obj = self.flatten_tiles(
resource_instance["_source"]["tiles"], self.datatype_factory, compact=self.compact, use_fieldname=use_fieldname
)
has_geom = resource_obj.pop("has_geometry")
skip_resource = self.format in ("shp",) and has_geom is False
if skip_resource is False:
try:
output[resource_instance["_source"]["graph_id"]]["output"].append(resource_obj)
except KeyError as e:
output[resource_instance["_source"]["graph_id"]] = {"output": []}
output[resource_instance["_source"]["graph_id"]]["output"].append(resource_obj)
for graph_id, resources in output.items():
graph = models.GraphModel.objects.get(pk=graph_id)
if (report_link == "true") and (format != "tilexl"):
for resource in resources["output"]:
report_url = reverse("resource_report", kwargs={"resourceid": resource["resourceid"]})
export_namespace = settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT.rstrip("/")
resource["Link"] = f"{export_namespace}{report_url}"
if format == "geojson":
if settings.EXPORT_DATA_FIELDS_IN_CARD_ORDER is True:
headers = self.return_ordered_header(graph_id, "csv")
else:
headers = list(graph.node_set.filter(exportable=True).values_list("name", flat=True))
if (report_link == "true") and ("Link" not in headers):
headers.append("Link")
ret = self.to_geojson(resources["output"], headers=headers, name=graph.name)
return ret, ""
if format == "tilecsv":
if settings.EXPORT_DATA_FIELDS_IN_CARD_ORDER is True:
headers = self.return_ordered_header(graph_id, "csv")
else:
headers = list(graph.node_set.filter(exportable=True).values_list("name", flat=True))
headers.append("resourceid")
if (report_link == "true") and ("Link" not in headers):
headers.append("Link")
ret.append(self.to_csv(resources["output"], headers=headers, name=graph.name))
if format == "shp":
if settings.EXPORT_DATA_FIELDS_IN_CARD_ORDER is True:
headers = self.return_ordered_header(graph_id, "shp")
else:
headers = graph.node_set.filter(exportable=True).values("fieldname", "datatype", "name")[::1]
headers.append({"fieldname": "resourceid", "datatype": "str"})
missing_field_names = []
for header in headers:
if not header["fieldname"]:
missing_field_names.append(header["name"])
header.pop("name")
if len(missing_field_names) > 0:
message = _("Shapefile are fieldnames required for the following nodes: {0}".format(", ".join(missing_field_names)))
logger.error(message)
raise (Exception(message))
if (report_link == "true") and ({"fieldname": "Link", "datatype": "str"} not in headers):
headers.append({"fieldname": "Link", "datatype": "str"})
else:
pass
ret += self.to_shp(resources["output"], headers=headers, name=graph.name)
if format == "tilexl":
headers = graph.node_set.filter(exportable=True).values("fieldname", "datatype", "name")[::1]
headers = graph.node_set.filter(exportable=True).values("fieldname", "datatype")[::1]
headers.append({"fieldname": "resourceid", "datatype": "str"})
ret += self.to_tilexl(resources["output"])
if format == "html":
ret += self.to_html(resources["output"], name=graph.name, graph_id=str(graph.pk))
full_path = self.search_request.get_full_path()
search_request_path = self.search_request.path if full_path is None else full_path
search_export_info = models.SearchExportHistory(
user=self.search_request.user, numberofinstances=len(instances), url=search_request_path
)
search_export_info.save()
return ret, search_export_info
