def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False):
try:
resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file)
except KeyError as e:
utils.print_message('{0} is not a valid export file format.'.format(file_format))
sys.exit()
except MissingConfigException as e:
utils.print_message('No mapping file specified. Please rerun this command with the \'-c\' parameter populated.')
sys.exit()
if data_dest != '':
try:
data = resource_exporter.export(graph_id=graph)
except MissingGraphException as e:
print utils.print_message('No resource graph specified. Please rerun this command with the \'-g\' parameter populated.')
sys.exit()
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.')
sys.exit()
def export_results(request):
search_results_dsl = build_search_results_dsl(request)
dsl = search_results_dsl['query']
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{
'id1': rr.entityid1,
'id2': rr.entityid2,
'type': rr.relationshiptype
} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest,
delimiter=',',
fieldnames=['id1', 'id2', 'type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow(
{k: v.encode('utf8')
for k, v in csv_record.items()})
results.append({'name': csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(
results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def handle(self, *args, **options):
model_ids = [
"f212980f-d534-11e7-8ca8-94659cf754d0",
"73889292-d536-11e7-b3b3-94659cf754d0",
"c67216bf-8cc2-11e7-883c-06ed184dc22c",
]
exportdir = os.path.join(os.path.dirname(settings.APP_ROOT), "exports")
exportresids = set()
for rm_id in model_ids:
print(rm_id)
resources = Resource.objects.filter(graph_id=rm_id)
print(f"total number of resources: {len(resources)}")
report_node = Node.objects.get(name="Scout Report", graph_id=rm_id)
report_toptiles = Tile.objects.filter(
nodegroup_id=report_node.nodegroup_id)
print(f"total number of reports: {len(report_toptiles)}")
resids = set([i.resourceinstance_id for i in report_toptiles])
exportresids.update(resids)
resrep = [i for i in resources if i.resourceinstanceid in resids]
print(f"resources with reports: {len(resrep)}")
resource_exporter = ResourceExporter('json', single_file=True)
data = resource_exporter.export(resourceinstanceids=list(exportresids))
for file in data:
file['name'] = "resources-with-scout-reports" + file['name'][-25:]
with open(os.path.join(exportdir, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
def export_resources(self, package_name, data_dest=None):
"""
Exports resources to archesjson
"""
if data_dest.endswith(".jsonl"):
format = "jsonl"
else:
format = "json"
resource_exporter = ResourceExporter(format)
resource_exporter.export(search_results=False, dest_dir=data_dest)
related_resources = [{
'RESOURCEID_FROM': rr.entityid1,
'RESOURCEID_TO': rr.entityid2,
'RELATION_TYPE': rr.relationshiptype,
'START_DATE': rr.datestarted,
'END_DATE': rr.dateended,
'NOTES': rr.notes
} for rr in models.RelatedResource.objects.all()]
relations_file = os.path.splitext(data_dest)[0] + '.relations'
with open(relations_file, 'w') as f:
csvwriter = csv.DictWriter(f,
delimiter='|',
fieldnames=[
'RESOURCEID_FROM', 'RESOURCEID_TO',
'START_DATE', 'END_DATE',
'RELATION_TYPE', 'NOTES'
])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow(
{k: str(v).encode('utf8')
for k, v in csv_record.items()})
def export_resources(self, package_name, data_dest=None):
"""
Exports resources to archesjson
"""
resource_exporter = ResourceExporter("json")
resource_exporter.export(search_results=False, dest_dir=data_dest)
related_resources = [
{
"RESOURCEID_FROM": rr.entityid1,
"RESOURCEID_TO": rr.entityid2,
"RELATION_TYPE": rr.relationshiptype,
"START_DATE": rr.datestarted,
"END_DATE": rr.dateended,
"NOTES": rr.notes,
}
for rr in models.RelatedResource.objects.all()
]
relations_file = os.path.splitext(data_dest)[0] + ".relations"
with open(relations_file, "w") as f:
csvwriter = csv.DictWriter(
f,
delimiter="|",
fieldnames=["RESOURCEID_FROM", "RESOURCEID_TO", "START_DATE", "END_DATE", "RELATION_TYPE", "NOTES"],
)
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k: str(v).encode("utf8") for k, v in csv_record.items()})
def export_results(request):
dsl = build_search_results_dsl(request)
search_related_resources = JSONDeserializer().deserialize(request.GET.get('searchRelatedResources'))
if search_related_resources:
results_from_prev_query = request.session['result-resource-ids']
related_resources_from_prev_query = list(get_related_resource_ids(results_from_prev_query, lang, start=0, limit=1000000))
ids_filter = Terms(field='entityid', terms=related_resources_from_prev_query)
dsl.add_filter(ids_filter)
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()})
results.append({'name':csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
if graph == None or graph == False:
print '*'*80
print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.'
print '*'*80
sys.exit()
config_file = None
elif file_format == 'csv':
graph = None
if config_file == None:
print '*'*80
print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.'
print '*'*80
sys.exit()
if data_dest != '':
data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph, single_file=single_file)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*'*80
print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.'
print '*'*80
sys.exit()
else:
print '*'*80
print '{0} is not a valid export file format.'.format(file_format)
print '*'*80
sys.exit()
def to_tilexl(self, instances):
resourceinstanceids = [
instance["resourceid"] for instance in instances
if "resourceid" in instance
]
tilexl_exporter = ResourceExporter(format="tilexl")
dest = tilexl_exporter.export(resourceinstanceids=resourceinstanceids)
return dest
def save_system_settings(self, data_dest=settings.SYSTEM_SETTINGS_LOCAL_PATH, file_format='json', config_file=None, graph=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID, single_file=False):
resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file)
if data_dest == '.':
data_dest = os.path.dirname(settings.SYSTEM_SETTINGS_LOCAL_PATH)
if data_dest != '':
data = resource_exporter.export(graph_id=graph)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.')
sys.exit()
def export_resources(self, package_name, data_dest=None):
"""
Exports resources to archesjson
"""
resource_exporter = ResourceExporter('json')
resource_exporter.export(search_results=False, dest_dir=data_dest)
related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()]
relations_file = os.path.splitext(data_dest)[0] + '.relations'
with open(relations_file, 'w') as f:
csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})
def export_resources(self, data_dest=None, resources='', file_format=None):
"""
Exports resources to specified format.
"""
# resource_exporter = ResourceExporter('json')
# resource_exporter.export(search_results=False, dest_dir=data_dest)
# related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()]
# relations_file = os.path.splitext(data_dest)[0] + '.relations'
# with open(relations_file, 'w') as f:
# csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES'])
# csvwriter.writeheader()
# for csv_record in related_resources:
# csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})
resource_exporter = ResourceExporter(file_format)
resource_exporter.export(resources=resources, dest_dir=data_dest)
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
config_file = None
elif file_format == 'csv':
graph = None
data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*'*80
print '{0} is not a valid export file format.'.format(file_format)
print '*'*80
sys.exit()
def export_results(request):
dsl = build_search_results_dsl(request)
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()})
results.append({'name':csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_results(request):
dsl = build_search_results_dsl(request)
search_related_resources = JSONDeserializer().deserialize(
request.GET.get('searchRelatedResources'))
if search_related_resources:
results_from_prev_query = request.session['result-resource-ids']
related_resources_from_prev_query = list(
get_related_resource_ids(results_from_prev_query,
lang,
start=0,
limit=1000000))
ids_filter = Terms(field='entityid',
terms=related_resources_from_prev_query)
dsl.add_filter(ids_filter)
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{
'id1': rr.entityid1,
'id2': rr.entityid2,
'type': rr.relationshiptype
} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest,
delimiter=',',
fieldnames=['id1', 'id2', 'type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow(
{k: v.encode('utf8')
for k, v in csv_record.items()})
results.append({'name': csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(
results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_business_data(self,
data_dest=None,
file_format=None,
config_file=None,
graph=None,
single_file=False):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
if graph == None or graph == False:
print '*' * 80
print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.'
print '*' * 80
sys.exit()
config_file = None
elif file_format == 'csv':
graph = None
if config_file == None:
print '*' * 80
print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.'
print '*' * 80
sys.exit()
if data_dest != '':
data = resource_exporter.export(data_dest=data_dest,
configs=config_file,
graph=graph,
single_file=single_file)
for file in data:
with open(os.path.join(data_dest, file['name']),
'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*' * 80
print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.'
print '*' * 80
sys.exit()
else:
print '*' * 80
print '{0} is not a valid export file format.'.format(file_format)
print '*' * 80
sys.exit()
def export_business_data(self,
data_dest=None,
file_format=None,
config_file=None,
graph=None):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
config_file = None
elif file_format == 'csv':
graph = None
data = resource_exporter.export(data_dest=data_dest,
configs=config_file,
graph=graph)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*' * 80
print '{0} is not a valid export file format.'.format(file_format)
print '*' * 80
sys.exit()
def to_shp(self, instances, headers, name):
shape_exporter = ResourceExporter(format="shp")
dest = shape_exporter.writer.create_shapefiles(instances, headers, name)
return dest
def export_results(request):
exporter = SearchResultsExporter(search_request=request)
resourceexporter = ResourceExporter(format="tilecsv")
return resourceexporter.zip_response(exporter.export(), zip_file_name="temp.zip")
return JSONResponse(exporter.export(), indent=4)
def get(self, request, resourceid=None, slug=None, graphid=None):
if user_can_read_resources(user=request.user):
allowed_formats = ['json', 'json-ld']
format = request.GET.get('format', 'json-ld')
if format not in allowed_formats:
return JSONResponse(
status=406, reason='incorrect format specified, only %s formats allowed' % allowed_formats
)
try:
indent = int(request.GET.get('indent', None))
except Exception:
indent = None
if resourceid:
if format == 'json-ld':
try:
exporter = ResourceExporter(format=format)
output = exporter.writer.write_resources(
resourceinstanceids=[resourceid], indent=indent, user=request.user)
out = output[0]['outputfile'].getvalue()
except models.ResourceInstance.DoesNotExist:
logger.exception(
_("The specified resource '{0}' does not exist. JSON-LD export failed.".format(
resourceid
))
)
return JSONResponse(status=404)
elif format == 'json':
out = Resource.objects.get(pk=resourceid)
out.load_tiles()
else:
#
# The following commented code would be what you would use if you wanted to use the rdflib module,
# the problem with using this is that items in the "ldp:contains" array don't maintain a consistent order
#
# archesproject = Namespace(settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT)
# ldp = Namespace('https://www.w3.org/ns/ldp/')
# g = Graph()
# g.bind('archesproject', archesproject, False)
# g.add((archesproject['resources'], RDF.type, ldp['BasicContainer']))
# base_url = "%s%s" % (settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT, reverse('resources',args=['']).lstrip('/'))
# for resourceid in list(Resource.objects.values_list('pk', flat=True).order_by('pk')[:10]):
# g.add((archesproject['resources'], ldp['contains'], URIRef("%s%s") % (base_url, resourceid) ))
# value = g.serialize(format='nt')
# out = from_rdf(str(value), options={format:'application/nquads'})
# framing = {
# "@omitDefault": True
# }
# out = frame(out, framing)
# context = {
# "@context": {
# 'ldp': 'https://www.w3.org/ns/ldp/',
# 'arches': settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT
# }
# }
# out = compact(out, context, options={'skipExpansion':False, 'compactArrays': False})
page_size = settings.API_MAX_PAGE_SIZE
try:
page = int(request.GET.get('page', None))
except Exception:
page = 1
start = ((page - 1) * page_size)
end = start + page_size
base_url = "%s%s" % (settings.ARCHES_NAMESPACE_FOR_DATA_EXPORT,
reverse('resources', args=['']).lstrip('/'))
out = {
"@context": "https://www.w3.org/ns/ldp/",
"@id": "",
"@type": "ldp:BasicContainer",
# Here we actually mean the name
#"label": str(model.name),
"ldp:contains": ["%s%s" % (base_url, resourceid) for resourceid in list(Resource.objects.values_list('pk', flat=True).
exclude(pk=settings.SYSTEM_SETTINGS_RESOURCE_ID).order_by('pk')[start:end])]
}
return JSONResponse(out, indent=indent)
else:
return JSONResponse(status=403)
def to_html(self, instances, name, graph_id):
resourceinstanceids = [instance["resourceid"] for instance in instances if "resourceid" in instance]
html_exporter = ResourceExporter(format="html")
dest = html_exporter.export(resourceinstanceids=resourceinstanceids)
return dest
