def export_resources(self, package_name, data_dest=None):
"""
Exports resources to archesjson
"""
resource_exporter = ResourceExporter("json")
resource_exporter.export(search_results=False, dest_dir=data_dest)
related_resources = [
{
"RESOURCEID_FROM": rr.entityid1,
"RESOURCEID_TO": rr.entityid2,
"RELATION_TYPE": rr.relationshiptype,
"START_DATE": rr.datestarted,
"END_DATE": rr.dateended,
"NOTES": rr.notes,
}
for rr in models.RelatedResource.objects.all()
]
relations_file = os.path.splitext(data_dest)[0] + ".relations"
with open(relations_file, "w") as f:
csvwriter = csv.DictWriter(
f,
delimiter="|",
fieldnames=["RESOURCEID_FROM", "RESOURCEID_TO", "START_DATE", "END_DATE", "RELATION_TYPE", "NOTES"],
)
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k: str(v).encode("utf8") for k, v in csv_record.items()})
def export_resources(self, package_name, data_dest=None):
"""
Exports resources to archesjson
"""
if data_dest.endswith(".jsonl"):
format = "jsonl"
else:
format = "json"
resource_exporter = ResourceExporter(format)
resource_exporter.export(search_results=False, dest_dir=data_dest)
related_resources = [{
'RESOURCEID_FROM': rr.entityid1,
'RESOURCEID_TO': rr.entityid2,
'RELATION_TYPE': rr.relationshiptype,
'START_DATE': rr.datestarted,
'END_DATE': rr.dateended,
'NOTES': rr.notes
} for rr in models.RelatedResource.objects.all()]
relations_file = os.path.splitext(data_dest)[0] + '.relations'
with open(relations_file, 'w') as f:
csvwriter = csv.DictWriter(f,
delimiter='|',
fieldnames=[
'RESOURCEID_FROM', 'RESOURCEID_TO',
'START_DATE', 'END_DATE',
'RELATION_TYPE', 'NOTES'
])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow(
{k: str(v).encode('utf8')
for k, v in csv_record.items()})
def export_resources(self, package_name, data_dest=None):
"""
Exports resources to archesjson
"""
resource_exporter = ResourceExporter('json')
resource_exporter.export(search_results=False, dest_dir=data_dest)
related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()]
relations_file = os.path.splitext(data_dest)[0] + '.relations'
with open(relations_file, 'w') as f:
csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})
def export_results(request):
dsl = build_search_results_dsl(request)
search_related_resources = JSONDeserializer().deserialize(request.GET.get('searchRelatedResources'))
if search_related_resources:
results_from_prev_query = request.session['result-resource-ids']
related_resources_from_prev_query = list(get_related_resource_ids(results_from_prev_query, lang, start=0, limit=1000000))
ids_filter = Terms(field='entityid', terms=related_resources_from_prev_query)
dsl.add_filter(ids_filter)
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()})
results.append({'name':csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
if graph == None or graph == False:
print '*'*80
print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.'
print '*'*80
sys.exit()
config_file = None
elif file_format == 'csv':
graph = None
if config_file == None:
print '*'*80
print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.'
print '*'*80
sys.exit()
if data_dest != '':
data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph, single_file=single_file)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*'*80
print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.'
print '*'*80
sys.exit()
else:
print '*'*80
print '{0} is not a valid export file format.'.format(file_format)
print '*'*80
sys.exit()
def export_resources(self, data_dest=None, resources='', file_format=None):
"""
Exports resources to specified format.
"""
# resource_exporter = ResourceExporter('json')
# resource_exporter.export(search_results=False, dest_dir=data_dest)
# related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()]
# relations_file = os.path.splitext(data_dest)[0] + '.relations'
# with open(relations_file, 'w') as f:
# csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES'])
# csvwriter.writeheader()
# for csv_record in related_resources:
# csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})
resource_exporter = ResourceExporter(file_format)
resource_exporter.export(resources=resources, dest_dir=data_dest)
def export_results(request):
search_results_dsl = build_search_results_dsl(request)
dsl = search_results_dsl['query']
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{
'id1': rr.entityid1,
'id2': rr.entityid2,
'type': rr.relationshiptype
} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest,
delimiter=',',
fieldnames=['id1', 'id2', 'type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow(
{k: v.encode('utf8')
for k, v in csv_record.items()})
results.append({'name': csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(
results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False):
try:
resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file)
except KeyError as e:
utils.print_message('{0} is not a valid export file format.'.format(file_format))
sys.exit()
except MissingConfigException as e:
utils.print_message('No mapping file specified. Please rerun this command with the \'-c\' parameter populated.')
sys.exit()
if data_dest != '':
try:
data = resource_exporter.export(graph_id=graph)
except MissingGraphException as e:
print utils.print_message('No resource graph specified. Please rerun this command with the \'-g\' parameter populated.')
sys.exit()
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.')
sys.exit()
def handle(self, *args, **options):
model_ids = [
"f212980f-d534-11e7-8ca8-94659cf754d0",
"73889292-d536-11e7-b3b3-94659cf754d0",
"c67216bf-8cc2-11e7-883c-06ed184dc22c",
]
exportdir = os.path.join(os.path.dirname(settings.APP_ROOT), "exports")
exportresids = set()
for rm_id in model_ids:
print(rm_id)
resources = Resource.objects.filter(graph_id=rm_id)
print(f"total number of resources: {len(resources)}")
report_node = Node.objects.get(name="Scout Report", graph_id=rm_id)
report_toptiles = Tile.objects.filter(
nodegroup_id=report_node.nodegroup_id)
print(f"total number of reports: {len(report_toptiles)}")
resids = set([i.resourceinstance_id for i in report_toptiles])
exportresids.update(resids)
resrep = [i for i in resources if i.resourceinstanceid in resids]
print(f"resources with reports: {len(resrep)}")
resource_exporter = ResourceExporter('json', single_file=True)
data = resource_exporter.export(resourceinstanceids=list(exportresids))
for file in data:
file['name'] = "resources-with-scout-reports" + file['name'][-25:]
with open(os.path.join(exportdir, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
def to_tilexl(self, instances):
resourceinstanceids = [
instance["resourceid"] for instance in instances
if "resourceid" in instance
]
tilexl_exporter = ResourceExporter(format="tilexl")
dest = tilexl_exporter.export(resourceinstanceids=resourceinstanceids)
return dest
def save_system_settings(self, data_dest=settings.SYSTEM_SETTINGS_LOCAL_PATH, file_format='json', config_file=None, graph=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID, single_file=False):
resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file)
if data_dest == '.':
data_dest = os.path.dirname(settings.SYSTEM_SETTINGS_LOCAL_PATH)
if data_dest != '':
data = resource_exporter.export(graph_id=graph)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.')
sys.exit()
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
config_file = None
elif file_format == 'csv':
graph = None
data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*'*80
print '{0} is not a valid export file format.'.format(file_format)
print '*'*80
sys.exit()
def export_results(request):
dsl = build_search_results_dsl(request)
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()})
results.append({'name':csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_results(request):
dsl = build_search_results_dsl(request)
search_related_resources = JSONDeserializer().deserialize(
request.GET.get('searchRelatedResources'))
if search_related_resources:
results_from_prev_query = request.session['result-resource-ids']
related_resources_from_prev_query = list(
get_related_resource_ids(results_from_prev_query,
lang,
start=0,
limit=1000000))
ids_filter = Terms(field='entityid',
terms=related_resources_from_prev_query)
dsl.add_filter(ids_filter)
search_results = dsl.search(index='entity', doc_type='')
response = None
format = request.GET.get('export', 'csv')
exporter = ResourceExporter(format)
results = exporter.export(search_results['hits']['hits'])
related_resources = [{
'id1': rr.entityid1,
'id2': rr.entityid2,
'type': rr.relationshiptype
} for rr in models.RelatedResource.objects.all()]
csv_name = 'resource_relationships.csv'
dest = StringIO()
csvwriter = csv.DictWriter(dest,
delimiter=',',
fieldnames=['id1', 'id2', 'type'])
csvwriter.writeheader()
for csv_record in related_resources:
csvwriter.writerow(
{k: v.encode('utf8')
for k, v in csv_record.items()})
results.append({'name': csv_name, 'outputfile': dest})
zipped_results = exporter.zip_response(
results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
return zipped_results
def export_business_data(self,
data_dest=None,
file_format=None,
config_file=None,
graph=None,
single_file=False):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
if graph == None or graph == False:
print '*' * 80
print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.'
print '*' * 80
sys.exit()
config_file = None
elif file_format == 'csv':
graph = None
if config_file == None:
print '*' * 80
print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.'
print '*' * 80
sys.exit()
if data_dest != '':
data = resource_exporter.export(data_dest=data_dest,
configs=config_file,
graph=graph,
single_file=single_file)
for file in data:
with open(os.path.join(data_dest, file['name']),
'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*' * 80
print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.'
print '*' * 80
sys.exit()
else:
print '*' * 80
print '{0} is not a valid export file format.'.format(file_format)
print '*' * 80
sys.exit()
def export_business_data(self,
data_dest=None,
file_format=None,
config_file=None,
graph=None):
if file_format in ['csv', 'json']:
resource_exporter = ResourceExporter(file_format)
if file_format == 'json':
config_file = None
elif file_format == 'csv':
graph = None
data = resource_exporter.export(data_dest=data_dest,
configs=config_file,
graph=graph)
for file in data:
with open(os.path.join(data_dest, file['name']), 'wb') as f:
f.write(file['outputfile'].getvalue())
else:
print '*' * 80
print '{0} is not a valid export file format.'.format(file_format)
print '*' * 80
sys.exit()
def to_html(self, instances, name, graph_id):
resourceinstanceids = [instance["resourceid"] for instance in instances if "resourceid" in instance]
html_exporter = ResourceExporter(format="html")
dest = html_exporter.export(resourceinstanceids=resourceinstanceids)
return dest