def export_resources(self, package_name, data_dest=None): """ Exports resources to archesjson """ resource_exporter = ResourceExporter("json") resource_exporter.export(search_results=False, dest_dir=data_dest) related_resources = [ { "RESOURCEID_FROM": rr.entityid1, "RESOURCEID_TO": rr.entityid2, "RELATION_TYPE": rr.relationshiptype, "START_DATE": rr.datestarted, "END_DATE": rr.dateended, "NOTES": rr.notes, } for rr in models.RelatedResource.objects.all() ] relations_file = os.path.splitext(data_dest)[0] + ".relations" with open(relations_file, "w") as f: csvwriter = csv.DictWriter( f, delimiter="|", fieldnames=["RESOURCEID_FROM", "RESOURCEID_TO", "START_DATE", "END_DATE", "RELATION_TYPE", "NOTES"], ) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow({k: str(v).encode("utf8") for k, v in csv_record.items()})
def export_resources(self, package_name, data_dest=None): """ Exports resources to archesjson """ if data_dest.endswith(".jsonl"): format = "jsonl" else: format = "json" resource_exporter = ResourceExporter(format) resource_exporter.export(search_results=False, dest_dir=data_dest) related_resources = [{ 'RESOURCEID_FROM': rr.entityid1, 'RESOURCEID_TO': rr.entityid2, 'RELATION_TYPE': rr.relationshiptype, 'START_DATE': rr.datestarted, 'END_DATE': rr.dateended, 'NOTES': rr.notes } for rr in models.RelatedResource.objects.all()] relations_file = os.path.splitext(data_dest)[0] + '.relations' with open(relations_file, 'w') as f: csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=[ 'RESOURCEID_FROM', 'RESOURCEID_TO', 'START_DATE', 'END_DATE', 'RELATION_TYPE', 'NOTES' ]) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow( {k: str(v).encode('utf8') for k, v in csv_record.items()})
def export_resources(self, package_name, data_dest=None): """ Exports resources to archesjson """ resource_exporter = ResourceExporter('json') resource_exporter.export(search_results=False, dest_dir=data_dest) related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()] relations_file = os.path.splitext(data_dest)[0] + '.relations' with open(relations_file, 'w') as f: csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES']) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})
def export_results(request): dsl = build_search_results_dsl(request) search_related_resources = JSONDeserializer().deserialize(request.GET.get('searchRelatedResources')) if search_related_resources: results_from_prev_query = request.session['result-resource-ids'] related_resources_from_prev_query = list(get_related_resource_ids(results_from_prev_query, lang, start=0, limit=1000000)) ids_filter = Terms(field='entityid', terms=related_resources_from_prev_query) dsl.add_filter(ids_filter) search_results = dsl.search(index='entity', doc_type='') response = None format = request.GET.get('export', 'csv') exporter = ResourceExporter(format) results = exporter.export(search_results['hits']['hits']) related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()] csv_name = 'resource_relationships.csv' dest = StringIO() csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type']) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()}) results.append({'name':csv_name, 'outputfile': dest}) zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format)) return zipped_results
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False): if file_format in ['csv', 'json']: resource_exporter = ResourceExporter(file_format) if file_format == 'json': if graph == None or graph == False: print '*'*80 print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.' print '*'*80 sys.exit() config_file = None elif file_format == 'csv': graph = None if config_file == None: print '*'*80 print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.' print '*'*80 sys.exit() if data_dest != '': data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph, single_file=single_file) for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: print '*'*80 print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.' print '*'*80 sys.exit() else: print '*'*80 print '{0} is not a valid export file format.'.format(file_format) print '*'*80 sys.exit()
def export_resources(self, data_dest=None, resources='', file_format=None): """ Exports resources to specified format. """ # resource_exporter = ResourceExporter('json') # resource_exporter.export(search_results=False, dest_dir=data_dest) # related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()] # relations_file = os.path.splitext(data_dest)[0] + '.relations' # with open(relations_file, 'w') as f: # csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES']) # csvwriter.writeheader() # for csv_record in related_resources: # csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()}) resource_exporter = ResourceExporter(file_format) resource_exporter.export(resources=resources, dest_dir=data_dest)
def export_results(request): search_results_dsl = build_search_results_dsl(request) dsl = search_results_dsl['query'] search_results = dsl.search(index='entity', doc_type='') response = None format = request.GET.get('export', 'csv') exporter = ResourceExporter(format) results = exporter.export(search_results['hits']['hits']) related_resources = [{ 'id1': rr.entityid1, 'id2': rr.entityid2, 'type': rr.relationshiptype } for rr in models.RelatedResource.objects.all()] csv_name = 'resource_relationships.csv' dest = StringIO() csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1', 'id2', 'type']) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow( {k: v.encode('utf8') for k, v in csv_record.items()}) results.append({'name': csv_name, 'outputfile': dest}) zipped_results = exporter.zip_response( results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format)) return zipped_results
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False): try: resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file) except KeyError as e: utils.print_message('{0} is not a valid export file format.'.format(file_format)) sys.exit() except MissingConfigException as e: utils.print_message('No mapping file specified. Please rerun this command with the \'-c\' parameter populated.') sys.exit() if data_dest != '': try: data = resource_exporter.export(graph_id=graph) except MissingGraphException as e: print utils.print_message('No resource graph specified. Please rerun this command with the \'-g\' parameter populated.') sys.exit() for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.') sys.exit()
def handle(self, *args, **options): model_ids = [ "f212980f-d534-11e7-8ca8-94659cf754d0", "73889292-d536-11e7-b3b3-94659cf754d0", "c67216bf-8cc2-11e7-883c-06ed184dc22c", ] exportdir = os.path.join(os.path.dirname(settings.APP_ROOT), "exports") exportresids = set() for rm_id in model_ids: print(rm_id) resources = Resource.objects.filter(graph_id=rm_id) print(f"total number of resources: {len(resources)}") report_node = Node.objects.get(name="Scout Report", graph_id=rm_id) report_toptiles = Tile.objects.filter( nodegroup_id=report_node.nodegroup_id) print(f"total number of reports: {len(report_toptiles)}") resids = set([i.resourceinstance_id for i in report_toptiles]) exportresids.update(resids) resrep = [i for i in resources if i.resourceinstanceid in resids] print(f"resources with reports: {len(resrep)}") resource_exporter = ResourceExporter('json', single_file=True) data = resource_exporter.export(resourceinstanceids=list(exportresids)) for file in data: file['name'] = "resources-with-scout-reports" + file['name'][-25:] with open(os.path.join(exportdir, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue())
def to_tilexl(self, instances): resourceinstanceids = [ instance["resourceid"] for instance in instances if "resourceid" in instance ] tilexl_exporter = ResourceExporter(format="tilexl") dest = tilexl_exporter.export(resourceinstanceids=resourceinstanceids) return dest
def save_system_settings(self, data_dest=settings.SYSTEM_SETTINGS_LOCAL_PATH, file_format='json', config_file=None, graph=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID, single_file=False): resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file) if data_dest == '.': data_dest = os.path.dirname(settings.SYSTEM_SETTINGS_LOCAL_PATH) if data_dest != '': data = resource_exporter.export(graph_id=graph) for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.') sys.exit()
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None): if file_format in ['csv', 'json']: resource_exporter = ResourceExporter(file_format) if file_format == 'json': config_file = None elif file_format == 'csv': graph = None data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph) for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: print '*'*80 print '{0} is not a valid export file format.'.format(file_format) print '*'*80 sys.exit()
def export_results(request): dsl = build_search_results_dsl(request) search_results = dsl.search(index='entity', doc_type='') response = None format = request.GET.get('export', 'csv') exporter = ResourceExporter(format) results = exporter.export(search_results['hits']['hits']) related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()] csv_name = 'resource_relationships.csv' dest = StringIO() csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type']) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()}) results.append({'name':csv_name, 'outputfile': dest}) zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format)) return zipped_results
def export_results(request): dsl = build_search_results_dsl(request) search_related_resources = JSONDeserializer().deserialize( request.GET.get('searchRelatedResources')) if search_related_resources: results_from_prev_query = request.session['result-resource-ids'] related_resources_from_prev_query = list( get_related_resource_ids(results_from_prev_query, lang, start=0, limit=1000000)) ids_filter = Terms(field='entityid', terms=related_resources_from_prev_query) dsl.add_filter(ids_filter) search_results = dsl.search(index='entity', doc_type='') response = None format = request.GET.get('export', 'csv') exporter = ResourceExporter(format) results = exporter.export(search_results['hits']['hits']) related_resources = [{ 'id1': rr.entityid1, 'id2': rr.entityid2, 'type': rr.relationshiptype } for rr in models.RelatedResource.objects.all()] csv_name = 'resource_relationships.csv' dest = StringIO() csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1', 'id2', 'type']) csvwriter.writeheader() for csv_record in related_resources: csvwriter.writerow( {k: v.encode('utf8') for k, v in csv_record.items()}) results.append({'name': csv_name, 'outputfile': dest}) zipped_results = exporter.zip_response( results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format)) return zipped_results
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False): if file_format in ['csv', 'json']: resource_exporter = ResourceExporter(file_format) if file_format == 'json': if graph == None or graph == False: print '*' * 80 print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.' print '*' * 80 sys.exit() config_file = None elif file_format == 'csv': graph = None if config_file == None: print '*' * 80 print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.' print '*' * 80 sys.exit() if data_dest != '': data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph, single_file=single_file) for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: print '*' * 80 print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.' print '*' * 80 sys.exit() else: print '*' * 80 print '{0} is not a valid export file format.'.format(file_format) print '*' * 80 sys.exit()
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None): if file_format in ['csv', 'json']: resource_exporter = ResourceExporter(file_format) if file_format == 'json': config_file = None elif file_format == 'csv': graph = None data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph) for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: print '*' * 80 print '{0} is not a valid export file format.'.format(file_format) print '*' * 80 sys.exit()
def to_html(self, instances, name, graph_id): resourceinstanceids = [instance["resourceid"] for instance in instances if "resourceid" in instance] html_exporter = ResourceExporter(format="html") dest = html_exporter.export(resourceinstanceids=resourceinstanceids) return dest