Esempio n. 1
0
 def export_resources(self, package_name, data_dest=None):
     """
     Exports resources to archesjson
     """
     resource_exporter = ResourceExporter("json")
     resource_exporter.export(search_results=False, dest_dir=data_dest)
     related_resources = [
         {
             "RESOURCEID_FROM": rr.entityid1,
             "RESOURCEID_TO": rr.entityid2,
             "RELATION_TYPE": rr.relationshiptype,
             "START_DATE": rr.datestarted,
             "END_DATE": rr.dateended,
             "NOTES": rr.notes,
         }
         for rr in models.RelatedResource.objects.all()
     ]
     relations_file = os.path.splitext(data_dest)[0] + ".relations"
     with open(relations_file, "w") as f:
         csvwriter = csv.DictWriter(
             f,
             delimiter="|",
             fieldnames=["RESOURCEID_FROM", "RESOURCEID_TO", "START_DATE", "END_DATE", "RELATION_TYPE", "NOTES"],
         )
         csvwriter.writeheader()
         for csv_record in related_resources:
             csvwriter.writerow({k: str(v).encode("utf8") for k, v in csv_record.items()})
Esempio n. 2
0
 def export_resources(self, package_name, data_dest=None):
     """
     Exports resources to archesjson
     """
     if data_dest.endswith(".jsonl"):
         format = "jsonl"
     else:
         format = "json"
     resource_exporter = ResourceExporter(format)
     resource_exporter.export(search_results=False, dest_dir=data_dest)
     related_resources = [{
         'RESOURCEID_FROM': rr.entityid1,
         'RESOURCEID_TO': rr.entityid2,
         'RELATION_TYPE': rr.relationshiptype,
         'START_DATE': rr.datestarted,
         'END_DATE': rr.dateended,
         'NOTES': rr.notes
     } for rr in models.RelatedResource.objects.all()]
     relations_file = os.path.splitext(data_dest)[0] + '.relations'
     with open(relations_file, 'w') as f:
         csvwriter = csv.DictWriter(f,
                                    delimiter='|',
                                    fieldnames=[
                                        'RESOURCEID_FROM', 'RESOURCEID_TO',
                                        'START_DATE', 'END_DATE',
                                        'RELATION_TYPE', 'NOTES'
                                    ])
         csvwriter.writeheader()
         for csv_record in related_resources:
             csvwriter.writerow(
                 {k: str(v).encode('utf8')
                  for k, v in csv_record.items()})
Esempio n. 3
0
 def export_resources(self, package_name, data_dest=None):
     """
     Exports resources to archesjson
     """
     resource_exporter = ResourceExporter('json')
     resource_exporter.export(search_results=False, dest_dir=data_dest)
     related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()] 
     relations_file = os.path.splitext(data_dest)[0] + '.relations'
     with open(relations_file, 'w') as f:
         csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES'])
         csvwriter.writeheader()
         for csv_record in related_resources:
             csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})
Esempio n. 4
0
def export_results(request):
    dsl = build_search_results_dsl(request)
    
    search_related_resources = JSONDeserializer().deserialize(request.GET.get('searchRelatedResources'))
    
    if search_related_resources:
        results_from_prev_query = request.session['result-resource-ids']
        related_resources_from_prev_query = list(get_related_resource_ids(results_from_prev_query, lang, start=0, limit=1000000))

        ids_filter = Terms(field='entityid', terms=related_resources_from_prev_query)
        dsl.add_filter(ids_filter)
    
    
    search_results = dsl.search(index='entity', doc_type='')

    response = None
    format = request.GET.get('export', 'csv')
    exporter = ResourceExporter(format)
    results = exporter.export(search_results['hits']['hits'])
    
    related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()] 
    csv_name = 'resource_relationships.csv'
    dest = StringIO()
    csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type'])
    csvwriter.writeheader()
    for csv_record in related_resources:
        csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()})
    results.append({'name':csv_name, 'outputfile': dest})
    zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
    return zipped_results
Esempio n. 5
0
    def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False):
        if file_format in ['csv', 'json']:
            resource_exporter = ResourceExporter(file_format)
            if file_format == 'json':
                if graph == None or graph == False:
                    print '*'*80
                    print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.'
                    print '*'*80
                    sys.exit()
                config_file = None
            elif file_format == 'csv':
                graph = None
                if config_file == None:
                    print '*'*80
                    print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.'
                    print '*'*80
                    sys.exit()
            if data_dest != '':
                data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph, single_file=single_file)

                for file in data:
                    with open(os.path.join(data_dest, file['name']), 'wb') as f:
                        f.write(file['outputfile'].getvalue())
            else:
                print '*'*80
                print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.'
                print '*'*80
                sys.exit()
        else:
            print '*'*80
            print '{0} is not a valid export file format.'.format(file_format)
            print '*'*80
            sys.exit()
Esempio n. 6
0
    def export_resources(self, data_dest=None, resources='', file_format=None):
        """
        Exports resources to specified format.
        """
        #     resource_exporter = ResourceExporter('json')
        #     resource_exporter.export(search_results=False, dest_dir=data_dest)
        #     related_resources = [{'RESOURCEID_FROM':rr.entityid1, 'RESOURCEID_TO':rr.entityid2,'RELATION_TYPE':rr.relationshiptype,'START_DATE':rr.datestarted,'END_DATE':rr.dateended,'NOTES':rr.notes} for rr in models.RelatedResource.objects.all()]
        #     relations_file = os.path.splitext(data_dest)[0] + '.relations'
        #     with open(relations_file, 'w') as f:
        #         csvwriter = csv.DictWriter(f, delimiter='|', fieldnames=['RESOURCEID_FROM','RESOURCEID_TO','START_DATE','END_DATE','RELATION_TYPE','NOTES'])
        #         csvwriter.writeheader()
        #         for csv_record in related_resources:
        #             csvwriter.writerow({k: str(v).encode('utf8') for k, v in csv_record.items()})

        resource_exporter = ResourceExporter(file_format)
        resource_exporter.export(resources=resources, dest_dir=data_dest)
Esempio n. 7
0
def export_results(request):
    search_results_dsl = build_search_results_dsl(request)
    dsl = search_results_dsl['query']
    search_results = dsl.search(index='entity', doc_type='')
    response = None
    format = request.GET.get('export', 'csv')
    exporter = ResourceExporter(format)
    results = exporter.export(search_results['hits']['hits'])

    related_resources = [{
        'id1': rr.entityid1,
        'id2': rr.entityid2,
        'type': rr.relationshiptype
    } for rr in models.RelatedResource.objects.all()]
    csv_name = 'resource_relationships.csv'
    dest = StringIO()
    csvwriter = csv.DictWriter(dest,
                               delimiter=',',
                               fieldnames=['id1', 'id2', 'type'])
    csvwriter.writeheader()
    for csv_record in related_resources:
        csvwriter.writerow(
            {k: v.encode('utf8')
             for k, v in csv_record.items()})
    results.append({'name': csv_name, 'outputfile': dest})
    zipped_results = exporter.zip_response(
        results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
    return zipped_results
Esempio n. 8
0
    def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False):
        try:
            resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file)
        except KeyError as e:
            utils.print_message('{0} is not a valid export file format.'.format(file_format))
            sys.exit()
        except MissingConfigException as e:
            utils.print_message('No mapping file specified. Please rerun this command with the \'-c\' parameter populated.')
            sys.exit()

        if data_dest != '':
            try:
                data = resource_exporter.export(graph_id=graph)
            except MissingGraphException as e:

                print utils.print_message('No resource graph specified. Please rerun this command with the \'-g\' parameter populated.')

                sys.exit()

            for file in data:
                with open(os.path.join(data_dest, file['name']), 'wb') as f:
                    f.write(file['outputfile'].getvalue())
        else:
            utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.')
            sys.exit()
Esempio n. 9
0
    def handle(self, *args, **options):

        model_ids = [
            "f212980f-d534-11e7-8ca8-94659cf754d0",
            "73889292-d536-11e7-b3b3-94659cf754d0",
            "c67216bf-8cc2-11e7-883c-06ed184dc22c",
        ]

        exportdir = os.path.join(os.path.dirname(settings.APP_ROOT), "exports")
        exportresids = set()

        for rm_id in model_ids:
            print(rm_id)
            resources = Resource.objects.filter(graph_id=rm_id)
            print(f"total number of resources: {len(resources)}")
            report_node = Node.objects.get(name="Scout Report", graph_id=rm_id)
            report_toptiles = Tile.objects.filter(
                nodegroup_id=report_node.nodegroup_id)
            print(f"total number of reports: {len(report_toptiles)}")
            resids = set([i.resourceinstance_id for i in report_toptiles])
            exportresids.update(resids)
            resrep = [i for i in resources if i.resourceinstanceid in resids]
            print(f"resources with reports: {len(resrep)}")

        resource_exporter = ResourceExporter('json', single_file=True)
        data = resource_exporter.export(resourceinstanceids=list(exportresids))
        for file in data:
            file['name'] = "resources-with-scout-reports" + file['name'][-25:]
            with open(os.path.join(exportdir, file['name']), 'wb') as f:
                f.write(file['outputfile'].getvalue())
Esempio n. 10
0
 def to_tilexl(self, instances):
     resourceinstanceids = [
         instance["resourceid"] for instance in instances
         if "resourceid" in instance
     ]
     tilexl_exporter = ResourceExporter(format="tilexl")
     dest = tilexl_exporter.export(resourceinstanceids=resourceinstanceids)
     return dest
Esempio n. 11
0
 def save_system_settings(self, data_dest=settings.SYSTEM_SETTINGS_LOCAL_PATH, file_format='json', config_file=None, graph=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID, single_file=False):
     resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file)
     if data_dest == '.':
         data_dest = os.path.dirname(settings.SYSTEM_SETTINGS_LOCAL_PATH)
     if data_dest != '':
         data = resource_exporter.export(graph_id=graph)
         for file in data:
             with open(os.path.join(data_dest, file['name']), 'wb') as f:
                 f.write(file['outputfile'].getvalue())
     else:
         utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.')
         sys.exit()
Esempio n. 12
0
    def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None):
        if file_format in ['csv', 'json']:
            resource_exporter = ResourceExporter(file_format)
            if file_format == 'json':
                config_file = None
            elif file_format == 'csv':
                graph = None
            data = resource_exporter.export(data_dest=data_dest, configs=config_file, graph=graph)

            for file in data:
                with open(os.path.join(data_dest, file['name']), 'wb') as f:
                    f.write(file['outputfile'].getvalue())
        else:
            print '*'*80
            print '{0} is not a valid export file format.'.format(file_format)
            print '*'*80
            sys.exit()
Esempio n. 13
0
def export_results(request):
    dsl = build_search_results_dsl(request)
    search_results = dsl.search(index='entity', doc_type='')
    response = None
    format = request.GET.get('export', 'csv')
    exporter = ResourceExporter(format)
    results = exporter.export(search_results['hits']['hits'])

    related_resources = [{'id1':rr.entityid1, 'id2':rr.entityid2, 'type':rr.relationshiptype} for rr in models.RelatedResource.objects.all()]
    csv_name = 'resource_relationships.csv'
    dest = StringIO()
    csvwriter = csv.DictWriter(dest, delimiter=',', fieldnames=['id1','id2','type'])
    csvwriter.writeheader()
    for csv_record in related_resources:
        csvwriter.writerow({k:v.encode('utf8') for k,v in csv_record.items()})
    results.append({'name':csv_name, 'outputfile': dest})
    zipped_results = exporter.zip_response(results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
    return zipped_results
Esempio n. 14
0
def export_results(request):
    dsl = build_search_results_dsl(request)

    search_related_resources = JSONDeserializer().deserialize(
        request.GET.get('searchRelatedResources'))

    if search_related_resources:
        results_from_prev_query = request.session['result-resource-ids']
        related_resources_from_prev_query = list(
            get_related_resource_ids(results_from_prev_query,
                                     lang,
                                     start=0,
                                     limit=1000000))

        ids_filter = Terms(field='entityid',
                           terms=related_resources_from_prev_query)
        dsl.add_filter(ids_filter)

    search_results = dsl.search(index='entity', doc_type='')

    response = None
    format = request.GET.get('export', 'csv')
    exporter = ResourceExporter(format)
    results = exporter.export(search_results['hits']['hits'])

    related_resources = [{
        'id1': rr.entityid1,
        'id2': rr.entityid2,
        'type': rr.relationshiptype
    } for rr in models.RelatedResource.objects.all()]
    csv_name = 'resource_relationships.csv'
    dest = StringIO()
    csvwriter = csv.DictWriter(dest,
                               delimiter=',',
                               fieldnames=['id1', 'id2', 'type'])
    csvwriter.writeheader()
    for csv_record in related_resources:
        csvwriter.writerow(
            {k: v.encode('utf8')
             for k, v in csv_record.items()})
    results.append({'name': csv_name, 'outputfile': dest})
    zipped_results = exporter.zip_response(
        results, '{0}_{1}_export.zip'.format(settings.PACKAGE_NAME, format))
    return zipped_results
Esempio n. 15
0
    def export_business_data(self,
                             data_dest=None,
                             file_format=None,
                             config_file=None,
                             graph=None,
                             single_file=False):
        if file_format in ['csv', 'json']:
            resource_exporter = ResourceExporter(file_format)
            if file_format == 'json':
                if graph == None or graph == False:
                    print '*' * 80
                    print 'No resource graph specified. Please rerun this command with the \'-g\' parameter populated.'
                    print '*' * 80
                    sys.exit()
                config_file = None
            elif file_format == 'csv':
                graph = None
                if config_file == None:
                    print '*' * 80
                    print 'No mapping file specified. Please rerun this command with the \'-c\' parameter populated.'
                    print '*' * 80
                    sys.exit()
            if data_dest != '':
                data = resource_exporter.export(data_dest=data_dest,
                                                configs=config_file,
                                                graph=graph,
                                                single_file=single_file)

                for file in data:
                    with open(os.path.join(data_dest, file['name']),
                              'wb') as f:
                        f.write(file['outputfile'].getvalue())
            else:
                print '*' * 80
                print 'No destination directory specified. Please rerun this command with the \'-d\' parameter populated.'
                print '*' * 80
                sys.exit()
        else:
            print '*' * 80
            print '{0} is not a valid export file format.'.format(file_format)
            print '*' * 80
            sys.exit()
Esempio n. 16
0
    def export_business_data(self,
                             data_dest=None,
                             file_format=None,
                             config_file=None,
                             graph=None):
        if file_format in ['csv', 'json']:
            resource_exporter = ResourceExporter(file_format)
            if file_format == 'json':
                config_file = None
            elif file_format == 'csv':
                graph = None
            data = resource_exporter.export(data_dest=data_dest,
                                            configs=config_file,
                                            graph=graph)

            for file in data:
                with open(os.path.join(data_dest, file['name']), 'wb') as f:
                    f.write(file['outputfile'].getvalue())
        else:
            print '*' * 80
            print '{0} is not a valid export file format.'.format(file_format)
            print '*' * 80
            sys.exit()
Esempio n. 17
0
 def to_html(self, instances, name, graph_id):
     resourceinstanceids = [instance["resourceid"] for instance in instances if "resourceid" in instance]
     html_exporter = ResourceExporter(format="html")
     dest = html_exporter.export(resourceinstanceids=resourceinstanceids)
     return dest