def save_system_settings(self, data_dest=settings.SYSTEM_SETTINGS_LOCAL_PATH, file_format='json', config_file=None, graph=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID, single_file=False): resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file) if data_dest == '.': data_dest = os.path.dirname(settings.SYSTEM_SETTINGS_LOCAL_PATH) if data_dest != '': data = resource_exporter.export(graph_id=graph) for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.') sys.exit()
def export_graphs(self, data_dest='', graphs=''): """ Exports graphs to arches.json. """ if data_dest != '': graphs = [graph.strip() for graph in graphs.split(',')] for graph in ResourceGraphExporter.get_graphs_for_export(graphids=graphs)['graph']: graph_name = graph['name'].replace('/', '-') with open(os.path.join(data_dest, graph_name + '.json'), 'wb') as f: f.write(JSONSerializer().serialize({'graph': [graph]}, indent=4)) else: utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.') sys.exit()
def export_business_data(self, data_dest=None, file_format=None, config_file=None, graph=None, single_file=False): try: resource_exporter = ResourceExporter(file_format, configs=config_file, single_file=single_file) except KeyError as e: utils.print_message('{0} is not a valid export file format.'.format(file_format)) sys.exit() except MissingConfigException as e: utils.print_message('No mapping file specified. Please rerun this command with the \'-c\' parameter populated.') sys.exit() if data_dest != '': try: data = resource_exporter.export(graph_id=graph) except MissingGraphException as e: print utils.print_message('No resource graph specified. Please rerun this command with the \'-g\' parameter populated.') sys.exit() for file in data: with open(os.path.join(data_dest, file['name']), 'wb') as f: f.write(file['outputfile'].getvalue()) else: utils.print_message('No destination directory specified. Please rerun this command with the \'-d\' parameter populated.') sys.exit()
def import_mapping_file(self, source=None): """ Imports export mapping files for resource models. """ if source == '': utils.print_message('No data source indicated. Please rerun command with \'-s\' parameter.') if isinstance(source, basestring): source = [source] for path in source: if os.path.isfile(os.path.join(path)): with open(path, 'rU') as f: mapping_file = json.load(f) graph_importer.import_mapping_file(mapping_file)
def import_business_data_relations(self, data_source): """ Imports business data relations """ if isinstance(data_source, basestring): data_source = [data_source] for path in data_source: if os.path.isabs(path): if os.path.isfile(os.path.join(path)): relations = csv.DictReader(open(path, 'rU')) RelationImporter().import_relations(relations) else: utils.print_message('No file found at indicated location: {0}'.format(path)) sys.exit() else: utils.print_message('ERROR: The specified file path appears to be relative. Please rerun command with an absolute file path.') sys.exit()
def import_node_value_data(self, data_source, overwrite=None): """ Imports node-value datatype business data only. """ if overwrite == '': utils.print_message('No overwrite option indicated. Please rerun command with \'-ow\' parameter.') sys.exit() if isinstance(data_source, basestring): data_source = [data_source] if len(data_source) > 0: for source in data_source: path = utils.get_valid_path(source) if path is not None: data = unicodecsv.DictReader(open(path, 'rU'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING') business_data = list(data) TileCsvReader(business_data).import_business_data(overwrite=None) else: utils.print_message('No file found at indicated location: {0}'.format(source)) sys.exit() else: utils.print_message('No BUSINESS_DATA_FILES locations specified in your settings file. Please rerun this command with BUSINESS_DATA_FILES locations specified or pass the locations in manually with the \'-s\' parameter.') sys.exit()
def import_business_data(self, data_source, config_file=None, overwrite=None, bulk_load=False, create_concepts=False): """ Imports business data from all formats. A config file (mapping file) is required for .csv format. """ if overwrite == '': utils.print_message('No overwrite option indicated. Please rerun command with \'-ow\' parameter.') sys.exit() if data_source == '': data_source = settings.BUSINESS_DATA_FILES if isinstance(data_source, basestring): data_source = [data_source] create_collections = False if create_concepts: create_concepts = str(create_concepts).lower() if create_concepts == 'create': create_collections = True print 'Creating new collections . . .' elif create_concepts == 'append': print 'Appending to existing collections . . .' create_concepts = True if len(data_source) > 0: for source in data_source: path = utils.get_valid_path(source) if path is not None: print 'Importing {0}. . .'.format(path) BusinessDataImporter(path, config_file).import_business_data(overwrite=overwrite, bulk=bulk_load, create_concepts=create_concepts, create_collections=create_collections) else: utils.print_message('No file found at indicated location: {0}'.format(source)) sys.exit() else: utils.print_message('No BUSINESS_DATA_FILES locations specified in your settings file. Please rerun this command with BUSINESS_DATA_FILES locations specified or pass the locations in manually with the \'-s\' parameter.') sys.exit()