def create_sqlfile(database_settings, path_to_file): db_directory = os.path.join(settings.ROOT_DIR, 'db') context = Context(database_settings) def source(file): return "\i \'" + file + "\'\n" # Generate a sql file that sources all necessary sql files into one file buffer = '' buffer = buffer + "\n-- Run all the sql scripts in the dependencies folder\n" for infile in glob.glob( posixpath.join(db_directory, 'install', 'dependencies', '*.sql')): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Reload all managed schemas\n" for infile in glob.glob(posixpath.join(db_directory, 'ddl', '*.sql')): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Add all the data in the dml folder\n" for infile in glob.glob(posixpath.join(db_directory, 'dml', '*.sql')): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Run all the sql in teh postdeployment folder\n" for infile in glob.glob( posixpath.join(db_directory, 'install', 'postdeployment', '*.sql')): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Spring cleaning\n" buffer = buffer + "VACUUM ANALYZE;\n" utils.write_to_file(path_to_file, buffer)
def create_sqlfile(database_settings, path_to_file): context = Context(database_settings) postgres_version = subprocess.check_output(["psql", "--version"]) if int(postgres_version.split('.')[1]) >= 2: context['PID'] = "pid" else: context['PID'] = "procpid" t = Template( "SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ NAME }}';\n" "SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ POSTGIS_TEMPLATE }}';\n" "\n" "DROP DATABASE IF EXISTS {{ NAME }};\n" "\n" "CREATE DATABASE {{ NAME }}\n" " WITH ENCODING='UTF8'\n" " OWNER={{ USER }}\n" " TEMPLATE={{POSTGIS_TEMPLATE}}\n" " CONNECTION LIMIT=-1;\n" "\n" ) utils.write_to_file(path_to_file, t.render(context));
def create_sqlfile(database_settings, path_to_file): db_directory = os.path.join(settings.ROOT_DIR,'db') context = Context(database_settings) def source(file): return "\i \'" + file + "\'\n" # Generate a sql file that sources all necessary sql files into one file buffer = '' buffer = buffer + "\n-- Run all the sql scripts in the dependencies folder\n" for infile in glob.glob(posixpath.join(db_directory, 'install', 'dependencies', '*.sql') ): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Reload all managed schemas\n" for infile in glob.glob( posixpath.join(db_directory, 'ddl', '*.sql') ): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Add all the data in the dml folder\n" for infile in glob.glob( posixpath.join(db_directory, 'dml', '*.sql') ): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Run all the sql in teh postdeployment folder\n" for infile in glob.glob( posixpath.join(db_directory, 'install', 'postdeployment', '*.sql') ): buffer = buffer + source(infile.replace("\\", posixpath.sep)) buffer = buffer + "\n-- Spring cleaning\n" buffer = buffer + "VACUUM ANALYZE;\n" utils.write_to_file(path_to_file, buffer)
def load_graphs(break_on_error=True, settings=None): """ Iterates through the resource node and edge files to add mappings in the database for nodes that don't collect data """ if not settings: from django.conf import settings suffix = '_nodes.csv' errors = [] #file_list = [] for path in settings.RESOURCE_GRAPH_LOCATIONS: if os.path.exists(path): print '\nLOADING GRAPHS (%s)' % (path) print '---------------' for f in listdir(path): if isfile(join(path,f)) and f.endswith(suffix): #file_list.append(join(path,f)) path_to_file = join(path,f) basepath = path_to_file[:-10] name = basepath.split(os.sep)[-1] if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or name in settings.LIMIT_ENTITY_TYPES_TO_LOAD): print 'updating mapping for %s' % name node_list = get_list_dict(basepath + '_nodes.csv', ['ID', 'LABEL', 'MERGENODE', 'BUSINESSTABLE']) edge_list = get_list_dict(basepath + '_edges.csv', ['SOURCE', 'TARGET', 'TYPE', 'ID', 'LABEL', 'WEIGHT']) mods = append_branch(os.path.join(settings.ROOT_DIR, 'management', 'resource_graphs', 'ARCHES_RECORD.E31'), node_list, edge_list) node_list = mods['node_list'] edge_list = mods['edge_list'] file_errors = validate_graph(node_list, edge_list) try: insert_mappings(node_list, edge_list) except Exception as e: file_errors.append('\nERROR: %s\n%s' % (str(e), traceback.format_exc())) pass if len(file_errors) > 0: file_errors.insert(0, 'ERRORS IN FILE: %s\n' % (basepath)) file_errors.append('\n\n\n\n') errors = errors + file_errors else: errors.append('\n\nPath in settings.RESOURCE_GRAPH_LOCATIONS doesn\'t exist (%s)' % (path)) utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '') if len(errors) > 0: utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors in some of the resource graphs." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt')) if break_on_error: sys.exit(101) else: print "\npatch '%s' successfully applied." % __name__
def create_setingsfile(self, data, path_to_file): context = Context(data) t = Template( "MODE = '{{ MODE }}'\n" "DEBUG = {{ DEBUG }}\n" "\n" "\n" ) utils.write_to_file(path_to_file, t.render(context));
def validate_file(self, arches_file, break_on_error=True): """ Creates row dictionaries from a csv file """ validator = Validator() with open(arches_file, 'rU') as f: fieldnames = [ 'RESOURCEID', 'RESOURCETYPE', 'ATTRIBUTENAME', 'ATTRIBUTEVALUE', 'GROUPID' ] rows = unicodecsv.DictReader(f, fieldnames=fieldnames, encoding='utf-8-sig', delimiter='|', restkey='ADDITIONAL', restval='MISSING') rows.next() # skip header row rownum = 2 start_time = time.time() for row in rows: validator.validate_row_syntax(row, rownum) validator.validate_entitytype(row, rownum) validator.valdiate_attribute_value(row, rownum) validator.validate_contiguousness(row, rownum) validator.unique_resourceids.add(row['RESOURCEID']) rownum += 1 validator.validate_relations_file(arches_file) duration = time.time() - start_time sorted_errors = [] attr_length = len(validator.validation_errors.__dict__) for attr, value in validator.validation_errors.__dict__.iteritems(): if value != []: sorted_errors.extend(value) sorted_errors.append('\n\n\n\n') if len(sorted_errors) > 1: del sorted_errors[-1] print 'Validation of your Arches file took: {0} seconds.'.format( str(duration)) if len(sorted_errors) > 0: utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'), '\n'.join(sorted_errors)) print "\n\nERROR: There were errors detected in your arches file." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % ( os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt')) if break_on_error: sys.exit(101)
def collect_resource_info(self, attr_mapping, auth_mapping, reader_output, geom_type, value_to_concept_label_mappings, break_on_error=True): dict_list=[] # list of dictionaries attr_names = reader_output[0] attr_vals = reader_output[1][0:-1] # get all attribute values except the geom_wkt values geom_values = reader_output[1][-1] # last index because we append wkt values at the end ''' first, add the attribute values to the dictionary and then add authority details. Because shapefile data does not contain authority details, the authority mapping is defined by the user and passed in a separate dictionary ''' errors = [] for record_index in range (0, len(attr_vals[0])): #len(attr_vals[0]) equals to the number of records record_dictionary= {} # i th dictionary for attr in attr_names[0]: # loops for (number of attributes) times #get the index of the selected attribute attr_index = attr_names[0].index(attr) #add the key/value pair retrieved from the attr_mapping entitytypeid = attr_mapping.get(attr) label = attr_vals[attr_index][record_index] found_match = False if type(entitytypeid) in [str, unicode]: if entitytypeid.endswith('.E55'): count = 0 for mapping in value_to_concept_label_mappings[entitytypeid]: count += 1 if unicode(label) == mapping.keys()[0]: label = mapping[label] count = 0 break if count == len(value_to_concept_label_mappings[entitytypeid]): errors.append('shapefile record {0}: "{1}", Does not match any available {2} concept value\n'.format(str(record_index), label, entitytypeid)) else: print str(record_index), label, entitytypeid record_dictionary[entitytypeid] = label record_dictionary.update(auth_mapping) record_dictionary[geom_type] = geom_values[record_index] dict_list.append(record_dictionary) if len(errors) > 0: utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'shapefile_loading_errors.txt'), '\n'.join(errors)) print 'There were errors matching some values to concepts, please see {0} for details'.format(os.path.join(settings.PACKAGE_ROOT, 'logs', 'shapefile_loading_errors.txt')) if break_on_error: sys.exit(101) return dict_list
def create_sqlfile(database_settings, path_to_file): context = Context(database_settings) if (split[0] >= 9) & (split[1] >= 2) & (split[2] >= 0): # 9.2.0 or above context['PID'] = "pid" else: context['PID'] = "procpid" t = Template( "SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ NAME }}';\n" "\n" "DROP DATABASE IF EXISTS {{ NAME }};\n" "\n" "CREATE DATABASE {{ NAME }}\n" " WITH ENCODING='UTF8'\n" " CONNECTION LIMIT=-1;\n") utils.write_to_file(path_to_file, t.render(context))
def generate_procfile(self, package_name): """ Generate a procfile for use with Honcho (https://honcho.readthedocs.org/en/latest/) """ python_exe = os.path.abspath(sys.executable) contents = [] contents.append('\nelasticsearch: %s' % os.path.join(self.get_elasticsearch_install_location(package_name), 'bin', 'elasticsearch')) contents.append('django: %s manage.py runserver' % (python_exe)) contents.append('livereload: %s manage.py packages --operation livereload' % (python_exe)) package_root = settings.PACKAGE_ROOT if hasattr(settings, 'SUBPACKAGE_ROOT'): package_root = settings.SUBPACKAGE_ROOT utils.write_to_file(os.path.join(package_root, '..', 'Procfile'), '\n'.join(contents))
def index_resources_by_type(resource_types, result_summary): """ Collects and indexes all resources """ errors = [] for resource_type in resource_types: resources = archesmodels.Entities.objects.filter( entitytypeid=resource_type) print "Indexing {0} {1} resources".format(len(resources), resource_type[0]) result_summary[resource_type[0]] = { 'database': len(resources), 'indexed': 0 } for resource in resources: try: resource = Resource().get(resource.entityid) resource.index() except Exception as e: msg = 'Could not index resource {}.\nERROR: {}'.format( resource.entityid, e) print msg errors.append(e) se = SearchEngineFactory().create() related_resource_records = archesmodels.RelatedResource.objects.all() for related_resource_record in related_resource_records: se.index_data(index='resource_relations', doc_type='all', body=model_to_dict(related_resource_record), idfield='resourcexid') if len(errors) > 0: print "Number of errors:", len(errors) log_file = os.path.join(settings.PACKAGE_ROOT, 'logs', 'indexing_errors.txt') utils.write_to_file(log_file, '\n'.join(errors), mode="wb") print " -- errors written to:", log_file return result_summary
def validate_file(self, archesjson, break_on_error=True): """ Going to validate the file and return errors similar to arches validation. Current only looks at the EAMENA_ID so isn't spun out into a different function/object """ start_time = time.time() with open(archesjson, 'r') as f: resources = JSONDeserializer().deserialize(f.read()) errors = [] for count, resource in enumerate(resources['resources']): if resource['entitytypeid'] in settings.EAMENA_RESOURCES: id_type = settings.EAMENA_RESOURCES[resource['entitytypeid']] else: id_type = resource['entitytypeid'].split('_')[0] for entity in resource['child_entities']: if entity['entitytypeid'] == 'EAMENA_ID.E42': eamena_id = entity['value'] num = int(eamena_id[len(id_type) + 1:]) found_ents = UniqueIds.objects.filter(val=num, id_type=id_type) if len(found_ents) > 0: errors.append( 'ERROR RESOURCE: {0} - {1} is a pre-existing unique ID.' .format(count + 1, eamena_id)) break duration = time.time() - start_time print 'Validation of your JSON file took: {0} seconds.'.format( str(duration)) if len(errors) > 0: utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors detected in your JSON file." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % ( os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt')) if break_on_error: sys.exit(101)
def validate_file(self, arches_file, break_on_error=True): """ Creates row dictionaries from a csv file """ validator = Validator() with open(arches_file, 'rU') as f: fieldnames = ['RESOURCEID','RESOURCETYPE','ATTRIBUTENAME','ATTRIBUTEVALUE','GROUPID'] rows = unicodecsv.DictReader(f, fieldnames=fieldnames, encoding='utf-8-sig', delimiter='|', restkey='ADDITIONAL', restval='MISSING') rows.next() # skip header row rownum = 2 start_time = time.time() for row in rows: validator.validate_row_syntax(row, rownum) validator.validate_entitytype(row, rownum) validator.valdiate_attribute_value(row, rownum) validator.validate_contiguousness(row, rownum) validator.unique_resourceids.add(row['RESOURCEID']) rownum += 1 validator.validate_relations_file(arches_file) duration = time.time() - start_time sorted_errors = [] attr_length = len(validator.validation_errors.__dict__) for attr, value in validator.validation_errors.__dict__.iteritems(): if value != []: sorted_errors.extend(value) sorted_errors.append('\n\n\n\n') if len(sorted_errors) > 1: del sorted_errors[-1] print 'Validation of your Arches file took: {0} seconds.'.format(str(duration)) if len(sorted_errors) > 0: send_mail_to_admins('Napaka pri uvozu datoteke ' + arches_file, '<br>'.join(sorted_errors)) utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'), '\n'.join(sorted_errors)) print "\n\nERROR: There were errors detected in your arches file." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt')) if break_on_error: sys.exit(101)
def generate_procfile(self, package_name): """ Generate a procfile for use with Honcho (https://honcho.readthedocs.org/en/latest/) """ python_exe = os.path.abspath(sys.executable) contents = [] contents.append( "\nelasticsearch: %s" % os.path.join(self.get_elasticsearch_install_location(package_name), "bin", "elasticsearch") ) contents.append("django: %s manage.py runserver" % (python_exe)) contents.append("livereload: %s manage.py packages --operation livereload" % (python_exe)) package_root = settings.PACKAGE_ROOT if hasattr(settings, "SUBPACKAGE_ROOT"): package_root = settings.SUBPACKAGE_ROOT utils.write_to_file(os.path.join(package_root, "..", "Procfile"), "\n".join(contents))
def load_authority_files(source=None, break_on_error=True): cursor = connection.cursor() file_list = [] errors = [] authority_files_path = list(settings.CONCEPT_SCHEME_LOCATIONS) if source != None: authority_files_path.append(source) for path in authority_files_path: if os.path.exists(path): print '\nLOADING AUTHORITY FILES (%s)' % (path) print '-----------------------' for f in listdir(path): if isfile(join(path,f)) and '.values.csv' not in f and f != 'ENTITY_TYPE_X_ADOC.csv' and f[-4:] == '.csv': file_list.append(f) file_list.sort() auth_file_to_entity_concept_mapping = entitytype_to_auth_doc_mapping(cursor, path) count = 1 for file_name in file_list: errors = errors + load_authority_file(cursor, path, file_name, auth_file_to_entity_concept_mapping) if count > 10: pass #break count = count + 1 #errors = errors + create_link_to_entity_types(cursor, path) else: errors.append('\n\nPath in settings.CONCEPT_SCHEME_LOCATIONS doesn\'t exist (%s)' % (path)) utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '') if len(errors) > 0: send_mail_to_admins('Napaka pri uvozu authority_files', '<br>'.join(errors)) utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors in some of the authority files." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt')) if break_on_error: sys.exit(101)
def load_authority_files(source=None, break_on_error=True): cursor = connection.cursor() file_list = [] errors = [] authority_files_path = list(settings.CONCEPT_SCHEME_LOCATIONS) if source != None: authority_files_path.append(source) for path in authority_files_path: if os.path.exists(path): print '\nLOADING AUTHORITY FILES (%s)' % (path) print '-----------------------' for f in listdir(path): if isfile(join(path,f)) and '.values.csv' not in f and f != 'ENTITY_TYPE_X_ADOC.csv' and f[-4:] == '.csv': file_list.append(f) file_list.sort() auth_file_to_entity_concept_mapping = entitytype_to_auth_doc_mapping(cursor, path) count = 1 for file_name in file_list: errors = errors + load_authority_file(cursor, path, file_name, auth_file_to_entity_concept_mapping) if count > 10: pass #break count = count + 1 #errors = errors + create_link_to_entity_types(cursor, path) else: errors.append('\n\nPath in settings.CONCEPT_SCHEME_LOCATIONS doesn\'t exist (%s)' % (path)) utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '') if len(errors) > 0: utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors in some of the authority files." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt')) if break_on_error: sys.exit(101)
def load_file(self, arches_file): '''Reads an arches data file and creates resource graphs''' resource_list = [] resource_id = '' group_id = '' resource = '' resource_info = csv.DictReader(open(arches_file, 'r'), delimiter='|') # Import package specific validation module. fully_qualified_modulename = settings.PACKAGE_VALIDATOR components = fully_qualified_modulename.split('.') classname = components[len(components)-1] modulename = ('.').join(components[0:len(components)]) validation_module = __import__(modulename, globals(), locals(), [classname], -1) start_time = time.time() for row in resource_info: # print row group_val = row['GROUPID'].strip() resource_type_val = row['RESOURCETYPE'].strip() resource_id_val = row['RESOURCEID'].strip() if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or resource_type_val in settings.LIMIT_ENTITY_TYPES_TO_LOAD): if resource_id_val != resource_id: if resource != '': validation_module.validate_resource(resource) resource = Resource(row) resource_list.append(resource) resource_id = resource_id_val group_id = '' if group_val != group_id: #create a new group of resouces resource.groups.append(Group(row)) group_id = group_val if group_val == group_id: resource.appendrow(Row(row), group_id=group_id) validation_module.validate_resource(resource) sorted_errors = [] attr_length = len(validation_module.validation_errors.__dict__) for attr, value in validation_module.validation_errors.__dict__.iteritems(): if value != []: sorted_errors.extend(value) sorted_errors.append('\n\n\n\n') if len(sorted_errors) > 1: del sorted_errors[-1] duration = time.time() - start_time print 'Validation of your business data took: {0} seconds.'.format(str(duration)) if len(sorted_errors) > 0: utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'), '\n'.join(sorted_errors)) print "\n\nERROR: There were errors detected in your business data." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt')) break_on_error = True if break_on_error: sys.exit(101) return resource_list
def load_graphs(break_on_error=True, settings=None): """ Iterates through the resource node and edge files to load entitytypes and mappings into the database. Generates node level permissions for each resourcetype/entitytype combination """ if not settings: from django.conf import settings suffix = '_nodes.csv' errors = [] #file_list = [] for path in settings.RESOURCE_GRAPH_LOCATIONS: if os.path.exists(path): print '\nLOADING GRAPHS (%s)' % (path) print '---------------' for f in listdir(path): if isfile(join(path, f)) and f.endswith(suffix): #file_list.append(join(path,f)) path_to_file = join(path, f) basepath = path_to_file[:-10] name = basepath.split(os.sep)[-1] if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or name in settings.LIMIT_ENTITY_TYPES_TO_LOAD): print name node_list = resource_graphs.get_list_dict( basepath + '_nodes.csv', ['ID', 'LABEL', 'MERGENODE', 'BUSINESSTABLE']) edge_list = resource_graphs.get_list_dict( basepath + '_edges.csv', [ 'SOURCE', 'TARGET', 'TYPE', 'ID', 'LABEL', 'WEIGHT' ]) mods = resource_graphs.append_branch( os.path.join(settings.ROOT_DIR, 'management', 'resource_graphs', 'ARCHES_RECORD.E31'), node_list, edge_list) node_list = mods['node_list'] edge_list = mods['edge_list'] file_errors = resource_graphs.validate_graph( node_list, edge_list) try: resource_graphs.insert_mappings( node_list, edge_list) resource_graphs.link_entitytypes_to_concepts( node_list) except Exception as e: file_errors.append( '\nERROR: %s\n%s' % (str(e), traceback.format_exc())) pass if len(file_errors) > 0: file_errors.insert( 0, 'ERRORS IN FILE: %s\n' % (basepath)) file_errors.append('\n\n\n\n') errors = errors + file_errors else: errors.append( '\n\nPath in settings.RESOURCE_GRAPH_LOCATIONS doesn\'t exist (%s)' % (path)) utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '') if len(errors) > 0: utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors in some of the resource graphs." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % ( os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt')) if break_on_error: sys.exit(101) print '\nADDING NODE LEVEL PERMISSIONS' print '-----------------------------' management.call_command('packages', operation='build_permissions')
def load_graphs(break_on_error=True, settings=None): """ Iterates through the resource node and edge files to add mappings in the database for nodes that don't collect data """ if not settings: from django.conf import settings suffix = '_nodes.csv' errors = [] #file_list = [] for path in settings.RESOURCE_GRAPH_LOCATIONS: if os.path.exists(path): print '\nLOADING GRAPHS (%s)' % (path) print '---------------' for f in listdir(path): if isfile(join(path, f)) and f.endswith(suffix): #file_list.append(join(path,f)) path_to_file = join(path, f) basepath = path_to_file[:-10] name = basepath.split(os.sep)[-1] if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or name in settings.LIMIT_ENTITY_TYPES_TO_LOAD): print 'updating mapping for %s' % name node_list = get_list_dict( basepath + '_nodes.csv', ['ID', 'LABEL', 'MERGENODE', 'BUSINESSTABLE']) edge_list = get_list_dict(basepath + '_edges.csv', [ 'SOURCE', 'TARGET', 'TYPE', 'ID', 'LABEL', 'WEIGHT' ]) mods = append_branch( os.path.join(settings.ROOT_DIR, 'management', 'resource_graphs', 'ARCHES_RECORD.E31'), node_list, edge_list) node_list = mods['node_list'] edge_list = mods['edge_list'] file_errors = validate_graph(node_list, edge_list) try: insert_mappings(node_list, edge_list) except Exception as e: file_errors.append( '\nERROR: %s\n%s' % (str(e), traceback.format_exc())) pass if len(file_errors) > 0: file_errors.insert( 0, 'ERRORS IN FILE: %s\n' % (basepath)) file_errors.append('\n\n\n\n') errors = errors + file_errors else: errors.append( '\n\nPath in settings.RESOURCE_GRAPH_LOCATIONS doesn\'t exist (%s)' % (path)) utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '') if len(errors) > 0: utils.write_to_file( os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '\n'.join(errors)) print "\n\nERROR: There were errors in some of the resource graphs." print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % ( os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt')) if break_on_error: sys.exit(101) else: print "\npatch '%s' successfully applied." % __name__