def create_sqlfile(database_settings, path_to_file):
db_directory = os.path.join(settings.ROOT_DIR, 'db')
context = Context(database_settings)
def source(file):
return "\i \'" + file + "\'\n"
# Generate a sql file that sources all necessary sql files into one file
buffer = ''
buffer = buffer + "\n-- Run all the sql scripts in the dependencies folder\n"
for infile in glob.glob(
posixpath.join(db_directory, 'install', 'dependencies', '*.sql')):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Reload all managed schemas\n"
for infile in glob.glob(posixpath.join(db_directory, 'ddl', '*.sql')):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Add all the data in the dml folder\n"
for infile in glob.glob(posixpath.join(db_directory, 'dml', '*.sql')):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Run all the sql in teh postdeployment folder\n"
for infile in glob.glob(
posixpath.join(db_directory, 'install', 'postdeployment',
'*.sql')):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Spring cleaning\n"
buffer = buffer + "VACUUM ANALYZE;\n"
utils.write_to_file(path_to_file, buffer)
def create_sqlfile(database_settings, path_to_file):
context = Context(database_settings)
postgres_version = subprocess.check_output(["psql", "--version"])
if int(postgres_version.split('.')[1]) >= 2:
context['PID'] = "pid"
else:
context['PID'] = "procpid"
t = Template(
"SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ NAME }}';\n"
"SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ POSTGIS_TEMPLATE }}';\n"
"\n"
"DROP DATABASE IF EXISTS {{ NAME }};\n"
"\n"
"CREATE DATABASE {{ NAME }}\n"
" WITH ENCODING='UTF8'\n"
" OWNER={{ USER }}\n"
" TEMPLATE={{POSTGIS_TEMPLATE}}\n"
" CONNECTION LIMIT=-1;\n"
"\n"
)
utils.write_to_file(path_to_file, t.render(context));
def create_sqlfile(database_settings, path_to_file):
context = Context(database_settings)
postgres_version = subprocess.check_output(["psql", "--version"])
if int(postgres_version.split('.')[1]) >= 2:
context['PID'] = "pid"
else:
context['PID'] = "procpid"
t = Template(
"SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ NAME }}';\n"
"SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ POSTGIS_TEMPLATE }}';\n"
"\n"
"DROP DATABASE IF EXISTS {{ NAME }};\n"
"\n"
"CREATE DATABASE {{ NAME }}\n"
" WITH ENCODING='UTF8'\n"
" OWNER={{ USER }}\n"
" TEMPLATE={{POSTGIS_TEMPLATE}}\n"
" CONNECTION LIMIT=-1;\n"
"\n"
)
utils.write_to_file(path_to_file, t.render(context));
def create_sqlfile(database_settings, path_to_file):
db_directory = os.path.join(settings.ROOT_DIR,'db')
context = Context(database_settings)
def source(file):
return "\i \'" + file + "\'\n"
# Generate a sql file that sources all necessary sql files into one file
buffer = ''
buffer = buffer + "\n-- Run all the sql scripts in the dependencies folder\n"
for infile in glob.glob(posixpath.join(db_directory, 'install', 'dependencies', '*.sql') ):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Reload all managed schemas\n"
for infile in glob.glob( posixpath.join(db_directory, 'ddl', '*.sql') ):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Add all the data in the dml folder\n"
for infile in glob.glob( posixpath.join(db_directory, 'dml', '*.sql') ):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Run all the sql in teh postdeployment folder\n"
for infile in glob.glob( posixpath.join(db_directory, 'install', 'postdeployment', '*.sql') ):
buffer = buffer + source(infile.replace("\\", posixpath.sep))
buffer = buffer + "\n-- Spring cleaning\n"
buffer = buffer + "VACUUM ANALYZE;\n"
utils.write_to_file(path_to_file, buffer)
def load_graphs(break_on_error=True, settings=None):
"""
Iterates through the resource node and edge files to add mappings in the database for nodes that don't collect data
"""
if not settings:
from django.conf import settings
suffix = '_nodes.csv'
errors = []
#file_list = []
for path in settings.RESOURCE_GRAPH_LOCATIONS:
if os.path.exists(path):
print '\nLOADING GRAPHS (%s)' % (path)
print '---------------'
for f in listdir(path):
if isfile(join(path,f)) and f.endswith(suffix):
#file_list.append(join(path,f))
path_to_file = join(path,f)
basepath = path_to_file[:-10]
name = basepath.split(os.sep)[-1]
if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or name in settings.LIMIT_ENTITY_TYPES_TO_LOAD):
print 'updating mapping for %s' % name
node_list = get_list_dict(basepath + '_nodes.csv', ['ID', 'LABEL', 'MERGENODE', 'BUSINESSTABLE'])
edge_list = get_list_dict(basepath + '_edges.csv', ['SOURCE', 'TARGET', 'TYPE', 'ID', 'LABEL', 'WEIGHT'])
mods = append_branch(os.path.join(settings.ROOT_DIR, 'management', 'resource_graphs', 'ARCHES_RECORD.E31'), node_list, edge_list)
node_list = mods['node_list']
edge_list = mods['edge_list']
file_errors = validate_graph(node_list, edge_list)
try:
insert_mappings(node_list, edge_list)
except Exception as e:
file_errors.append('\nERROR: %s\n%s' % (str(e), traceback.format_exc()))
pass
if len(file_errors) > 0:
file_errors.insert(0, 'ERRORS IN FILE: %s\n' % (basepath))
file_errors.append('\n\n\n\n')
errors = errors + file_errors
else:
errors.append('\n\nPath in settings.RESOURCE_GRAPH_LOCATIONS doesn\'t exist (%s)' % (path))
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '')
if len(errors) > 0:
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'), '\n'.join(errors))
print "\n\nERROR: There were errors in some of the resource graphs."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'resource_graph_errors.txt'))
if break_on_error:
sys.exit(101)
else:
print "\npatch '%s' successfully applied." % __name__
def create_setingsfile(self, data, path_to_file):
context = Context(data)
t = Template(
"MODE = '{{ MODE }}'\n"
"DEBUG = {{ DEBUG }}\n"
"\n"
"\n"
)
utils.write_to_file(path_to_file, t.render(context));
def validate_file(self, arches_file, break_on_error=True):
"""
Creates row dictionaries from a csv file
"""
validator = Validator()
with open(arches_file, 'rU') as f:
fieldnames = [
'RESOURCEID', 'RESOURCETYPE', 'ATTRIBUTENAME',
'ATTRIBUTEVALUE', 'GROUPID'
]
rows = unicodecsv.DictReader(f,
fieldnames=fieldnames,
encoding='utf-8-sig',
delimiter='|',
restkey='ADDITIONAL',
restval='MISSING')
rows.next() # skip header row
rownum = 2
start_time = time.time()
for row in rows:
validator.validate_row_syntax(row, rownum)
validator.validate_entitytype(row, rownum)
validator.valdiate_attribute_value(row, rownum)
validator.validate_contiguousness(row, rownum)
validator.unique_resourceids.add(row['RESOURCEID'])
rownum += 1
validator.validate_relations_file(arches_file)
duration = time.time() - start_time
sorted_errors = []
attr_length = len(validator.validation_errors.__dict__)
for attr, value in validator.validation_errors.__dict__.iteritems():
if value != []:
sorted_errors.extend(value)
sorted_errors.append('\n\n\n\n')
if len(sorted_errors) > 1:
del sorted_errors[-1]
print 'Validation of your Arches file took: {0} seconds.'.format(
str(duration))
if len(sorted_errors) > 0:
utils.write_to_file(
os.path.join(settings.PACKAGE_ROOT, 'logs',
'validation_errors.txt'),
'\n'.join(sorted_errors))
print "\n\nERROR: There were errors detected in your arches file."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (
os.path.join(settings.PACKAGE_ROOT, 'logs',
'validation_errors.txt'))
if break_on_error:
sys.exit(101)
def collect_resource_info(self, attr_mapping, auth_mapping, reader_output, geom_type, value_to_concept_label_mappings, break_on_error=True):
dict_list=[] # list of dictionaries
attr_names = reader_output[0]
attr_vals = reader_output[1][0:-1] # get all attribute values except the geom_wkt values
geom_values = reader_output[1][-1] # last index because we append wkt values at the end
'''
first, add the attribute values to the dictionary
and then add authority details. Because shapefile data does not
contain authority details, the authority mapping
is defined by the user and passed in a separate dictionary
'''
errors = []
for record_index in range (0, len(attr_vals[0])): #len(attr_vals[0]) equals to the number of records
record_dictionary= {} # i th dictionary
for attr in attr_names[0]: # loops for (number of attributes) times
#get the index of the selected attribute
attr_index = attr_names[0].index(attr)
#add the key/value pair retrieved from the attr_mapping
entitytypeid = attr_mapping.get(attr)
label = attr_vals[attr_index][record_index]
found_match = False
if type(entitytypeid) in [str, unicode]:
if entitytypeid.endswith('.E55'):
count = 0
for mapping in value_to_concept_label_mappings[entitytypeid]:
count += 1
if unicode(label) == mapping.keys()[0]:
label = mapping[label]
count = 0
break
if count == len(value_to_concept_label_mappings[entitytypeid]):
errors.append('shapefile record {0}: "{1}", Does not match any available {2} concept value\n'.format(str(record_index), label, entitytypeid))
else:
print str(record_index), label, entitytypeid
record_dictionary[entitytypeid] = label
record_dictionary.update(auth_mapping)
record_dictionary[geom_type] = geom_values[record_index]
dict_list.append(record_dictionary)
if len(errors) > 0:
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'shapefile_loading_errors.txt'), '\n'.join(errors))
print 'There were errors matching some values to concepts, please see {0} for details'.format(os.path.join(settings.PACKAGE_ROOT, 'logs', 'shapefile_loading_errors.txt'))
if break_on_error:
sys.exit(101)
return dict_list
def create_sqlfile(database_settings, path_to_file):
context = Context(database_settings)
if (split[0] >= 9) & (split[1] >= 2) & (split[2] >= 0): # 9.2.0 or above
context['PID'] = "pid"
else:
context['PID'] = "procpid"
t = Template(
"SELECT pg_terminate_backend({{ PID }}) from pg_stat_activity where datname='{{ NAME }}';\n"
"\n"
"DROP DATABASE IF EXISTS {{ NAME }};\n"
"\n"
"CREATE DATABASE {{ NAME }}\n"
" WITH ENCODING='UTF8'\n"
" CONNECTION LIMIT=-1;\n")
utils.write_to_file(path_to_file, t.render(context))
def generate_procfile(self, package_name):
"""
Generate a procfile for use with Honcho (https://honcho.readthedocs.org/en/latest/)
"""
python_exe = os.path.abspath(sys.executable)
contents = []
contents.append('\nelasticsearch: %s' % os.path.join(self.get_elasticsearch_install_location(package_name), 'bin', 'elasticsearch'))
contents.append('django: %s manage.py runserver' % (python_exe))
contents.append('livereload: %s manage.py packages --operation livereload' % (python_exe))
package_root = settings.PACKAGE_ROOT
if hasattr(settings, 'SUBPACKAGE_ROOT'):
package_root = settings.SUBPACKAGE_ROOT
utils.write_to_file(os.path.join(package_root, '..', 'Procfile'), '\n'.join(contents))
def generate_procfile(self, package_name):
"""
Generate a procfile for use with Honcho (https://honcho.readthedocs.org/en/latest/)
"""
python_exe = os.path.abspath(sys.executable)
contents = []
contents.append('\nelasticsearch: %s' % os.path.join(self.get_elasticsearch_install_location(package_name), 'bin', 'elasticsearch'))
contents.append('django: %s manage.py runserver' % (python_exe))
contents.append('livereload: %s manage.py packages --operation livereload' % (python_exe))
package_root = settings.PACKAGE_ROOT
if hasattr(settings, 'SUBPACKAGE_ROOT'):
package_root = settings.SUBPACKAGE_ROOT
utils.write_to_file(os.path.join(package_root, '..', 'Procfile'), '\n'.join(contents))
def index_resources_by_type(resource_types, result_summary):
"""
Collects and indexes all resources
"""
errors = []
for resource_type in resource_types:
resources = archesmodels.Entities.objects.filter(
entitytypeid=resource_type)
print "Indexing {0} {1} resources".format(len(resources),
resource_type[0])
result_summary[resource_type[0]] = {
'database': len(resources),
'indexed': 0
}
for resource in resources:
try:
resource = Resource().get(resource.entityid)
resource.index()
except Exception as e:
msg = 'Could not index resource {}.\nERROR: {}'.format(
resource.entityid, e)
print msg
errors.append(e)
se = SearchEngineFactory().create()
related_resource_records = archesmodels.RelatedResource.objects.all()
for related_resource_record in related_resource_records:
se.index_data(index='resource_relations',
doc_type='all',
body=model_to_dict(related_resource_record),
idfield='resourcexid')
if len(errors) > 0:
print "Number of errors:", len(errors)
log_file = os.path.join(settings.PACKAGE_ROOT, 'logs',
'indexing_errors.txt')
utils.write_to_file(log_file, '\n'.join(errors), mode="wb")
print " -- errors written to:", log_file
return result_summary
def validate_file(self, archesjson, break_on_error=True):
"""
Going to validate the file and return errors similar to arches validation.
Current only looks at the EAMENA_ID so isn't spun out into a different function/object
"""
start_time = time.time()
with open(archesjson, 'r') as f:
resources = JSONDeserializer().deserialize(f.read())
errors = []
for count, resource in enumerate(resources['resources']):
if resource['entitytypeid'] in settings.EAMENA_RESOURCES:
id_type = settings.EAMENA_RESOURCES[resource['entitytypeid']]
else:
id_type = resource['entitytypeid'].split('_')[0]
for entity in resource['child_entities']:
if entity['entitytypeid'] == 'EAMENA_ID.E42':
eamena_id = entity['value']
num = int(eamena_id[len(id_type) + 1:])
found_ents = UniqueIds.objects.filter(val=num,
id_type=id_type)
if len(found_ents) > 0:
errors.append(
'ERROR RESOURCE: {0} - {1} is a pre-existing unique ID.'
.format(count + 1, eamena_id))
break
duration = time.time() - start_time
print 'Validation of your JSON file took: {0} seconds.'.format(
str(duration))
if len(errors) > 0:
utils.write_to_file(
os.path.join(settings.PACKAGE_ROOT, 'logs',
'validation_errors.txt'), '\n'.join(errors))
print "\n\nERROR: There were errors detected in your JSON file."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (
os.path.join(settings.PACKAGE_ROOT, 'logs',
'validation_errors.txt'))
if break_on_error:
sys.exit(101)
def validate_file(self, arches_file, break_on_error=True):
"""
Creates row dictionaries from a csv file
"""
validator = Validator()
with open(arches_file, 'rU') as f:
fieldnames = ['RESOURCEID','RESOURCETYPE','ATTRIBUTENAME','ATTRIBUTEVALUE','GROUPID']
rows = unicodecsv.DictReader(f, fieldnames=fieldnames,
encoding='utf-8-sig', delimiter='|', restkey='ADDITIONAL', restval='MISSING')
rows.next() # skip header row
rownum = 2
start_time = time.time()
for row in rows:
validator.validate_row_syntax(row, rownum)
validator.validate_entitytype(row, rownum)
validator.valdiate_attribute_value(row, rownum)
validator.validate_contiguousness(row, rownum)
validator.unique_resourceids.add(row['RESOURCEID'])
rownum += 1
validator.validate_relations_file(arches_file)
duration = time.time() - start_time
sorted_errors = []
attr_length = len(validator.validation_errors.__dict__)
for attr, value in validator.validation_errors.__dict__.iteritems():
if value != []:
sorted_errors.extend(value)
sorted_errors.append('\n\n\n\n')
if len(sorted_errors) > 1:
del sorted_errors[-1]
print 'Validation of your Arches file took: {0} seconds.'.format(str(duration))
if len(sorted_errors) > 0:
send_mail_to_admins('Napaka pri uvozu datoteke ' + arches_file, '<br>'.join(sorted_errors))
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'), '\n'.join(sorted_errors))
print "\n\nERROR: There were errors detected in your arches file."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'))
if break_on_error:
sys.exit(101)
def generate_procfile(self, package_name):
"""
Generate a procfile for use with Honcho (https://honcho.readthedocs.org/en/latest/)
"""
python_exe = os.path.abspath(sys.executable)
contents = []
contents.append(
"\nelasticsearch: %s"
% os.path.join(self.get_elasticsearch_install_location(package_name), "bin", "elasticsearch")
)
contents.append("django: %s manage.py runserver" % (python_exe))
contents.append("livereload: %s manage.py packages --operation livereload" % (python_exe))
package_root = settings.PACKAGE_ROOT
if hasattr(settings, "SUBPACKAGE_ROOT"):
package_root = settings.SUBPACKAGE_ROOT
utils.write_to_file(os.path.join(package_root, "..", "Procfile"), "\n".join(contents))
def load_authority_files(source=None, break_on_error=True):
cursor = connection.cursor()
file_list = []
errors = []
authority_files_path = list(settings.CONCEPT_SCHEME_LOCATIONS)
if source != None:
authority_files_path.append(source)
for path in authority_files_path:
if os.path.exists(path):
print '\nLOADING AUTHORITY FILES (%s)' % (path)
print '-----------------------'
for f in listdir(path):
if isfile(join(path,f)) and '.values.csv' not in f and f != 'ENTITY_TYPE_X_ADOC.csv' and f[-4:] == '.csv':
file_list.append(f)
file_list.sort()
auth_file_to_entity_concept_mapping = entitytype_to_auth_doc_mapping(cursor, path)
count = 1
for file_name in file_list:
errors = errors + load_authority_file(cursor, path, file_name, auth_file_to_entity_concept_mapping)
if count > 10:
pass
#break
count = count + 1
#errors = errors + create_link_to_entity_types(cursor, path)
else:
errors.append('\n\nPath in settings.CONCEPT_SCHEME_LOCATIONS doesn\'t exist (%s)' % (path))
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '')
if len(errors) > 0:
send_mail_to_admins('Napaka pri uvozu authority_files', '<br>'.join(errors))
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '\n'.join(errors))
print "\n\nERROR: There were errors in some of the authority files."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'))
if break_on_error:
sys.exit(101)
def load_authority_files(source=None, break_on_error=True):
cursor = connection.cursor()
file_list = []
errors = []
authority_files_path = list(settings.CONCEPT_SCHEME_LOCATIONS)
if source != None:
authority_files_path.append(source)
for path in authority_files_path:
if os.path.exists(path):
print '\nLOADING AUTHORITY FILES (%s)' % (path)
print '-----------------------'
for f in listdir(path):
if isfile(join(path,f)) and '.values.csv' not in f and f != 'ENTITY_TYPE_X_ADOC.csv' and f[-4:] == '.csv':
file_list.append(f)
file_list.sort()
auth_file_to_entity_concept_mapping = entitytype_to_auth_doc_mapping(cursor, path)
count = 1
for file_name in file_list:
errors = errors + load_authority_file(cursor, path, file_name, auth_file_to_entity_concept_mapping)
if count > 10:
pass
#break
count = count + 1
#errors = errors + create_link_to_entity_types(cursor, path)
else:
errors.append('\n\nPath in settings.CONCEPT_SCHEME_LOCATIONS doesn\'t exist (%s)' % (path))
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '')
if len(errors) > 0:
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'), '\n'.join(errors))
print "\n\nERROR: There were errors in some of the authority files."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'authority_file_errors.txt'))
if break_on_error:
sys.exit(101)
def load_file(self, arches_file):
'''Reads an arches data file and creates resource graphs'''
resource_list = []
resource_id = ''
group_id = ''
resource = ''
resource_info = csv.DictReader(open(arches_file, 'r'), delimiter='|')
# Import package specific validation module.
fully_qualified_modulename = settings.PACKAGE_VALIDATOR
components = fully_qualified_modulename.split('.')
classname = components[len(components)-1]
modulename = ('.').join(components[0:len(components)])
validation_module = __import__(modulename, globals(), locals(), [classname], -1)
start_time = time.time()
for row in resource_info:
# print row
group_val = row['GROUPID'].strip()
resource_type_val = row['RESOURCETYPE'].strip()
resource_id_val = row['RESOURCEID'].strip()
if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None or resource_type_val in settings.LIMIT_ENTITY_TYPES_TO_LOAD):
if resource_id_val != resource_id:
if resource != '':
validation_module.validate_resource(resource)
resource = Resource(row)
resource_list.append(resource)
resource_id = resource_id_val
group_id = ''
if group_val != group_id: #create a new group of resouces
resource.groups.append(Group(row))
group_id = group_val
if group_val == group_id:
resource.appendrow(Row(row), group_id=group_id)
validation_module.validate_resource(resource)
sorted_errors = []
attr_length = len(validation_module.validation_errors.__dict__)
for attr, value in validation_module.validation_errors.__dict__.iteritems():
if value != []:
sorted_errors.extend(value)
sorted_errors.append('\n\n\n\n')
if len(sorted_errors) > 1:
del sorted_errors[-1]
duration = time.time() - start_time
print 'Validation of your business data took: {0} seconds.'.format(str(duration))
if len(sorted_errors) > 0:
utils.write_to_file(os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'), '\n'.join(sorted_errors))
print "\n\nERROR: There were errors detected in your business data."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (os.path.join(settings.PACKAGE_ROOT, 'logs', 'validation_errors.txt'))
break_on_error = True
if break_on_error:
sys.exit(101)
return resource_list
def load_graphs(break_on_error=True, settings=None):
"""
Iterates through the resource node and edge files to load entitytypes and mappings into the database.
Generates node level permissions for each resourcetype/entitytype combination
"""
if not settings:
from django.conf import settings
suffix = '_nodes.csv'
errors = []
#file_list = []
for path in settings.RESOURCE_GRAPH_LOCATIONS:
if os.path.exists(path):
print '\nLOADING GRAPHS (%s)' % (path)
print '---------------'
for f in listdir(path):
if isfile(join(path, f)) and f.endswith(suffix):
#file_list.append(join(path,f))
path_to_file = join(path, f)
basepath = path_to_file[:-10]
name = basepath.split(os.sep)[-1]
if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None
or name in settings.LIMIT_ENTITY_TYPES_TO_LOAD):
print name
node_list = resource_graphs.get_list_dict(
basepath + '_nodes.csv',
['ID', 'LABEL', 'MERGENODE', 'BUSINESSTABLE'])
edge_list = resource_graphs.get_list_dict(
basepath + '_edges.csv', [
'SOURCE', 'TARGET', 'TYPE', 'ID', 'LABEL',
'WEIGHT'
])
mods = resource_graphs.append_branch(
os.path.join(settings.ROOT_DIR, 'management',
'resource_graphs',
'ARCHES_RECORD.E31'), node_list,
edge_list)
node_list = mods['node_list']
edge_list = mods['edge_list']
file_errors = resource_graphs.validate_graph(
node_list, edge_list)
try:
resource_graphs.insert_mappings(
node_list, edge_list)
resource_graphs.link_entitytypes_to_concepts(
node_list)
except Exception as e:
file_errors.append(
'\nERROR: %s\n%s' %
(str(e), traceback.format_exc()))
pass
if len(file_errors) > 0:
file_errors.insert(
0, 'ERRORS IN FILE: %s\n' % (basepath))
file_errors.append('\n\n\n\n')
errors = errors + file_errors
else:
errors.append(
'\n\nPath in settings.RESOURCE_GRAPH_LOCATIONS doesn\'t exist (%s)'
% (path))
utils.write_to_file(
os.path.join(settings.PACKAGE_ROOT, 'logs',
'resource_graph_errors.txt'), '')
if len(errors) > 0:
utils.write_to_file(
os.path.join(settings.PACKAGE_ROOT, 'logs',
'resource_graph_errors.txt'), '\n'.join(errors))
print "\n\nERROR: There were errors in some of the resource graphs."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (
os.path.join(settings.PACKAGE_ROOT, 'logs',
'resource_graph_errors.txt'))
if break_on_error:
sys.exit(101)
print '\nADDING NODE LEVEL PERMISSIONS'
print '-----------------------------'
management.call_command('packages', operation='build_permissions')
def load_graphs(break_on_error=True, settings=None):
"""
Iterates through the resource node and edge files to add mappings in the database for nodes that don't collect data
"""
if not settings:
from django.conf import settings
suffix = '_nodes.csv'
errors = []
#file_list = []
for path in settings.RESOURCE_GRAPH_LOCATIONS:
if os.path.exists(path):
print '\nLOADING GRAPHS (%s)' % (path)
print '---------------'
for f in listdir(path):
if isfile(join(path, f)) and f.endswith(suffix):
#file_list.append(join(path,f))
path_to_file = join(path, f)
basepath = path_to_file[:-10]
name = basepath.split(os.sep)[-1]
if (settings.LIMIT_ENTITY_TYPES_TO_LOAD == None
or name in settings.LIMIT_ENTITY_TYPES_TO_LOAD):
print 'updating mapping for %s' % name
node_list = get_list_dict(
basepath + '_nodes.csv',
['ID', 'LABEL', 'MERGENODE', 'BUSINESSTABLE'])
edge_list = get_list_dict(basepath + '_edges.csv', [
'SOURCE', 'TARGET', 'TYPE', 'ID', 'LABEL', 'WEIGHT'
])
mods = append_branch(
os.path.join(settings.ROOT_DIR, 'management',
'resource_graphs',
'ARCHES_RECORD.E31'), node_list,
edge_list)
node_list = mods['node_list']
edge_list = mods['edge_list']
file_errors = validate_graph(node_list, edge_list)
try:
insert_mappings(node_list, edge_list)
except Exception as e:
file_errors.append(
'\nERROR: %s\n%s' %
(str(e), traceback.format_exc()))
pass
if len(file_errors) > 0:
file_errors.insert(
0, 'ERRORS IN FILE: %s\n' % (basepath))
file_errors.append('\n\n\n\n')
errors = errors + file_errors
else:
errors.append(
'\n\nPath in settings.RESOURCE_GRAPH_LOCATIONS doesn\'t exist (%s)'
% (path))
utils.write_to_file(
os.path.join(settings.PACKAGE_ROOT, 'logs',
'resource_graph_errors.txt'), '')
if len(errors) > 0:
utils.write_to_file(
os.path.join(settings.PACKAGE_ROOT, 'logs',
'resource_graph_errors.txt'), '\n'.join(errors))
print "\n\nERROR: There were errors in some of the resource graphs."
print "Please review the errors at %s, \ncorrect the errors and then rerun this script." % (
os.path.join(settings.PACKAGE_ROOT, 'logs',
'resource_graph_errors.txt'))
if break_on_error:
sys.exit(101)
else:
print "\npatch '%s' successfully applied." % __name__
