def __init__(self, file=None): self.graphs = "" self.reference_data = "" self.business_data = "" if not file: file = settings.RESOURCE_GRAPH_LOCATIONS else: file = [file] for path in file: if os.path.exists(path): if isfile(join(path)): with open(file[0], "rU") as f: archesfile = JSONDeserializer().deserialize(f) if "graph" in archesfile.keys(): self.graphs = archesfile["graph"] if "reference_data" in archesfile.keys(): self.reference_data = archesfile["reference_data"] if "business_data" in archesfile.keys(): self.business_data = archesfile["business_data"] else: print file + " is not a valid file" else: print path + " is not a valid path"
def __init__(self, file=None, mapping_file=None): self.graphs = '' self.reference_data = '' self.business_data = '' self.mapping = '' if not file: file = settings.RESOURCE_GRAPH_LOCATIONS else: file = [file] if mapping_file == None: try: mapping_file = [file[0].split('.')[0] + '.mapping'] except: print "mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your archesjson file and the extension .mapping" else: try: mapping_file = [mapping_file] except: print "mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your archesjson file and the extension .mapping" for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, 'r')) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): with open(file[0], 'rU') as f: archesfile = JSONDeserializer().deserialize(f) if 'graph' in archesfile.keys(): self.graphs = archesfile['graph'] if 'reference_data' in archesfile.keys(): self.reference_data = archesfile['reference_data'] if 'business_data' in archesfile.keys(): self.business_data = archesfile['business_data'] else: print str(file) + ' is not a valid file' else: print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None): self.business_data = '' self.mapping = None self.graphs = '' self.reference_data = '' self.business_data = '' self.file_format = '' self.relations = '' csv.field_size_limit(sys.maxint) if not file: file = settings.BUSINESS_DATA_FILES else: file = [file] if mapping_file == None: try: mapping_file = [file[0].split('.')[0] + '.mapping'] except: print '*'*80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*'*80 sys.exit() else: try: mapping_file = [mapping_file] except: print '*'*80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*'*80 sys.exit() if relations_file == None: try: relations_file = [file[0].split('.')[0] + '.relations'] except: pass for path in relations_file: if os.path.exists(path): if isfile(join(path)): self.relations = csv.DictReader(open(relations_file[0], 'r')) for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, 'r')) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): self.file_format = file[0].split('.')[-1] if self.file_format == 'json': with open(file[0], 'rU') as f: archesfile = JSONDeserializer().deserialize(f) if 'graph' in archesfile.keys(): self.graphs = archesfile['graph'] if 'reference_data' in archesfile.keys(): self.reference_data = archesfile['reference_data'] if 'business_data' in archesfile.keys(): self.business_data = archesfile['business_data'] elif self.file_format == 'csv': data = unicodecsv.DictReader(open(file[0], 'rU'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING') self.business_data = list(data) elif self.file_format == 'zip': zipfile = ZipFile(StringIO(open(file[0], 'r').read())) filenames = [y for y in sorted(zipfile.namelist()) for ending in ['dbf', 'prj', 'shp', 'shx'] if y.endswith(ending)] dbf, prj, shp, shx = [StringIO(zipfile.read(filename)) for filename in filenames] shape_file = shapefile.Reader(shp=shp, shx=shx, dbf=dbf) self.business_data = self.shape_to_csv(shape_file) elif self.file_format == 'shp': self.business_data = self.shape_to_csv(shapefile.Reader(file[0])) else: print str(file) + ' is not a valid file' else: print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None): self.business_data = '' self.mapping = None self.graphs = '' self.reference_data = '' self.business_data = '' self.file_format = '' self.relations = '' csv.field_size_limit(sys.maxint) if not file: file = settings.BUSINESS_DATA_FILES else: file = [file] if mapping_file == None: try: mapping_file = [file[0].split('.')[0] + '.mapping'] except: print '*' * 80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*' * 80 sys.exit() else: try: mapping_file = [mapping_file] except: print '*' * 80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*' * 80 sys.exit() if relations_file == None: try: relations_file = [file[0].split('.')[0] + '.relations'] except: pass for path in relations_file: if os.path.exists(path): if isfile(join(path)): self.relations = csv.DictReader( open(relations_file[0], 'r')) for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, 'r')) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): self.file_format = file[0].split('.')[-1] if self.file_format == 'json': with open(file[0], 'rU') as f: archesfile = JSONDeserializer().deserialize(f) if 'graph' in archesfile.keys(): self.graphs = archesfile['graph'] if 'reference_data' in archesfile.keys(): self.reference_data = archesfile[ 'reference_data'] if 'business_data' in archesfile.keys(): self.business_data = archesfile[ 'business_data'] elif self.file_format == 'csv': data = unicodecsv.DictReader(open(file[0], 'rU'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING') self.business_data = list(data) elif self.file_format == 'zip': shp_zipfile = os.path.basename(path) shp_zipfile_name = os.path.splitext(shp_zipfile)[0] unzip_dir = os.path.join(os.path.dirname(path), shp_zipfile_name) unzip_file(path, unzip_dir) shp = [ i for i in os.listdir(unzip_dir) if i.endswith(".shp") ] if len(shp) == 0: print '*' * 80 print "ERROR: There is no shapefile in this zipfile." print '*' * 80 exit() elif len(shp) > 1: print '*' * 80 print "ERROR: There are multiple shapefiles in this zipfile. Please load each individually:" for s in shp: print "\npython manage.py packages -o import_business_data -s {0} -c {1} -ow [append or overwrite]".format( os.path.join(unzip_dir, s), mapping_file[0]) print '*' * 80 exit() shp_path = os.path.join(unzip_dir, shp[0]) self.business_data = self.shape_to_csv(shp_path) elif self.file_format == 'shp': self.business_data = self.shape_to_csv(path) else: print str(file) + ' is not a valid file' else: print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None): self.business_data = '' self.mapping = None self.graphs = '' self.reference_data = '' self.business_data = '' self.file_format = '' self.relations = '' csv.field_size_limit(sys.maxint) if not file: file = settings.BUSINESS_DATA_FILES else: file = [file] if mapping_file == None: try: mapping_file = [file[0].split('.')[0] + '.mapping'] except: print '*'*80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*'*80 sys.exit() else: try: mapping_file = [mapping_file] except: print '*'*80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*'*80 sys.exit() if relations_file == None: try: relations_file = [file[0].split('.')[0] + '.relations'] except: pass for path in relations_file: if os.path.exists(path): if isfile(join(path)): self.relations = csv.DictReader(open(relations_file[0], 'r')) for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, 'r')) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): self.file_format = file[0].split('.')[-1] if self.file_format == 'json': with open(file[0], 'rU') as f: archesfile = JSONDeserializer().deserialize(f) if 'graph' in archesfile.keys(): self.graphs = archesfile['graph'] if 'reference_data' in archesfile.keys(): self.reference_data = archesfile['reference_data'] if 'business_data' in archesfile.keys(): self.business_data = archesfile['business_data'] elif self.file_format == 'csv': data = unicodecsv.DictReader(open(file[0], 'r'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING') self.business_data = list(data) else: print str(file) + ' is not a valid file' else: print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None): self.business_data = "" self.mapping = None self.graphs = "" self.reference_data = "" self.business_data = "" self.file_format = "" self.relations = "" try: csv.field_size_limit(sys.maxsize) except: csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2)) if not file: file = settings.BUSINESS_DATA_FILES else: file = [file] self.file = file if mapping_file is None: try: mapping_file_base = os.path.splitext(file[0])[0] mapping_file = [f"{mapping_file_base}.mapping"] except: print("*" * 80) print( "ERROR: Mapping file is missing or improperly named. Make sure you have \ mapping file with the same basename as your business data file and the extension .mapping" ) print("*" * 80) sys.exit() else: try: mapping_file = [mapping_file] except: print("*" * 80) print( "ERROR: Mapping file is missing or improperly named. Make sure you have \ mapping file with the same basename as your business data file and the extension .mapping" ) print("*" * 80) sys.exit() if relations_file is None: try: relations_file_base = os.path.splitext(file[0])[0] relations_file = [f"{relations_file_base}.relations"] except: pass for path in relations_file: if os.path.exists(path): if isfile(join(path)): self.relations = csv.DictReader( open(relations_file[0], "r")) for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, "r")) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): self.file_format = os.path.splitext(file[0])[1].strip(".") if self.file_format == "json": with open(file[0], "rU") as f: archesfile = JSONDeserializer().deserialize(f) if "graph" in list(archesfile.keys()): self.graphs = archesfile["graph"] if "reference_data" in list(archesfile.keys()): self.reference_data = archesfile[ "reference_data"] if "business_data" in list(archesfile.keys()): self.business_data = archesfile[ "business_data"] elif self.file_format == "csv": data = csv.DictReader(open(file[0], encoding="utf-8")) self.business_data = list(data) elif self.file_format == "zip": shp_zipfile = os.path.basename(path) shp_zipfile_name = os.path.splitext(shp_zipfile)[0] unzip_dir = os.path.join(os.path.dirname(path), shp_zipfile_name) unzip_file(path, unzip_dir) shp = [ i for i in os.listdir(unzip_dir) if i.endswith(".shp") ] if len(shp) == 0: print("*" * 80) print( "ERROR: There is no shapefile in this zipfile." ) print("*" * 80) exit() elif len(shp) > 1: print("*" * 80) print( "ERROR: There are multiple shapefiles in this zipfile. Please load each individually:" ) for s in shp: print( "\npython manage.py packages -o import_business_data -s {0} -c {1} -ow [append or overwrite]" .format(os.path.join(unzip_dir, s), mapping_file[0])) print("*" * 80) exit() shp_path = os.path.join(unzip_dir, shp[0]) self.business_data = self.shape_to_csv(shp_path) elif self.file_format == "shp": self.business_data = self.shape_to_csv(path) else: print(str(file) + " is not a valid file") else: print(path + " is not a valid path")
def __init__(self, file=None, mapping_file=None, relations_file=None): self.business_data = '' self.mapping = None self.graphs = '' self.reference_data = '' self.business_data = '' self.file_format = '' self.relations = '' if not file: file = settings.BUSINESS_DATA_FILES else: file = [file] if mapping_file == None: try: mapping_file = [file[0].split('.')[0] + '.mapping'] except: print '*' * 80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*' * 80 sys.exit() else: try: mapping_file = [mapping_file] except: print '*' * 80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*' * 80 sys.exit() if relations_file == None: try: relations_file = [file[0].split('.')[0] + '.relations'] except: pass for path in relations_file: if os.path.exists(path): if isfile(join(path)): self.relations = csv.DictReader( open(relations_file[0], 'r')) for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, 'r')) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): self.file_format = file[0].split('.')[1] if self.file_format == 'json': with open(file[0], 'rU') as f: archesfile = JSONDeserializer().deserialize(f) if 'graph' in archesfile.keys(): self.graphs = archesfile['graph'] if 'reference_data' in archesfile.keys(): self.reference_data = archesfile[ 'reference_data'] if 'business_data' in archesfile.keys(): self.business_data = archesfile[ 'business_data'] elif self.file_format == 'csv': data = unicodecsv.DictReader(open(file[0], 'r'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING') self.business_data = list(data) else: print str(file) + ' is not a valid file' else: print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None): self.business_data = '' self.mapping = None self.graphs = '' self.reference_data = '' self.business_data = '' self.file_format = '' self.relations = '' csv.field_size_limit(sys.maxint) if not file: file = settings.BUSINESS_DATA_FILES else: file = [file] if mapping_file == None: try: mapping_file = [file[0].split('.')[0] + '.mapping'] except: print '*'*80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*'*80 sys.exit() else: try: mapping_file = [mapping_file] except: print '*'*80 print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping" print '*'*80 sys.exit() if relations_file == None: try: relations_file = [file[0].split('.')[0] + '.relations'] except: pass for path in relations_file: if os.path.exists(path): if isfile(join(path)): self.relations = csv.DictReader(open(relations_file[0], 'r')) for path in mapping_file: if os.path.exists(path): if isfile(join(path)): self.mapping = json.load(open(path, 'r')) else: self.mapping = None for path in file: if os.path.exists(path): if isfile(join(path)): self.file_format = file[0].split('.')[-1] if self.file_format == 'json': with open(file[0], 'rU') as f: archesfile = JSONDeserializer().deserialize(f) if 'graph' in archesfile.keys(): self.graphs = archesfile['graph'] if 'reference_data' in archesfile.keys(): self.reference_data = archesfile['reference_data'] if 'business_data' in archesfile.keys(): self.business_data = archesfile['business_data'] elif self.file_format == 'csv': data = unicodecsv.DictReader(open(file[0], 'rU'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING') self.business_data = list(data) elif self.file_format == 'zip': shp_zipfile = os.path.basename(path) shp_zipfile_name = os.path.splitext(shp_zipfile)[0] unzip_dir = os.path.join(os.path.dirname(path),shp_zipfile_name) unzip_file(path,unzip_dir) shp = [i for i in os.listdir(unzip_dir) if i.endswith(".shp")] if len(shp) == 0: print '*'*80 print "ERROR: There is no shapefile in this zipfile." print '*'*80 exit() elif len(shp) > 1: print '*'*80 print "ERROR: There are multiple shapefiles in this zipfile. Please load each individually:" for s in shp: print "\npython manage.py packages -o import_business_data -s {0} -c {1} -ow [append or overwrite]".format( os.path.join(unzip_dir,s),mapping_file[0]) print '*'*80 exit() shp_path = os.path.join(unzip_dir,shp[0]) self.business_data = self.shape_to_csv(shp_path) elif self.file_format == 'shp': self.business_data = self.shape_to_csv(path) else: print str(file) + ' is not a valid file' else: print path + ' is not a valid path'