def __init__(self, file=None):
self.graphs = ""
self.reference_data = ""
self.business_data = ""
if not file:
file = settings.RESOURCE_GRAPH_LOCATIONS
else:
file = [file]
for path in file:
if os.path.exists(path):
if isfile(join(path)):
with open(file[0], "rU") as f:
archesfile = JSONDeserializer().deserialize(f)
if "graph" in archesfile.keys():
self.graphs = archesfile["graph"]
if "reference_data" in archesfile.keys():
self.reference_data = archesfile["reference_data"]
if "business_data" in archesfile.keys():
self.business_data = archesfile["business_data"]
else:
print file + " is not a valid file"
else:
print path + " is not a valid path"
def __init__(self, file=None, mapping_file=None):
self.graphs = ''
self.reference_data = ''
self.business_data = ''
self.mapping = ''
if not file:
file = settings.RESOURCE_GRAPH_LOCATIONS
else:
file = [file]
if mapping_file == None:
try:
mapping_file = [file[0].split('.')[0] + '.mapping']
except:
print "mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your archesjson file and the extension .mapping"
else:
try:
mapping_file = [mapping_file]
except:
print "mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your archesjson file and the extension .mapping"
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, 'r'))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
with open(file[0], 'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
if 'graph' in archesfile.keys():
self.graphs = archesfile['graph']
if 'reference_data' in archesfile.keys():
self.reference_data = archesfile['reference_data']
if 'business_data' in archesfile.keys():
self.business_data = archesfile['business_data']
else:
print str(file) + ' is not a valid file'
else:
print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None):
self.business_data = ''
self.mapping = None
self.graphs = ''
self.reference_data = ''
self.business_data = ''
self.file_format = ''
self.relations = ''
csv.field_size_limit(sys.maxint)
if not file:
file = settings.BUSINESS_DATA_FILES
else:
file = [file]
if mapping_file == None:
try:
mapping_file = [file[0].split('.')[0] + '.mapping']
except:
print '*'*80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*'*80
sys.exit()
else:
try:
mapping_file = [mapping_file]
except:
print '*'*80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*'*80
sys.exit()
if relations_file == None:
try:
relations_file = [file[0].split('.')[0] + '.relations']
except:
pass
for path in relations_file:
if os.path.exists(path):
if isfile(join(path)):
self.relations = csv.DictReader(open(relations_file[0], 'r'))
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, 'r'))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
self.file_format = file[0].split('.')[-1]
if self.file_format == 'json':
with open(file[0], 'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
if 'graph' in archesfile.keys():
self.graphs = archesfile['graph']
if 'reference_data' in archesfile.keys():
self.reference_data = archesfile['reference_data']
if 'business_data' in archesfile.keys():
self.business_data = archesfile['business_data']
elif self.file_format == 'csv':
data = unicodecsv.DictReader(open(file[0], 'rU'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING')
self.business_data = list(data)
elif self.file_format == 'zip':
zipfile = ZipFile(StringIO(open(file[0], 'r').read()))
filenames = [y for y in sorted(zipfile.namelist()) for ending in ['dbf', 'prj', 'shp', 'shx'] if y.endswith(ending)]
dbf, prj, shp, shx = [StringIO(zipfile.read(filename)) for filename in filenames]
shape_file = shapefile.Reader(shp=shp, shx=shx, dbf=dbf)
self.business_data = self.shape_to_csv(shape_file)
elif self.file_format == 'shp':
self.business_data = self.shape_to_csv(shapefile.Reader(file[0]))
else:
print str(file) + ' is not a valid file'
else:
print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None):
self.business_data = ''
self.mapping = None
self.graphs = ''
self.reference_data = ''
self.business_data = ''
self.file_format = ''
self.relations = ''
csv.field_size_limit(sys.maxint)
if not file:
file = settings.BUSINESS_DATA_FILES
else:
file = [file]
if mapping_file == None:
try:
mapping_file = [file[0].split('.')[0] + '.mapping']
except:
print '*' * 80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*' * 80
sys.exit()
else:
try:
mapping_file = [mapping_file]
except:
print '*' * 80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*' * 80
sys.exit()
if relations_file == None:
try:
relations_file = [file[0].split('.')[0] + '.relations']
except:
pass
for path in relations_file:
if os.path.exists(path):
if isfile(join(path)):
self.relations = csv.DictReader(
open(relations_file[0], 'r'))
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, 'r'))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
self.file_format = file[0].split('.')[-1]
if self.file_format == 'json':
with open(file[0], 'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
if 'graph' in archesfile.keys():
self.graphs = archesfile['graph']
if 'reference_data' in archesfile.keys():
self.reference_data = archesfile[
'reference_data']
if 'business_data' in archesfile.keys():
self.business_data = archesfile[
'business_data']
elif self.file_format == 'csv':
data = unicodecsv.DictReader(open(file[0], 'rU'),
encoding='utf-8-sig',
restkey='ADDITIONAL',
restval='MISSING')
self.business_data = list(data)
elif self.file_format == 'zip':
shp_zipfile = os.path.basename(path)
shp_zipfile_name = os.path.splitext(shp_zipfile)[0]
unzip_dir = os.path.join(os.path.dirname(path),
shp_zipfile_name)
unzip_file(path, unzip_dir)
shp = [
i for i in os.listdir(unzip_dir)
if i.endswith(".shp")
]
if len(shp) == 0:
print '*' * 80
print "ERROR: There is no shapefile in this zipfile."
print '*' * 80
exit()
elif len(shp) > 1:
print '*' * 80
print "ERROR: There are multiple shapefiles in this zipfile. Please load each individually:"
for s in shp:
print "\npython manage.py packages -o import_business_data -s {0} -c {1} -ow [append or overwrite]".format(
os.path.join(unzip_dir, s),
mapping_file[0])
print '*' * 80
exit()
shp_path = os.path.join(unzip_dir, shp[0])
self.business_data = self.shape_to_csv(shp_path)
elif self.file_format == 'shp':
self.business_data = self.shape_to_csv(path)
else:
print str(file) + ' is not a valid file'
else:
print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None):
self.business_data = ''
self.mapping = None
self.graphs = ''
self.reference_data = ''
self.business_data = ''
self.file_format = ''
self.relations = ''
csv.field_size_limit(sys.maxint)
if not file:
file = settings.BUSINESS_DATA_FILES
else:
file = [file]
if mapping_file == None:
try:
mapping_file = [file[0].split('.')[0] + '.mapping']
except:
print '*'*80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*'*80
sys.exit()
else:
try:
mapping_file = [mapping_file]
except:
print '*'*80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*'*80
sys.exit()
if relations_file == None:
try:
relations_file = [file[0].split('.')[0] + '.relations']
except:
pass
for path in relations_file:
if os.path.exists(path):
if isfile(join(path)):
self.relations = csv.DictReader(open(relations_file[0], 'r'))
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, 'r'))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
self.file_format = file[0].split('.')[-1]
if self.file_format == 'json':
with open(file[0], 'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
if 'graph' in archesfile.keys():
self.graphs = archesfile['graph']
if 'reference_data' in archesfile.keys():
self.reference_data = archesfile['reference_data']
if 'business_data' in archesfile.keys():
self.business_data = archesfile['business_data']
elif self.file_format == 'csv':
data = unicodecsv.DictReader(open(file[0], 'r'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING')
self.business_data = list(data)
else:
print str(file) + ' is not a valid file'
else:
print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None):
self.business_data = ""
self.mapping = None
self.graphs = ""
self.reference_data = ""
self.business_data = ""
self.file_format = ""
self.relations = ""
try:
csv.field_size_limit(sys.maxsize)
except:
csv.field_size_limit(int(ctypes.c_ulong(-1).value // 2))
if not file:
file = settings.BUSINESS_DATA_FILES
else:
file = [file]
self.file = file
if mapping_file is None:
try:
mapping_file_base = os.path.splitext(file[0])[0]
mapping_file = [f"{mapping_file_base}.mapping"]
except:
print("*" * 80)
print(
"ERROR: Mapping file is missing or improperly named. Make sure you have \
mapping file with the same basename as your business data file and the extension .mapping"
)
print("*" * 80)
sys.exit()
else:
try:
mapping_file = [mapping_file]
except:
print("*" * 80)
print(
"ERROR: Mapping file is missing or improperly named. Make sure you have \
mapping file with the same basename as your business data file and the extension .mapping"
)
print("*" * 80)
sys.exit()
if relations_file is None:
try:
relations_file_base = os.path.splitext(file[0])[0]
relations_file = [f"{relations_file_base}.relations"]
except:
pass
for path in relations_file:
if os.path.exists(path):
if isfile(join(path)):
self.relations = csv.DictReader(
open(relations_file[0], "r"))
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, "r"))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
self.file_format = os.path.splitext(file[0])[1].strip(".")
if self.file_format == "json":
with open(file[0], "rU") as f:
archesfile = JSONDeserializer().deserialize(f)
if "graph" in list(archesfile.keys()):
self.graphs = archesfile["graph"]
if "reference_data" in list(archesfile.keys()):
self.reference_data = archesfile[
"reference_data"]
if "business_data" in list(archesfile.keys()):
self.business_data = archesfile[
"business_data"]
elif self.file_format == "csv":
data = csv.DictReader(open(file[0], encoding="utf-8"))
self.business_data = list(data)
elif self.file_format == "zip":
shp_zipfile = os.path.basename(path)
shp_zipfile_name = os.path.splitext(shp_zipfile)[0]
unzip_dir = os.path.join(os.path.dirname(path),
shp_zipfile_name)
unzip_file(path, unzip_dir)
shp = [
i for i in os.listdir(unzip_dir)
if i.endswith(".shp")
]
if len(shp) == 0:
print("*" * 80)
print(
"ERROR: There is no shapefile in this zipfile."
)
print("*" * 80)
exit()
elif len(shp) > 1:
print("*" * 80)
print(
"ERROR: There are multiple shapefiles in this zipfile. Please load each individually:"
)
for s in shp:
print(
"\npython manage.py packages -o import_business_data -s {0} -c {1} -ow [append or overwrite]"
.format(os.path.join(unzip_dir, s),
mapping_file[0]))
print("*" * 80)
exit()
shp_path = os.path.join(unzip_dir, shp[0])
self.business_data = self.shape_to_csv(shp_path)
elif self.file_format == "shp":
self.business_data = self.shape_to_csv(path)
else:
print(str(file) + " is not a valid file")
else:
print(path + " is not a valid path")
def __init__(self, file=None, mapping_file=None, relations_file=None):
self.business_data = ''
self.mapping = None
self.graphs = ''
self.reference_data = ''
self.business_data = ''
self.file_format = ''
self.relations = ''
if not file:
file = settings.BUSINESS_DATA_FILES
else:
file = [file]
if mapping_file == None:
try:
mapping_file = [file[0].split('.')[0] + '.mapping']
except:
print '*' * 80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*' * 80
sys.exit()
else:
try:
mapping_file = [mapping_file]
except:
print '*' * 80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*' * 80
sys.exit()
if relations_file == None:
try:
relations_file = [file[0].split('.')[0] + '.relations']
except:
pass
for path in relations_file:
if os.path.exists(path):
if isfile(join(path)):
self.relations = csv.DictReader(
open(relations_file[0], 'r'))
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, 'r'))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
self.file_format = file[0].split('.')[1]
if self.file_format == 'json':
with open(file[0], 'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
if 'graph' in archesfile.keys():
self.graphs = archesfile['graph']
if 'reference_data' in archesfile.keys():
self.reference_data = archesfile[
'reference_data']
if 'business_data' in archesfile.keys():
self.business_data = archesfile[
'business_data']
elif self.file_format == 'csv':
data = unicodecsv.DictReader(open(file[0], 'r'),
encoding='utf-8-sig',
restkey='ADDITIONAL',
restval='MISSING')
self.business_data = list(data)
else:
print str(file) + ' is not a valid file'
else:
print path + ' is not a valid path'
def __init__(self, file=None, mapping_file=None, relations_file=None):
self.business_data = ''
self.mapping = None
self.graphs = ''
self.reference_data = ''
self.business_data = ''
self.file_format = ''
self.relations = ''
csv.field_size_limit(sys.maxint)
if not file:
file = settings.BUSINESS_DATA_FILES
else:
file = [file]
if mapping_file == None:
try:
mapping_file = [file[0].split('.')[0] + '.mapping']
except:
print '*'*80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*'*80
sys.exit()
else:
try:
mapping_file = [mapping_file]
except:
print '*'*80
print "ERROR: Mapping file is missing or improperly named. Make sure you have mapping file with the same basename as your business data file and the extension .mapping"
print '*'*80
sys.exit()
if relations_file == None:
try:
relations_file = [file[0].split('.')[0] + '.relations']
except:
pass
for path in relations_file:
if os.path.exists(path):
if isfile(join(path)):
self.relations = csv.DictReader(open(relations_file[0], 'r'))
for path in mapping_file:
if os.path.exists(path):
if isfile(join(path)):
self.mapping = json.load(open(path, 'r'))
else:
self.mapping = None
for path in file:
if os.path.exists(path):
if isfile(join(path)):
self.file_format = file[0].split('.')[-1]
if self.file_format == 'json':
with open(file[0], 'rU') as f:
archesfile = JSONDeserializer().deserialize(f)
if 'graph' in archesfile.keys():
self.graphs = archesfile['graph']
if 'reference_data' in archesfile.keys():
self.reference_data = archesfile['reference_data']
if 'business_data' in archesfile.keys():
self.business_data = archesfile['business_data']
elif self.file_format == 'csv':
data = unicodecsv.DictReader(open(file[0], 'rU'), encoding='utf-8-sig', restkey='ADDITIONAL', restval='MISSING')
self.business_data = list(data)
elif self.file_format == 'zip':
shp_zipfile = os.path.basename(path)
shp_zipfile_name = os.path.splitext(shp_zipfile)[0]
unzip_dir = os.path.join(os.path.dirname(path),shp_zipfile_name)
unzip_file(path,unzip_dir)
shp = [i for i in os.listdir(unzip_dir) if i.endswith(".shp")]
if len(shp) == 0:
print '*'*80
print "ERROR: There is no shapefile in this zipfile."
print '*'*80
exit()
elif len(shp) > 1:
print '*'*80
print "ERROR: There are multiple shapefiles in this zipfile. Please load each individually:"
for s in shp:
print "\npython manage.py packages -o import_business_data -s {0} -c {1} -ow [append or overwrite]".format(
os.path.join(unzip_dir,s),mapping_file[0])
print '*'*80
exit()
shp_path = os.path.join(unzip_dir,shp[0])
self.business_data = self.shape_to_csv(shp_path)
elif self.file_format == 'shp':
self.business_data = self.shape_to_csv(path)
else:
print str(file) + ' is not a valid file'
else:
print path + ' is not a valid path'
