def get_by_code(code,
path,
area_type,
catalog_path,
with_attrs=False,
epsilon=5,
coord_out='EPSG:3857',
output="output",
display=False,
with_log=True):
area = Area(code,
media_path=path,
area_type=area_type,
epsilon=epsilon,
with_log=with_log,
catalog=catalog_path,
coord_out=coord_out)
abspath = os.path.abspath(output)
geojson = area.to_geojson_poly(with_attrs=with_attrs)
if geojson:
filename = '%s.geojson' % area.file_name
geojson_path = os.path.join(abspath, "geojson")
if not os.path.isdir(geojson_path):
os.makedirs(geojson_path)
file_path = os.path.join(geojson_path, filename)
f = open(file_path, 'w')
f.write(geojson)
f.close()
print(file_path)
if display:
area.show_plot()
return area
def get_by_code(code, output, display, **kwargs):
area = Area(code, **kwargs)
abspath = os.path.abspath(output)
geojson = area.to_geojson_poly()
if geojson:
filename = '%s.geojson' % area.file_name.replace(":", "_")
geojson_path = os.path.join(abspath, "geojson")
if not os.path.isdir(geojson_path):
os.makedirs(geojson_path)
file_path = os.path.join(geojson_path, filename)
f = open(file_path, 'w')
f.write(geojson)
f.close()
print("Vector - {}".format(file_path))
if display:
area.show_plot()
return area
def get_by_code(code, path, area_type, catalog_path, with_attrs=False, epsilon=5,
coord_out='EPSG:3857', output="output", display=False, center_only=False, with_log=True, with_proxy=False):
area = Area(code, media_path=path, area_type=area_type, epsilon=epsilon, with_log=with_log, catalog=catalog_path,
coord_out=coord_out, center_only=center_only, with_proxy=with_proxy)
abspath = os.path.abspath(output)
geojson = area.to_geojson_poly(with_attrs=with_attrs)
if geojson:
filename = '%s.geojson' % area.file_name.replace(":", "_")
geojson_path = os.path.join(abspath, "geojson")
if not os.path.isdir(geojson_path):
os.makedirs(geojson_path)
file_path = os.path.join(geojson_path, filename)
f = open(file_path, 'w')
f.write(geojson)
f.close()
print(file_path)
if display:
area.show_plot()
return area
def get_by_code(code, path, area_type, catalog_path, with_attrs=False, epsilon=5,
coord_out='EPSG:3857', output="output", display=False, center_only=False, with_log=True,
with_proxy=False, static_proxy="none", tile_mode="direct"):
area = Area(code, media_path=path, area_type=area_type, epsilon=epsilon, with_log=with_log, catalog=catalog_path,
coord_out=coord_out, center_only=center_only, with_proxy=with_proxy, static_proxy=static_proxy, tile_mode=tile_mode)
abspath = os.path.abspath(output)
geojson = area.to_geojson_poly(with_attrs=with_attrs)
if geojson:
filename = '%s.geojson' % area.file_name.replace(":", "_")
geojson_path = os.path.join(abspath, "geojson")
if not os.path.isdir(geojson_path):
os.makedirs(geojson_path)
file_path = os.path.join(geojson_path, filename)
f = open(file_path, 'w')
f.write(geojson)
f.close()
print(file_path)
if display:
area.show_plot()
return area
def batch_parser(codes,
area_type=1,
media_path="",
with_log=False,
catalog_path="",
coord_out="EPSG:3857",
file_name="example",
output=os.path.join("output"),
repeat=0,
areas=None,
with_attrs=False,
delay=1,
center_only=False,
with_proxy=False):
if areas is None:
areas = []
try:
catalog = Catalog(catalog_path)
except:
print("Catalog is required for batch mode!")
return
restores = []
with_error = []
with_no_coord = []
success = 0
from_catalog = 0
print("================================")
print("Launched parsing of %i areas:" % len(codes))
print("================================")
need_sleep = 0
features = []
for c in codes:
area = None
code = c.strip()
print("%s" % code, end="")
restore = catalog.find(code)
if not restore:
try:
sleep(need_sleep)
area = Area(code,
media_path=media_path,
area_type=area_type,
with_log=with_log,
coord_out=coord_out,
center_only=center_only,
with_proxy=with_proxy)
need_sleep = delay
restore = catalog.refresh(area)
if not (len(area.get_coord()) > 0):
print(" - no coord", end="")
with_no_coord.append(area)
else:
print(" - ok", end="")
success += 1
except TimeoutException:
print(" - error")
print("Your IP is probably blocked. Try later or use proxy")
break
except Exception as er:
print(" - error", end="")
with_error.append(code)
else:
from_catalog += 1
area = restore_area(restore,
media_path=media_path,
area_type=area_type,
with_log=with_log,
coord_out=coord_out,
center_only=center_only,
with_proxy=with_proxy)
if restore["image_path"]:
print(" - ok, from catalog", end="")
success += 1
else:
print(" - no_coord, from catalog", end="")
with_no_coord.append(area)
percent = ((success + len(with_error) + len(with_no_coord)) /
len(codes)) * 100
print(", %i%%" % percent)
restores.append(restore)
if area:
areas.append(area)
feature = area_json_output(output, area, with_attrs)
if feature:
features.append(feature)
area_csv_output(output, area)
catalog.close()
print("=================")
print("Parsing complate:")
print(" success : %i" % success)
print(" error : %i" % len(with_error))
print(" no_coord : %i" % len(with_no_coord))
print(" from catalog: %i" % from_catalog)
print("-----------------")
if len(with_error) and repeat:
print("Retries parse areas with error")
batch_parser(with_error,
area_type=area_type,
media_path=media_path,
with_log=with_log,
file_name=file_name,
catalog_path=catalog_path,
coord_out=coord_out,
repeat=repeat - 1,
areas=areas,
output=output,
delay=delay,
with_proxy=with_proxy)
else:
path = batch_csv_output(output, areas, file_name)
print("Create output complete: %s" % path)
if len(with_no_coord):
path = batch_csv_output(output, with_no_coord,
"%s_no_coord" % file_name)
print("Create output for no_coord complete: %s" % path)
if len(features):
batch_json_output(output, areas, file_name, with_attrs, coord_out)
if len(with_error):
print("-----------------")
print("Error list:")
for e in with_error:
print(e)
def batch_parser(codes, area_type=1, media_path="", with_log=False, catalog_path="", coord_out="EPSG:3857",
file_name="example", output=os.path.join("output"), repeat=0, areas=None, with_attrs=False, delay=1,
center_only=False, with_proxy=False):
if areas is None:
areas = []
try:
catalog = Catalog(catalog_path)
except:
print("Catalog is required for batch mode!")
return
restores = []
with_error = []
with_no_coord = []
success = 0
from_catalog = 0
print("================================")
print("Launched parsing of %i areas:" % len(codes))
print("================================")
need_sleep = 0
features = []
for c in codes:
area = None
code = c.strip()
print("%s" % code, end="")
restore = catalog.find(code)
if not restore:
try:
sleep(need_sleep)
area = Area(code, media_path=media_path, area_type=area_type, with_log=with_log, coord_out=coord_out,
center_only=center_only, with_proxy=with_proxy)
need_sleep = delay
restore = catalog.refresh(area)
if not (len(area.get_coord()) > 0):
print(" - no coord", end="")
with_no_coord.append(area)
else:
print(" - ok", end="")
success += 1
except TimeoutException:
print(" - error")
print("Your IP is probably blocked. Try later or use proxy")
break
except Exception as er:
print(" - error", end="")
with_error.append(code)
else:
from_catalog += 1
area = restore_area(restore, media_path=media_path, area_type=area_type, with_log=with_log, coord_out=coord_out,
center_only=center_only, with_proxy=with_proxy)
if restore["image_path"]:
print(" - ok, from catalog", end="")
success += 1
else:
print(" - no_coord, from catalog", end="")
with_no_coord.append(area)
percent = ((success + len(with_error) + len(with_no_coord)) / len(codes)) * 100
print(", %i%%" % percent)
restores.append(restore)
if area:
areas.append(area)
feature = area_json_output(output, area, with_attrs)
if feature:
features.append(feature)
area_csv_output(output, area)
catalog.close()
print("=================")
print("Parsing complate:")
print(" success : %i" % success)
print(" error : %i" % len(with_error))
print(" no_coord : %i" % len(with_no_coord))
print(" from catalog: %i" % from_catalog)
print("-----------------")
if len(with_error) and repeat:
print("Retries parse areas with error")
batch_parser(with_error, area_type=area_type, media_path=media_path, with_log=with_log, file_name=file_name,
catalog_path=catalog_path, coord_out=coord_out, repeat=repeat - 1, areas=areas, output=output,
delay=delay, with_proxy=with_proxy)
else:
path = batch_csv_output(output, areas, file_name)
print("Create output complete: %s" % path)
if len(with_no_coord):
path = batch_csv_output(output, with_no_coord, "%s_no_coord" % file_name)
print("Create output for no_coord complete: %s" % path)
if len(features):
batch_json_output(output, areas, file_name, with_attrs, coord_out)
if len(with_error):
print("-----------------")
print("Error list:")
for e in with_error:
print(e)
def batch_parser(codes,
with_log=False,
file_name="example",
areas=None,
output=os.path.join("output"),
repeat=0,
delay=1,
**kwargs):
if areas is None:
areas = []
with_error = []
with_no_coord = []
success = 0
print("================================")
print("Launched parsing of %i areas:" % len(codes))
print("================================")
need_sleep = 0
features = []
for c in codes:
area = None
code = c.strip()
print("%s" % code, end="")
try:
sleep(need_sleep)
area = Area(code, with_log=with_log, **kwargs)
need_sleep = delay
if not (len(area.get_coord()) > 0):
print(" - no coord", end="")
with_no_coord.append(area)
else:
print(" - ok", end="")
success += 1
except TimeoutException:
print(" - error")
print("Your IP is probably blocked. Try later or use proxy")
break
except Exception as er:
print(" - error", end="")
with_error.append(code)
percent = ((success + len(with_error) + len(with_no_coord)) /
len(codes)) * 100
print(", %i%%" % percent)
if area:
areas.append(area)
feature = area_json_output(output, area)
if feature:
features.append(feature)
# area_csv_output(output, area)
print("=================")
print("Parsing complete:")
print(" success : %i" % success)
print(" error : %i" % len(with_error))
print(" no_coord : %i" % len(with_no_coord))
print("-----------------")
if len(with_error) and repeat:
print("Retries parse areas with error")
batch_parser(with_error,
with_log=with_log,
file_name=file_name,
repeat=repeat - 1,
areas=areas,
output=output,
delay=delay,
**kwargs)
else:
path = batch_csv_output(output, areas, file_name)
if len(with_no_coord):
path = batch_csv_output(output, with_no_coord,
"%s_no_coord" % file_name)
print("Create output for no_coord complete: %s" % path)
if len(features):
path = batch_json_output(output, areas, file_name)
print("Create output complete: %s" % path)
if len(with_error):
print("-----------------")
print("Error list:")
for e in with_error:
print(e)
def batch_parser(codes, area_type=1, media_path="", with_log=False, catalog_path="", coord_out="EPSG:3857",
file_name="example", output=os.path.join("output"), repeat=5, areas=None, with_attrs=False):
if areas is None:
areas = []
catalog = Catalog(catalog_path)
restores = []
with_error = []
success = 0
from_catalog = 0
print("================================")
print("Launched parsing of %i areas:" % len(codes))
print("================================")
for c in codes:
code = c.strip()
print("%s" % code, end="")
restore = catalog.find(code)
if not restore:
try:
area = Area(code, media_path=media_path, area_type=area_type, with_log=with_log, coord_out=coord_out)
assert (len(area.get_coord()) > 0)
restore = catalog.update(area)
print(" - ok", end="")
success += 1
except Exception:
area = None
print(" - error", end="")
with_error.append(code)
else:
print(" - ok, from catalog", end="")
success += 1
from_catalog += 1
area = restore_area(restore, coord_out)
percent = ((success + len(with_error)) / len(codes)) * 100
print(", %i%%" % percent)
restores.append(restore)
if area:
areas.append(area)
area_json_output(output, area, with_attrs)
area_csv_output(output, area)
catalog.close()
print("=================")
print("Parsing complate:")
print(" success : %i" % success)
print(" error : %i" % len(with_error))
print(" from catalog: %i" % from_catalog)
print("-----------------")
if len(with_error) and repeat:
print("Retries parse areas with error")
batch_parser(with_error, area_type=area_type, media_path=media_path, with_log=with_log, file_name=file_name,
catalog_path=catalog_path, coord_out=coord_out, repeat=repeat - 1, areas=areas, output=output)
else:
path = batch_csv_output(output, areas, file_name)
print("Create output complete: %s" % path)
if len(with_error):
print("-----------------")
print("Error list:")
for e in with_error:
print(e)