def main(args):
dataset = load_config(args.dataset)
classes = dataset["common"]["classes"]
colors = dataset["common"]["colors"]
assert len(classes) == len(colors), "classes and colors coincide"
assert len(colors) == 2, "only binary models supported right now"
bg = colors[0]
fg = colors[1]
os.makedirs(args.out, exist_ok=True)
# We can only rasterize all tiles at a single zoom.
assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles))
with open(args.features) as f:
fc = json.load(f)
# Find all tiles the features cover and make a map object for quick lookup.
feature_map = collections.defaultdict(list)
for i, feature in enumerate(
tqdm(fc["features"], ascii=True, unit="feature")):
if feature["geometry"]["type"] != "Polygon":
continue
try:
for tile in burntiles.burn([feature], zoom=args.zoom):
feature_map[mercantile.Tile(*tile)].append(feature)
except ValueError as e:
print("Warning: invalid feature {}, skipping".format(i),
file=sys.stderr)
continue
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True,
unit="tile"):
if tile in feature_map:
out = burn(tile, feature_map[tile], args.size)
else:
out = np.zeros(shape=(args.size, args.size), dtype=np.uint8)
out_dir = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_dir, exist_ok=True)
out_path = os.path.join(out_dir, "{}.png".format(tile.y))
if os.path.exists(out_path):
prev = np.array(Image.open(out_path))
out = np.maximum(out, prev)
out = Image.fromarray(out, mode="P")
palette = make_palette(bg, fg)
out.putpalette(palette)
out.save(out_path, optimize=True)
def main(args):
dataset = load_config(args.dataset)
classes = dataset['common']['classes']
colors = dataset['common']['colors']
assert len(classes) == len(colors), 'classes and colors coincide'
assert len(colors) == 2, 'only binary models supported right now'
bg = colors[0]
fg = colors[1]
os.makedirs(args.out, exist_ok=True)
# We can only rasterize all tiles at a single zoom.
assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles))
with open(args.features) as f:
fc = json.load(f)
# Find all tiles the features cover and make a map object for quick lookup.
feature_map = collections.defaultdict(list)
for i, feature in enumerate(
tqdm(fc['features'], ascii=True, unit='feature')):
if feature['geometry']['type'] != 'Polygon':
continue
try:
for tile in burntiles.burn([feature], zoom=args.zoom):
feature_map[mercantile.Tile(*tile)].append(feature)
except ValueError as e:
print('Warning: invalid feature {}, skipping'.format(i),
file=sys.stderr)
continue
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True,
unit='tile'):
if tile in feature_map:
out = burn(tile, feature_map[tile], args.size)
else:
out = Image.fromarray(np.zeros(shape=(args.size,
args.size)).astype(int),
mode='P')
palette = make_palette(bg, fg)
out.putpalette(palette)
out_path = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_path, exist_ok=True)
out.save(os.path.join(out_path, '{}.png'.format(tile.y)),
optimize=True)
def test_read_tiles(self):
filename = "tests/fixtures/tiles.csv"
tiles = [tile for tile in tiles_from_csv(filename)]
tiles.sort()
self.assertEqual(len(tiles), 3)
self.assertEqual(tiles[1], mercantile.Tile(69623, 104945, 18))
def multi_burning(args, feature_map, bg, fg):
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True,
unit="tile"):
if tile in feature_map:
out = burn(tile, feature_map[tile], args.size, args.multicolors)
else:
out = np.zeros(shape=(args.size, args.size), dtype=np.uint8)
out_dir = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_dir, exist_ok=True)
out_path = os.path.join(out_dir, "{}.png".format(tile.y))
if os.path.exists(out_path):
prev = np.array(Image.open(out_path))
out = np.maximum(out, prev)
out = Image.fromarray(out, mode="P")
if args.multicolors is True:
random_colors = []
for graycolor in randomgrayscale():
random_colors.append(graycolor)
palette = make_palette_with_random(bg, random_colors=random_colors)
else:
palette = make_palette(bg, fg)
out.putpalette(palette)
out.save(out_path, optimize=True)
def main(args):
def difference_set(x, y):
"""
x包含y(set)
:param x:
:param y:
:return:
"""
if y is not None:
return x - y, None
else:
return x, None
path = args.tiles
label = ['training', 'validation', 'evaluation']
data_set = {'training': None, 'validation': None, 'evaluation': None}
data_rate = {
'training': args.training,
'validation': args.validation,
'evaluation': args.evaluation
}
tiles = set(tiles_from_csv(path))
num = len(tiles)
for data_label in tqdm(label, desc="split", ascii=True):
out = os.path.join(args.out, 'csv_' + data_label + '.tiles')
rate = data_rate[data_label]
for i in data_set.keys():
if i != data_label:
tiles, data_set[i] = difference_set(tiles, data_set[i])
tiles_list = list(tiles)
tiles_out = random.sample(tiles_list, int(num * rate))
with open(out, "w") as fp:
writer = csv.writer(fp)
writer.writerows(tiles_out)
def main(args):
log = Log(os.path.join(args.out, "log"), out=sys.stderr)
tiles = set(tiles_from_csv(args.cover))
extension = ""
for tile in tqdm(tiles, desc="Subset", unit="tiles", ascii=True):
paths = glob(os.path.join(args.dir, str(tile.z), str(tile.x), "{}.*".format(tile.y)))
if len(paths) != 1:
log.log("Warning: {} skipped.".format(tile))
continue
src = paths[0]
try:
extension = os.path.splitext(src)[1][1:]
dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}.{}".format(tile.y, extension))
if not os.path.isdir(os.path.join(args.out, str(tile.z), str(tile.x))):
os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
if args.move:
assert os.path.isfile(src)
shutil.move(src, dst)
else:
shutil.copyfile(src, dst)
except:
sys.exit("Error: Unable to process {}".format(tile))
if args.web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
web_ui(args.out, args.web_ui, tiles, tiles, extension, template)
def main(args):
dataset = load_config(args.dataset)
classes = dataset['common']['classes']
colors = dataset['common']['colors']
assert len(classes) == len(colors), 'classes and colors coincide'
assert len(colors) == 2, 'only binary models supported right now'
bg = colors[0]
fg = colors[1]
os.makedirs(args.out, exist_ok=True)
# We can only rasterize all tiles at a single zoom.
assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles))
with open(args.features) as f:
fc = json.load(f)
# Find all tiles the features cover and make a map object for quick lookup.
feature_map = collections.defaultdict(list)
for i, feature in enumerate(tqdm(fc['features'], ascii=True, unit='feature')):
if feature['geometry']['type'] != 'Polygon':
continue
try:
for tile in burntiles.burn([feature], zoom=args.zoom):
feature_map[mercantile.Tile(*tile)].append(feature)
except ValueError as e:
print('Warning: invalid feature {}, skipping'.format(i), file=sys.stderr)
continue
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True, unit='tile'):
if tile in feature_map:
out = burn(tile, feature_map[tile], args.size)
else:
out = Image.fromarray(np.zeros(shape=(args.size, args.size)).astype(int), mode='P')
palette = make_palette(bg, fg)
out.putpalette(palette)
out_path = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_path, exist_ok=True)
out.save(os.path.join(out_path, '{}.png'.format(tile.y)), optimize=True)
def main(args):
tiles = list(tiles_from_csv(args.tiles))
with requests.Session() as session:
num_workers = args.rate
# tqdm has problems with concurrent.futures.ThreadPoolExecutor; explicitly call `.update`
# https://github.com/tqdm/tqdm/issues/97
progress = tqdm(total=len(tiles), ascii=True, unit="image")
with futures.ThreadPoolExecutor(num_workers) as executor:
def worker(tile):
tick = time.monotonic()
x, y, z = map(str, [tile.x, tile.y, tile.z])
os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
path = os.path.join(args.out, z, x,
"{}.{}".format(y, args.ext))
if os.path.isfile(path):
return tile, True
url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
res = fetch_image(session, url)
if not res:
return tile, False
try:
image = Image.open(res)
image.save(path, optimize=True)
except OSError:
return tile, False
tock = time.monotonic()
time_for_req = tock - tick
time_per_worker = num_workers / args.rate
if time_for_req < time_per_worker:
time.sleep(time_per_worker - time_for_req)
progress.update()
return tile, True
for tile, ok in executor.map(worker, tiles):
if not ok:
print("Warning: {} failed, skipping".format(tile),
file=sys.stderr)
def single_burning(args, feature_map, bg, fg):
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.tiles)), ascii=True,
unit="tile"):
if tile in feature_map:
index = 0
for feature in feature_map[tile]:
out = burn_single(tile, feature, args.size, args.multicolors)
save_out(out, args, tile, bg, fg, index)
index = index + 1
else:
out = np.zeros(shape=(args.size, args.size), dtype=np.uint8)
save_out(out, args, tile, bg, fg, 0)
def main(args):
images = tiles_from_slippy_map(args.images)
tiles = set(tiles_from_csv(args.tiles))
for tile, src in tqdm(list(images), desc="Subset", unit="image", ascii=True):
if tile not in tiles:
continue
# The extention also includes the period.
extention = os.path.splitext(src)[1]
os.makedirs(os.path.join(args.out, str(tile.z), str(tile.x)), exist_ok=True)
dst = os.path.join(args.out, str(tile.z), str(tile.x), "{}{}".format(tile.y, extention))
shutil.copyfile(src, dst)
def main(args):
dataset = load_config(args.dataset)
classes = dataset["common"]["classes"]
colors = dataset["common"]["colors"]
assert len(classes) == len(colors), "classes and colors coincide"
assert len(colors) == 2, "only binary models supported right now"
bg = colors[0]
fg = colors[1]
os.makedirs(args.out, exist_ok=True)
# We can only rasterize all tiles at a single zoom.
assert all(tile.z == args.zoom for tile in tiles_from_csv(args.tiles))
with open(args.features) as f:
fc = json.load(f)
# Find all tiles the features cover and make a map object for quick lookup.
feature_map = collections.defaultdict(list)
for i, feature in enumerate(
tqdm(fc["features"], ascii=True, unit="feature")):
if feature["geometry"]["type"] != "Polygon":
continue
try:
for tile in burntiles.burn([feature], zoom=args.zoom):
feature_map[mercantile.Tile(*tile)].append(feature)
except ValueError as e:
print("Warning: invalid feature {}, skipping".format(i),
file=sys.stderr)
continue
single_burning(args, feature_map, bg, fg)
def test_read_tiles(self):
filename = 'tests/fixtures/tiles.csv'
tiles = [tile for tile in tiles_from_csv(filename)]
self.assertEqual(len(tiles), 3)
self.assertEqual(tiles[0], mercantile.Tile(69623, 104945, 18))
def main(args):
tiles = list(tiles_from_csv(args.tiles))
already_dl = 0
dl = 0
with requests.Session() as session:
num_workers = args.rate
os.makedirs(os.path.join(args.out), exist_ok=True)
log = Log(os.path.join(args.out, "log"), out=sys.stderr)
log.log("Begin download from {}".format(args.url))
# tqdm has problems with concurrent.futures.ThreadPoolExecutor; explicitly call `.update`
# https://github.com/tqdm/tqdm/issues/97
progress = tqdm(total=len(tiles), ascii=True, unit="image")
with futures.ThreadPoolExecutor(num_workers) as executor:
def worker(tile):
tick = time.monotonic()
x, y, z = map(str, [tile.x, tile.y, tile.z])
os.makedirs(os.path.join(args.out, z, x), exist_ok=True)
path = os.path.join(args.out, z, x, "{}.{}".format(y, args.ext))
if os.path.isfile(path):
return tile, None, True
if args.type == "XYZ":
url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
elif args.type == "TMS":
tile.y = (2 ** tile.z) - tile.y - 1
url = args.url.format(x=tile.x, y=tile.y, z=tile.z)
elif args.type == "WMS":
xmin, ymin, xmax, ymax = xy_bounds(tile)
url = args.url.format(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax)
res = fetch_image(session, url, args.timeout)
if not res:
return tile, url, False
try:
image = Image.open(res)
image.save(path, optimize=True)
except OSError:
return tile, url, False
tock = time.monotonic()
time_for_req = tock - tick
time_per_worker = num_workers / args.rate
if time_for_req < time_per_worker:
time.sleep(time_per_worker - time_for_req)
progress.update()
return tile, url, True
for tile, url, ok in executor.map(worker, tiles):
if url and ok:
dl += 1
elif not url and ok:
already_dl += 1
else:
log.log("Warning:\n {} failed, skipping.\n {}\n".format(tile, url))
if already_dl:
log.log("Notice:\n {} tiles were already downloaded previously, and so skipped now.".format(already_dl))
if already_dl + dl == len(tiles):
log.log(" Coverage is fully downloaded.")
if args.web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
web_ui(args.out, args.web_ui, tiles, tiles, args.ext, template)
def main(args):
config = load_config(args.config)
classes = config["classes"]["titles"]
colors = config["classes"]["colors"]
assert len(classes) == len(colors), "classes and colors coincide"
assert len(colors) == 2, "only binary models supported right now"
os.makedirs(args.out, exist_ok=True)
# We can only rasterize all tiles at a single zoom.
assert all(tile.z == args.zoom for tile in tiles_from_csv(args.cover))
# Find all tiles the features cover and make a map object for quick lookup.
feature_map = collections.defaultdict(list)
log = Log(os.path.join(args.out, "log"), out=sys.stderr)
def parse_polygon(feature_map, polygon, i):
try:
for i, ring in enumerate(
polygon["coordinates"]
): # GeoJSON coordinates could be N dimensionals
polygon["coordinates"][i] = [[
x, y
] for point in ring for x, y in zip([point[0]], [point[1]])]
for tile in burntiles.burn([{
"type": "feature",
"geometry": polygon
}],
zoom=args.zoom):
feature_map[mercantile.Tile(*tile)].append({
"type": "feature",
"geometry": polygon
})
except ValueError as e:
log.log("Warning: invalid feature {}, skipping".format(i))
return feature_map
def parse_geometry(feature_map, geometry, i):
if geometry["type"] == "Polygon":
feature_map = parse_polygon(feature_map, geometry, i)
elif geometry["type"] == "MultiPolygon":
for polygon in geometry["coordinates"]:
feature_map = parse_polygon(feature_map, {
"type": "Polygon",
"coordinates": polygon
}, i)
else:
log.log(
"Notice: {} is a non surfacic geometry type, skipping feature {}"
.format(geometry["type"], i))
return feature_map
for feature in args.features:
with open(feature) as f:
fc = json.load(f)
for i, feature in enumerate(
tqdm(fc["features"], ascii=True, unit="feature")):
if feature["geometry"]["type"] == "GeometryCollection":
for geometry in feature["geometry"]["geometries"]:
feature_map = parse_geometry(feature_map, geometry, i)
else:
feature_map = parse_geometry(feature_map,
feature["geometry"], i)
# Burn features to tiles and write to a slippy map directory.
for tile in tqdm(list(tiles_from_csv(args.cover)), ascii=True,
unit="tile"):
if tile in feature_map:
out = burn(tile, feature_map[tile], args.size)
else:
out = np.zeros(shape=(args.size, args.size), dtype=np.uint8)
out_dir = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_dir, exist_ok=True)
out_path = os.path.join(out_dir, "{}.png".format(tile.y))
if os.path.exists(out_path):
prev = np.array(Image.open(out_path))
out = np.maximum(out, prev)
out = Image.fromarray(out, mode="P")
out_path = os.path.join(args.out, str(tile.z), str(tile.x))
os.makedirs(out_path, exist_ok=True)
out.putpalette(
complementary_palette(make_palette(colors[0], colors[1])))
out.save(os.path.join(out_path, "{}.png".format(tile.y)),
optimize=True)
if args.web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
tiles = [tile for tile in tiles_from_csv(args.cover)]
web_ui(args.out, args.web_ui, tiles, tiles, "png", template)
