def draw_robot_error(self): robot_pos = self.inches_to_pixels( flip_y( align_with_origin(self.log_info[self.step].actual_pos.pos, self.alliance))) target_x, target_y, target_h = v3_align_with_origin( self.log_info[self.step].abs_target.v3, self.alliance) target_pos = self.inches_to_pixels(flip_y((target_x, target_y))) heading_target_vector = self.inches_to_pixels( rotate_vector((0, 5), target_h + 180, True)) abs_heading_target = (target_pos[0] + heading_target_vector[0], target_pos[1] + heading_target_vector[1]) pygame.draw.line(self.screen, (255, 0, 0), robot_pos, target_pos, 5) pygame.draw.line(self.screen, (0, 255, 0), target_pos, abs_heading_target, 5)
def grid(self, z, x, y, callback=None): if self.tilescheme == 'tms': y = flip_y(y, z) rows = self._query( '''SELECT grid FROM grids WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, y)) t = rows.fetchone() if not t: raise ExtractionError( _("Could not extract grid %s from %s") % ((z, x, y), self.filename)) grid_json = json.loads(zlib.decompress(t[0])) rows = self._query( '''SELECT key_name, key_json FROM grid_data WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) # join up with the grid 'data' which is in pieces when stored in mbtiles file grid_json['data'] = {} grid_data = rows.fetchone() while grid_data: grid_json['data'][grid_data[0]] = json.loads(grid_data[1]) grid_data = rows.fetchone() serialized = json.dumps(grid_json) if callback is not None: return '%s(%s);' % (callback, serialized) return serialized
def draw_robot(self): x, y, angle = v3_align_with_origin( self.log_info[self.step].actual_pos.v3, self.alliance) robot_rect = self.robot_surface.get_rect() robot_rect.center = self.inches_to_pixels(flip_y((x, y))) self.screen.blit(pygame.transform.rotate(self.robot_surface, angle), robot_rect)
class Disk(Cache): def __init__(self, basename, folder, **kwargs): super(Disk, self).__init__(**kwargs) self._basename = None self._basefolder = folder self.folder = folder self.basename = basename @property def basename(self): return self._basename @basename.setter def basename(self, basename): self._basename = basename subfolder = re.sub(r'[^a-z^A-Z^0-9^_]+', '', basename.replace("/", "_").lower()) self.folder = os.path.join(self._basefolder, subfolder) @Cache.scheme.setter def scheme(self, scheme): assert scheme in ('wmts', 'xyz', 'tms'), "Unknown scheme %s" % scheme self._scheme = 'xyz' if (scheme == 'wmts') else scheme def tile_file(self, (z, x, y)): tile_dir = os.path.join("%s" % z, "%s" % x) if (self.scheme != 'xyz'): y = flip_y(y, z) tile_name = "%s%s" % (y, self.extension) return tile_dir, tile_name
def convert_tile_to_bbox(tile_z, tile_x, tile_y, flip_tile_y): if flip_tile_y: tile_y = flip_y(tile_z, tile_y) min_x, min_y = tile_to_coordinate(tile_x - 0.5, tile_y + 0.5, tile_z) max_x, max_y = tile_to_coordinate(tile_x + 0.5, tile_y - 0.5, tile_z) return [min_x, min_y, max_x, max_y]
def tile(self, z, x, y): logger.debug(_("Extract tile %s") % ((z, x, y),)) tms_y = flip_y(int(y), int(z)) rows = self._query('''SELECT tile_data FROM tiles WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) t = rows.fetchone() if not t: raise ExtractionError(_("Could not extract tile %s from %s") % ((z, x, y), self.filename)) return t[0]
class Cache(object): def __init__(self, **kwargs): self.extension = kwargs.get('extension', '.png') self._scheme = 'tms' def tile_file(self, (z, x, y)): tile_dir = os.path.join("%s" % z, "%s" % x) y = flip_y(y, z) tile_name = "%s%s" % (y, self.extension) return tile_dir, tile_name
def tiles_for_bbox(left, bottom, right, top, tile_z, flip_tile_y): min_x, min_y = coordinate_to_tile(left, bottom, tile_z) max_x, max_y = coordinate_to_tile(right, top, tile_z) if min_y > max_y: min_y, max_y = max_y, min_y for tile_x in range(min_x, max_x+1): for tile_y in range(min_y, max_y+1): if flip_tile_y: tile_y = flip_y(tile_z, tile_y) yield [tile_z, tile_x, tile_y]
def display_zebra_motionworks(self): data = self.zmw.data index = self.zmw.closest_time_index(self.stopwatch.get_time()) robots = [] for alliance, teams in data.items(): robots.extend([(alliance, team, coords_list[index]) for team, coords_list in teams.items()]) for robot in robots: color = (255, 0, 0) if robot[0] == "red" else (0, 0, 255) try: pygame.draw.circle( self.screen, color, self.inches_to_pixels( flip_y([(coord * 12) for coord in robot[2]])), 7) except TypeError: pass
def mbtiles_tilelist(mbtiles_file, **kwargs): flip_tile_y = kwargs.get('flip_y', False) as_bboxes = kwargs.get('as_bboxes', False) scale = kwargs.get('tile_scale', None) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) if zoom >= 0: min_zoom = max_zoom = zoom con = mbtiles_connect(mbtiles_file, auto_commit, journal_mode, synchronous_off, False, True) zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Tile list for %s (%s)" % (prettify_connect_string(con.connect_string), zoom_level_string)) for tile_z in range(min_zoom, max_zoom+1): logger.debug("Starting zoom level %d" % (tile_z)) for t in con.columns_and_rows_for_zoom_level(tile_z, scale): tile_x, tile_y = int(t[0]), int(t[1]) if as_bboxes: convert_tile_to_bbox(tile_z, tile_x, tile_y, flip_tile_y) else: if flip_tile_y: tile_y = flip_y(tile_z, tile_y) sys.stdout.write("%d/%d/%d\n" % (tile_z, tile_x, tile_y)) con.close()
def grid(self, z, x, y, callback=None): tms_y = flip_y(int(y), int(z)) rows = self._query('''SELECT grid FROM grids WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) t = rows.fetchone() if not t: raise ExtractionError(_("Could not extract grid %s from %s") % ((z, x, y), self.filename)) grid_json = json.loads(zlib.decompress(t[0])) rows = self._query('''SELECT key_name, key_json FROM grid_data WHERE zoom_level=? AND tile_column=? AND tile_row=?;''', (z, x, tms_y)) # join up with the grid 'data' which is in pieces when stored in mbtiles file grid_json['data'] = {} grid_data = rows.fetchone() while grid_data: grid_json['data'][grid_data[0]] = json.loads(grid_data[1]) grid_data = rows.fetchone() serialized = json.dumps(grid_json) if callback is not None: return '%s(%s);' % (callback, serialized) return serialized
def __init__(self, filename, tilesize=None, tilescheme='xyz'): super(MBTilesReader, self).__init__(tilesize) self.filename = filename self.basename = os.path.basename(self.filename) self._con = None self._cur = None self.tilescheme = tilescheme # read the scheme from the metadata metadata = self.metadata() if 'scheme' in metadata: self.tilescheme = metadata['scheme'] if self.tilescheme != 'xyz' and self.tilescheme != 'tms': raise InvalidFormatError(_("unknown scheme: ") + scheme) # read tile size from file from StringIO import StringIO try: import Image except ImportError: from PIL import Image for z in self.zoomlevels(): query = self._query( '''SELECT zoom_level, tile_column, tile_row FROM tiles WHERE zoom_level=? ;''', (z, )) try: t = query.fetchone() if self.tilescheme == 'tms': t = t[0], t[1], flip_y(t[2], t[0]) img = Image.open(StringIO(apply(self.tile, t))) if img.width != img.height: raise InvalidFormatError(_("tile not square!") + t) self.tilesize = img.width return except IOError: print 'invalid tile at zoom level', z print 'no valid first tiles found in any zoom level!'
def __init__(self, filename, tilesize=None, tilescheme='xyz'): super(MBTilesReader, self).__init__(tilesize) self.filename = filename self.basename = os.path.basename(self.filename) self._con = None self._cur = None self.tilescheme = tilescheme # read the scheme from the metadata metadata = self.metadata() if 'scheme' in metadata: self.tilescheme = metadata['scheme'] if self.tilescheme != 'xyz' and self.tilescheme != 'tms': raise InvalidFormatError(_("unknown scheme: ") + scheme) # read tile size from file from StringIO import StringIO try: import Image except ImportError: from PIL import Image for z in self.zoomlevels(): query = self._query('''SELECT zoom_level, tile_column, tile_row FROM tiles WHERE zoom_level=? ;''', (z, )) try: t = query.fetchone() if self.tilescheme == 'tms': t = t[0], t[1], flip_y(t[2], t[0]) img = Image.open(StringIO(apply(self.tile, t))) if img.width != img.height: raise InvalidFormatError(_("tile not square!") + t) self.tilesize = img.width return except IOError: print 'invalid tile at zoom level', z print 'no valid first tiles found in any zoom level!'
def disk_to_mbtiles(directory_path, mbtiles_file, **kwargs): auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) print_progress = kwargs.get('progress', False) flip_tile_y = kwargs.get('flip_y', False) scale = kwargs.get('tile_scale', None) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) tmp_dir = kwargs.get('tmp_dir', None) if tmp_dir and not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) if zoom >= 0: min_zoom = max_zoom = zoom con = mbtiles_connect(mbtiles_file, auto_commit, journal_mode, synchronous_off, False) con.mbtiles_setup() # if not con.is_compacted(): # con.close() # logger.info("The mbtiles database must be compacted, exiting...") # return con.mbtiles_setup() zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Importing path:'%s' --> %s (%s)" % (directory_path, prettify_connect_string(con.connect_string), zoom_level_string)) image_format = 'png' try: metadata = json.load(open(os.path.join(directory_path, 'metadata.json'), 'r')) image_format = metadata.get('format', 'png') # Check that the old and new image formats are the same receiving_metadata = con.metadata() if receiving_metadata != None and len(receiving_metadata) > 0: original_format = receiving_metadata.get('format') if original_format != None and image_format != original_format: sys.stderr.write('The databases to merge must use the same image format (png or jpg)\n') sys.exit(1) for name, value in metadata.items(): con.update_metadata(name, value) logger.info('metadata from metadata.json restored') except IOError: logger.warning('metadata.json not found') count = 0 start_time = time.time() if print_progress: sys.stdout.write("0 tiles imported (0 tiles/sec)") sys.stdout.flush() known_tile_ids = set() tmp_images_list = [] tmp_row_list = [] tmp_tiles_list = [] for r1, zs, ignore in os.walk(os.path.join(directory_path, "tiles")): for tile_z in zs: if int(tile_z) < min_zoom or int(tile_z) > max_zoom: continue for r2, xs, ignore in os.walk(os.path.join(r1, tile_z)): for tile_x in xs: for r2, ignore, ys in os.walk(os.path.join(r1, tile_z, tile_x)): for tile_y in ys: tile_y, extension = tile_y.split('.') f = open(os.path.join(r1, tile_z, tile_x, tile_y) + '.' + extension, 'rb') tile_data = f.read() f.close() if flip_tile_y: tile_y = str(flip_y(tile_z, tile_y)) # Execute commands if kwargs.get('command_list'): tile_data = execute_commands_on_tile(kwargs['command_list'], image_format, tile_data, tmp_dir) if con.is_compacted(): m = hashlib.md5() m.update(tile_data) tile_id = m.hexdigest() if tile_id not in known_tile_ids: tmp_images_list.append( (tile_id, tile_data) ) known_tile_ids.add(tile_id) tmp_row_list.append( (tile_z, tile_x, tile_y, 1, tile_id, int(time.time())) ) else: tmp_tiles_list.append( (tile_z, tile_x, tile_y, 1, tile_data, int(time.time())) ) count = count + 1 if (count % 100) == 0: logger.debug("%d tiles imported (%.1f tiles/sec)" % (count, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d tiles imported (%.1f tiles/sec)" % (count, count / (time.time() - start_time))) sys.stdout.flush() if len(tmp_images_list) > 250: con.insert_tiles_to_images(tmp_images_list) tmp_images_list = [] if len(tmp_row_list) > 250: con.insert_tiles_to_map(tmp_row_list) tmp_row_list = [] if len(tmp_tiles_list) > 250: con.insert_tiles(tmp_tiles_list) tmp_tiles_list = [] # Push the remaining rows to the database if len(tmp_images_list) > 0: con.insert_tiles_to_images(tmp_images_list) if len(tmp_row_list) > 0: con.insert_tiles_to_map(tmp_row_list) if len(tmp_tiles_list) > 0: con.insert_tiles(tmp_tiles_list) if print_progress: sys.stdout.write('\n') logger.info("%d tiles imported." % (count)) if print_progress: sys.stdout.write("%d tiles imported.\n" % (count)) sys.stdout.flush() con.optimize_database(kwargs.get('skip_analyze', False), kwargs.get('skip_vacuum', False)) con.close()
def merge_mbtiles(mbtiles_file1, mbtiles_file2, **kwargs): scale = kwargs.get('tile_scale', None) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) tmp_dir = kwargs.get('tmp_dir', None) auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) min_timestamp = kwargs.get('min_timestamp', 0) max_timestamp = kwargs.get('max_timestamp', 0) delete_after_export = kwargs.get('delete_after_export', False) print_progress = kwargs.get('progress', False) delete_vanished_tiles = kwargs.get('delete_vanished_tiles', False) flip_tile_y = kwargs.get('flip_y', False) debug = kwargs.get('debug', False) if tmp_dir and not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) if zoom >= 0: min_zoom = max_zoom = zoom check_before_merge = kwargs.get('check_before_merge', False) if check_before_merge and not check_mbtiles(mbtiles_file2, **kwargs): sys.stderr.write("The pre-merge check on %s failed\n" % (mbtiles_file2)) sys.exit(1) con1 = mbtiles_connect(mbtiles_file1, auto_commit, journal_mode, synchronous_off, False, False) con2 = mbtiles_connect(mbtiles_file2, auto_commit, journal_mode, synchronous_off, False, True) con1.mbtiles_setup() # if not con1.is_compacted(): # sys.stderr.write('To merge two mbtiles databases, the receiver must already be compacted\n') # con1.close() # con2.close() # sys.exit(1) if not con2.is_compacted() and (min_timestamp != 0 or max_timestamp != 0): con1.close() con2.close() sys.stderr.write('min-timestamp/max-timestamp can only be used with compacted databases.\n') sys.exit(1) zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Merging %s --> %s (%s)" % (prettify_connect_string(con2.connect_string), prettify_connect_string(con1.connect_string), zoom_level_string)) # Check that the old and new image formats are the same original_format = new_format = None try: original_format = con1.metadata().get('format') except: pass try: new_format = con2.metadata().get('format') except: pass if new_format == None: logger.info("No image format found in the sending database, assuming 'png'") new_format = "png" if original_format != None and new_format != original_format: con1.close() con2.close() sys.stderr.write('The files to merge must use the same image format (png or jpg)\n') sys.exit(1) if original_format == None and new_format != None: con1.update_metadata("format", new_format) if new_format == None: new_format = original_format count = 0 start_time = time.time() chunk = 1000 total_tiles = 1 if print_progress or debug: total_tiles = con2.tiles_count(min_zoom, max_zoom, min_timestamp, max_timestamp, scale) if total_tiles == 0: con1.close() con2.close() sys.stderr.write('No tiles to merge, exiting...\n') return logger.debug("%d tiles to merge" % (total_tiles)) if print_progress: sys.stdout.write("%d tiles to merge\n" % (total_tiles)) sys.stdout.write("0 tiles merged (0% @ 0 tiles/sec)") sys.stdout.flush() # merge and process (--merge --execute) if con2.is_compacted() and kwargs['command_list']: default_pool_size = kwargs.get('poolsize', -1) if default_pool_size < 1: default_pool_size = None logger.debug("Using default pool size") else: logger.debug("Using pool size = %d" % (default_pool_size)) pool = Pool(default_pool_size) multiprocessing.log_to_stderr(logger.level) tiles_to_process = [] known_tile_ids = {} for t in con2.tiles_with_tile_id(min_zoom, max_zoom, min_timestamp, max_timestamp, scale): tile_z = t[0] tile_x = t[1] tile_y = t[2] tile_scale = t[3] tile_data = str(t[4]) tile_id = t[5] if flip_tile_y: tile_y = flip_y(tile_z, tile_y) new_tile_id = known_tile_ids.get(tile_id) if new_tile_id is None: tmp_file_fd, tmp_file_name = tempfile.mkstemp(suffix=".%s" % (new_format), prefix="tile_", dir=tmp_dir) tmp_file = os.fdopen(tmp_file_fd, "w") tmp_file.write(tile_data) tmp_file.close() tiles_to_process.append({ 'tile_id':tile_id, 'filename':tmp_file_name, 'format':new_format, 'size':len(tile_data), 'command_list':kwargs['command_list'], 'tile_x':tile_x, 'tile_y':tile_y, 'tile_z':tile_z, 'tile_scale':tile_scale }) else: con1.insert_tile_to_map(tile_z, tile_x, tile_y, tile_scale, new_tile_id) count = count + 1 if (count % 100) == 0: logger.debug("%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) sys.stdout.flush() if len(tiles_to_process) < chunk: continue count = process_tiles(pool, tiles_to_process, con1, count, total_tiles, start_time, print_progress, delete_vanished_tiles, known_tile_ids) tiles_to_process = [] if len(tiles_to_process) > 0: count = process_tiles(pool, tiles_to_process, con1, count, total_tiles, start_time, print_progress, delete_vanished_tiles, known_tile_ids) # merge from a compacted database (--merge) elif con2.is_compacted(): known_tile_ids = set() tmp_images_list = [] tmp_row_list = [] tmp_tiles_list = [] for t in con2.tiles_with_tile_id(min_zoom, max_zoom, min_timestamp, max_timestamp, scale): tile_z = t[0] tile_x = t[1] tile_y = t[2] tile_scale = t[3] tile_data = str(t[4]) tile_id = t[5] if flip_tile_y: tile_y = flip_y(tile_z, tile_y) if con1.is_compacted(): if tile_id not in known_tile_ids: tmp_images_list.append( (tile_id, tile_data) ) known_tile_ids.add(tile_id) tmp_row_list.append( (tile_z, tile_x, tile_y, tile_scale, tile_id, int(time.time())) ) else: tmp_tiles_list.append( (tile_z, tile_x, tile_y, tile_scale, tile_data, int(time.time())) ) count = count + 1 if (count % 100) == 0: logger.debug("%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) sys.stdout.flush() if len(tmp_images_list) > 250: con1.insert_tiles_to_images(tmp_images_list) tmp_images_list = [] if len(tmp_row_list) > 250: con1.insert_tiles_to_map(tmp_row_list) tmp_row_list = [] if len(tmp_tiles_list) > 250: con1.insert_tiles(tmp_tiles_list) tmp_tiles_list = [] # Push the remaining rows to the database if len(tmp_images_list) > 0: con1.insert_tiles_to_images(tmp_images_list) if len(tmp_row_list) > 0: con1.insert_tiles_to_map(tmp_row_list) if len(tmp_tiles_list) > 0: con1.insert_tiles(tmp_tiles_list) # merge an uncompacted database (--merge) else: known_tile_ids = set() tmp_images_list = [] tmp_row_list = [] tmp_tiles_list = [] for t in con2.tiles(min_zoom, max_zoom, min_timestamp, max_timestamp, scale): tile_z = t[0] tile_x = t[1] tile_y = t[2] tile_scale = t[3] tile_data = str(t[4]) if flip_tile_y: tile_y = flip_y(tile_z, tile_y) # Execute commands if kwargs.get('command_list'): tile_data = execute_commands_on_tile(kwargs['command_list'], new_format, tile_data, tmp_dir) if con1.is_compacted(): m = hashlib.md5() m.update(tile_data) tile_id = m.hexdigest() if tile_id not in known_tile_ids: tmp_images_list.append( (tile_id, tile_data) ) known_tile_ids.add(tile_id) tmp_row_list.append( (tile_z, tile_x, tile_y, tile_scale, tile_id, int(time.time())) ) else: tmp_tiles_list.append( (tile_z, tile_x, tile_y, tile_scale, tile_data, int(time.time())) ) count = count + 1 if (count % 100) == 0: logger.debug("%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) sys.stdout.flush() if len(tmp_images_list) > 250: con1.insert_tiles_to_images(tmp_images_list) tmp_images_list = [] if len(tmp_row_list) > 250: con1.insert_tiles_to_map(tmp_row_list) tmp_row_list = [] if len(tmp_tiles_list) > 250: con1.insert_tiles(tmp_tiles_list) tmp_tiles_list = [] # Push the remaining rows to the database if len(tmp_images_list) > 0: con1.insert_tiles_to_images(tmp_images_list) if len(tmp_row_list) > 0: con1.insert_tiles_to_map(tmp_row_list) if len(tmp_tiles_list) > 0: con1.insert_tiles(tmp_tiles_list) if print_progress: sys.stdout.write('\n') logger.info("%d tiles merged (100.0%% @ %.1f tiles/sec)" % (count, count / (time.time() - start_time))) if print_progress: sys.stdout.write("%d tiles merged (100.0%% @ %.1f tiles/sec)\n" % (count, count / (time.time() - start_time))) sys.stdout.flush() if delete_after_export: logger.debug("WARNING: Removing merged tiles from %s" % (mbtiles_file2)) con2.delete_tiles(min_zoom, max_zoom, min_timestamp, max_timestamp, scale) con2.optimize_database(kwargs.get('skip_analyze', False), kwargs.get('skip_vacuum', False)) con1.close() con2.close()
def fill_mbtiles(mbtiles_file, image_filename, **kwargs): zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) flip_tile_y = kwargs.get('flip_y', False) bbox = kwargs.get('bbox', None) tile_bbox = kwargs.get('tile_bbox', None) auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) print_progress = kwargs.get('progress', False) if zoom >= 0: min_zoom = max_zoom = zoom elif min_zoom == max_zoom: zoom = min_zoom if tile_bbox != None and zoom < 0: logger.info("--tile-bbox can only be used with --zoom, exiting...") return if tile_bbox == None and bbox == None: logger.info("Either --tile-bbox or --bbox must be given, exiting...") return con = mbtiles_connect(mbtiles_file, auto_commit, journal_mode, synchronous_off, False) con.mbtiles_setup() if not con.is_compacted(): con.close() logger.info("The mbtiles database must be compacted, exiting...") return zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Filling %s (%s)" % (prettify_connect_string(con.connect_string), zoom_level_string)) # Insert an image tmp_file = open(image_filename, "r") tile_data = tmp_file.read() tmp_file.close() m = hashlib.md5() m.update(tile_data) tile_id = m.hexdigest() con.insert_tile_to_images(tile_id, tile_data) count = 0 start_time = time.time() for tile_z in range(min_zoom, max_zoom+1): min_x = min_y = max_x = max_y = 0 if tile_bbox != None: match = re.match(r'(\d+),(\d+),(\d+),(\d+)', tile_bbox, re.I) if match: min_x, min_y, max_x, max_y = int(match.group(1)), int(match.group(2)), int(match.group(3)), int(match.group(4)) elif bbox != None: match = re.match(r'([-0-9\.]+),([-0-9\.]+),([-0-9\.]+),([-0-9\.]+)', bbox, re.I) if match: left, bottom, right, top = float(match.group(1)), float(match.group(2)), float(match.group(3)), float(match.group(4)) min_x, min_y = coordinate_to_tile(left, bottom, tile_z) max_x, max_y = coordinate_to_tile(right, top, tile_z) if min_y > max_y: min_y, max_y = max_y, min_y for tile_x in range(min_x, max_x+1): for tile_y in range(min_y, max_y+1): if flip_tile_y: tile_y = flip_y(tile_z, tile_y) # z, x, y con.insert_tile_to_map(tile_z, tile_x, tile_y, tile_id, False) # Don't overwrite existing tiles count = count + 1 if (count % 100) == 0: logger.debug("%d tiles inserted (%.1f tiles/sec)" % (count, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d tiles inserted (%.1f tiles/sec)" % (count, count / (time.time() - start_time))) sys.stdout.flush() if print_progress: sys.stdout.write('\n') logger.info("%d tiles inserted (100.0%% @ %.1f tiles/sec)" % (count, count / (time.time() - start_time))) if print_progress: sys.stdout.write("%d tiles inserted (100.0%% @ %.1f tiles/sec)\n" % (count, count / (time.time() - start_time))) sys.stdout.flush() con.optimize_database(kwargs.get('skip_analyze', False), kwargs.get('skip_vacuum', False)) con.close()
def disk_to_mbtiles(directory_path, mbtiles_file, **kwargs): logger.info("Importing from disk to database: %s --> %s" % (directory_path, mbtiles_file)) import_into_existing_mbtiles = os.path.isfile(mbtiles_file) existing_mbtiles_is_compacted = True no_overwrite = kwargs.get('no_overwrite', False) auto_commit = kwargs.get('auto_commit', False) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 255) if zoom >= 0: min_zoom = max_zoom = zoom con = mbtiles_connect(mbtiles_file, auto_commit) cur = con.cursor() optimize_connection(cur, False) if import_into_existing_mbtiles: existing_mbtiles_is_compacted = (con.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='images'").fetchone()[0] > 0) else: mbtiles_setup(cur) image_format = 'png' try: metadata = json.load(open(os.path.join(directory_path, 'metadata.json'), 'r')) image_format = metadata.get('format', 'png') # Check that the old and new image formats are the same if import_into_existing_mbtiles: original_format = None try: original_format = cur.execute("SELECT value FROM metadata WHERE name='format'").fetchone()[0] except: pass if original_format != None and image_format != original_format: sys.stderr.write('The files to merge must use the same image format (png or jpg)\n') sys.exit(1) if not import_into_existing_mbtiles: for name, value in metadata.items(): cur.execute('INSERT OR IGNORE INTO metadata (name, value) VALUES (?, ?)', (name, value)) con.commit() logger.info('metadata from metadata.json restored') except IOError: logger.warning('metadata.json not found') existing_tiles = {} if no_overwrite: tiles = cur.execute("""SELECT zoom_level, tile_column, tile_row FROM tiles WHERE zoom_level>=? AND zoom_level<=?""", (min_zoom, max_zoom)) t = tiles.fetchone() while t: z = str(t[0]) x = str(t[1]) y = str(t[2]) zoom = existing_tiles.get(z, None) if not zoom: zoom = {} existing_tiles[z] = zoom row = zoom.get(y, None) if not row: row = set() zoom[y] = row row.add(x) t = tiles.fetchone() count = 0 start_time = time.time() for r1, zs, ignore in os.walk(os.path.join(directory_path, "tiles")): for z in zs: if int(z) < min_zoom or int(z) > max_zoom: continue for r2, xs, ignore in os.walk(os.path.join(r1, z)): for x in xs: for r2, ignore, ys in os.walk(os.path.join(r1, z, x)): for y in ys: y, extension = y.split('.') if no_overwrite: if x in existing_tiles.get(z, {}).get(y, set()): logging.debug("Ignoring tile (%s, %s, %s)" % (z, x, y)) continue if kwargs.get('flip_y', False) == True: y = flip_y(z, y) f = open(os.path.join(r1, z, x, y) + '.' + extension, 'rb') tile_data = f.read() f.close() # Execute commands if kwargs.get('command_list'): tile_data = execute_commands_on_tile(kwargs['command_list'], image_format, tile_data) if existing_mbtiles_is_compacted: m = hashlib.md5() m.update(tile_data) tile_id = m.hexdigest() cur.execute("""INSERT OR IGNORE INTO images (tile_id, tile_data) VALUES (?, ?)""", (tile_id, sqlite3.Binary(tile_data))) cur.execute("""REPLACE INTO map (zoom_level, tile_column, tile_row, tile_id) VALUES (?, ?, ?, ?)""", (z, x, y, tile_id)) else: cur.execute("""REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) VALUES (?, ?, ?, ?)""", (z, x, y.split('.')[0], sqlite3.Binary(tile_data))) count = count + 1 if (count % 100) == 0: logger.debug("%s tiles imported (%d tiles/sec)" % (count, count / (time.time() - start_time))) logger.info("%d tiles imported." % (count)) con.commit() con.close()
def mbtiles_to_disk(mbtiles_file, directory_path, **kwargs): delete_after_export = kwargs.get('delete_after_export', False) auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) tmp_dir = kwargs.get('tmp_dir', None) print_progress = kwargs.get('progress', False) flip_tile_y = kwargs.get('flip_y', False) min_timestamp = kwargs.get('min_timestamp', 0) max_timestamp = kwargs.get('max_timestamp', 0) if tmp_dir and not os.path.isdir(tmp_dir): os.mkdir(tmp_dir) if zoom >= 0: min_zoom = max_zoom = zoom con = mbtiles_connect(mbtiles_file, auto_commit, journal_mode, synchronous_off, False, True) zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Exporting %s --> path:'%s' (%s)" % (prettify_connect_string(con.connect_string), directory_path, zoom_level_string)) if not os.path.isdir(directory_path): os.mkdir(directory_path) base_path = os.path.join(directory_path, "tiles") if not os.path.isdir(base_path): os.makedirs(base_path) metadata = con.metadata() json.dump(metadata, open(os.path.join(directory_path, 'metadata.json'), 'w'), indent=4) count = 0 start_time = time.time() image_format = metadata.get('format', 'png') sending_mbtiles_is_compacted = con.is_compacted() if not sending_mbtiles_is_compacted and (min_timestamp != 0 or max_timestamp != 0): con.close() sys.stderr.write('min-timestamp/max-timestamp can only be used with compacted databases.\n') sys.exit(1) total_tiles = con.tiles_count(min_zoom, max_zoom, min_timestamp, max_timestamp) logger.debug("%d tiles to export" % (total_tiles)) if print_progress: sys.stdout.write("%d tiles to export\n" % (total_tiles)) sys.stdout.write("%d / %d tiles exported (0%% @ 0 tiles/sec)" % (count, total_tiles)) sys.stdout.flush() for t in con.tiles(min_zoom, max_zoom, min_timestamp, max_timestamp): tile_z = t[0] tile_x = t[1] tile_y = t[2] tile_data = str(t[3]) # Execute commands if kwargs.get('command_list'): tile_data = execute_commands_on_tile(kwargs['command_list'], image_format, tile_data, tmp_dir) if flip_tile_y: tile_y = flip_y(tile_z, tile_y) tile_dir = os.path.join(base_path, str(tile_z), str(tile_x)) if not os.path.isdir(tile_dir): os.makedirs(tile_dir) tile_file = os.path.join(tile_dir, '%s.%s' % (tile_y, metadata.get('format', 'png'))) f = open(tile_file, 'wb') f.write(tile_data) f.close() count = count + 1 if (count % 100) == 0: logger.debug("%d / %d tiles exported (%.1f%% @ %.1f tiles/sec)" % (count, total_tiles, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d / %d tiles exported (%.1f%% @ %.1f tiles/sec)" % (count, total_tiles, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) sys.stdout.flush() if print_progress: sys.stdout.write('\n') logger.info("%d / %d tiles exported (100.0%% @ %.1f tiles/sec)" % (count, total_tiles, count / (time.time() - start_time))) if print_progress: sys.stdout.write("%d / %d tiles exported (100.0%% @ %.1f tiles/sec)\n" % (count, total_tiles, count / (time.time() - start_time))) sys.stdout.flush() if delete_after_export: logger.debug("WARNING: Removing exported tiles from %s" % (mbtiles_file)) con.delete_tiles(min_zoom, max_zoom, min_timestamp, max_timestamp) con.optimize_database(kwargs.get('skip_analyze', False), kwargs.get('skip_vacuum', False)) con.close()
def update_mbtiles(mbtiles_file1, mbtiles_file2, **kwargs): scale = kwargs.get('tile_scale', None) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) print_progress = kwargs.get('progress', False) flip_tile_y = kwargs.get('flip_y', False) con1 = mbtiles_connect(mbtiles_file1, auto_commit, journal_mode, synchronous_off, False, False) con2 = mbtiles_connect(mbtiles_file2, auto_commit, journal_mode, synchronous_off, False, True) con1.mbtiles_setup() if not con1.is_compacted() or not con2.is_compacted: con1.close() con2.close() sys.stderr.write('To update mbtiles databases, both databases must already be compacted\n') sys.exit(1) zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Updating %s --> %s (%s)" % (prettify_connect_string(con2.connect_string), prettify_connect_string(con1.connect_string), zoom_level_string)) # Check that the old and new image formats are the same original_format = new_format = None try: original_format = con1.metadata().get('format') except: pass try: new_format = con2.metadata().get('format') except: pass if new_format == None: logger.info("No image format found in the sending database, assuming 'png'") new_format = "png" if original_format != None and new_format != original_format: con1.close() con2.close() sys.stderr.write('The files to merge must use the same image format (png or jpg)\n') sys.exit(1) if original_format == None and new_format != None: con1.update_metadata("format", new_format) if new_format == None: new_format = original_format count = 0 start_time = time.time() min_timestamp = con1.max_timestamp() if min_timestamp is None: min_timestamp = 0 max_timestamp = int(time.time()) total_tiles = con2.updates_count(min_zoom, max_zoom, min_timestamp, max_timestamp) if total_tiles == 0: con1.close() con2.close() sys.stderr.write('No tiles to update, exiting...\n') return logger.debug("%d tiles to update" % (total_tiles)) if print_progress: sys.stdout.write("%d tiles to update\n" % (total_tiles)) sys.stdout.write("0 tiles updated (0% @ 0 tiles/sec)") sys.stdout.flush() known_tile_ids = set() tmp_images_list = [] tmp_row_list = [] deleted_tiles_count = 0 for t in con2.updates(min_zoom, max_zoom, min_timestamp, max_timestamp): tile_z = t[0] tile_x = t[1] tile_y = t[2] tile_scale = t[3] tile_data = str(t[4]) tile_id = t[5] if flip_tile_y: tile_y = flip_y(tile_z, tile_y) if tile_id is None: deleted_tiles_count += 1 if tile_id and tile_id not in known_tile_ids: tmp_images_list.append( (tile_id, tile_data) ) known_tile_ids.add(tile_id) tmp_row_list.append( (tile_z, tile_x, tile_y, tile_scale, tile_id, int(time.time())) ) count = count + 1 if (count % 100) == 0: logger.debug("%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) if print_progress: sys.stdout.write("\r%d tiles merged (%.1f%% @ %.1f tiles/sec)" % (count, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) sys.stdout.flush() if len(tmp_images_list) > 250: con1.insert_tiles_to_images(tmp_images_list) tmp_images_list = [] if len(tmp_row_list) > 250: con1.insert_tiles_to_map(tmp_row_list) tmp_row_list = [] # Push the remaining rows to the database if len(tmp_images_list) > 0: con1.insert_tiles_to_images(tmp_images_list) if len(tmp_row_list) > 0: con1.insert_tiles_to_map(tmp_row_list) if print_progress: sys.stdout.write('\n') logger.info("%d tiles merged (100.0%% @ %.1f tiles/sec)" % (count, count / (time.time() - start_time))) if print_progress: sys.stdout.write("%d tiles merged (100.0%% @ %.1f tiles/sec)\n" % (count, count / (time.time() - start_time))) sys.stdout.flush() if deleted_tiles_count > 0: logger.info("Deleting %d orphaned image(s)" % (deleted_tiles_count)) if print_progress: sys.stdout.write("Deleting %d orphaned image(s)\n" % (deleted_tiles_count)) sys.stdout.flush() con1.delete_orphaned_images() con1.close() con2.close()
def test_mbtiles(mbtiles_file, **kwargs): scale = kwargs.get('tile_scale', None) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 18) tmp_dir = kwargs.get('tmp_dir', None) flip_tile_y = kwargs.get('flip_y', False) min_timestamp = kwargs.get('min_timestamp', 0) max_timestamp = kwargs.get('max_timestamp', 0) revert_test = kwargs.get('revert_test', False) auto_commit = kwargs.get('auto_commit', False) journal_mode = kwargs.get('journal_mode', 'wal') synchronous_off = kwargs.get('synchronous_off', False) default_pool_size = kwargs.get('poolsize', -1) if zoom >= 0: min_zoom = max_zoom = zoom con = mbtiles_connect(mbtiles_file, auto_commit, journal_mode, synchronous_off, False, True) zoom_level_string = None if min_zoom == max_zoom: zoom_level_string = "zoom level %d" % (min_zoom) else: zoom_level_string = "zoom levels %d -> %d" % (min_zoom, max_zoom) logger.info("Testing %s (%s)" % (prettify_connect_string(con.connect_string), zoom_level_string)) image_format = 'png' metadata = con.metadata() if metadata.has_key('format'): image_format = metadata['format'] if default_pool_size < 1: default_pool_size = None logger.debug("Using default pool size") else: logger.debug("Using pool size = %d" % (default_pool_size)) pool = Pool(default_pool_size) multiprocessing.log_to_stderr(logger.level) chunk = 1000 tiles_to_process = [] for t in con.tiles(min_zoom, max_zoom, min_timestamp, max_timestamp, scale): tile_z = t[0] tile_x = t[1] tile_y = t[2] tile_scale = t[3] tile_data = str(t[4]) if flip_tile_y: tile_y = flip_y(tile_z, tile_y) tmp_file_fd, tmp_file_name = tempfile.mkstemp(suffix=".%s" % (image_format), prefix="tile_", dir=tmp_dir) tmp_file = os.fdopen(tmp_file_fd, "w") tmp_file.write(tile_data) tmp_file.close() tiles_to_process.append({ 'tile_x' : tile_x, 'tile_y' : tile_y, 'tile_z' : tile_z, 'filename' : tmp_file_name, 'format' : image_format, 'revert_test' : revert_test, 'command_list' : kwargs.get('command_list', []) }) if len(tiles_to_process) < chunk: continue process_tiles(pool, tiles_to_process) tiles_to_process = [] if len(tiles_to_process) > 0: process_tiles(pool, tiles_to_process) pool.close() con.close()
def mbtiles_to_disk(mbtiles_file, directory_path, **kwargs): logger.info("Exporting database to disk: %s --> %s" % (mbtiles_file, directory_path)) delete_after_export = kwargs.get('delete_after_export', False) no_overwrite = kwargs.get('no_overwrite', False) zoom = kwargs.get('zoom', -1) min_zoom = kwargs.get('min_zoom', 0) max_zoom = kwargs.get('max_zoom', 255) if zoom >= 0: min_zoom = max_zoom = zoom con = mbtiles_connect(mbtiles_file) cur = con.cursor() optimize_connection(cur) if not os.path.isdir(directory_path): os.mkdir(directory_path) base_path = os.path.join(directory_path, "tiles") if not os.path.isdir(base_path): os.makedirs(base_path) metadata = dict(con.execute('SELECT name, value FROM metadata').fetchall()) json.dump(metadata, open(os.path.join(directory_path, 'metadata.json'), 'w'), indent=4) count = 0 start_time = time.time() image_format = metadata.get('format', 'png') total_tiles = con.execute("""SELECT count(zoom_level) FROM tiles WHERE zoom_level>=? AND zoom_level<=?""", (min_zoom, max_zoom)).fetchone()[0] sending_mbtiles_is_compacted = (con.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='images'").fetchone()[0] > 0) tiles = cur.execute("""SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles WHERE zoom_level>=? AND zoom_level<=?""", (min_zoom, max_zoom)) t = tiles.fetchone() while t: z = t[0] x = t[1] y = t[2] tile_data = t[3] # Execute commands if kwargs.get('command_list'): tile_data = execute_commands_on_tile(kwargs['command_list'], image_format, tile_data) if kwargs.get('flip_y', False) == True: y = flip_y(z, y) tile_dir = os.path.join(base_path, str(z), str(x)) if not os.path.isdir(tile_dir): os.makedirs(tile_dir) tile_file = os.path.join(tile_dir, '%s.%s' % (y, metadata.get('format', 'png'))) if no_overwrite == False or not os.path.isfile(tile_file): f = open(tile_file, 'wb') f.write(tile_data) f.close() count = count + 1 if (count % 100) == 0: logger.debug("%s / %s tiles exported (%.1f%%, %.1f tiles/sec)" % (count, total_tiles, (float(count) / float(total_tiles)) * 100.0, count / (time.time() - start_time))) t = tiles.fetchone() logger.info("%s / %s tiles exported (100.0%%, %.1f tiles/sec)" % (count, total_tiles, count / (time.time() - start_time))) if delete_after_export: logger.debug("WARNING: Removing exported tiles from %s" % (mbtiles_file)) if sending_mbtiles_is_compacted: cur.execute("""DELETE FROM images WHERE tile_id IN (SELECT tile_id FROM map WHERE zoom_level>=? AND zoom_level<=?)""", (min_zoom, max_zoom)) cur.execute("""DELETE FROM map WHERE zoom_level>=? AND zoom_level<=?""", (min_zoom, max_zoom)) else: cur.execute("""DELETE FROM tiles WHERE zoom_level>=? AND zoom_level<=?""", (min_zoom, max_zoom)) optimize_database(cur, kwargs.get('skip_analyze', False), kwargs.get('skip_vacuum', False)) con.commit() con.close()