def create_cropped_image(path: str, filename: str, crop_options: str): full_path = os.path.join(path, filename) log.info('Cropping image %s', full_path) crop_config = ['-crop', crop_options, '+repage'] with stats.timer_context(['transform', 'crop']): transform(full_path, full_path, crop_config)
def put(self, key, path): with open(path, 'rb') as file: with stats.timer_context(['storage', 's3', 'put']): self.bucket().put_object( ACL=self.default_acl, Body=file, ContentType=mimetypes.guess_type(key)[0], Key=key)
def get_format(path: str, filename: str) -> str: # imagemagick returns 'PNG' for SVG files base, ext = os.path.splitext(filename) if ext.upper() == '.SVG': return 'SVG' with stats.timer_context(['upload', 'get_format']): with Image(filename=path) as image: return image.format
def create_resized_image(path: str, original: str, config: Dict) -> str: original_path = os.path.join(path, original) resized = resized_key(original, config) resized_path = os.path.join(path, resized) resize_config = config['convert'] log.info('Creating resized image %s with options %s', resized_path, resize_config) with stats.timer_context(['transform', 'resize', _get_size(config)]): transform(original_path, resized_path, resize_config) return resized
def check(request: pyramid.request.Request) -> None: with binding as session: if stats.USE_TAGS: key = ["sql", "manual", "health_check", "db"] tags: Optional[Dict[str, str]] = dict(con=binding.name()) else: key = ["sql", "manual", "health_check", "db", binding.name()] tags = None with stats.timer_context(key, tags): return query_cb(session)
def get_status(gene: TileGeneration) -> List[str]: """Get the tile generation status.""" config = gene.get_main_config() store = get_queue_store(config, False) prefix = "redis" if "redis" in config.config else "sqs" conf = config.config[ "redis"] if "redis" in config.config else config.config["sqs"] stats_prefix = [prefix, conf["queue"]] with stats.timer_context(stats_prefix + ["get_stats"]): status_ = store.get_status() return [name + ": " + str(value) for name, value in status_.items()]
def fetch(self): try: self._is_loaded = False with stats.timer_context(["source", self.get_id(), "fetch"]): self._do_fetch() self._eval_templates() except Exception: LOG.exception("Error with source %s", self.get_id()) stats.increment_counter(["source", self.get_id(), "error"]) raise finally: self._is_loaded = True
def refresh(self): LOG.info("Doing a refresh of %s", self.get_id()) try: self._is_loaded = False with stats.timer_context(["source", self.get_id(), "refresh"]): self._do_refresh() self._eval_templates() except Exception: LOG.exception("Error with source %s", self.get_id()) stats.increment_counter(["source", self.get_id(), "error"]) raise finally: self._is_loaded = True
def _redis_status(gene): config = gene.config['redis'] queue = config['queue'] if not queue.startswith('queue_'): queue = 'queue_' + queue stats_prefix = ['redis', queue] with stats.timer_context(stats_prefix + ['get_stats']): con = redis.StrictRedis.from_url(config['url']) nb_messages = con.llen(queue) print("Approximate number of tiles to generate: {nb_messages}".format( nb_messages=nb_messages)) stats.set_gauge(stats_prefix + ['nb_messages'], nb_messages)
def __call__(self, tile: Tile) -> Tile: if tile is None: logger.warning("The tile is None") return None if tile.error: action = "error" elif tile.data: action = "create" else: action = "delete" layer = tile.metadata.get("layer", "- No layer -") run = tile.metadata.get("run", -1) with stats.timer_context(["db_logger", "insert"]): with self.connection.cursor() as cursor: try: cursor.execute( psycopg2.sql.SQL( "INSERT INTO {} (layer, run, action, tile) " "VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)" ).format(psycopg2.sql.Identifier(self.schema), psycopg2.sql.Identifier(self.table)), { "layer": layer, "action": action, "tile": str(tile.tilecoord), "run": run }, ) except psycopg2.IntegrityError: self.connection.rollback() cursor.execute( psycopg2.sql.SQL( "UPDATE {} SET action = %(action)s " "WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s" ).format(psycopg2.sql.Identifier(self.schema), psycopg2.sql.Identifier(self.table)), { "layer": layer, "action": action, "tile": str(tile.tilecoord), "run": run }, ) self.connection.commit() return tile
def copy(self, key, other_storage): with stats.timer_context(['storage', 's3', 'copy']): if isinstance(other_storage, S3Storage): new_object = other_storage.object(key) new_object.copy_from( ACL=other_storage.default_acl, ContentType=mimetypes.guess_type(key)[0], CopySource={ 'Bucket': self._bucket_name, 'Key': key}) elif isinstance(other_storage, LocalStorage): path = os.path.join(other_storage.path(), key) self.get(key, path) else: raise NotImplementedError()
def put(self, key, path): with open(path, 'rb') as file: kwargs = {} if self._should_expire: now = datetime.datetime.now() expires = now + datetime.timedelta(hours=EXPIRE_HOURS) kwargs['Expires'] = expires with stats.timer_context(['storage', 's3', 'put']): self.bucket().put_object( ACL=self.default_acl, Body=file, ContentType=mimetypes.guess_type(key)[0], CacheControl=CACHE_CONTROL, Key=key, **kwargs)
def check(_request: pyramid.request.Request) -> None: prev_bind = session.bind try: session.bind = bind if stats.USE_TAGS: key = ['sql', 'manual', 'health_check', 'db'] tags: Optional[Dict[str, str]] = dict(con=bind.c2c_name) else: key = [ 'sql', 'manual', 'health_check', 'db', bind.c2c_name ] tags = None with stats.timer_context(key, tags): return query_cb(session) finally: session.bind = prev_bind
def copy(self, key, other_storage): with stats.timer_context(['storage', 's3', 'copy']): if isinstance(other_storage, S3Storage): new_object = other_storage.object(key) new_object.copy_from(ACL=other_storage.default_acl, ContentType=mimetypes.guess_type(key)[0], CacheControl=CACHE_CONTROL, CopySource={ 'Bucket': self._bucket_name, 'Key': key }) elif isinstance(other_storage, LocalStorage): path = os.path.join(other_storage.path(), key) self.get(key, path) else: raise NotImplementedError()
def get_geoms(self, layer, extent=None): if layer['name'] in self._layers_geoms: # pragma: no cover # already build return self._layers_geoms[layer['name']] layer_geoms = {} self._layers_geoms[layer['name']] = layer_geoms if extent: geom = Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]), )) for z, r in enumerate(layer['grid_ref']['resolutions']): layer_geoms[z] = geom if self.options is None or (self.options.near is None and self.options.geom): for g in layer['geoms']: with stats.timer_context(['geoms_get', layer['name']]): connection = psycopg2.connect(g['connection']) cursor = connection.cursor() sql = 'SELECT ST_AsBinary(geom) FROM (SELECT {}) AS g'.format( g['sql']) logger.info('Execute SQL: {}.'.format(sql)) cursor.execute(sql) geoms = [ loads_wkb(binary_type(r[0])) for r in cursor.fetchall() ] geom = cascaded_union(geoms) if extent: geom = geom.intersection( Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]), ))) for z, r in enumerate(layer['grid_ref']['resolutions']): if ('min_resolution' not in g or g['min_resolution'] <= r) and \ ('max_resolution' not in g or g['max_resolution'] >= r): layer_geoms[z] = geom cursor.close() connection.close() return layer_geoms
def create_resized_images(path, key): base, ext = os.path.splitext(key) raster_file = False if ext == '.svg': svg_key = key jpg_key = '{}{}'.format(base, '.jpg') with stats.timer_context(['transform', 'rasterize_svg']): rasterize_svg(os.path.join(path, svg_key), os.path.join(path, jpg_key)) raster_file = os.path.join(path, jpg_key) key = jpg_key for config in RESIZING_CONFIG: create_resized_image(path, key, config) if raster_file: os.unlink(raster_file)
def upload(request): pre_key = create_pseudo_unique_key() log.debug('%s - received upload request', pre_key) # Store the original image as raw file raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key) with stats.timer_context(['upload', 'read']): input_file = request.POST['file'].file input_file.seek(0) with open(raw_file, 'wb') as output_file: shutil.copyfileobj(input_file, output_file) log.debug('%s - copied raw file to %s', pre_key, output_file) try: kind = get_format(raw_file, request.POST['file'].filename) log.debug('%s - detected format is %s', pre_key, kind) except Exception: raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename) if kind == 'JPEG': kind = 'jpg' elif kind in ('PNG', 'GIF', 'SVG'): kind = kind.lower() else: raise HTTPBadRequest('Unsupported image format %s' % kind) # Rename to official extension original_key = "{}.{}".format(pre_key, kind) os.rename(raw_file, temp_storage.object_path(original_key)) if AUTO_ORIENT_ORIGINAL and kind == 'jpg': auto_orient(temp_storage.object_path(original_key)) create_resized_images(temp_storage.path(), original_key) log.debug('%s - uploading original file', pre_key) temp_storage.move(original_key, incoming_storage) for key in resized_keys(original_key): log.debug('%s - uploading resized image %s', pre_key, key) temp_storage.move(key, incoming_storage) log.debug('%s - returning response', pre_key) return {'filename': pre_key + '.' + kind}
def get_geoms(self, layer, extent=None): if layer['name'] in self._layers_geoms: # pragma: no cover # already build return self._layers_geoms[layer['name']] layer_geoms = {} self._layers_geoms[layer['name']] = layer_geoms if extent: geom = Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]), )) for z, r in enumerate(layer['grid_ref']['resolutions']): layer_geoms[z] = geom if self.options is None or ( self.options.near is None and self.options.geom ): for g in layer['geoms']: with stats.timer_context(['geoms_get', layer['name']]): connection = psycopg2.connect(g['connection']) cursor = connection.cursor() sql = 'SELECT ST_AsBinary(geom) FROM (SELECT {}) AS g'.format(g['sql']) logger.info('Execute SQL: {}.'.format(sql)) cursor.execute(sql) geoms = [loads_wkb(binary_type(r[0])) for r in cursor.fetchall()] geom = cascaded_union(geoms) if extent: geom = geom.intersection(Polygon(( (extent[0], extent[1]), (extent[0], extent[3]), (extent[2], extent[3]), (extent[2], extent[1]), ))) for z, r in enumerate(layer['grid_ref']['resolutions']): if ('min_resolution' not in g or g['min_resolution'] <= r) and \ ('max_resolution' not in g or g['max_resolution'] >= r): layer_geoms[z] = geom cursor.close() connection.close() return layer_geoms
def _eval_templates(self): if mode.is_master_with_slaves(): # masters with slaves don't need to evaluate templates return # We get the list of files only once to avoid consecutive template engines eating the output of # the previous template engines. This method is always called with a root_dir that is clean from # all the files that are created by template engines (see the --delete rsync flag in # BaseSource._copy). root_dir = self.get_path() files = [ os.path.relpath(str(p), root_dir) for p in pathlib.Path(root_dir).glob("**/*") ] for engine in self._template_engines: with stats.timer_context( ["source", self.get_id(), "template", engine.get_type()]): engine.evaluate(root_dir, files)
def upload(request): pre_key = create_pseudo_unique_key() log.debug('%s - received upload request', pre_key) # Store the original image as raw file raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key) with stats.timer_context(['upload', 'read']): input_file = request.POST['file'].file input_file.seek(0) with open(raw_file, 'wb') as output_file: shutil.copyfileobj(input_file, output_file) log.debug('%s - copied raw file to %s', pre_key, output_file) try: kind = get_format(raw_file, request.POST['file'].filename) log.debug('%s - detected format is %s', pre_key, kind) except: raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename) if kind == 'JPEG': kind = 'jpg' elif kind in ('PNG', 'GIF', 'SVG'): kind = kind.lower() else: raise HTTPBadRequest('Unsupported image format %s' % kind) # Rename to official extension original_key = "{}.{}".format(pre_key, kind) os.rename(raw_file, temp_storage.object_path(original_key)) create_resized_images(temp_storage.path(), original_key) log.debug('%s - uploading original file', pre_key) temp_storage.move(original_key, incoming_storage) for key in resized_keys(original_key): log.debug('%s - uploading resized image %s', pre_key, key) temp_storage.move(key, incoming_storage) log.debug('%s - returning response', pre_key) return {'filename': pre_key + '.' + kind}
def _copy(self, source, excludes=None): os.makedirs(self.get_path(), exist_ok=True) cmd = [ "rsync", "--recursive", "--links", "--devices", "--specials", "--delete", "--verbose", "--checksum", ] if excludes is not None: cmd += ["--exclude=" + exclude for exclude in excludes] if "excludes" in self._config: cmd += [ "--exclude=" + exclude for exclude in self._config["excludes"] ] cmd += [source + "/", self.get_path()] with stats.timer_context(["source", self.get_id(), "copy"]): self._exec(*cmd)
def status(gene): # pragma: no cover # get SQS status stats_prefix = ['SQS', gene.config.get('sqs', {}).get('queue', 'unknown')] with stats.timer_context(stats_prefix + ['get_stats']): queue = gene.get_sqs_queue() queue.load() attributes = dict(queue.attributes) attributes["CreatedTimestamp"] = time.ctime(int(attributes["CreatedTimestamp"])) attributes["LastModifiedTimestamp"] = time.ctime(int(attributes["LastModifiedTimestamp"])) print( """Approximate number of tiles to generate: {ApproximateNumberOfMessages} Approximate number of generating tiles: {ApproximateNumberOfMessagesNotVisible} Delay in seconds: {DelaySeconds} Receive message wait time in seconds: {ReceiveMessageWaitTimeSeconds} Visibility timeout in seconds: {VisibilityTimeout} Queue creation date: {CreatedTimestamp} Last modification in tile queue: {LastModifiedTimestamp}""".format(**attributes) ) stats.set_gauge(stats_prefix + ['nb_messages'], int(attributes['ApproximateNumberOfMessages']))
def check(_request: Any) -> str: for binding in _get_bindings(session): prev_bind = session.bind try: session.bind = binding if stats.USE_TAGS: key = ['sql', 'manual', 'health_check', 'alembic'] tags: Optional[Dict[str, str]] = dict(conf=alembic_ini_path, con=binding.c2c_name) else: key = [ 'sql', 'manual', 'health_check', 'alembic', alembic_ini_path, binding.c2c_name ] tags = None with stats.timer_context(key, tags): quote = session.bind.dialect.identifier_preparer.quote actual_version, = session.execute( "SELECT version_num FROM {schema}.{table}" . # nosec format(schema=quote(version_schema), table=quote(version_table))).fetchone() if stats.USE_TAGS: stats.increment_counter(['alembic_version'], 1, tags=dict( version=actual_version, name=name)) else: stats.increment_counter( ['alembic_version', name, actual_version], 1) if actual_version != version_: raise Exception( "Invalid alembic version: %s != %s" % (actual_version, version_)) finally: session.bind = prev_bind return version_
def exists(self, key): try: with stats.timer_context(['storage', 's3', 'exists']): object = self.object(key) object.load() mimetype = mimetypes.guess_type(key)[0] if object.content_type != mimetype: object.copy_from(ACL=self.default_acl, ContentType=mimetype, CacheControl=CACHE_CONTROL, CopySource={ 'Bucket': self._bucket_name, 'Key': key }) return True except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": return False else: raise e
def exists(self, key): try: with stats.timer_context(['storage', 's3', 'exists']): object = self.object(key) object.load() mimetype = mimetypes.guess_type(key)[0] if object.content_type != mimetype: object.copy_from( ACL=self.default_acl, ContentType=mimetype, CopySource={ 'Bucket': self._bucket_name, 'Key': key } ) return True except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": return False else: raise e return True
def __call__(self, tile): if tile is None: logger.warning("The tile is None") return None if tile.error: action = 'error' elif tile.data: action = 'create' else: action = 'delete' layer = tile.metadata.get('layer', '- No layer -') run = tile.metadata.get('run', -1) with stats.timer_context(['db_logger', 'insert']): with self.connection.cursor() as cursor: try: cursor.execute( 'INSERT INTO {} (layer, run, action, tile) ' 'VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)'. format(self.full_table), {'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run} ) except psycopg2.IntegrityError: self.connection.rollback() cursor.execute( 'UPDATE {} SET action = %(action)s ' 'WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s'. format(self.full_table), {'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run} ) self.connection.commit() return tile
def __contains__(self, tile): with stats.timer_context(self._get_stats_name("contains", tile)): return self._tile_store.__contains__(tile)
def get_one(self, tile): with stats.timer_context(self._get_stats_name('get_one', tile)): return self._tile_store.get_one(tile)
def last_modified(self, key): with stats.timer_context(['storage', 's3', 'last_modified']): return self.object(key).last_modified
def move(self, key, other_storage): with stats.timer_context(['storage', 's3', 'move']): self.copy(key, other_storage) self.delete(key)
def auto_orient(filename: str): log.info('Change image orientation image %s', filename) with stats.timer_context(['transform', 'auto_orient']): transform(filename, filename, ['-auto-orient'])
def delete(self, key): with stats.timer_context(['storage', 's3', 'delete']): self.object(key).delete()
def get(self, key, path): with stats.timer_context(['storage', 's3', 'get']): self.object(key).download_file(path)
def put_one(self, tile): with stats.timer_context(self._get_stats_name("put_one", tile)): return self._tile_store.put_one(tile)
def __len__(self): with stats.timer_context(self._get_stats_name("len")): return self._tile_store.__len__()
def __len__(self): with stats.timer_context(self._get_stats_name('len')): return self._tile_store.__len__()
def create_cropped_image(path: str, filename: str, crop_options: str): full_path = os.path.join(path, filename) crop_config = ['-crop', crop_options, '+repage'] log.info('Cropping image %s with options %s', full_path, crop_config) with stats.timer_context(['transform', 'crop']): transform(full_path, full_path, crop_config)
def delete_one(self, tile): with stats.timer_context(self._get_stats_name('delete_one', tile)): return self._tile_store.delete_one(tile)
def put_one(self, tile): with stats.timer_context(self._get_stats_name('put_one', tile)): return self._tile_store.put_one(tile)
def delete_one(self, tile): with stats.timer_context(self._get_stats_name("delete_one", tile)): return self._tile_store.delete_one(tile)
def __contains__(self, tile): with stats.timer_context(self._get_stats_name('contains', tile)): return self._tile_store.__contains__(tile)