def create_cropped_image(path: str, filename: str, crop_options: str):
full_path = os.path.join(path, filename)
log.info('Cropping image %s', full_path)
crop_config = ['-crop', crop_options, '+repage']
with stats.timer_context(['transform', 'crop']):
transform(full_path,
full_path,
crop_config)
def put(self, key, path):
with open(path, 'rb') as file:
with stats.timer_context(['storage', 's3', 'put']):
self.bucket().put_object(
ACL=self.default_acl,
Body=file,
ContentType=mimetypes.guess_type(key)[0],
Key=key)
def get_format(path: str, filename: str) -> str:
# imagemagick returns 'PNG' for SVG files
base, ext = os.path.splitext(filename)
if ext.upper() == '.SVG':
return 'SVG'
with stats.timer_context(['upload', 'get_format']):
with Image(filename=path) as image:
return image.format
def create_resized_image(path: str, original: str, config: Dict) -> str:
original_path = os.path.join(path, original)
resized = resized_key(original, config)
resized_path = os.path.join(path, resized)
resize_config = config['convert']
log.info('Creating resized image %s with options %s', resized_path,
resize_config)
with stats.timer_context(['transform', 'resize', _get_size(config)]):
transform(original_path, resized_path, resize_config)
return resized
def check(request: pyramid.request.Request) -> None:
with binding as session:
if stats.USE_TAGS:
key = ["sql", "manual", "health_check", "db"]
tags: Optional[Dict[str, str]] = dict(con=binding.name())
else:
key = ["sql", "manual", "health_check", "db", binding.name()]
tags = None
with stats.timer_context(key, tags):
return query_cb(session)
def get_status(gene: TileGeneration) -> List[str]:
"""Get the tile generation status."""
config = gene.get_main_config()
store = get_queue_store(config, False)
prefix = "redis" if "redis" in config.config else "sqs"
conf = config.config[
"redis"] if "redis" in config.config else config.config["sqs"]
stats_prefix = [prefix, conf["queue"]]
with stats.timer_context(stats_prefix + ["get_stats"]):
status_ = store.get_status()
return [name + ": " + str(value) for name, value in status_.items()]
def fetch(self):
try:
self._is_loaded = False
with stats.timer_context(["source", self.get_id(), "fetch"]):
self._do_fetch()
self._eval_templates()
except Exception:
LOG.exception("Error with source %s", self.get_id())
stats.increment_counter(["source", self.get_id(), "error"])
raise
finally:
self._is_loaded = True
def refresh(self):
LOG.info("Doing a refresh of %s", self.get_id())
try:
self._is_loaded = False
with stats.timer_context(["source", self.get_id(), "refresh"]):
self._do_refresh()
self._eval_templates()
except Exception:
LOG.exception("Error with source %s", self.get_id())
stats.increment_counter(["source", self.get_id(), "error"])
raise
finally:
self._is_loaded = True
def _redis_status(gene):
config = gene.config['redis']
queue = config['queue']
if not queue.startswith('queue_'):
queue = 'queue_' + queue
stats_prefix = ['redis', queue]
with stats.timer_context(stats_prefix + ['get_stats']):
con = redis.StrictRedis.from_url(config['url'])
nb_messages = con.llen(queue)
print("Approximate number of tiles to generate: {nb_messages}".format(
nb_messages=nb_messages))
stats.set_gauge(stats_prefix + ['nb_messages'], nb_messages)
def __call__(self, tile: Tile) -> Tile:
if tile is None:
logger.warning("The tile is None")
return None
if tile.error:
action = "error"
elif tile.data:
action = "create"
else:
action = "delete"
layer = tile.metadata.get("layer", "- No layer -")
run = tile.metadata.get("run", -1)
with stats.timer_context(["db_logger", "insert"]):
with self.connection.cursor() as cursor:
try:
cursor.execute(
psycopg2.sql.SQL(
"INSERT INTO {} (layer, run, action, tile) "
"VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)"
).format(psycopg2.sql.Identifier(self.schema),
psycopg2.sql.Identifier(self.table)),
{
"layer": layer,
"action": action,
"tile": str(tile.tilecoord),
"run": run
},
)
except psycopg2.IntegrityError:
self.connection.rollback()
cursor.execute(
psycopg2.sql.SQL(
"UPDATE {} SET action = %(action)s "
"WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s"
).format(psycopg2.sql.Identifier(self.schema),
psycopg2.sql.Identifier(self.table)),
{
"layer": layer,
"action": action,
"tile": str(tile.tilecoord),
"run": run
},
)
self.connection.commit()
return tile
def copy(self, key, other_storage):
with stats.timer_context(['storage', 's3', 'copy']):
if isinstance(other_storage, S3Storage):
new_object = other_storage.object(key)
new_object.copy_from(
ACL=other_storage.default_acl,
ContentType=mimetypes.guess_type(key)[0],
CopySource={
'Bucket': self._bucket_name,
'Key': key})
elif isinstance(other_storage, LocalStorage):
path = os.path.join(other_storage.path(), key)
self.get(key, path)
else:
raise NotImplementedError()
def put(self, key, path):
with open(path, 'rb') as file:
kwargs = {}
if self._should_expire:
now = datetime.datetime.now()
expires = now + datetime.timedelta(hours=EXPIRE_HOURS)
kwargs['Expires'] = expires
with stats.timer_context(['storage', 's3', 'put']):
self.bucket().put_object(
ACL=self.default_acl,
Body=file,
ContentType=mimetypes.guess_type(key)[0],
CacheControl=CACHE_CONTROL,
Key=key,
**kwargs)
def check(_request: pyramid.request.Request) -> None:
prev_bind = session.bind
try:
session.bind = bind
if stats.USE_TAGS:
key = ['sql', 'manual', 'health_check', 'db']
tags: Optional[Dict[str, str]] = dict(con=bind.c2c_name)
else:
key = [
'sql', 'manual', 'health_check', 'db', bind.c2c_name
]
tags = None
with stats.timer_context(key, tags):
return query_cb(session)
finally:
session.bind = prev_bind
def copy(self, key, other_storage):
with stats.timer_context(['storage', 's3', 'copy']):
if isinstance(other_storage, S3Storage):
new_object = other_storage.object(key)
new_object.copy_from(ACL=other_storage.default_acl,
ContentType=mimetypes.guess_type(key)[0],
CacheControl=CACHE_CONTROL,
CopySource={
'Bucket': self._bucket_name,
'Key': key
})
elif isinstance(other_storage, LocalStorage):
path = os.path.join(other_storage.path(), key)
self.get(key, path)
else:
raise NotImplementedError()
def get_geoms(self, layer, extent=None):
if layer['name'] in self._layers_geoms: # pragma: no cover
# already build
return self._layers_geoms[layer['name']]
layer_geoms = {}
self._layers_geoms[layer['name']] = layer_geoms
if extent:
geom = Polygon((
(extent[0], extent[1]),
(extent[0], extent[3]),
(extent[2], extent[3]),
(extent[2], extent[1]),
))
for z, r in enumerate(layer['grid_ref']['resolutions']):
layer_geoms[z] = geom
if self.options is None or (self.options.near is None
and self.options.geom):
for g in layer['geoms']:
with stats.timer_context(['geoms_get', layer['name']]):
connection = psycopg2.connect(g['connection'])
cursor = connection.cursor()
sql = 'SELECT ST_AsBinary(geom) FROM (SELECT {}) AS g'.format(
g['sql'])
logger.info('Execute SQL: {}.'.format(sql))
cursor.execute(sql)
geoms = [
loads_wkb(binary_type(r[0]))
for r in cursor.fetchall()
]
geom = cascaded_union(geoms)
if extent:
geom = geom.intersection(
Polygon((
(extent[0], extent[1]),
(extent[0], extent[3]),
(extent[2], extent[3]),
(extent[2], extent[1]),
)))
for z, r in enumerate(layer['grid_ref']['resolutions']):
if ('min_resolution' not in g or g['min_resolution'] <= r) and \
('max_resolution' not in g or g['max_resolution'] >= r):
layer_geoms[z] = geom
cursor.close()
connection.close()
return layer_geoms
def create_resized_images(path, key):
base, ext = os.path.splitext(key)
raster_file = False
if ext == '.svg':
svg_key = key
jpg_key = '{}{}'.format(base, '.jpg')
with stats.timer_context(['transform', 'rasterize_svg']):
rasterize_svg(os.path.join(path, svg_key),
os.path.join(path, jpg_key))
raster_file = os.path.join(path, jpg_key)
key = jpg_key
for config in RESIZING_CONFIG:
create_resized_image(path, key, config)
if raster_file:
os.unlink(raster_file)
def upload(request):
pre_key = create_pseudo_unique_key()
log.debug('%s - received upload request', pre_key)
# Store the original image as raw file
raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key)
with stats.timer_context(['upload', 'read']):
input_file = request.POST['file'].file
input_file.seek(0)
with open(raw_file, 'wb') as output_file:
shutil.copyfileobj(input_file, output_file)
log.debug('%s - copied raw file to %s', pre_key, output_file)
try:
kind = get_format(raw_file, request.POST['file'].filename)
log.debug('%s - detected format is %s', pre_key, kind)
except Exception:
raise HTTPBadRequest('Bad format for %s' %
request.POST['file'].filename)
if kind == 'JPEG':
kind = 'jpg'
elif kind in ('PNG', 'GIF', 'SVG'):
kind = kind.lower()
else:
raise HTTPBadRequest('Unsupported image format %s' % kind)
# Rename to official extension
original_key = "{}.{}".format(pre_key, kind)
os.rename(raw_file, temp_storage.object_path(original_key))
if AUTO_ORIENT_ORIGINAL and kind == 'jpg':
auto_orient(temp_storage.object_path(original_key))
create_resized_images(temp_storage.path(), original_key)
log.debug('%s - uploading original file', pre_key)
temp_storage.move(original_key, incoming_storage)
for key in resized_keys(original_key):
log.debug('%s - uploading resized image %s', pre_key, key)
temp_storage.move(key, incoming_storage)
log.debug('%s - returning response', pre_key)
return {'filename': pre_key + '.' + kind}
def get_geoms(self, layer, extent=None):
if layer['name'] in self._layers_geoms: # pragma: no cover
# already build
return self._layers_geoms[layer['name']]
layer_geoms = {}
self._layers_geoms[layer['name']] = layer_geoms
if extent:
geom = Polygon((
(extent[0], extent[1]),
(extent[0], extent[3]),
(extent[2], extent[3]),
(extent[2], extent[1]),
))
for z, r in enumerate(layer['grid_ref']['resolutions']):
layer_geoms[z] = geom
if self.options is None or (
self.options.near is None and self.options.geom
):
for g in layer['geoms']:
with stats.timer_context(['geoms_get', layer['name']]):
connection = psycopg2.connect(g['connection'])
cursor = connection.cursor()
sql = 'SELECT ST_AsBinary(geom) FROM (SELECT {}) AS g'.format(g['sql'])
logger.info('Execute SQL: {}.'.format(sql))
cursor.execute(sql)
geoms = [loads_wkb(binary_type(r[0])) for r in cursor.fetchall()]
geom = cascaded_union(geoms)
if extent:
geom = geom.intersection(Polygon((
(extent[0], extent[1]),
(extent[0], extent[3]),
(extent[2], extent[3]),
(extent[2], extent[1]),
)))
for z, r in enumerate(layer['grid_ref']['resolutions']):
if ('min_resolution' not in g or g['min_resolution'] <= r) and \
('max_resolution' not in g or g['max_resolution'] >= r):
layer_geoms[z] = geom
cursor.close()
connection.close()
return layer_geoms
def _eval_templates(self):
if mode.is_master_with_slaves():
# masters with slaves don't need to evaluate templates
return
# We get the list of files only once to avoid consecutive template engines eating the output of
# the previous template engines. This method is always called with a root_dir that is clean from
# all the files that are created by template engines (see the --delete rsync flag in
# BaseSource._copy).
root_dir = self.get_path()
files = [
os.path.relpath(str(p), root_dir)
for p in pathlib.Path(root_dir).glob("**/*")
]
for engine in self._template_engines:
with stats.timer_context(
["source",
self.get_id(), "template",
engine.get_type()]):
engine.evaluate(root_dir, files)
def upload(request):
pre_key = create_pseudo_unique_key()
log.debug('%s - received upload request', pre_key)
# Store the original image as raw file
raw_file = '%s/%s_raw' % (temp_storage.path(), pre_key)
with stats.timer_context(['upload', 'read']):
input_file = request.POST['file'].file
input_file.seek(0)
with open(raw_file, 'wb') as output_file:
shutil.copyfileobj(input_file, output_file)
log.debug('%s - copied raw file to %s', pre_key, output_file)
try:
kind = get_format(raw_file, request.POST['file'].filename)
log.debug('%s - detected format is %s', pre_key, kind)
except:
raise HTTPBadRequest('Bad format for %s' % request.POST['file'].filename)
if kind == 'JPEG':
kind = 'jpg'
elif kind in ('PNG', 'GIF', 'SVG'):
kind = kind.lower()
else:
raise HTTPBadRequest('Unsupported image format %s' % kind)
# Rename to official extension
original_key = "{}.{}".format(pre_key, kind)
os.rename(raw_file, temp_storage.object_path(original_key))
create_resized_images(temp_storage.path(), original_key)
log.debug('%s - uploading original file', pre_key)
temp_storage.move(original_key, incoming_storage)
for key in resized_keys(original_key):
log.debug('%s - uploading resized image %s', pre_key, key)
temp_storage.move(key, incoming_storage)
log.debug('%s - returning response', pre_key)
return {'filename': pre_key + '.' + kind}
def _copy(self, source, excludes=None):
os.makedirs(self.get_path(), exist_ok=True)
cmd = [
"rsync",
"--recursive",
"--links",
"--devices",
"--specials",
"--delete",
"--verbose",
"--checksum",
]
if excludes is not None:
cmd += ["--exclude=" + exclude for exclude in excludes]
if "excludes" in self._config:
cmd += [
"--exclude=" + exclude for exclude in self._config["excludes"]
]
cmd += [source + "/", self.get_path()]
with stats.timer_context(["source", self.get_id(), "copy"]):
self._exec(*cmd)
def status(gene): # pragma: no cover
# get SQS status
stats_prefix = ['SQS', gene.config.get('sqs', {}).get('queue', 'unknown')]
with stats.timer_context(stats_prefix + ['get_stats']):
queue = gene.get_sqs_queue()
queue.load()
attributes = dict(queue.attributes)
attributes["CreatedTimestamp"] = time.ctime(int(attributes["CreatedTimestamp"]))
attributes["LastModifiedTimestamp"] = time.ctime(int(attributes["LastModifiedTimestamp"]))
print(
"""Approximate number of tiles to generate: {ApproximateNumberOfMessages}
Approximate number of generating tiles: {ApproximateNumberOfMessagesNotVisible}
Delay in seconds: {DelaySeconds}
Receive message wait time in seconds: {ReceiveMessageWaitTimeSeconds}
Visibility timeout in seconds: {VisibilityTimeout}
Queue creation date: {CreatedTimestamp}
Last modification in tile queue: {LastModifiedTimestamp}""".format(**attributes)
)
stats.set_gauge(stats_prefix + ['nb_messages'], int(attributes['ApproximateNumberOfMessages']))
def check(_request: Any) -> str:
for binding in _get_bindings(session):
prev_bind = session.bind
try:
session.bind = binding
if stats.USE_TAGS:
key = ['sql', 'manual', 'health_check', 'alembic']
tags: Optional[Dict[str,
str]] = dict(conf=alembic_ini_path,
con=binding.c2c_name)
else:
key = [
'sql', 'manual', 'health_check', 'alembic',
alembic_ini_path, binding.c2c_name
]
tags = None
with stats.timer_context(key, tags):
quote = session.bind.dialect.identifier_preparer.quote
actual_version, = session.execute(
"SELECT version_num FROM {schema}.{table}"
. # nosec
format(schema=quote(version_schema),
table=quote(version_table))).fetchone()
if stats.USE_TAGS:
stats.increment_counter(['alembic_version'],
1,
tags=dict(
version=actual_version,
name=name))
else:
stats.increment_counter(
['alembic_version', name, actual_version], 1)
if actual_version != version_:
raise Exception(
"Invalid alembic version: %s != %s" %
(actual_version, version_))
finally:
session.bind = prev_bind
return version_
def exists(self, key):
try:
with stats.timer_context(['storage', 's3', 'exists']):
object = self.object(key)
object.load()
mimetype = mimetypes.guess_type(key)[0]
if object.content_type != mimetype:
object.copy_from(ACL=self.default_acl,
ContentType=mimetype,
CacheControl=CACHE_CONTROL,
CopySource={
'Bucket': self._bucket_name,
'Key': key
})
return True
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
return False
else:
raise e
def exists(self, key):
try:
with stats.timer_context(['storage', 's3', 'exists']):
object = self.object(key)
object.load()
mimetype = mimetypes.guess_type(key)[0]
if object.content_type != mimetype:
object.copy_from(
ACL=self.default_acl,
ContentType=mimetype,
CopySource={
'Bucket': self._bucket_name,
'Key': key
}
)
return True
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "404":
return False
else:
raise e
return True
def __call__(self, tile):
if tile is None:
logger.warning("The tile is None")
return None
if tile.error:
action = 'error'
elif tile.data:
action = 'create'
else:
action = 'delete'
layer = tile.metadata.get('layer', '- No layer -')
run = tile.metadata.get('run', -1)
with stats.timer_context(['db_logger', 'insert']):
with self.connection.cursor() as cursor:
try:
cursor.execute(
'INSERT INTO {} (layer, run, action, tile) '
'VALUES (%(layer)s, %(run)s, %(action)s::varchar(7), %(tile)s)'.
format(self.full_table),
{'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run}
)
except psycopg2.IntegrityError:
self.connection.rollback()
cursor.execute(
'UPDATE {} SET action = %(action)s '
'WHERE layer = %(layer)s AND run = %(run)s AND tile = %(tile)s'.
format(self.full_table),
{'layer': layer, 'action': action, 'tile': str(tile.tilecoord), 'run': run}
)
self.connection.commit()
return tile
def __contains__(self, tile):
with stats.timer_context(self._get_stats_name("contains", tile)):
return self._tile_store.__contains__(tile)
def get_one(self, tile):
with stats.timer_context(self._get_stats_name('get_one', tile)):
return self._tile_store.get_one(tile)
def last_modified(self, key):
with stats.timer_context(['storage', 's3', 'last_modified']):
return self.object(key).last_modified
def move(self, key, other_storage):
with stats.timer_context(['storage', 's3', 'move']):
self.copy(key, other_storage)
self.delete(key)
def auto_orient(filename: str):
log.info('Change image orientation image %s', filename)
with stats.timer_context(['transform', 'auto_orient']):
transform(filename, filename, ['-auto-orient'])
def delete(self, key):
with stats.timer_context(['storage', 's3', 'delete']):
self.object(key).delete()
def get(self, key, path):
with stats.timer_context(['storage', 's3', 'get']):
self.object(key).download_file(path)
def put_one(self, tile):
with stats.timer_context(self._get_stats_name("put_one", tile)):
return self._tile_store.put_one(tile)
def __len__(self):
with stats.timer_context(self._get_stats_name("len")):
return self._tile_store.__len__()
def __len__(self):
with stats.timer_context(self._get_stats_name('len')):
return self._tile_store.__len__()
def create_cropped_image(path: str, filename: str, crop_options: str):
full_path = os.path.join(path, filename)
crop_config = ['-crop', crop_options, '+repage']
log.info('Cropping image %s with options %s', full_path, crop_config)
with stats.timer_context(['transform', 'crop']):
transform(full_path, full_path, crop_config)
def delete_one(self, tile):
with stats.timer_context(self._get_stats_name('delete_one', tile)):
return self._tile_store.delete_one(tile)
def put_one(self, tile):
with stats.timer_context(self._get_stats_name('put_one', tile)):
return self._tile_store.put_one(tile)
def last_modified(self, key):
with stats.timer_context(['storage', 's3', 'last_modified']):
return self.object(key).last_modified
def move(self, key, other_storage):
with stats.timer_context(['storage', 's3', 'move']):
self.copy(key, other_storage)
self.delete(key)
def delete_one(self, tile):
with stats.timer_context(self._get_stats_name("delete_one", tile)):
return self._tile_store.delete_one(tile)
def get(self, key, path):
with stats.timer_context(['storage', 's3', 'get']):
self.object(key).download_file(path)
def __contains__(self, tile):
with stats.timer_context(self._get_stats_name('contains', tile)):
return self._tile_store.__contains__(tile)
def delete(self, key):
with stats.timer_context(['storage', 's3', 'delete']):
self.object(key).delete()
