from cachetools import cached
from cachetools.keys import hashkey
from numpy.linalg import norm
from pandas import np
from transforms3d.euler import mat2euler
from transforms3d.quaternions import mat2quat
import matplotlib.pyplot as plt
from scripts.steps.load_data import load_image
from scripts.steps.ransac import ransac
ATTRIBUTE = "Amplitude"
detector = cv2.xfeatures2d.SIFT_create(nfeatures=1000, sigma=0.5)
@cached(cache={}, key=lambda image_name, img_x: hashkey(image_name))
def cachedDetectAndCompute(image_name, img_x):
return detector.detectAndCompute(img_x, None)
def predict_pose_change(data_1, data_2, settings, real_change, CALCULATE_ERROR, print_image=False):
#
# Load Amplitude image and 3D point cloud ✓
#
img1, depth_img1 = load_image(data_1, ATTRIBUTE, settings["MEDIAN_BLUR"])
img2, depth_img2 = load_image(data_2, ATTRIBUTE, settings["MEDIAN_BLUR"])
#
# Apply Image Blurring ✓
#
if settings["GAUSSIAN_BLUR"]:
img1 = cv2.GaussianBlur(img1, (5, 5), 0)
class BaseBackend(BaseReader):
"""Base Class for cogeo-mosaic backend storage.
Attributes:
path (str): mosaic path.
mosaic_def (MosaicJSON, optional): mosaicJSON document.
reader (rio_tiler.io.BaseReader): Dataset reader. Defaults to `rio_tiler.io.COGReader`.
reader_options (dict): Options to forward to the reader config.
backend_options (dict): Global backend options.
tms (morecantile.TileMatrixSet, optional): TileMatrixSet grid definition. **READ ONLY attribute**. Defaults to `WebMercatorQuad`.
bbox (tuple): mosaic bounds (left, bottom, right, top). **READ ONLY attribute**. Defaults to `(-180, -90, 180, 90)`.
minzoom (int): mosaic Min zoom level. **READ ONLY attribute**. Defaults to `0`.
maxzoom (int): mosaic Max zoom level. **READ ONLY attribute**. Defaults to `30`
"""
path: str = attr.ib()
mosaic_def: MosaicJSON = attr.ib(default=None,
converter=_convert_to_mosaicjson)
reader: Type[BaseReader] = attr.ib(default=COGReader)
reader_options: Dict = attr.ib(factory=dict)
backend_options: Dict = attr.ib(factory=dict)
# TMS is outside the init because mosaicJSON and cogeo-mosaic only
# works with WebMercator (mercantile) for now.
tms: TileMatrixSet = attr.ib(init=False, default=WEB_MERCATOR_TMS)
# default values for bounds and zoom
bounds: Tuple[float, float, float,
float] = attr.ib(init=False, default=(-180, -90, 180, 90))
minzoom: int = attr.ib(init=False, default=0)
maxzoom: int = attr.ib(init=False, default=30)
_backend_name: str
_file_byte_size: Optional[int] = 0
def __attrs_post_init__(self):
"""Post Init: if not passed in init, try to read from self.path."""
self.mosaic_def = self.mosaic_def or self._read(**self.backend_options)
self.minzoom = self.mosaic_def.minzoom
self.maxzoom = self.mosaic_def.maxzoom
self.bounds = self.mosaic_def.bounds
@abc.abstractmethod
def _read(self) -> MosaicJSON:
"""Fetch mosaic definition"""
@abc.abstractmethod
def write(self, overwrite: bool = True):
"""Upload new MosaicJSON to backend."""
def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
new_mosaic = MosaicJSON.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
for quadkey, new_assets in new_mosaic.tiles.items():
tile = mercantile.quadkey_to_tile(quadkey)
assets = self.assets_for_tile(*tile)
assets = [*new_assets, *assets
] if add_first else [*assets, *new_assets]
# add custom sorting algorithm (e.g based on path name)
self.mosaic_def.tiles[quadkey] = assets
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
self.bounds = bounds
self.write(overwrite=True)
def assets_for_tile(self, x: int, y: int, z: int) -> List[str]:
"""Retrieve assets for tile."""
return self.get_assets(x, y, z)
def assets_for_point(self, lng: float, lat: float) -> List[str]:
"""Retrieve assets for point."""
tile = mercantile.tile(lng, lat, self.quadkey_zoom)
return self.get_assets(tile.x, tile.y, tile.z)
@cached(
TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl),
key=lambda self, x, y, z: hashkey(self.path, x, y, z, self.mosaicid),
)
def get_assets(self, x: int, y: int, z: int) -> List[str]:
"""Find assets."""
mercator_tile = mercantile.Tile(x=x, y=y, z=z)
quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom)
return list(
itertools.chain.from_iterable(
[self.mosaic_def.tiles.get(qk, []) for qk in quadkeys]))
def tile( # type: ignore
self,
x: int,
y: int,
z: int,
reverse: bool = False,
**kwargs: Any,
) -> Tuple[ImageData, List[str]]:
"""Get Tile from multiple observation."""
mosaic_assets = self.assets_for_tile(x, y, z)
if not mosaic_assets:
raise NoAssetFoundError(f"No assets found for tile {z}-{x}-{y}")
if reverse:
mosaic_assets = list(reversed(mosaic_assets))
def _reader(asset: str, x: int, y: int, z: int,
**kwargs: Any) -> ImageData:
with self.reader(asset, **self.reader_options) as src_dst:
return src_dst.tile(x, y, z, **kwargs)
return mosaic_reader(mosaic_assets, _reader, x, y, z, **kwargs)
def point(
self,
lon: float,
lat: float,
threads=MAX_THREADS,
reverse: bool = False,
**kwargs: Any,
) -> List[Dict]:
"""Get Point value from multiple observation."""
mosaic_assets = self.assets_for_point(lon, lat)
if not mosaic_assets:
raise NoAssetFoundError(f"No assets found for point ({lon},{lat})")
if reverse:
mosaic_assets = list(reversed(mosaic_assets))
def _reader(asset: str, lon: float, lat: float, **kwargs) -> Dict:
with self.reader(asset, **self.reader_options) as src_dst:
return src_dst.point(lon, lat, **kwargs)
tasks = create_tasks(_reader, mosaic_assets, threads, lon, lat,
**kwargs)
return [{
"asset": asset,
"values": pt
} for pt, asset in filter_tasks(
tasks, allowed_exceptions=(PointOutsideBounds, ))]
def info(self, quadkeys: bool = False) -> Info: # type: ignore
"""Mosaic info."""
return Info(
bounds=self.mosaic_def.bounds,
center=self.mosaic_def.center,
maxzoom=self.mosaic_def.maxzoom,
minzoom=self.mosaic_def.minzoom,
name=self.mosaic_def.name if self.mosaic_def.name else "mosaic",
quadkeys=[] if not quadkeys else self._quadkeys,
)
@property
def metadata(self) -> Metadata: # type: ignore
"""Retrieve Mosaic metadata
Returns
-------
MosaicJSON as dict without `tiles` key.
"""
return Metadata(**self.mosaic_def.dict())
@property
def center(self):
"""Return center from the mosaic definition."""
return self.mosaic_def.center
@property
def mosaicid(self) -> str:
"""Return sha224 id of the mosaicjson document."""
return get_hash(**self.mosaic_def.dict(exclude_none=True))
@property
def _quadkeys(self) -> List[str]:
"""Return the list of quadkey tiles."""
return list(self.mosaic_def.tiles)
@property
def quadkey_zoom(self) -> int:
"""Return Quadkey zoom property."""
return self.mosaic_def.quadkey_zoom or self.mosaic_def.minzoom
############################################################################
# Not Implemented methods
# BaseReader required those method to be implemented
def stats(self):
"""PlaceHolder for BaseReader.stats."""
raise NotImplementedError
def preview(self):
"""PlaceHolder for BaseReader.preview."""
raise NotImplementedError
def part(self):
"""PlaceHolder for BaseReader.part."""
raise NotImplementedError
def feature(self):
"""PlaceHolder for BaseReader.feature."""
raise NotImplementedError
class DynamoDBBackend(BaseBackend):
"""DynamoDB Backend Adapter."""
client: Any = attr.ib(default=None)
region: str = attr.ib(default=os.getenv("AWS_REGION", "us-east-1"))
table_name: str = attr.ib(init=False)
mosaic_name: str = attr.ib(init=False)
table: Any = attr.ib(init=False)
_backend_name = "AWS DynamoDB"
_metadata_quadkey: str = "-1"
def __attrs_post_init__(self):
"""Post Init: parse path, create client and connect to Table.
A path looks like
dynamodb://{region}/{table_name}:{mosaic_name}
dynamodb:///{table_name}:{mosaic_name}
"""
assert boto3 is not None, "'boto3' must be installed to use DynamoDBBackend"
logger.debug(f"Using DynamoDB backend: {self.input}")
if not re.match(
r"^dynamodb://([a-z]{2}\-[a-z]+\-[0-9])?\/[a-zA-Z0-9\_\-\.]+\:[a-zA-Z0-9\_\-\.]+$",
self.input,
):
raise ValueError(f"Invalid DynamoDB path: {self.input}")
parsed = urlparse(self.input)
mosaic_info = parsed.path.lstrip("/").split(":")
self.table_name = mosaic_info[0]
self.mosaic_name = mosaic_info[1]
logger.debug(f"Table: {self.table_name}")
logger.debug(f"Mosaic: {self.mosaic_name}")
self.region = parsed.netloc or self.region
self.client = self.client or boto3.resource("dynamodb", region_name=self.region)
self.table = self.client.Table(self.table_name)
super().__attrs_post_init__()
@cached(
TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl),
key=lambda self: hashkey(self.input),
)
def _read(self) -> MosaicJSON: # type: ignore
"""Get Mosaic definition info."""
meta = self._fetch_dynamodb(self._metadata_quadkey)
if not meta:
raise MosaicNotFoundError(
f"Mosaic {self.mosaic_name} not found in table {self.table_name}"
)
# Numeric values are loaded from DynamoDB as Decimal types
# Convert maxzoom, minzoom, quadkey_zoom to int
for key in ["minzoom", "maxzoom", "quadkey_zoom"]:
if meta.get(key):
meta[key] = int(meta[key])
# Convert bounds, center to float
for key in ["bounds", "center"]:
if meta.get(key):
meta[key] = list(map(float, meta[key]))
# Create pydantic class
# For now, a tiles key must exist
meta["tiles"] = {}
return MosaicJSON(**meta)
def write(self, overwrite: bool = False, **kwargs: Any):
"""Write mosaicjson document to AWS DynamoDB.
Args:
overwrite (bool): delete old mosaic items inthe Table.
**kwargs (any): Options forwarded to `dynamodb.create_table`
Raises:
MosaicExistsError: If mosaic already exists in the Table.
"""
if not self._table_exists():
self._create_table(**kwargs)
if self._mosaic_exists():
if not overwrite:
raise MosaicExistsError(
f"Mosaic already exists in {self.table_name}, use `overwrite=True`."
)
self.delete()
items: List[Dict[str, Any]] = []
# Create Metadata item
# Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be
# in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/)
meta = json.loads(self.mosaic_def.json(exclude={"tiles"}), parse_float=Decimal)
items.append(
{"quadkey": self._metadata_quadkey, "mosaicId": self.mosaic_name, **meta}
)
# Create Tile items
for quadkey, assets in self.mosaic_def.tiles.items():
items.append(
{"mosaicId": self.mosaic_name, "quadkey": quadkey, "assets": assets}
)
self._write_items(items)
def update(
self,
features: Sequence[Dict],
add_first: bool = True,
quiet: bool = False,
**kwargs,
):
"""Update existing MosaicJSON on backend."""
logger.debug(f"Updating {self.mosaic_name}...")
new_mosaic = MosaicJSON.from_features(
features,
self.mosaic_def.minzoom,
self.mosaic_def.maxzoom,
quadkey_zoom=self.quadkey_zoom,
quiet=quiet,
**kwargs,
)
bounds = bbox_union(new_mosaic.bounds, self.mosaic_def.bounds)
self.mosaic_def._increase_version()
self.mosaic_def.bounds = bounds
self.mosaic_def.center = (
(bounds[0] + bounds[2]) / 2,
(bounds[1] + bounds[3]) / 2,
self.mosaic_def.minzoom,
)
self.bounds = bounds
items: List[Dict[str, Any]] = []
# Create Metadata item
# Note: `parse_float=Decimal` is required because DynamoDB requires all numbers to be
# in Decimal type (ref: https://blog.ruanbekker.com/blog/2019/02/05/convert-float-to-decimal-data-types-for-boto3-dynamodb-using-python/)
meta = json.loads(self.mosaic_def.json(exclude={"tiles"}), parse_float=Decimal)
items.append(
{"quadkey": self._metadata_quadkey, "mosaicId": self.mosaic_name, **meta}
)
# Create Tile items
for quadkey, new_assets in new_mosaic.tiles.items():
tile = mercantile.quadkey_to_tile(quadkey)
assets = self.assets_for_tile(*tile)
assets = [*new_assets, *assets] if add_first else [*assets, *new_assets]
items.append(
{"mosaicId": self.mosaic_name, "quadkey": quadkey, "assets": assets}
)
self._write_items(items)
@cached(
TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl),
key=lambda self, x, y, z: hashkey(self.input, x, y, z, self.mosaicid),
)
def get_assets(self, x: int, y: int, z: int) -> List[str]:
"""Find assets."""
mercator_tile = mercantile.Tile(x=x, y=y, z=z)
quadkeys = find_quadkeys(mercator_tile, self.quadkey_zoom)
return list(
dict.fromkeys(
itertools.chain.from_iterable(
[self._fetch_dynamodb(qk).get("assets", []) for qk in quadkeys]
)
)
)
@property
def _quadkeys(self) -> List[str]:
"""Return the list of quadkey tiles."""
resp = self.table.query(
KeyConditionExpression=Key("mosaicId").eq(self.mosaic_name),
ProjectionExpression="quadkey",
)
return [
item["quadkey"]
for item in resp["Items"]
if item["quadkey"] != self._metadata_quadkey
]
def _create_table(self, billing_mode: str = "PAY_PER_REQUEST", **kwargs: Any):
"""Create DynamoDB Table.
Args:
billing_mode (str): DynamoDB billing mode (default set to PER_REQUEST).
**kwargs (any): Options forwarded to `dynamodb.create_table`
"""
logger.debug(f"Creating {self.table_name} Table.")
# Define schema for primary key
# Non-keys don't need a schema
attr_defs = [
{"AttributeName": "mosaicId", "AttributeType": "S"},
{"AttributeName": "quadkey", "AttributeType": "S"},
]
key_schema = [
{"AttributeName": "mosaicId", "KeyType": "HASH"},
{"AttributeName": "quadkey", "KeyType": "RANGE"},
]
# Note: errors if table already exists
try:
self.client.create_table(
AttributeDefinitions=attr_defs,
TableName=self.table.table_name,
KeySchema=key_schema,
BillingMode=billing_mode,
**kwargs,
)
# If outside try/except block, could wait forever if unable to
# create table
self.table.wait_until_exists()
except self.table.meta.client.exceptions.ResourceNotFoundException:
warnings.warn("Unable to create table.")
return
def _write_items(self, items: List[Dict]):
with self.table.batch_writer() as batch:
with click.progressbar(
items,
length=len(items),
show_percent=True,
label=f"Uploading mosaic {self.table_name}:{self.mosaic_name} to DynamoDB",
) as progitems:
for item in progitems:
batch.put_item(item)
def _fetch_dynamodb(self, quadkey: str) -> Dict:
try:
return self.table.get_item(
Key={"mosaicId": self.mosaic_name, "quadkey": quadkey}
).get("Item", {})
except ClientError as e:
status_code = e.response["ResponseMetadata"]["HTTPStatusCode"]
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response["Error"]["Message"]) from e
def _table_exists(self) -> bool:
"""Check if the Table already exists."""
try:
_ = self.table.table_status
return True
except self.table.meta.client.exceptions.ResourceNotFoundException:
return False
def _mosaic_exists(self) -> bool:
"""Check if the mosaic already exists in the Table."""
item = self.table.get_item(
Key={"mosaicId": self.mosaic_name, "quadkey": self._metadata_quadkey}
).get("Item", {})
return bool(item)
def delete(self):
"""Delete all items for a specific mosaic in the dynamoDB Table."""
logger.debug(f"Deleting all items for mosaic {self.mosaic_name}...")
quadkey_list = self._quadkeys + [self._metadata_quadkey]
with self.table.batch_writer() as batch_writer:
for item in quadkey_list:
batch_writer.delete_item(
Key={"mosaicId": self.mosaic_name, "quadkey": item}
)
nb_slices_of_pizza = tuple([int(s) for s in lines[1].split(" ")])
return {
"max_slice": max_slice,
"nb_types": nb_types,
"nb_slices_of_pizza": nb_slices_of_pizza,
}
def solve(problem):
return solve_rec(problem["max_slice"], problem["nb_slices_of_pizza"],
tuple([]))
@cached(
cache={},
key=lambda max_slice, pizza_list, pizza_selected: hashkey(
max_slice, pizza_list),
)
def solve_rec(max_slice, pizza_list, pizza_selected):
if max_slice == 0:
return pizza_selected
without_this_pizza = solve_rec(max_slice, pizza_list[1:], pizza_selected)
sum_without_this_pizza = sum(without_this_pizza)
pizza_list = list(
filter(lambda pizza: (pizza + sum_without_this_pizza) > max_slice,
pizza_list))
if len(pizza_list) == 0:
return pizza_selected
nb_slice = pizza_list[0]
with_this_pizza = solve_rec(max_slice - nb_slice, pizza_list[1:],
pizza_selected + (nb_slice, ))
if sum(with_this_pizza) > sum_without_this_pizza:
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from cachetools import cached, LRUCache
from cachetools.keys import hashkey
@cached(cache=LRUCache(maxsize=256), key=lambda file_path, mtime, profile: hashkey(file_path, mtime))
def get_time_range(file_path, mtime, profile):
start_time = None
end_time = None
for row in profile:
if row['ph'] != 'M':
start_time = row['ts']
break
for row in reversed(profile):
if row['ph'] != 'M':
end_time = row['ts']
break
return (start_time, end_time)
cache = LRUCache(maxsize=_config['database']['cache_size'])
_caches.append(cache)
wrapped = cached(cache, key=key)(func)
return wrapped
return decorator
def clear_cache():
"""
Clear ALL the caches!
"""
for cache in _caches:
cache.clear()
@_cache(key=lambda namespace, cursor=None: hashkey(namespace))
def repo_object_namespace_id(namespace, cursor=None):
"""
Get a repo object namespace, ID creating it if necessary.
"""
cursor = object_reader.namespace_id(namespace, cursor=cursor)
if not cursor.rowcount:
# @XXX burns the first PID in a namespace.
object_writer.get_pid_id(namespace, cursor=cursor)
return cursor.fetchone()['id']
@_cache(key=lambda namespace, cursor=None: hashkey(namespace))
def rdf_namespace_id(namespace, cursor=None):
diff = np.array([
domega,
ddelta,
deqp,
deqpp,
dedp,
dedpp,
],
dtype=np.float64)
return diff
# https://stackoverflow.com/a/32655449/8899565
@cached(cache=LRUCache(maxsize=128), key=lambda t, *args, **kwargs: hashkey(t))
def get_ybus_inv(t, ybus_og, ybus_states, d=1e-6):
ybus = ybus_og
for event_t, event_ybus in ybus_states:
factor = 1 / (1 + np.exp(np.clip(-(t - event_t) / d, -50, 50)))
ybus = ybus + factor * event_ybus # don't use in-place += as it mutates.
ybus_inv = np.linalg.inv(ybus)
return ybus_inv
# @profile
def residual(t, x, xdot, result, machs, ybus_og, ybus_states):
""" Aggregate machine residual functions. """
t1 = time.perf_counter()
class S3Backend(BaseBackend):
"""S3 Backend Adapter"""
client: Any = attr.ib(default=None)
bucket: str = attr.ib(init=False)
key: str = attr.ib(init=False)
_backend_name = "AWS S3"
def __attrs_post_init__(self):
"""Post Init: parse path and create client."""
assert boto3_session is not None, "'boto3' must be installed to use S3Backend"
parsed = urlparse(self.path)
self.bucket = parsed.netloc
self.key = parsed.path.strip("/")
self.client = self.client or boto3_session().client("s3")
super().__attrs_post_init__()
def write(self, overwrite: bool = False, **kwargs: Any):
"""Write mosaicjson document to AWS S3."""
if not overwrite and self._head_object(self.key, self.bucket):
raise MosaicExistsError(
"Mosaic file already exist, use `overwrite=True`.")
mosaic_doc = self.mosaic_def.dict(exclude_none=True)
if self.key.endswith(".gz"):
body = _compress_gz_json(mosaic_doc)
else:
body = json.dumps(mosaic_doc).encode("utf-8")
self._put_object(self.key, self.bucket, body, **kwargs)
@cached(
TTLCache(maxsize=cache_config.maxsize, ttl=cache_config.ttl),
key=lambda self: hashkey(self.path),
)
def _read(self) -> MosaicJSON: # type: ignore
"""Get mosaicjson document."""
body = self._get_object(self.key, self.bucket)
self._file_byte_size = len(body)
if self.key.endswith(".gz"):
body = _decompress_gz(body)
return MosaicJSON(**json.loads(body))
def _get_object(self, key: str, bucket: str) -> bytes:
try:
response = self.client.get_object(Bucket=bucket, Key=key)
except ClientError as e:
status_code = e.response["ResponseMetadata"]["HTTPStatusCode"]
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response["Error"]["Message"]) from e
return response["Body"].read()
def _put_object(self, key: str, bucket: str, body: bytes, **kwargs) -> str:
try:
self.client.put_object(Bucket=bucket, Key=key, Body=body, **kwargs)
except ClientError as e:
status_code = e.response["ResponseMetadata"]["HTTPStatusCode"]
exc = _HTTP_EXCEPTIONS.get(status_code, MosaicError)
raise exc(e.response["Error"]["Message"]) from e
return key
def _head_object(self, key: str, bucket: str) -> bool:
try:
return self.client.head_object(Bucket=bucket, Key=key)
except ClientError:
return False