def test_rfc3339_parse_datetime():
assert rfc3339.parse_datetime("2011-12-13T14:15:16Z") == datetime(
2011, 12, 13, 14, 15, 16)
# `datetime.strptime` is apparently case insensitive about non-placeholder bits
assert rfc3339.parse_datetime("2011-12-13t14:15:16z") == datetime(
2011, 12, 13, 14, 15, 16)
# `datetime.strptime` does not require leading zeros for month, day, hour, minutes, seconds
assert rfc3339.parse_datetime("0001-2-3T4:5:6Z") == datetime(
1, 2, 3, 4, 5, 6)
def get_jobs_in(
get_path: Callable[[Union[str, None], Union[str, None]], str]
) -> List[Dict]:
user_ids = self._zk.get_children(get_path(None, None))
jobs_before = []
for user_id in user_ids:
user_job_ids = self._zk.get_children(get_path(user_id, None))
for job_id in user_job_ids:
path = get_path(user_id, job_id)
data, stat = self._zk.get(path)
job_info = json.loads(data.decode())
updated = job_info.get('updated')
job_date = (rfc3339.parse_datetime(updated) if updated else
datetime.utcfromtimestamp(stat.last_modified))
if job_date < upper:
_log.debug(
"job {j}'s job_date {d} is before {u}".format(
j=job_id, d=job_date, u=upper))
jobs_before.append(job_info)
return jobs_before
def from_dict(cls, d: dict) -> 'ServiceMetadata':
"""Load ServiceMetadata from dict (e.g. parsed JSON dump)."""
d = extract_namedtuple_fields_from_dict(d, ServiceMetadata)
created = d.get("created")
if isinstance(created, str):
d["created"] = rfc3339.parse_datetime(created)
return cls(**d)
def extract_namedtuple_fields_from_dict(
d: dict, named_tuple_class: typing.Type[typing.NamedTuple],
convert_datetime: bool = False, convert_timedelta: bool = False,
) -> dict:
"""
Extract `typing.NamedTuple` fields from given dictionary,
silently skipping items not defined as field.
:param d: dictionary
:param named_tuple_class: `typing.NamedTuple` subclass
:return: subset of given dictionary (only containing fields defined by named tuple class)
"""
field_names = set(named_tuple_class.__annotations__.keys())
result = {k: v for k, v in d.items() if k in field_names}
required = set(f for f in field_names if f not in named_tuple_class._field_defaults)
missing = set(f for f in required if f not in result)
if missing:
raise KeyError(
f"Missing {named_tuple_class.__name__} field{'s' if len(missing) > 1 else ''}: {', '.join(sorted(missing))}."
)
# Additional auto-conversions (by type annotation)
converters = {}
if convert_datetime:
converters[datetime.datetime] = lambda v: rfc3339.parse_datetime(v)
if convert_timedelta:
converters[datetime.timedelta] = lambda v: datetime.timedelta(seconds=v)
if converters:
for k in result:
converter = converters.get(named_tuple_class.__annotations__.get(k))
if converter:
result[k] = converter(result[k])
return result
def test_rfc3339_parse_datetime_invalid(date):
with pytest.raises(ValueError):
rfc3339.parse_datetime(date)
def test_rfc3339_parse_datetime_none():
with pytest.raises(ValueError):
rfc3339.parse_datetime(None)
assert Rfc3339(propagate_none=True).parse_datetime(None) is None
def test_rfc3339_parse_datetime():
assert rfc3339.parse_datetime("2011-12-13T14:15:16Z") == datetime(
2011, 12, 13, 14, 15, 16)
def load_test_collection(
collection_id: str,
collection_metadata: GeopysparkCubeMetadata,
extent,
srs: str,
from_date: str,
to_date: str,
bands=None,
correlation_id: str = "NA",
) -> Dict[int, geopyspark.TiledRasterLayer]:
"""
Load synthetic data as test collection
:param collection_id:
:param collection_metadata:
:param extent:
:param srs:
:param from_date:
:param to_date:
:param bands:
:param correlation_id:
:return:
"""
# TODO: support more test collections
assert collection_id == "TestCollection-LonLat4x4"
grid_size: float = 1.0
tile_size = 4
# TODO: support other srs'es?
assert srs == "EPSG:4326"
# Get bounds of tiling layout
extent = geopyspark.Extent(extent.xmin(), extent.ymin(), extent.xmax(),
extent.ymax())
col_min = int(math.floor(extent.xmin / grid_size))
row_min = int(math.floor(extent.ymin / grid_size))
col_max = int(math.ceil(extent.xmax / grid_size) - 1)
row_max = int(math.ceil(extent.ymax / grid_size) - 1)
# Simulate sparse range of observation dates
from_date = rfc3339.parse_datetime(rfc3339.datetime(from_date))
to_date = rfc3339.parse_datetime(rfc3339.datetime(to_date))
dates = dates_between(from_date, to_date)
# Build RDD of tiles with requested bands.
tile_builder = TestCollectionLonLat(tile_size=tile_size,
grid_size=grid_size)
bands = bands or [b.name for b in collection_metadata.bands]
rdd_data = [(SpaceTimeKey(col, row, date),
tile_builder.get_tile(bands=bands,
col=col,
row=row,
date=date))
for col in range(col_min, col_max + 1)
for row in range(row_min, row_max + 1) for date in dates]
rdd = SparkContext.getOrCreate().parallelize(rdd_data)
metadata = Metadata(
bounds=Bounds(SpaceTimeKey(col_min, row_min, min(dates)),
SpaceTimeKey(col_max, row_max, max(dates))),
crs="+proj=longlat +datum=WGS84 +no_defs ",
cell_type=CellType.FLOAT64,
extent=extent,
layout_definition=LayoutDefinition(
extent=geopyspark.Extent(col_min * grid_size, row_min * grid_size,
(col_max + 1) * grid_size,
(row_max + 1) * grid_size),
tileLayout=TileLayout(layoutCols=col_max - col_min + 1,
layoutRows=row_max - row_min + 1,
tileCols=tile_size,
tileRows=tile_size)))
layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata)
return {0: layer}
