def test_rfc3339_parse_datetime(): assert rfc3339.parse_datetime("2011-12-13T14:15:16Z") == datetime( 2011, 12, 13, 14, 15, 16) # `datetime.strptime` is apparently case insensitive about non-placeholder bits assert rfc3339.parse_datetime("2011-12-13t14:15:16z") == datetime( 2011, 12, 13, 14, 15, 16) # `datetime.strptime` does not require leading zeros for month, day, hour, minutes, seconds assert rfc3339.parse_datetime("0001-2-3T4:5:6Z") == datetime( 1, 2, 3, 4, 5, 6)
def get_jobs_in( get_path: Callable[[Union[str, None], Union[str, None]], str] ) -> List[Dict]: user_ids = self._zk.get_children(get_path(None, None)) jobs_before = [] for user_id in user_ids: user_job_ids = self._zk.get_children(get_path(user_id, None)) for job_id in user_job_ids: path = get_path(user_id, job_id) data, stat = self._zk.get(path) job_info = json.loads(data.decode()) updated = job_info.get('updated') job_date = (rfc3339.parse_datetime(updated) if updated else datetime.utcfromtimestamp(stat.last_modified)) if job_date < upper: _log.debug( "job {j}'s job_date {d} is before {u}".format( j=job_id, d=job_date, u=upper)) jobs_before.append(job_info) return jobs_before
def from_dict(cls, d: dict) -> 'ServiceMetadata': """Load ServiceMetadata from dict (e.g. parsed JSON dump).""" d = extract_namedtuple_fields_from_dict(d, ServiceMetadata) created = d.get("created") if isinstance(created, str): d["created"] = rfc3339.parse_datetime(created) return cls(**d)
def extract_namedtuple_fields_from_dict( d: dict, named_tuple_class: typing.Type[typing.NamedTuple], convert_datetime: bool = False, convert_timedelta: bool = False, ) -> dict: """ Extract `typing.NamedTuple` fields from given dictionary, silently skipping items not defined as field. :param d: dictionary :param named_tuple_class: `typing.NamedTuple` subclass :return: subset of given dictionary (only containing fields defined by named tuple class) """ field_names = set(named_tuple_class.__annotations__.keys()) result = {k: v for k, v in d.items() if k in field_names} required = set(f for f in field_names if f not in named_tuple_class._field_defaults) missing = set(f for f in required if f not in result) if missing: raise KeyError( f"Missing {named_tuple_class.__name__} field{'s' if len(missing) > 1 else ''}: {', '.join(sorted(missing))}." ) # Additional auto-conversions (by type annotation) converters = {} if convert_datetime: converters[datetime.datetime] = lambda v: rfc3339.parse_datetime(v) if convert_timedelta: converters[datetime.timedelta] = lambda v: datetime.timedelta(seconds=v) if converters: for k in result: converter = converters.get(named_tuple_class.__annotations__.get(k)) if converter: result[k] = converter(result[k]) return result
def test_rfc3339_parse_datetime_invalid(date): with pytest.raises(ValueError): rfc3339.parse_datetime(date)
def test_rfc3339_parse_datetime_none(): with pytest.raises(ValueError): rfc3339.parse_datetime(None) assert Rfc3339(propagate_none=True).parse_datetime(None) is None
def test_rfc3339_parse_datetime(): assert rfc3339.parse_datetime("2011-12-13T14:15:16Z") == datetime( 2011, 12, 13, 14, 15, 16)
def load_test_collection( collection_id: str, collection_metadata: GeopysparkCubeMetadata, extent, srs: str, from_date: str, to_date: str, bands=None, correlation_id: str = "NA", ) -> Dict[int, geopyspark.TiledRasterLayer]: """ Load synthetic data as test collection :param collection_id: :param collection_metadata: :param extent: :param srs: :param from_date: :param to_date: :param bands: :param correlation_id: :return: """ # TODO: support more test collections assert collection_id == "TestCollection-LonLat4x4" grid_size: float = 1.0 tile_size = 4 # TODO: support other srs'es? assert srs == "EPSG:4326" # Get bounds of tiling layout extent = geopyspark.Extent(extent.xmin(), extent.ymin(), extent.xmax(), extent.ymax()) col_min = int(math.floor(extent.xmin / grid_size)) row_min = int(math.floor(extent.ymin / grid_size)) col_max = int(math.ceil(extent.xmax / grid_size) - 1) row_max = int(math.ceil(extent.ymax / grid_size) - 1) # Simulate sparse range of observation dates from_date = rfc3339.parse_datetime(rfc3339.datetime(from_date)) to_date = rfc3339.parse_datetime(rfc3339.datetime(to_date)) dates = dates_between(from_date, to_date) # Build RDD of tiles with requested bands. tile_builder = TestCollectionLonLat(tile_size=tile_size, grid_size=grid_size) bands = bands or [b.name for b in collection_metadata.bands] rdd_data = [(SpaceTimeKey(col, row, date), tile_builder.get_tile(bands=bands, col=col, row=row, date=date)) for col in range(col_min, col_max + 1) for row in range(row_min, row_max + 1) for date in dates] rdd = SparkContext.getOrCreate().parallelize(rdd_data) metadata = Metadata( bounds=Bounds(SpaceTimeKey(col_min, row_min, min(dates)), SpaceTimeKey(col_max, row_max, max(dates))), crs="+proj=longlat +datum=WGS84 +no_defs ", cell_type=CellType.FLOAT64, extent=extent, layout_definition=LayoutDefinition( extent=geopyspark.Extent(col_min * grid_size, row_min * grid_size, (col_max + 1) * grid_size, (row_max + 1) * grid_size), tileLayout=TileLayout(layoutCols=col_max - col_min + 1, layoutRows=row_max - row_min + 1, tileCols=tile_size, tileRows=tile_size))) layer = TiledRasterLayer.from_numpy_rdd(LayerType.SPACETIME, rdd, metadata) return {0: layer}