def _stac_to_sns(sns_arn, stac): """ Publish our STAC document to an SNS """ bbox = stac["bbox"] client = boto3.client("sns") client.publish( TopicArn=sns_arn, Message=json.dumps(stac, indent=4, default=json_fallback), MessageAttributes={ "action": {"DataType": "String", "StringValue": "ADDED"}, "datetime": { "DataType": "String", "StringValue": str(dicttoolz.get_in(["properties", "datetime"], stac)), }, "product": { "DataType": "String", "StringValue": dicttoolz.get_in(["properties", "odc:product"], stac), }, "maturity": { "DataType": "String", "StringValue": dicttoolz.get_in( ["properties", "dea:dataset_maturity"], stac ), }, "bbox.ll_lon": {"DataType": "Number", "StringValue": str(bbox.left)}, "bbox.ll_lat": {"DataType": "Number", "StringValue": str(bbox.bottom)}, "bbox.ur_lon": {"DataType": "Number", "StringValue": str(bbox.right)}, "bbox.ur_lat": {"DataType": "Number", "StringValue": str(bbox.top)}, }, )
def ownership_build_node(ownership): return ( None, relationship_filter_empty( [ { "subject": get_in(["interestedParty", "@id"], ownership, None), "predicate": { "key": f"{SINAR_NS_MOCK}ownershipOrControlStatement", "attributes": predicate_attribute_filter_empty( { "interest_level": get_in( ["interest_level", "token"], ownership, None ), "interest_type": get_in( ["interest_type", "token"], ownership, None ), } ), }, "object": ownership["bods_subject"]["@id"], } ] ), )
def get_album_artist_track_info(self): return [ (get_in(['id'], i[0], []), get_in(['id'], j, [])) for i in map(partial(get_in, ['tracks', 'items']), get_in(['albums'], loads(self.spotify.albums), [])) for j in get_in(['artists'], i[0], []) ]
def organization_build_node(organization): return ( attribute_filter_empty( { "id": organization["@id"], "name": organization["name"], "classficication": get_in( ["classification", "token"], organization, None ), }, ), relationship_filter_empty( [ { "subject": organization["@id"], "predicate": { "key": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "attributes": {}, }, "object": TYPE_ORGANIZATION, }, { "subject": organization["@id"], "predicate": { "key": "http://www.w3.org/ns/org#subOrganizationOf", "attributes": {}, }, "object": get_in( ["parent_organization", "@id"], organization, None ), }, ] ), )
def handle_bucket_notification_message(message, metadata: dict, record_path: tuple) -> Tuple[dict, str]: """[summary] Args: message (Message resource) metadata (dict): [description] record_path (tuple): [PATH for selectingthe s3 key path from the JSON message document] Raises: IndexingException: [Catch s3 ] Returns: Tuple[dict, str]: [description] """ data = None uri = None if metadata.get("Records"): for record in metadata.get("Records"): bucket_name = dicttoolz.get_in(["s3", "bucket", "name"], record) key = dicttoolz.get_in(["s3", "object", "key"], record) # Check for bucket name and key, and fail if there isn't one if not (bucket_name and key): # Not deleting this message, as it's non-conforming. Check this logic raise IndexingException( "No bucket name or key in message, are you sure this is a bucket notification?" ) # If you specific a list of record paths, and there's no # match in them for the key, then we skip this one forever if record_path is not None and not any( [PurePath(key).match(p) for p in record_path]): logging.warning( f"Key: {key} not in specified list of record_paths, deleting message from the queue." ) # This will return Nones, which will flag the message to be ignored return None, None # We have enough information to proceed, get the key and extract # the contents... try: s3 = boto3.resource("s3") obj = s3.Object(bucket_name, key).get(ResponseCacheControl="no-cache") data = load(obj["Body"].read()) uri = f"s3://{bucket_name}/{key}" except Exception as e: raise IndexingException( f"Exception thrown when trying to load s3 object: {e}") else: raise IndexingException( "Attempted to get metadata from record when no record key exists in message." ) return data, uri
def get_common_message_attributes(stac_doc: Dict) -> Dict: """ Returns common message attributes dict :param stac_doc: STAC dict :return: common message attributes dict """ msg_attributes = {} product = dicttoolz.get_in(["properties", "odc:product"], stac_doc) if product: msg_attributes["product"] = { "DataType": "String", "StringValue": product, } datetime = dicttoolz.get_in(["properties", "datetime"], stac_doc) if datetime: msg_attributes["datetime"] = { "DataType": "String", "StringValue": datetime, } cloudcover = dicttoolz.get_in(["properties", "eo:cloud_cover"], stac_doc) if cloudcover: msg_attributes["cloudcover"] = { "DataType": "Number", "StringValue": str(cloudcover), } maturity = dicttoolz.get_in(["properties", "dea:dataset_maturity"], stac_doc) if maturity: msg_attributes["maturity"] = { "DataType": "String", "StringValue": maturity, } bbox = dicttoolz.get_in(["bbox"], stac_doc) if bbox and len(bbox) > 3: msg_attributes["bbox.ll_lon"] = { "DataType": "Number", "StringValue": str(bbox[0]), } msg_attributes["bbox.ll_lat"] = { "DataType": "Number", "StringValue": str(bbox[1]), } msg_attributes["bbox.ur_lon"] = { "DataType": "Number", "StringValue": str(bbox[2]), } msg_attributes["bbox.ur_lat"] = { "DataType": "Number", "StringValue": str(bbox[3]), } return msg_attributes
def test_get_in(): # Test object support: o = C() a = C() a.b = 1 o.a = a assert get_in(['a', 'b'], o) == 1 assert get_in(['a', 'b', 'c'], o, 2) == 2 assert raises(AttributeError, lambda: get_in(['a', 'b', 'c'], o, no_default=True))
def test_dicttoolz(): d1 = {'foo': 'bar'} d2 = {'baz': 'quux'} assert_that(merge(d1, d2)).is_equal_to({'foo': 'bar', 'baz': 'quux'}) assert_that(d1).is_equal_to({'foo': 'bar'}) assert_that(assoc(d1, 'a', 1)).is_equal_to({'foo': 'bar', 'a': 1}) assert_that(dissoc(d2, 'baz')).is_equal_to({}) struct = {'a': [{'c': 'hello'}]} assert_that(get_in(['a', 0, 'c'], struct)).is_equal_to(struct['a'][0]['c']) assert_that(get_in(['a', 0, 'd'], struct, 'not found')).is_equal_to('not found')
def membership_build_node(membership): return ( attribute_filter_empty( {"id": membership["@id"], "label": membership.get("label", None)} ), relationship_filter_empty( [ { "subject": membership["@id"], "predicate": { "key": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "attributes": {}, }, "object": TYPE_MEMBERSHIP, }, { "subject": membership["@id"], "predicate": { "key": "http://www.w3.org/ns/org#member", "attributes": {}, }, "object": get_in(["person", "@id"], membership, None), }, { "subject": membership["@id"], "predicate": { "key": "http://www.w3.org/ns/org#organization", "attributes": {}, }, "object": get_in(["organization", "@id"], membership, None), }, { "subject": membership["@id"], "predicate": { "key": "http://www.w3.org/ns/opengov#post", "attributes": {}, }, "object": get_in(["post", "@id"], membership, None), }, { "subject": membership["@id"], "predicate": { "key": "http://www.w3.org/ns/opengov#onBehalfOf", "attributes": {}, }, "object": get_in(["on_behalf_of", "@id"], membership, None), }, ] ), )
def to_dataset_elem(cars_meta, cars_annos): description = get_in([0], cars_meta) classes = str(get_in([0, 0, 0], cars_annos)) split_desc = ''.join(description).split(" ") return { str(cars_meta[0]): { 'id': classes, 'make': split_desc[0], 'model': split_desc[1], 'body': split_desc[2], 'year': split_desc[3] } }
def post_build_node(post): return ( attribute_filter_empty( {"id": post["@id"], "label": post["label"], "role": post.get("role", None)}, ), relationship_filter_empty( [ { "subject": post["@id"], "predicate": { "key": "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", "attributes": {}, }, "object": TYPE_POST, }, { "subject": post["@id"], "predicate": { "key": "http://www.w3.org/ns/org#organization", "attributes": {}, }, "object": get_in(["organization", "@id"], post, None), }, ] ), )
def from_server_details_json(cls, server_json): """ Create a :obj:`NovaServer` instance from a server details JSON dictionary, although without any 'server' or 'servers' initial resource key. See http://docs.rackspace.com/servers/api/v2/cs-devguide/content/ Get_Server_Details-d1e2623.html :return: :obj:`NovaServer` instance """ try: server_state = ServerState.lookupByName(server_json['status']) except ValueError: server_state = ServerState.UNKNOWN_TO_OTTER if server_json.get("OS-EXT-STS:task_state", "") == "deleting": server_state = ServerState.DELETED metadata = server_json.get('metadata', {}) return cls( id=server_json['id'], state=server_state, created=timestamp_to_epoch(server_json['created']), image_id=get_in(["image", "id"], server_json), flavor_id=server_json['flavor']['id'], links=freeze(server_json['links']), desired_lbs=_lbs_from_metadata(metadata), servicenet_address=_servicenet_address(server_json), json=freeze(server_json))
def explode(d, keys): values = get_in(keys, d) if isinstance(values, list): for v in values: yield assoc_in(d, keys, v) else: yield d
def setup_selfheal_service(clock, config, dispatcher, health_checker, log): """ Setup selfheal timer service and return it. :param clock: :obj:`IReactorTime` provider :param dict config: Configuration dict containing selfheal info :param dispatcher: Effect dispatcher :param health_checker: ``HealthChecker`` object where SelfHeal's health check will be added :param log: :obj:`BoundLog` logger used by service :return: selfheal service or None if relevant config is not found :rtype: :obj:`IService` """ if "selfheal" not in config: return None interval = get_in(["selfheal", "interval"], config, no_default=True) selfheal = SelfHeal(clock, dispatcher, config_value, interval, log) func, lock = zk.locked_logged_func( dispatcher, "/selfheallock", log, "selfheal-lock-acquired", selfheal.setup) health_checker.checks["selfheal"] = zk.create_health_check(lock) sh_timer = TimerService(interval, func) sh_timer.clock = clock return sh_timer
def append_stack_uuid(stack_config, uuid): """ Append the given uuid to the `stack_name` value in `stack_config`. """ name_key = ('stack_name',) name = get_in(name_key, stack_config) return set_in(stack_config, name_key, name + '_%s' % uuid)
def _image_ref(self): """ Get group config's imageRef """ return get_in( self.group_config, ["launchConfiguration", "args", "server", "imageRef"])
def from_server_details_json(cls, server_json): """ Create a :obj:`NovaServer` instance from a server details JSON dictionary, although without any 'server' or 'servers' initial resource key. See http://docs.rackspace.com/servers/api/v2/cs-devguide/content/ Get_Server_Details-d1e2623.html :return: :obj:`NovaServer` instance """ try: server_state = ServerState.lookupByName(server_json['status']) except ValueError: server_state = ServerState.UNKNOWN_TO_OTTER if server_json.get("OS-EXT-STS:task_state", "") == "deleting": server_state = ServerState.DELETED metadata = server_json.get('metadata', {}) return cls(id=server_json['id'], state=server_state, created=timestamp_to_epoch(server_json['created']), image_id=get_in(["image", "id"], server_json), flavor_id=server_json['flavor']['id'], links=freeze(server_json['links']), desired_lbs=_lbs_from_metadata(metadata), servicenet_address=_servicenet_address(server_json), json=freeze(server_json))
def setup_selfheal_service(clock, config, dispatcher, health_checker, log): """ Setup selfheal timer service and return it. :param clock: :obj:`IReactorTime` provider :param dict config: Configuration dict containing selfheal info :param dispatcher: Effect dispatcher :param health_checker: ``HealthChecker`` object where SelfHeal's health check will be added :param log: :obj:`BoundLog` logger used by service :return: selfheal service or None if relevant config is not found :rtype: :obj:`IService` """ if "selfheal" not in config: return None interval = get_in(["selfheal", "interval"], config, no_default=True) selfheal = SelfHeal(clock, dispatcher, config_value, interval, log) func, lock = zk.locked_logged_func(dispatcher, "/selfheallock", log, "selfheal-lock-acquired", selfheal.setup) health_checker.checks["selfheal"] = zk.create_health_check(lock) sh_timer = TimerService(interval, func) sh_timer.clock = clock return sh_timer
def on_listing_nodes(rcv3_description, lbnodes_result): _, body = lbnodes_result return [ RCv3Node(node_id=node['id'], description=rcv3_description, cloud_server_id=get_in(('cloud_server', 'id'), node)) for node in body ]
def handle_json_message(metadata, transform, odc_metadata_link): odc_yaml_uri = None uri = None if odc_metadata_link: if odc_metadata_link.startswith("STAC-LINKS-REL:"): rel_val = odc_metadata_link.replace("STAC-LINKS-REL:", "") odc_yaml_uri = get_uri(metadata, rel_val) else: # if odc_metadata_link is provided, it will look for value with dict path provided odc_yaml_uri = dicttoolz.get_in(odc_metadata_link.split("/"), metadata) # if odc_yaml_uri exist, it will load the metadata content from that URL if odc_yaml_uri: try: content = requests.get(odc_yaml_uri).content metadata = documents.parse_yaml(content) uri = odc_yaml_uri except requests.RequestException as err: raise IndexingException( f"Failed to load metadata from the link provided - {err}") else: raise IndexingException("ODC EO3 metadata link not found") else: # if no odc_metadata_link provided, it will look for metadata dict "href" value with "rel==self" uri = get_uri(metadata, "self") if transform: metadata = transform(metadata) return metadata, uri
def store(): d = doi.pkg_doi(c.pkg_dict) c.pkg_dict.update({doi_key: d}) date = '{:%Y-%m-%d %H:%M:%S}'.format(datetime.now()) k = get_in([dara, 0], a) c.pkg_dict[k] = date tk.get_action('package_update')(context, c.pkg_dict)
def accum(d, item): d['count'] += 1 if item['car_exterior_present'] and get_in( ['car_detected_probability'], item, 0.0) >= p_threshold: d['true_positive'] += 1 elif not item['car_exterior_present'] and get_in( ['car_detected_probability'], item, 0.0) >= p_threshold: d['false_positive'] += 1 elif not item['car_exterior_present'] and not get_in( ['car_detected_probability'], item, 0.0) >= p_threshold: d['true_negative'] += 1 elif item['car_exterior_present'] and not get_in( ['car_detected_probability'], item, 0.0) >= p_threshold: d['false_negative'] += 1 return d
def validated_schedule( schedule: dict, getters: t.Optional[t.List[dict]] = None, ) -> dict: """Validates ``schedule`` with schemas and returns modified schedule dict if some getters provided. Getters are special functions that can be attached to get some schedule value from external source by key-path:: getters = [ { 'getter': payload_getter, 'params': { 'path': ['start', 'relative_timeshift', 'delay'], 'source': payload_source, } } ] In this case, :func:`payload_getter` will receive value by ``path`` from the ``payload_source``. And ``schedule`` will be modified with the result from ``payload_getter`` by the same path. """ if getters: getters = GettersSchema(getters) for getter in getters: modifier = getter['getter'] params = getter.get('params', {}) path_to_value = params.get('path') if path_to_value: path_value = get_in(path_to_value, schedule) if path_value is not None: val = modifier(path_value, **params) schedule = assoc_in(schedule, path_to_value, val) relative_params = ( get_in(['periodical', 'relative_day'], schedule) and get_in(['periodical', 'relative_day_index'], schedule) ) if relative_params: schedule = RelativeScheduleSchema(schedule) else: schedule = ScheduleSchema(schedule) return schedule
def config_value(name): """ :param str name: Name is a . separated path to a configuration value stored in a nested dictionary. :returns: The value specificed in the configuration file, or None. """ return get_in(name.split('.'), _config_data)
def _private_ipv4_addresses(server): """ Get all private IPv4 addresses from the addresses section of a server. :param dict server: A server dict. :return: List of IP addresses as strings. """ private_addresses = get_in(["addresses", "private"], server, []) return [addr['addr'] for addr in private_addresses if addr['version'] == 4]
def response(): if dara in a.iterkeys(): store() h.flash_success(get_in([dara, 1], a)) else: h.flash_error("ERROR! Sorry, dataset has not been registered or\ updated. Please contact your friendly sysadmin. ({})\ ".format(dara)) tk.redirect_to('dara_doi', id=id)
def get_accuracy(key): make, model = key features = ['make', 'model', 'year', 'color', 'body'] return { feature: [ get_in([f'{make}/{model}', f'{n}', 'accuracy', feature], accuracy) for n in ["1", "3", "5", "10"] ] for feature in features }
def copy_evaluation_to_spreadsheet(meta, evaluation, spreadsheet_id, sheet_name, row): for feature, feature_meta in meta.items(): pprint(feature) data_keys = feature_meta['data_keys'] pprint(data_keys) values = list(map(lambda key: get_in([key, 'accuracy', feature], evaluation), data_keys)) pprint(values) col_begin = feature_meta['col_begin'] col_end = feature_meta['col_end'] rangeName = f'{sheet_name}!{col_begin}{row}:{col_end}{row}' pprint(rangeName) update_cells(spreadsheet_id, rangeName, [values])
def try_json_with_keys(maybe_json_error, keys): """ Attemp to grab the message body from possibly a JSON error body. If invalid JSON, or if the JSON is of an unexpected format (keys are not found), `None` is returned. """ try: error_body = json.loads(maybe_json_error) except (ValueError, TypeError): return None else: return get_in(keys, error_body, None)
def fn(d): match_make = not makes or d['make'] in makes match_model = not models or d['model'] in models match_x_image_ids = not x_image_ids or d['image_id'] not in x_image_ids match_seller = not seller or d['seller'] == seller match_probability = get_in(['bounding_box', 'probability'], d, 1.0) >= probability return (match_make and match_model and match_x_image_ids and match_seller and match_probability)
def format_single_event(response): start = response['start'] end = response['end'] return { 'creator': get_in(['creator', 'displayName'], response), 'status': response.get('status'), 'summary': split_by(response.get('summary', '')), 'start': start.get('date') or start.get('dateTime'), 'end': end.get('date') or end.get('dateTime'), 'kind': EVENT_KIND_MAPPING.get(response.get('kind'), 'unknown'), 'all_day': 'Yes' if response['start'].get('date') else 'No' }
def _match_errors(code_keys_exc_mapping, status_code, response_dict): """ Take a list of tuples of: (status code, json keys, regex pattern (optional), exception callable), and attempt to match them against the given status code and response dict. If a match is found raises the given exception type with the exception callable, passing along the message. """ for code, keys, pattern, make_exc in code_keys_exc_mapping: if code == status_code: message = get_in(keys, response_dict, None) if message is not None and (not pattern or pattern.match(message)): raise make_exc(message)
def match_errors(code_keys_exc_mapping, status_code, response_dict): """ Take a list of tuples of: (status code, json keys, regex pattern (optional), exception callable), and attempt to match them against the given status code and response dict. If a match is found raises the given exception type with the exception callable, passing along the message. """ for code, keys, pattern, make_exc in code_keys_exc_mapping: if code == status_code: message = get_in(keys, response_dict, None) if message is not None and (not pattern or pattern.match(message)): raise make_exc(message)
def _predict_action(state): mine = shapely.geometry.Polygon(state['desc']['mine_shell']) obstacles = [shapely.geometry.Polygon(sh) for sh in state['desc']['obstacle_shells']] obstacle = shapely.ops.unary_union(obstacles) situable = mine.difference(obstacle) wrappeds = [shapely.geometry.Polygon(sh) for sh in state['wrapped_shells']] wrapped = shapely.ops.unary_union(wrappeds) not_wrapped = situable.difference(wrapped) if not_wrapped.area < 1.0: return None, state last_move = state.get('last_move', 'W') for move in [last_move, 'W', 'S', 'A', 'D']: proj = _move_projection_center(state['worker']['pos'], move) if not_wrapped.contains(proj): return move, tzd.dissoc(state, 'path_pts_to_not_wrapped') if not state.get('path_pts_to_not_wrapped'): target_tile = tzf.thread_first(not_wrapped.representative_point(), _shapely_point2pt, _snap_to_tile) print('Finding shortest path from tile {} to {}'.format(state['worker']['pos'], target_tile)) if tzd.get_in(['cache', 'incidence_m'], state) is None: incidence_m = _incidence_matrix(situable) state = tzd.assoc_in(state, ['cache', 'incidence_m'], incidence_m) else: incidence_m = state['cache']['incidence_m'] target_vertex_ind = _incidence_ind(target_tile[0], target_tile[1], x_size=math.ceil(situable.bounds[2])) path_dists, path_predecessors = sp.sparse.csgraph.shortest_path(csgraph=incidence_m, directed=False, return_predecessors=True, unweighted=True, indices=target_vertex_ind) start_vertex_ind = _incidence_ind(state['worker']['pos'][0], state['worker']['pos'][1], x_size=math.ceil(situable.bounds[2])) path_inds = _path_inds(path_predecessors, start_vertex_ind) path_pts = [_incidence_pt(ind, x_size=math.ceil(situable.bounds[2])) for ind in path_inds] print('Found path: {}'.format(path_pts)) state = tzd.assoc(state, 'path_pts_to_not_wrapped', path_pts) path_move = _projection_pt_move(state['worker']['pos'], state['path_pts_to_not_wrapped'][0]) if path_move is not None: return path_move, tzd.update_in(state, ['path_pts_to_not_wrapped'], lambda p: p[1:]) return 'Z', state
async def update(self, dto: UpdateTodoItemDto, id_: int): item = get_in([id_], self.items) if not item: return None self.items, new_item = pipe( (item, dto), lambda items: { **items[0].dict(), **items[1].dict(exclude_defaults=True) }, lambda data: TodoItem(**data), lambda todo: (assoc(self.items, id_, todo), todo), ) return new_item
def prepare_server_launch_config(group_id, server_config, lb_descriptions): """ Prepare a server config (the server part of the Group's launch config) with any necessary dynamic data. :param str group_id: The group ID :param PMap server_config: The server part of the Group's launch config, as per :obj:`otter.json_schema.group_schemas.server` except as the value of a one-element PMap with key "server". :param iterable lb_descriptions: iterable of :class:`ILBDescription` providers """ updated_metadata = merge( get_in(('server', 'metadata'), server_config, {}), generate_metadata(group_id, lb_descriptions)) return set_in(server_config, ('server', 'metadata'), updated_metadata)
def _is_server_in_group(group, server_id): """ Given a group and server ID, determines if the server is a member of the group. If it isn't, it raises a :class:`ServerNotFoundError`. """ try: response, server_info = yield Effect( TenantScope( retry_effect(get_server_details(server_id), retry_times(3), exponential_backoff_interval(2)), group.tenant_id, ) ) except NoSuchServerError: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id) group_id = group_id_from_metadata(get_in(("server", "metadata"), server_info, {})) if group_id != group.uuid: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id)
def orcid_map(k): return (k, get_in(mapping[k], profile, default=author_orig[k]))