コード例 #1
0
def _validate_jmespath_compiles(expression):
    try:
        jmespath.compile(expression)
    except JMESPathError as e:
        raise AssertionError("Invalid JMESPath expression used "
                             "in pagination config: %s\nerror: %s"
                             % (expression, e))
コード例 #2
0
ファイル: policy.py プロジェクト: JJediny/cloud-custodian
 def validate(self):
     events = self.policy.data['mode'].get('events')
     assert events, "cloud trail mode requires specifiying events to subscribe"
     for e in events:
         if isinstance(e, six.string_types):
             assert e in CloudWatchEvents.trail_events, "event shortcut not defined: %s" % e
         if isinstance(e, dict):
             jmespath.compile(e['ids'])
コード例 #3
0
ファイル: disk.py プロジェクト: apache/aurora
  def __init__(
      self,
      http_api_url=DEFAULT_AGENT_CONTAINERS_ENDPOINT,
      executor_id_json_path=DEFAULT_EXECUTOR_ID_PATH,
      disk_usage_json_path=DEFAULT_DISK_USAGE_PATH,
      disk_collection_timeout=DEFAULT_DISK_COLLECTION_TIMEOUT,
      disk_collection_interval=DISK_COLLECTION_INTERVAL):

    self._http_api_url = http_api_url
    # We compile the JMESpath here for speed and also to detect bad JMESPaths immediately
    self._executor_id_json_expression = compile(executor_id_json_path)
    self._disk_usage_json_expression = compile(disk_usage_json_path)
    self._disk_collection_interval = disk_collection_interval
    self._disk_collection_timeout = disk_collection_timeout
コード例 #4
0
ファイル: policy.py プロジェクト: joshuaroot/cloud-custodian
 def validate(self):
     events = self.policy.data['mode'].get('events')
     assert events, "cloud trail mode requires specifiying events to subscribe"
     for e in events:
         if isinstance(e, six.string_types):
             assert e in CloudWatchEvents.trail_events, "event shortcut not defined: %s" % e
         if isinstance(e, dict):
             jmespath.compile(e['ids'])
     if isinstance(self.policy.resource_manager, query.ChildResourceManager):
         if not getattr(self.policy.resource_manager.resource_type,
                        'supports_trailevents', False):
             raise ValueError(
                 "resource:%s does not support cloudtrail mode policies" % (
                     self.policy.resource_type))
コード例 #5
0
ファイル: core.py プロジェクト: capitalone/cloud-custodian
 def get_resource_value(self, k, i):
     if k.startswith('tag:'):
         tk = k.split(':', 1)[1]
         r = None
         if 'Tags' in i:
             for t in i.get("Tags", []):
                 if t.get('Key') == tk:
                     r = t.get('Value')
                     break
         # GCP schema: 'labels': {'key': 'value'}
         elif 'labels' in i:
             r = i.get('labels', {}).get(tk, None)
         # GCP has a secondary form of labels called tags
         # as labels without values.
         # Azure schema: 'tags': {'key': 'value'}
         elif 'tags' in i:
             r = i.get('tags', {}).get(tk, None)
     elif k in i:
         r = i.get(k)
     elif k not in self.expr:
         self.expr[k] = jmespath.compile(k)
         r = self.expr[k].search(i)
     else:
         r = self.expr[k].search(i)
     return r
コード例 #6
0
ファイル: waiter.py プロジェクト: Bmitchem/RiotGames
    def _create_path_matcher(self):
        expression = jmespath.compile(self.argument)
        expected = self.expected

        def acceptor_matches(response):
            return expression.search(response) == expected
        return acceptor_matches
コード例 #7
0
ファイル: query.py プロジェクト: ssato/python-anyconfig
def query(data, **options):
    """
    Filter data with given JMESPath expression.

    See also: https://github.com/jmespath/jmespath.py and http://jmespath.org.

    :param data: Target object (a dict or a dict-like object) to query
    :param options:
        Keyword option may include 'ac_query' which is a string represents
        JMESPath expression.

    :return: Maybe queried result data, primitive (int, str, ...) or dict
    """
    expression = options.get("ac_query", None)
    if expression is None or not expression:
        return data

    try:
        pexp = jmespath.compile(expression)
        return pexp.search(data)
    except ValueError as exc:  # jmespath.exceptions.*Error inherit from it.
        LOGGER.warning("Failed to compile or search: exp=%s, exc=%r",
                       expression, exc)
    except (NameError, AttributeError):
        LOGGER.warning("Filter module (jmespath) is not available. "
                       "Do nothing.")

    return data
コード例 #8
0
 def _get_result_keys(self, config):
     result_key = config.get('result_key')
     if result_key is not None:
         if not isinstance(result_key, list):
             result_key = [result_key]
         result_key = [jmespath.compile(rk) for rk in result_key]
         return result_key
コード例 #9
0
    def search(self, expression):
        """Applies a JMESPath expression to a paginator

        Each page of results is searched using the provided JMESPath
        expression. If the result is not a list, it is yielded
        directly. If the result is a list, each element in the result
        is yielded individually (essentially implementing a flatmap in
        which the JMESPath search is the mapping function).

        :type expression: str
        :param expression: JMESPath expression to apply to each page.

        :return: Returns an iterator that yields the individual
            elements of applying a JMESPath expression to each page of
            results.
        """
        compiled = jmespath.compile(expression)
        for page in self:
            results = compiled.search(page)
            if isinstance(results, list):
                for element in results:
                    yield element
            else:
                # Yield result directly if it is not a list.
                yield results
コード例 #10
0
    def test_distribution_set_ssl(self):
        factory = self.replay_flight_data("test_distrbution_set_ssl")

        k = "DefaultCacheBehavior.ViewerProtocolPolicy"

        p = self.load_policy(
            {
                "name": "distribution-set-ssl",
                "resource": "distribution",
                "filters": [
                    {"type": "value", "key": k, "value": "allow-all", "op": "contains"}
                ],
                "actions": [
                    {"type": "set-protocols", "ViewerProtocolPolicy": "https-only"}
                ],
            },
            session_factory=factory,
        )
        resources = p.run()
        self.assertEqual(len(resources), 1)
        expr = jmespath.compile(k)
        r = expr.search(resources[0])
        self.assertTrue("allow-all" in r)

        client = local_session(factory).client("cloudfront")
        resp = client.list_distributions()
        self.assertEqual(
            resp["DistributionList"]["Items"][0]["DefaultCacheBehavior"][
                "ViewerProtocolPolicy"
            ],
            "https-only",
        )
コード例 #11
0
ファイル: index.py プロジェクト: AlvinChiou/aws-shell
def extract_field_from_jmespath(expression):
    result = jmespath.compile(expression)
    current = result.parsed
    while current['children']:
        current = current['children'][0]
    if current['type'] == 'field':
        return current['value']
コード例 #12
0
    def process(self, functions):
        client = local_session(self.manager.session_factory).client('lambda')
        is_expr = self.data.get('expr', False)
        value = self.data['value']
        if is_expr:
            value = jmespath.compile(value)

        none_type = type(None)

        for function in functions:
            fvalue = value
            if is_expr:
                fvalue = value.search(function)
                if isinstance(fvalue, float):
                    fvalue = int(fvalue)
                if isinstance(value, int) or isinstance(value, none_type):
                    self.policy.log.warning(
                        "Function: %s Invalid expression value for concurrency: %s",
                        function['FunctionName'], fvalue)
                    continue
            if fvalue is None:
                client.delete_function_concurrency(
                    FunctionName=function['FunctionName'])
            else:
                client.put_function_concurrency(
                    FunctionName=function['FunctionName'],
                    ReservedConcurrentExecutions=fvalue)
コード例 #13
0
ファイル: __init__.py プロジェクト: josh-paul/skew
 def _build_components_from_string(self, arn_string):
     if '|' in arn_string:
         arn_string, query = arn_string.split('|')
         self.query = jmespath.compile(query)
     pairs = zip_longest(
         self.ComponentClasses, arn_string.split(':', 6), fillvalue='*')
     self._components = [c(n, self) for c, n in pairs]
コード例 #14
0
ファイル: util.py プロジェクト: iopipe/iopipe-python
def _get_parser(path):
    path = _format_path(path)
    parser = parsers.get(path)
    if parser is None:
        parser = jmespath.compile(path)
        parsers[path] = parser
    return parser
コード例 #15
0
    def match(self, i):
        if self.v is None and len(self.data) == 1:
            [(self.k, self.v)] = self.data.items()
        elif self.v is None:
            self.k = self.data.get('key')
            self.op = self.data.get('op')
            self.v = self.data.get('value')

        # Value extract
        if self.k.startswith('tag:'):
            tk = self.k.split(':', 1)[1]
            r = None
            for t in i.get("Tags", []):
                if t.get('Key') == tk:
                    r = t.get('Value')
                    break
        elif not '.' in self.k and not '[' in self.k and not '(' in self.k:
            r = i.get(self.k)
        elif self.expr:
            r = self.expr.search(i)
        else:
            self.expr = jmespath.compile(self.k)
            r = self.expr.search(i)

        # Value match
        if r is None and self.v == 'absent':
            return True
        elif self.v == 'not-null' and r:
            return True
        elif self.op:
            op = OPERATORS[self.op]
            return op(r, self.v)
        elif r == self.v:
            return True
        return False
コード例 #16
0
    def test_distribution_set_ssl(self):
        factory = self.replay_flight_data('test_distrbution_set_ssl')

        k = 'DefaultCacheBehavior.ViewerProtocolPolicy'

        p = self.load_policy({
            'name': 'distribution-set-ssl',
            'resource': 'distribution',
            'filters': [{
                'type': 'value',
                'key': k,
                'value': 'allow-all',
                'op': 'contains'
            }],
            'actions': [{
                'type': 'set-protocols',
                'ViewerProtocolPolicy': 'https-only'
            }]
        }, session_factory=factory)
        resources = p.run()
        self.assertEqual(len(resources), 1)
        expr = jmespath.compile(k)
        r = expr.search(resources[0])
        self.assertTrue('allow-all' in r)

        client = local_session(factory).client('cloudfront')
        resp = client.list_distributions()
        self.assertEqual(
            resp['DistributionList']['Items'][0]['DefaultCacheBehavior']['ViewerProtocolPolicy'],
            'https-only')
コード例 #17
0
    def test_distribution_custom_origin(self):
        factory = self.replay_flight_data('test_distrbution_custom_origin')

        k = 'Origins.Items[].CustomOriginConfig.OriginSslProtocols.Items[]'

        p = self.load_policy({
            'name': 'distribution-set-ssl',
            'resource': 'distribution',
            'filters': [{
                'type': 'value',
                'key': k,
                'value': 'TLSv1',
                'op': 'contains'
            }],
            'actions': [{
                'type': 'set-protocols',
                'OriginSslProtocols': ['TLSv1.1','TLSv1.2'],
                'OriginProtocolPolicy': 'https-only'
            }]
        }, session_factory=factory)
        resources = p.run()
        self.assertEqual(len(resources), 1)
        expr = jmespath.compile(k)
        r = expr.search(resources[0])
        self.assertTrue('TLSv1' in r)

        client = local_session(factory).client('cloudfront')
        resp = client.list_distributions()
        self.assertEqual(
            resp['DistributionList']['Items'][0]['Origins']['Items'][0]['CustomOriginConfig']['OriginProtocolPolicy'],
            'https-only')
        self.assertTrue('TLSv1.2' in
            resp['DistributionList']['Items'][0]['Origins']['Items'][0]['CustomOriginConfig']['OriginSslProtocols']['Items'])
        self.assertFalse('TLSv1' in
            resp['DistributionList']['Items'][0]['Origins']['Items'][0]['CustomOriginConfig']['OriginSslProtocols']['Items'])
コード例 #18
0
ファイル: retrieve_MPDS.py プロジェクト: matk86/MatMiner
    def get_data(self, search, phases=[], fields=default_fields):
        """
        Retrieve data in JSON.
        JSON is expected to be valid against the schema
        at http://developer.mpds.io/mpds.schema.json

        Args:
            search: (dict) Search query like {"categ_A": "val_A", "categ_B": "val_B"},
                documented at http://developer.mpds.io/#Categories
            phases: (list) Phase IDs, according to the MPDS distinct phases concept
            fields: (dict) Data of interest for C-, S-, and P-entries,
                e.g. for phase diagrams: {'C': ['naxes', 'arity', 'shapes']},
                documented at http://developer.mpds.io/#JSON-schemata

        Returns:
            List of dicts: C-, S-, and P-entries, the format is
            documented at http://developer.mpds.io/#JSON-schemata
        """
        output = []
        counter, hits_count = 0, 0
        fields = {
            key: [jmespath.compile(item) if isinstance(item, six.string_types) else item() for item in value]
            for key, value in fields.items()
        } if fields else None

        while True:
            result = self._request(search, phases=phases, page=counter)
            if result['error']:
                raise APIError(result['error'], result.get('code', 0))

            if result['npages'] > MPDSDataRetrieval.maxnpages:
                raise APIError(
                    "Too much hits (%s > %s), please, be more specific" % \
                    (result['count'], MPDSDataRetrieval.maxnpages * MPDSDataRetrieval.pagesize),
                    1
                )
            assert result['npages'] > 0

            output.extend(self._massage(result['out'], fields))

            if hits_count and hits_count != result['count']:
                raise APIError("API error: hits count has been changed during the query")
            hits_count = result['count']

            if counter == result['npages'] - 1:
                break

            counter += 1
            time.sleep(MPDSDataRetrieval.chillouttime)

            sys.stdout.write("\r\t%d%%" % ((counter / result['npages']) * 100))
            sys.stdout.flush()

        if len(output) != hits_count:
            raise APIError("API error: collected and declared counts of hits differ")

        sys.stdout.write("\r\nGot %s hits\r\n" % hits_count)
        sys.stdout.flush()
        return output
コード例 #19
0
ファイル: filter.py プロジェクト: walkinreeds/eliottree
def filter_by_jmespath(query):
    """
    Produce a function for filtering a task by a jmespath query expression.
    """
    def _filter(task):
        return bool(expn.search(task))
    expn = jmespath.compile(query)
    return _filter
コード例 #20
0
ファイル: jmespath_example.py プロジェクト: vhnuuh/pyutil
def example1():
    path = jmespath.search('foo.bar', {'foo': {'bar': 'baz'}})
    print(path)

    expression = jmespath.compile('foo.bar')
    print(expression.search({'foo': {'bar': 'baz'}}))

    print(expression.search({'foo': {'bar': 'other'}}))
コード例 #21
0
ファイル: test_compliance.py プロジェクト: issuu/jmespath
def _test_expression(given, expression, expected, filename):
    parsed = jmespath.compile(expression)
    actual = parsed.search(given)
    expected_repr = json.dumps(expected, indent=4)
    actual_repr = json.dumps(actual, indent=4)
    error_msg = ("\n(%s) The expression '%s' was suppose to give: %s.\n"
                 "Instead it matched: %s" % (filename, expression, expected_repr, actual_repr))
    assert_equal(actual, expected, error_msg)
コード例 #22
0
ファイル: test_cwe.py プロジェクト: ewbankkit/cloud-custodian
 def test_cloud_trail_resource(self):
     self.assertEqual(
         CloudWatchEvents.match(event_data("event-cloud-trail-s3.json")),
         {
             "source": "s3.amazonaws.com",
             "ids": jmespath.compile("detail.requestParameters.bucketName"),
         },
     )
コード例 #23
0
ファイル: waiter.py プロジェクト: Abixer/croomcroom
 def _process_config(self, acceptor_config):
     if acceptor_config is None:
         return {}
     new_config = acceptor_config.copy()
     if new_config['type'] == 'output' and \
             new_config.get('path') is not None:
         new_config['path'] = jmespath.compile(acceptor_config['path'])
     return new_config
コード例 #24
0
 def _get_output_tokens(self, config):
     output = []
     output_token = config['output_token']
     if not isinstance(output_token, list):
         output_token = [output_token]
     for config in output_token:
         output.append(jmespath.compile(config))
     return output
コード例 #25
0
ファイル: command_state.py プロジェクト: globus/globus-cli
    def jmespath_callback(ctx, param, value):
        if value is None:
            return

        state = ctx.ensure_object(CommandState)
        state.jmespath_expr = jmespath.compile(value)

        if state.output_format == TEXT_FORMAT:
            state.output_format = JSON_FORMAT
コード例 #26
0
    def test_distribution_custom_origin(self):
        factory = self.replay_flight_data("test_distrbution_custom_origin")

        k = "Origins.Items[].CustomOriginConfig.OriginSslProtocols.Items[]"

        p = self.load_policy(
            {
                "name": "distribution-set-ssl",
                "resource": "distribution",
                "filters": [
                    {"type": "value", "key": k, "value": "TLSv1", "op": "contains"}
                ],
                "actions": [
                    {
                        "type": "set-protocols",
                        "OriginSslProtocols": ["TLSv1.1", "TLSv1.2"],
                        "OriginProtocolPolicy": "https-only",
                    }
                ],
            },
            session_factory=factory,
        )
        resources = p.run()
        self.assertEqual(len(resources), 1)
        expr = jmespath.compile(k)
        r = expr.search(resources[0])
        self.assertTrue("TLSv1" in r)

        client = local_session(factory).client("cloudfront")
        resp = client.list_distributions()
        self.assertEqual(
            resp["DistributionList"]["Items"][0]["Origins"]["Items"][0][
                "CustomOriginConfig"
            ][
                "OriginProtocolPolicy"
            ],
            "https-only",
        )
        self.assertTrue(
            "TLSv1.2" in resp["DistributionList"]["Items"][0]["Origins"]["Items"][0][
                "CustomOriginConfig"
            ][
                "OriginSslProtocols"
            ][
                "Items"
            ]
        )
        self.assertFalse(
            "TLSv1" in resp["DistributionList"]["Items"][0]["Origins"]["Items"][0][
                "CustomOriginConfig"
            ][
                "OriginSslProtocols"
            ][
                "Items"
            ]
        )
コード例 #27
0
ファイル: aws_api.py プロジェクト: 314l5926/gimel
def aws(service, action, **kwargs):
    client = boto_session().client(service)
    query = kwargs.pop('query', None)
    if client.can_paginate(action):
        paginator = client.get_paginator(action)
        result = paginator.paginate(**kwargs).build_full_result()
    else:
        result = getattr(client, action)(**kwargs)
    if query:
        result = jmespath.compile(query).search(result)
    return result
コード例 #28
0
    def configure(self):
        super(SimpleJSON, self).configure()

        self.url = self.config.get('url', None)
        self.polling_timeout = self.config.get('polling_timeout', 20)
        self.verify_cert = self.config.get('verify_cert', True)

        self.extractor = jmespath.compile(self.config.get('extractor', '@'))
        self.indicator = self.config.get('indicator', 'indicator')
        self.prefix = self.config.get('prefix', 'json')
        self.fields = self.config.get('fields', None)
コード例 #29
0
ファイル: awsclient.py プロジェクト: Gifflen/skew
    def call(self, op_name, query=None, **kwargs):
        """
        Make a request to a method in this client.  The response data is
        returned from this call as native Python data structures.

        This method differs from just calling the client method directly
        in the following ways:

          * It automatically handles the pagination rather than
            relying on a separate pagination method call.
          * You can pass an optional jmespath query and this query
            will be applied to the data returned from the low-level
            call.  This allows you to tailor the returned data to be
            exactly what you want.

        :type op_name: str
        :param op_name: The name of the request you wish to make.

        :type query: str
        :param query: A jmespath query that will be applied to the
            data returned by the operation prior to returning
            it to the user.

        :type kwargs: keyword arguments
        :param kwargs: Additional keyword arguments you want to pass
            to the method when making the request.
        """
        LOG.debug(kwargs)
        if query:
            query = jmespath.compile(query)
        if self._client.can_paginate(op_name):
            paginator = self._client.get_paginator(op_name)
            results = paginator.paginate(**kwargs)
            data = results.build_full_result()
        else:
            op = getattr(self._client, op_name)
            done = False
            data = {}
            while not done:
                try:
                    data = op(**kwargs)
                    done = True
                except ClientError as e:
                    LOG.debug(e, kwargs)
                    if 'Throttling' in str(e):
                        time.sleep(1)
                    elif 'AccessDenied' in str(e):
                        done = True
                except Exception:
                    done = True
        if query:
            data = query.search(data)
        return data
コード例 #30
0
def _eval_expression(eb, i):
    ce = jmespath.compile(eb.expression)

    try:
        r = ce.search(i)
    except jmespath.exceptions.JMESPathError:
        LOG.exception("Exception in searching")
        r = None

    LOG.info("r: %s value: %s", r, eb.value)

    return eb.comparator(r, eb.value)
コード例 #31
0
ファイル: __init__.py プロジェクト: Danielhiversen/brainfm
         "type": str
     }],
     "response":
     None,
     "exceptions": {}
 },
 # setTestCompleted  # unknown; not seen
 # setFeedback
 "get_stations": {
     "name":
     "getExploreStations",
     "method":
     "post",
     "parameters": [],
     "response":
     jmespath.compile(
         "*[*].{station_id:id, name:name, canonical_name:string_id}[]"),
     "exceptions": {}
 },
 # getMainStations  # not implemented for now, use get_stations
 # getMainStations_NoPlans  # not implemented for now, use get_stations
 "get_stations_by_id": {
     "name":
     "getStationsByID",
     "method":
     "post",
     "parameters": [{
         "name": "pid",
         "alias": "parent_id",
         "required": True,
         "type": int
     }],
コード例 #32
0
if (True and "iterate"):
    vcc = cerberus.Validator(allow_unknown=True)
    for dataroot in aadocuments:

        print(
            "## -------------------------------------------------------------------"
        )
        print("## {person_fname} {person_lname}".format(**dataroot))
        pass

        for myruleset in validationrules_table:
            print("----")
            ddresult = dict()
            ddresult.update(myruleset)
            ddresult['rule_vpath_hasdata'] = (not jmespath.compile(
                myruleset['rule_vpath']).search(dataroot) is None)
            ddresult['rule_vpath_dataval'] = (jmespath.compile(
                myruleset['rule_vpath']).search(dataroot))
            pass
            if (ddresult['rule_vpath_hasdata']):
                vcc.schema = myruleset['validation_schema']
                ddresult['validation_result'] = vcc.validate(dataroot)
                ddresult['validation_errors'] = vcc.errors
            elif (True):
                ddresult['validation_result'] = None
            pprint.pprint(ddresult)
        pass

    pass
##endif
コード例 #33
0
class TypeMeta(type):

    def __repr__(cls):
        return "<TypeInfo service:%s component:%s scope:%s version:%s>" % (
            cls.service,
            cls.component,
            cls.scope,
            cls.version)


@six.add_metaclass(TypeMeta)
class TypeInfo(object):

    service = None
    version = None
    scope = 'project'
    enum_spec = ('list', 'items', None)


ERROR_REASON = jmespath.compile('error.errors[0].reason')


def extract_error(e):

    try:
        edata = json.loads(e.content)
    except Exception:
        return None
    return ERROR_REASON.search(edata)
コード例 #34
0
#!/usr/bin/env python
import jmespath
import json
import sys

expression = sys.argv[1]
parsed = jmespath.compile(expression)
print(json.dumps(parsed.parsed, indent=2))
コード例 #35
0
class FhirApi(PatientApi):

    extraction_functions: Dict[str, path.parser.ParsedResult] = {
        'resources': path.compile('entry[*].resource'),
        'next': path.compile("link[?relation=='next'].url | [0]")
    }

    def page_parameter(self, page: int) -> str:
        pass

    def get_fhir_bundle(
            self,
            endpoint: str,
            params=None,
            count=100) -> Iterable[Dict[str, Union[str, list, dict]]]:
        url: str = f"{self.base_url}{endpoint}?patient={self.id}&_count={count}"
        logging.info(f"Getting resource at {url}")
        bundle = self.get(url, params)
        total = bundle.get('total', 0)
        logging.info(f"Total {total}, received {url}")
        for resource in self.extraction_functions['resources'].search(bundle):
            yield resource
        if total > count:
            gpool = pool.Pool(40)
            next_url = self.extraction_functions['next'].search(bundle)
            logging.info(f"Next url would be {next_url}")
            final_page = ((total - 1) // count) + 1
            pages = {}
            for page_num in range(2, final_page + 1):
                page_param = self.page_parameter(page_num)
                url_page = f"{url}{page_param}"
                logging.info(f"Getting resource at {url_page}")
                gpool.wait_available()
                pages[gpool.spawn(self.get, url_page, params)] = url_page
            if len(pages) == 0:
                logging.warn("Unexpected issue, pages empty")
            else:
                for page in iwait(pages):
                    logging.info(f"Received resource at {pages[page]}")
                    resources = self.extraction_functions['resources'].search(
                        page.value)
                    if resources is not None:
                        for resource in resources:
                            logging.info(
                                f"Returning {endpoint} resources {resource['id']}"
                            )
                            yield resource
                    else:
                        logging.warn(f"Unexpected resource: {str(page.value)}")

        # while url is not None:
        #     logging.info(f"Getting resource at {url}")
        #     bundle = self.get(url, params)
        #     logging.info(f"Total {bundle['total']}, received {url}")
        #     for resource in self.extraction_functions['resources'].search(bundle):
        #         yield resource
        #     url = self.extraction_functions['next'].search(bundle)

    def get_demographics(self) -> fhir.Demographics:
        url = f"{self.base_url}Patient/{self.id}"
        return fhir.Demographics(self.get(url))
コード例 #36
0
ファイル: autoscaling.py プロジェクト: xu001186/skew
 def __init__(self, client, data, query=None):
     super(AutoScalingGroup, self).__init__(client, data, query)
     self._arn_query = jmespath.compile('AutoScalingGroupARN')
コード例 #37
0
class NACPDeclaration(DocType, AbstractDeclaration):
    """NACP Declaration document.
    Assumes there's a dynamic mapping with all fields not indexed by default."""

    persons = Text(analyzer="ukrainian", copy_to="all")
    countries = Text(analyzer="ukrainian", copy_to="all")
    companies = Text(analyzer="ukrainian", copy_to="all")
    names_autocomplete = Text(
        analyzer="namesAutocompleteAnalyzer",
        search_analyzer="namesAutocompleteSearchAnalyzer",
        fields={"raw": Text(index=True)},
        term_vector="with_positions_offsets",
    )

    all = Text(analyzer="ukrainian")

    general = Object(
        properties={
            "full_name_suggest": Completion(preserve_separators=False),
            "full_name": Text(index=True, analyzer="ukrainian"),
            "full_name_for_sorting": Keyword(
                index=True, ignore_above=100
            ),  # only for sorting purposes
            "name": Text(index=True, analyzer="ukrainian"),
            "patronymic": Text(index=True, analyzer="ukrainian"),
            "last_name": Text(index=True, analyzer="ukrainian"),
            "post": Object(
                properties={
                    "actual_region": Text(
                        index=True,
                        analyzer="ukrainian",
                        fields={"raw": Keyword(index=True)},
                    ),
                    "region": Text(
                        index=True,
                        analyzer="ukrainian",
                        fields={"raw": Keyword(index=True)},
                    ),
                    "office": Text(
                        index=True,
                        analyzer="ukrainian",
                        fields={"raw": Keyword(index=True)},
                    ),
                    "post_type": Text(
                        index=True,
                        analyzer="ukrainian",
                        fields={"raw": Keyword(index=True)},
                    ),
                    "post": Text(
                        index=True,
                        analyzer="ukrainian",
                        fields={"raw": Keyword(index=True)},
                    ),
                }
            ),
        }
    )
    declaration = Object(properties={"date": NoneAwareDate()})
    estate = Object(
        properties={
            "region": Text(
                index=True, analyzer="ukrainian", fields={"raw": Keyword(index=True)}
            )
        }
    )
    intro = Object(
        properties={
            "declaration_year": Keyword(index=True),
            "declaration_year_to": NoneAwareDate(),
            "declaration_year_from": NoneAwareDate(),
            "doc_type": Keyword(index=True),
            "date": NoneAwareDate(index=True),
        }
    )

    ft_src = Text(index=True, analyzer="ukrainian", copy_to="all")
    nacp_orig = Object(include_in_all=False, enabled=False)

    # concatinated from set of fields for regular search (not deepsearch mode)
    index_card = Text(index=True, analyzer="ukrainian")

    INDEX_CARD_FIELDS = [
        "general.last_name",
        "general.name",
        "general.patronymic",
        "general.full_name",
        "general.post.post",
        "general.post.office",
        "general.post.region",
        "general.post.actual_region",
        "intro.declaration_year",
        "intro.doc_type",
        "declaration.source",
        "declaration.url",
    ]

    def raw_html(self):
        fname = os.path.join(
            settings.NACP_DECLARATIONS_PATH,
            self.meta.id[5:7],
            os.path.basename(self.declaration.basename) + ".html",
        )

        try:
            with open(fname, "r") as fp:
                d = fp.read()
        except FileNotFoundError:
            return "<h2>Вибачте, декларація тимчасово відсутня, але ми вже працюємо над вирішенням проблеми</h2>"

        m = re.search(r"<\/style>(.*)</body>", d)
        declaration_html = m.group(1)

        # OH LORD, THAT'S NOT WHAT I'VE BEEN TAUGHT IN UNIVERSITY
        doc = declaration_html.replace(
            "</div></div></div><header><h2>", "</div></div><header><h2>"
        )
        # MY ASS IS ON FIRE
        doc = re.sub(r"</table>\s*<header>", "</table></div><header>", doc)

        companies = self._all_companies()

        codes = [c.lstrip("0") for c in companies if c.isdigit() and 4 < len(c) < 9]

        for c in codes:
            if c:
                full_code = c.rjust(8, "0")
                doc = re.sub(
                    r"\b0*{}\b".format(c),
                    ' <a href="https://ring.org.ua/edr/uk/company/{}" target="_blank">{}</a>'.format(
                        full_code, full_code
                    ),
                    doc,
                )

        return doc

    def prepare_translations(self, language):
        if language == "en":
            self.translator = HTMLTranslator(
                self.raw_html(),
                "h2 span, legend i, label strong, label, th, header, span.block b, b, p, span, td",
            )

    def raw_en_html(self):
        assert hasattr(self, "translator"), "You should call prepare_translations first"
        return self.translator.get_translated_html()

    af_paths = [
        jmespath.compile("step_7.*.emitent_ua_company_code"),
        jmespath.compile("step_7.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_8.*.corporate_rights_company_code"),
        jmespath.compile("step_8.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_9.*.beneficial_owner_company_code"),
    ]

    def _is_change_form(self):
        return self.intro.doc_type and self.intro.doc_type == "Форма змін"

    def _affiliated_companies(self, src=None):
        # For now
        if self._is_change_form():
            return []

        results = []
        if src is None:
            src = self.nacp_orig.to_dict()

        for path in self.af_paths:
            results += path.search(src) or []

        return set(filter(None, results))

    rl_paths = {
        "step_11": jmespath.compile("step_11.*"),
        "step_12": jmespath.compile("step_12.*"),
    }

    def _related_companies(self, src=None):
        # For now
        if self._is_change_form():
            return []

        results = []
        if src is None:
            src = self.nacp_orig.to_dict()

        for section in self.rl_paths["step_11"].search(src) or []:
            try:
                section = section or {}
                obj_type = section.get("objectType", "").lower()
                other_obj_type = section.get("otherObjectType", "").lower()

                if obj_type in INCOME_TYPES or other_obj_type in INCOME_TYPES:
                    results += [section.get("source_ua_company_code", "")]
            except AttributeError:
                pass

        for section in self.rl_paths["step_12"].search(src) or []:
            try:
                section = section or {}
                obj_type = section.get("objectType", "").lower()

                if obj_type in MONETARY_ASSETS_TYPES:
                    results += [section.get("organization_ua_company_code", "")]
            except AttributeError:
                pass

        return set(filter(None, results))

    ac_paths = [
        jmespath.compile("step_2.*.source_ua_company_code[]"),
        jmespath.compile("step_3.*.beneficial_owner_company_code[]"),
        jmespath.compile("step_3.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_4.*.addition_company_code[]"),
        jmespath.compile("step_4.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_4.undefined.rights[].*.ua_company_code[]"),
        jmespath.compile("step_5.*.emitent_ua_company_code[]"),
        jmespath.compile("step_5.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_6.*.corporate_rights_company_code[]"),
        jmespath.compile("step_6.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_10.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_11.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_11.*.rights[].*.ua_company_name[]"),
        jmespath.compile("step_12.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_13.*.emitent_ua_company_code[]"),
        jmespath.compile("step_13.*.emitent_ua_company_name[]"),
        jmespath.compile("step_13.*.guarantor[].*.guarantor_ua_company_code[]"),
        jmespath.compile(
            "step_13.*.guarantor_realty[].*.realty_rights_ua_company_code[]"
        ),
        jmespath.compile(
            "step_13.*.guarantor_realty[].*.realty_rights_ua_company_code[]"
        ),
        jmespath.compile("step_15.*.emitent_ua_company_code[]"),
        jmespath.compile("step_16.org.*.reestrCode[]"),
        jmespath.compile("step_16.part_org.*.reestrCode[]"),
        jmespath.compile("step_7.*.emitent_ua_company_code"),
        jmespath.compile("step_7.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_8.*.corporate_rights_company_code"),
        jmespath.compile("step_8.*.rights[].*.ua_company_code[]"),
        jmespath.compile("step_9.*.beneficial_owner_company_code"),
        jmespath.compile("step_11.*.source_ua_company_code"),
        jmespath.compile("step_12.*.organization_ua_company_code"),
    ]

    def _all_companies(self, src=None):
        # For now
        if self._is_change_form():
            return []

        results = []
        if src is None:
            src = self.nacp_orig.to_dict()

        for path in self.ac_paths:
            results += path.search(src) or []

        return set(filter(None, results))

    def related_companies(self, affiliated_only=True):
        """
        Prepares data to use with procurement dataset
        """
        src = self.nacp_orig.to_dict()

        res = self._affiliated_companies(src)
        if not affiliated_only:
            res += self._related_companies(src)

        res = filter(None, map(lambda x: x.strip().lstrip("0"), set(res)))

        return list(set(res) - BANK_EDRPOUS)

    def get_procurement_earnings_by_year(self, affiliated_only=True):
        # Safety valve against transactions with malformed dates
        next_year_dt = date(date.today().year + 1, 1, 1)

        return (
            Transactions.objects.select_related("seller")
            .filter(
                seller__code__in=self.related_companies(affiliated_only),
                date__lt=next_year_dt,
            )
            .annotate(year=ExtractYear("date"))
            .values("year")
            .annotate(count=Count("pk"), sum_uah=Sum("volume_uah"))
        )

    def get_procurement_earnings_by_company(self, affiliated_only=True):
        # Safety valve against transactions with malformed dates
        next_year_dt = date(date.today().year + 1, 1, 1)

        return (
            Transactions.objects.select_related("seller")
            .filter(
                seller__code__in=self.related_companies(affiliated_only),
                date__lt=next_year_dt,
            )
            .values("seller__code", "seller__pk", "seller__name")
            .annotate(count=Count("pk"), sum_uah=Sum("volume_uah"))
        )

    def infocard(self):
        return {
            "first_name": self.general.name,
            "patronymic": self.general.patronymic,
            "last_name": self.general.last_name,
            "office": self.general.post.office,
            "position": self.general.post.post,
            "source": self.declaration.source,
            "id": self.meta.id,
            "url": settings.SITE_URL
            + reverse("details", kwargs={"declaration_id": self.meta.id}),
            "document_type": self.intro.doc_type,
            "is_corrected": self.intro.corrected,
            "created_date": self.intro.date,
            "declaration_year": getattr(self.intro, "declaration_year"),
        }

    def raw_source(self):
        return {
            "url": "https://public-api.nazk.gov.ua/v1/declaration/%s"
            % self.meta.id.replace("nacp_", "")
        }

    def related_entities(self):
        src = self.nacp_orig.to_dict()
        owned_companies = self._affiliated_companies(src)
        related_companies = self._related_companies(src)
        all_companies = self._all_companies(src)

        return {
            "people": {"family": list(self.get_family_members())},
            "documents": {
                "corrected": list(getattr(self, "corrected_declarations", []) or []),
                "originals": list(getattr(self, "original_declarations", []) or []),
            },
            "companies": {
                "owned": list(owned_companies),
                "related": list(related_companies),
                "all": list(all_companies),
            },
        }

    def unified_source(self):
        return self.nacp_orig.to_dict()

    def aggregated_data(self):
        if hasattr(self, "aggregated"):
            return self.aggregated.to_dict()
        else:
            return {}

    class Meta:
        doc_type = "nacp_declaration_doctype"
コード例 #38
0
ファイル: paginate.py プロジェクト: lolo-pop/dast
 def _get_more_results_token(self, config):
     more_results = config.get('more_results')
     if more_results is not None:
         return jmespath.compile(more_results)
コード例 #39
0
 def _compile_selection(self):
     if not self.selection_compiled:
         self.selection_compiled = dict(
             (name, jmespath.compile(path))
             for name, path in self.selection.items())
コード例 #40
0
    def __init__(
        self,
        jmes=None,
        jmes_ignore_none=None,
        output=None,
        expand=None,
        depth=None,
        include_node=None,
        exclude_node=None,
        include_path=None,
        exclude_path=None,
        fmt=None,
        sep=None,
        prefix=None,
        suffix=None,
        expand_array=None,
        fmt_array=None,
    ):
        """
        :param jmes:  jmes filter to select or generate new field
        :param jmes_ignore_none: if jmes filter is null, ignore it (default). Or else consider it as "". default is
        :param output: put the value parsed from jmes filter to this field
        :param expand: If jmes filter is configure, expand the result or not (Default is False in this case), If jmes is not configured, directly expand the field passed or not (Default is True in this case).
        :param depth: depth to scan, 1 means first layer, default is 100.
        :param include_node: keys to expand and include. regex string. using '|' for multiple ones. default is all.
        :param exclude_node: keys to skip, regex string. using '|' for multiple ones. default is nothing.
        :param include_path: path to expand and include. regex string to match from begin. using '|' for multiple ones. default is all. e.g. r"data\.k1", all sub-keys in data.k1 will be included.
        :param exclude_path: path to skip, regex string to match from begin. using '|' for multiple ones. default is nothing. . e.g. r"data\.k2", all sub-keys in data.k2 will be excluded.
        :param fmt: during expansion, how to format the key name, there're several types or customized as described in FMT_MAP
        :param sep: sep used in formatting during expansion
        :param prefix: prefix used in formatting during expansion
        :param suffix: suffix used in formatting during expansion
        :param expand_array: if expand array or just render it. default is True. item in array will be with name index, e.g. [1,2,3] will be considered as {"0": 1, "1": 2, "2": 3}
        :param fmt_array: format string for key name of each array element, default is "{parent_rlist[0]}_{index}", can be custom formatting string using placehodler: parent_list, parent_list, current
        """
        self.expand = expand
        if expand is None:
            # when jmes is not configured or configure but no output configured
            self.expand = not jmes or not output

        # self.level = level or 1
        self.jmes = self._u(jmes or "")
        self.prefix = self._u("" if prefix is None else prefix)
        self.suffix = self._u("" if suffix is None else suffix)
        self.sep = self._u(self.DEFAULT_SEP if sep is None else sep)
        self.output = self._u(output or "")
        self.jmes_filter = None
        self.jmes_ignore_none = True if jmes_ignore_none is None else jmes_ignore_none
        if jmes:
            try:
                self.jmes_filter = jmespath.compile(jmes)
            except jmespath.exceptions.ParseError as ex:
                raise SettingError(ex=ex,
                                   msg="Invalid JMES filter setting",
                                   settings=jmes)
        elif self.output:
            logger.warning(
                u"json_transformer: parameter output '{0}' will be ignored as there's no filter is selected."
                .format(output))

        self.depth = min((depth or self.DEFAULT_DEPTH), self.DEFAULT_DEPTH)
        self.include_node = self._u(include_node or self.DEFAULT_INCLUDE_NODE)
        self.exclude_node = self._u(exclude_node or self.DEFAULT_EXCLUDE_NODE)
        self.include_path = self._u(include_path or self.DEFAULT_INCLUDE_PATH)
        self.exclude_path = self._u(exclude_path or self.DEFAULT_EXCLUDE_PATH)
        self.fmt = self._u(fmt or self.DEFAULT_FMT)

        try:
            self.include_node_match = get_re_full_match(self.include_node)
            self.exclude_node_match = get_re_full_match(self.exclude_node)
            self.include_path_match = re.compile(
                self.include_path).match  # use match instead of full match
            self.exclude_path_match = re.compile(
                self.exclude_path).match  # use match instead of full match
        except Exception as ex:
            raise SettingError(ex=ex,
                               msg="Invalid regex string for include/exclude")

        self.expand_array = True if expand_array is None else expand_array
        self.format_array = self._u(fmt_array or self.DEFAULT_FMT_ARRAY)
コード例 #41
0
ファイル: cli.py プロジェクト: rocketinventor/brainfm
#!/usr/bin/env python
import shlex
import subprocess
import webbrowser

import click
import jmespath
import terminaltables

import brainfm

ENVKEY_SID = "BRAINFM_SID"
ENVKEY_API_ENDPOINT = "BRAINFM_API_ENDPOINT"
ENVKEY_STREAM_ENDPOINT = "BRAINFM_STREAM_ENDPOINT"
STATIONS_PATTERN = jmespath.compile("[*].[id, name, string_id, length]")


def validate_client(client: brainfm.Connection):
    if not client.sid:
        raise click.UsageError(
            "missing option --sid or environment variable {}\n".format(
                ENVKEY_SID) +
            "Did you run `brain init` and export the variable?")


@click.group()
@click.version_option()
@click.option("--sid", envvar=ENVKEY_SID)
@click.option("--api-endpoint", envvar=ENVKEY_API_ENDPOINT)
@click.option("--stream-endpoint", envvar=ENVKEY_STREAM_ENDPOINT)
@click.pass_context
コード例 #42
0
class UmlsApi(Api):

    url_config = 'UMLS_BASE_URL'

    extraction_functions = {
        'crosswalk':
        path.compile(
            "result[?ui != 'TCGA' && ui != 'OMFAQ' && ui != 'MPN-SAF'].{code: ui, description:name} | [0]"
        )
    }

    def __init__(self):
        self.auth = umls.Authentication(app.config['UMLS_API_KEY'])
        self.tgt = g.setdefault('tgt', self.auth.gettgt())
        super().__init__()

    def get_crosswalk(self, orig_code: str,
                      codeset: str) -> Tuple[Optional[str], Optional[str]]:
        tik = self.auth.getst(self.tgt)
        params = {"targetSource": "NCI", "ticket": tik}
        route = "/crosswalk/current/source/"
        if orig_code[-1] == '.':
            logging.warn(f"Original Code = {orig_code}")
            orig_code = orig_code[:-1]
            logging.warn(f"New code = {orig_code}")
        url = f"{self.base_url}{route}{codeset}/{orig_code}"
        response = self._get_response(url, params=params)
        if response.status_code != 200:
            return None, None
        result = response.json()
        crosswalk = self.extraction_functions['crosswalk'].search(result)
        return (crosswalk['code'], crosswalk['description'])

    def get_code(self, description: str) -> List[Tuple[str, str]]:
        tik = self.auth.getst(self.tgt)
        params = {
            "string": description,
            "ticket": tik,
            "searchType": "words",
            "returnIdType": "sourceUi",
            "sabs": "NCI",
            "page_size": "1000"
        }
        route = "/search/current"
        url = f"{self.base_url}{route}"
        response = self._get_response(url, params=params)
        if response.status_code != 200:
            return []
        res = response.json()
        results = []
        for result in res["result"]["results"]:
            if "rootSource" in result and result["rootSource"] == "NCI":
                results.append((result["ui"], result["name"]))
        return results

    def get_code_exact(
            self,
            description: str) -> Union[Tuple[str, str], Tuple[None, None]]:
        tik = self.auth.getst(self.tgt)
        params = {
            "string": description,
            "ticket": tik,
            "searchType": "exact",
            "returnIdType": "sourceUi",
            "sabs": "NCI",
            "page_size": "1000"
        }
        route = "/search/current"
        url = f"{self.base_url}{route}"
        response = self._get_response(url, params=params)
        if response.status_code != 200:
            return None, None
        res = response.json()
        for result in res["result"]["results"]:
            if "rootSource" in result and result["rootSource"] == "NCI":
                return result["ui"], result["name"]
        return None, None

    def get_matches(
        self, conditions_by_code: Dict[str, Dict[str, str]]
    ) -> Iterable[Tuple[str, Optional[Dict[str, str]]]]:
        matches: Dict[Greenlet, str] = {}
        for orig_code, condition in conditions_by_code.items():
            logging.info(
                f"Getting match for {condition['codeset']} code {orig_code} [{condition['description']}] "
            )
            matches[spawn(self.get_crosswalk, orig_code,
                          condition['codeset'])] = orig_code
        for match in iwait(matches):
            orig_code = matches[match]
            ncit_code, ncit_desc = cast(Tuple[Optional[str], Optional[str]],
                                        match.value)
            if ncit_code and ncit_desc:
                logging.info(f"Match for {orig_code} is {ncit_code}")
                yield orig_code, {'match': ncit_code, 'description': ncit_desc}
            else:
                logging.info(f"No match for {orig_code}")
                yield orig_code, None

    def perform_query(self, route, body):
        tik = self.auth.getst(self.tgt)
        body['ticket'] = tik
        url = f"{self.base_url}{route}"
        response = self._get_response(url, params=body)
        if response.status_code != 200:
            return None
        return response.json()
コード例 #43
0
class MyriaPlan(object):
    expressions = {
        'type': jmespath.compile('plan.type || plan.body.type'),
        'subplans': jmespath.compile('plan.plans[] || plans[] || plan.body[] || body[]'),
        'fragments': jmespath.compile('plan.fragments[] || fragments[]'),
        'fragment-list': jmespath.compile('plan.fragments || fragments'),
        'operators': jmespath.compile('plan.fragments[].operators[]'),
        'shuffles': jmespath.compile('plan.fragments[].operators[?contains(opType, `Shuffle`)][contains(opType, `Producer`)]')
        }

    def __init__(self, data, parent=None):
        self.parent = parent
        self.data = data

    @property
    def type(self):
        return self.expressions['type'].search(self.data)

    @property
    def language(self):
        return self.data['language']

    @property
    def profiling_mode(self):
        return self.data['profilingMode']

    @property
    def text(self):
        return self.data['rawQuery']

    @property
    def logicalRa(self):
        return self.data['logicalRa']

    @property
    def root(self):
        return self.parent.root if self.parent else self

    @property
    def subplans(self):
        return (MyriaPlan(subplan, parent=self) for subplan in self.expressions['subplans'].search(self.data) or [])

    @property
    def fragments(self, descendants=True):
        return chain((MyriaFragment(self, fragment) for fragment in self.expressions['fragments'].search(self.data) or []),
                     (fragment for subplan in self.subplans for fragment in subplan.fragments) if descendants else [])

    @property
    def operators(self, descendants=True):
        return chain((operator for fragment in self.fragments for operator in fragment.operators),
                     (operator for subplan in self.subplans for operator in subplan.operators) if descendants else [])

    @property
    def shuffles(self):
        return (operator for fragment in self.fragments for operator in fragment.shuffles)

    def search(self, path):
        return jmespath.search(path, self.data)

    @property
    def _fragment_list(self):
        return self.expressions['fragment-list'].search(self.data)

    def __str__(self):
        return "MyriaPlan(%s)" % (self.text or self.data)

    def __repr__(self):
        return self.__str__()
コード例 #44
0
ファイル: tagging.py プロジェクト: zelfick/cloud-custodian
class AutoTagUser(AutoTagBase):
    """Attempts to tag a resource with the first user who created/modified it.

    :example:

    This policy will tag all existing resource groups with the 'CreatorEmail' tag

    .. code-block:: yaml

      policies:
        - name: azure-auto-tag-creator
          resource: azure.resourcegroup
          description: |
            Tag all existing resource groups with the 'CreatorEmail' tag
          actions:
           - type: auto-tag-user
             tag: CreatorEmail

    This action searches from the earliest 'write' operation's caller
    in the activity logs for a particular resource.

    Note: activity logs are only held for the last 90 days.

    """

    schema = type_schema('auto-tag-user', rinherit=AutoTagBase.schema)
    log = logging.getLogger('custodian.azure.tagging.AutoTagUser')

    # compiled JMES paths
    service_admin_jmes_path = jmespath.compile(constants.EVENT_GRID_SERVICE_ADMIN_JMES_PATH)
    sp_jmes_path = jmespath.compile(constants.EVENT_GRID_SP_NAME_JMES_PATH)
    upn_jmes_path = jmespath.compile(constants.EVENT_GRID_UPN_CLAIM_JMES_PATH)
    principal_role_jmes_path = jmespath.compile(constants.EVENT_GRID_PRINCIPAL_ROLE_JMES_PATH)
    principal_type_jmes_path = jmespath.compile(constants.EVENT_GRID_PRINCIPAL_TYPE_JMES_PATH)

    def __init__(self, data=None, manager=None, log_dir=None):
        super(AutoTagUser, self).__init__(data, manager, log_dir)

    def _get_tag_value_from_event(self, event):
        principal_role = self.principal_role_jmes_path.search(event)
        principal_type = self.principal_type_jmes_path.search(event)
        user = None
        # The Subscription Admins role does not have a principal type
        if StringUtils.equal(principal_role, 'Subscription Admin'):
            user = self.service_admin_jmes_path.search(event)
        # ServicePrincipal type
        elif StringUtils.equal(principal_type, 'ServicePrincipal'):
            user = self.sp_jmes_path.search(event)

        # Other types and main fallback (e.g. User, Office 365 Groups, and Security Groups)
        if not user and self.upn_jmes_path.search(event):
            user = self.upn_jmes_path.search(event)

        # Last effort search for an email address in the claims
        if not user:
            claims = event['data']['claims']
            for c in claims:
                value = claims[c]
                if self._is_email(value):
                    user = value

        if not user:
            self.log.error('Principal could not be determined.')

        return user

    def _is_email(self, target):
        if target is None:
            return False
        elif parseaddr(target)[1] and '@' in target and '.' in target:
            return True
        else:
            return False

    def _get_tag_value_from_resource(self, resource):
        first_op = self._get_first_event(resource)
        return first_op.caller if first_op else None
コード例 #45
0
 def __init__(self, endpoint, data):
     super(LaunchConfiguration, self).__init__(endpoint, data)
     self._arn_query = jmespath.compile('LaunchConfigurationARN')
コード例 #46
0
ファイル: constants.py プロジェクト: gooddata/better-aws-cli
    '--no-sign-request': False,
    '--ca-bundle': True,
    '--cli-read-timeout': True,
    '--cli-connect-timeout': True
}

CACHED_OPTIONS = {
    '--bucket', '--user-name', '--group-name', '--role-name', '--instance-ids'
}

CONFIG_FILE = 'AWS_CONFIG_FILE'
CONFIG_PATH = '~/.aws/config'
CREDS_FILE = 'AWS_SHARED_CREDENTIALS_FILE'
CREDS_PATH = '~/.aws/credentials'

EC2_REGIONS_JMES = jmespath.compile('Regions[].RegionName')

IGNORED_ENV_VARS = {
    'AWS_ACCESS_KEY_ID', 'AWS_PROFILE', 'AWS_ROLE_SESSION_NAME',
    'AWS_SECRET_ACCESS_KEY', 'AWS_SESSION_TOKEN'
}

PROFILE_OPTIONS = {'-p', '--profile'}

PROFILE_MANAGER_COMMANDS = {
    'list-available-profiles': 'List all available named profiles',
    'list-available-accounts': 'List all available accounts',
    'list-available-regions': 'List all available regions',
    'list-active-profiles': 'List all currently active named profiles',
    'list-active-regions': 'List all currently active regions',
    'switch-profiles': 'Mark all currently active profiles as disabled'
コード例 #47
0
 def __init__(self, path, expression):
     self.expression = jmespath.compile(expression)
     self.path = path
コード例 #48
0
def _resolve_query(value):
    try:
        return jmespath.compile(value)
    except Exception as e:
        raise ValueError("Bad value for --query %s: %s" % (value, str(e)))
コード例 #49
0
ファイル: paginate.py プロジェクト: lolo-pop/dast
 def _get_non_aggregate_keys(self, config):
     keys = []
     for key in config.get('non_aggregate_keys', []):
         keys.append(jmespath.compile(key))
     return keys
コード例 #50
0
class AutoTagUser(AzureEventAction):
    """Attempts to tag a resource with the first user who created/modified it.

    :example:
    This policy will tag all existing resource groups with the 'CreatorEmail' tag

    .. code-block:: yaml

      policies:
        - name: azure-auto-tag-creator
          resource: azure.resourcegroup
          description: |
            Tag all existing resource groups with the 'CreatorEmail' tag
          actions:
           - type: auto-tag-user
             tag: CreatorEmail

    This action searches from the earliest 'write' operation's caller
    in the activity logs for a particular resource.

    Note: activity logs are only held for the last 90 days.

    """
    default_user = "******"
    query_select = "eventTimestamp, operationName, caller"
    max_query_days = 90

    # compiled JMES paths
    service_admin_jmes_path = jmespath.compile(
        constants.EVENT_GRID_SERVICE_ADMIN_JMES_PATH)
    sp_jmes_path = jmespath.compile(constants.EVENT_GRID_SP_NAME_JMES_PATH)
    upn_jmes_path = jmespath.compile(constants.EVENT_GRID_UPN_CLAIM_JMES_PATH)
    principal_role_jmes_path = jmespath.compile(
        constants.EVENT_GRID_PRINCIPAL_ROLE_JMES_PATH)
    principal_type_jmes_path = jmespath.compile(
        constants.EVENT_GRID_PRINCIPAL_TYPE_JMES_PATH)

    schema = utils.type_schema('auto-tag-user',
                               required=['tag'],
                               **{
                                   'update': {
                                       'type': 'boolean'
                                   },
                                   'tag': {
                                       'type': 'string'
                                   },
                                   'days': {
                                       'type': 'integer'
                                   }
                               })
    schema_alias = True

    def __init__(self, data=None, manager=None, log_dir=None):
        super(AutoTagUser, self).__init__(data, manager, log_dir)
        self.log = logging.getLogger('custodian.azure.actions.auto-tag-user')

    def validate(self):

        if self.manager.action_registry.get('tag') is None:
            raise FilterValidationError("Resource does not support tagging")

        if self.manager.data.get('mode', {}).get('type') == 'azure-event-grid' \
                and self.data.get('days') is not None:
            raise PolicyValidationError(
                "Auto tag user in event mode does not use days.")

        if (self.data.get('days') is not None
                and (self.data.get('days') < 1 or self.data.get('days') > 90)):
            raise FilterValidationError("Days must be between 1 and 90")

        return self

    def _prepare_processing(self):
        self.session = self.manager.get_session()
        self.client = self.manager.get_client(
            'azure.mgmt.monitor.MonitorManagementClient')
        self.tag_key = self.data['tag']
        self.should_update = self.data.get('update', False)

    def _process_resource(self, resource, event):
        # if the auto-tag-user policy set update to False (or it's unset) then we
        # will skip writing their UserName tag and not overwrite pre-existing values
        if not self.should_update and resource.get('tags', {}).get(
                self.tag_key, None):
            return

        user = self.default_user
        if event:
            user = self._get_user_from_event(event) or user
        else:
            user = self._get_user_from_resource_logs(resource) or user

        # issue tag action to label user
        TagHelper.add_tags(self, resource, {self.tag_key: user})

    def _get_user_from_event(self, event):
        principal_role = self.principal_role_jmes_path.search(event)
        principal_type = self.principal_type_jmes_path.search(event)
        user = None
        # The Subscription Admins role does not have a principal type
        if StringUtils.equal(principal_role, 'Subscription Admin'):
            user = self.service_admin_jmes_path.search(event)
        # ServicePrincipal type
        elif StringUtils.equal(principal_type, 'ServicePrincipal'):
            user = self.sp_jmes_path.search(event)

        # Other types and main fallback (e.g. User, Office 365 Groups, and Security Groups)
        if not user and self.upn_jmes_path.search(event):
            user = self.upn_jmes_path.search(event)

        # Last effort search for an email address in the claims
        if not user:
            claims = event['data']['claims']
            for c in claims:
                value = claims[c]
                if self._is_email(value):
                    user = value

        if not user:
            self.log.error('Principal could not be determined.')

        return user

    def _is_email(self, target):
        if target is None:
            return False
        elif parseaddr(target)[1] and '@' in target and '.' in target:
            return True
        else:
            return False

    def _get_user_from_resource_logs(self, resource):
        # Makes patching this easier
        from c7n_azure.utils import utcnow

        # Calculate start time
        delta_days = self.data.get('days', self.max_query_days)
        start_time = utcnow() - datetime.timedelta(days=delta_days)

        # resource group type
        if self.manager.type == 'resourcegroup':
            resource_type = "Microsoft.Resources/subscriptions/resourcegroups"
            query_filter = " and ".join([
                "eventTimestamp ge '%s'" % start_time,
                "resourceGroupName eq '%s'" % resource['name'],
                "eventChannels eq 'Operation'"
            ])
        # other Azure resources
        else:
            resource_type = resource['type']
            query_filter = " and ".join([
                "eventTimestamp ge '%s'" % start_time,
                "resourceUri eq '%s'" % resource['id'],
                "eventChannels eq 'Operation'"
            ])

        # fetch activity logs
        logs = self.client.activity_logs.list(filter=query_filter,
                                              select=self.query_select)

        # get the user who issued the first operation
        operation_name = "%s/write" % resource_type
        first_op = self.get_first_operation(logs, operation_name)
        return first_op.caller if first_op else None

    @staticmethod
    def get_first_operation(logs, operation_name):
        first_operation = None
        for l in logs:
            if l.operation_name.value and l.operation_name.value.lower(
            ) == operation_name.lower():
                first_operation = l

        return first_operation
コード例 #51
0
ファイル: jpath.py プロジェクト: Kintyre/jmespath
def jpath():
    try:
        keywords, options = si.getKeywordsAndOptions()
        legacy_args_fixer(options)

        defaultval = options.get('default', None)
        fn_input = options.get('input', options.get('field', '_raw'))
        fn_output = options.get('output', 'jpath')
        if len(keywords) != 1:
            si.generateErrorResults('Requires exactly one path argument.')
            sys.exit(0)
        path = keywords[0]

        # Handle literal (escaped) quotes.  Presumably necessary because of raw args?
        path = path.replace(r'\"', '"')

        if "*" in fn_output:
            apply_output = output_to_wildcard
        else:
            apply_output = output_to_field

        try:
            jp = jmespath.compile(path)
        except ParseError as e:
            # Todo:  Consider stripping off the last line "  ^" pointing to the issue.
            # Not helpful since Splunk wraps the error message in a really ugly way.
            si.generateErrorResults(
                "Invalid JMESPath expression '{}'. {}".format(path, e))
            sys.exit(0)

        results, dummyresults, settings = si.getOrganizedResults()
        # for each results
        for result in results:
            # get field value
            ojson = result.get(fn_input, None)
            added = False
            if ojson is not None:
                if isinstance(ojson, (list, tuple)):
                    # XXX: Add proper support for multivalue input fields.  Just use first value for now
                    ojson = ojson[0]
                try:
                    json_obj = json.loads(ojson)
                except ValueError:
                    # Invalid JSON.  Move on, nothing to see here.
                    continue
                try:
                    values = jp.search(json_obj, options=jp_options)
                    apply_output(values, fn_output, result)
                    result[ERROR_FIELD] = None
                    added = True
                except UnknownFunctionError as e:
                    # Can't detect invalid function names during the compile, but we want to treat
                    # these like syntax errors:  Stop processing immediately
                    si.generateErrorResults(
                        "Issue with JMESPath expression. {}".format(e))
                    sys.exit(0)
                except JMESPathError as e:
                    # Not 100% sure I understand what these errors mean. Should they halt?
                    result[ERROR_FIELD] = "JMESPath error: {}".format(e)
                except Exception as e:
                    result[ERROR_FIELD] = "Exception: {}".format(e)

            if not added and defaultval is not None:
                result[fn_output] = defaultval

        si.outputResults(results)
    except Exception as e:
        import traceback

        stack = traceback.format_exc()
        si.generateErrorResults("Error '%s'. %s" % (e, stack))
コード例 #52
0
 def get_compiled_jmespath_file_names(cls):
     """Return a compiled JMESPath expression for file names, using the class name as part of the key."""
     return jmespath.compile("nitpick.{}.file_names".format(cls.__name__))
コード例 #53
0
from converter.items import LomTechnicalItem, LicenseItem, LomGeneralItem, ValuespaceItem
from .base_classes.mediawiki_base import MediaWikiBase, jmes_pageids
import scrapy
import json
import jmespath
from ..constants import Constants

jmes_continue = jmespath.compile('"query-continue".allpages')


class ZUMSpider(MediaWikiBase, scrapy.Spider):
    name = "zum_klexikon_spider"
    friendlyName = "ZUM-Klexikon"
    url = "https://klexikon.zum.de/"
    version = "0.1.0"
    license = Constants.LICENSE_CC_BY_SA_30

    def parse_page_query(self, response: scrapy.http.Response):
        """
        @url https://klexikon.zum.de/api.php?format=json&action=query&list=allpages&aplimit=100&apfilterredir=nonredirects
        @returns requests 101 101
        """
        data = json.loads(response.body)
        pageids = jmes_pageids.search(data)
        for pageid in pageids:
            yield scrapy.FormRequest(
                url=self.api_url,
                formdata=self._parse_params | {'pageid': str(pageid)},
                callback=self.parse_page_data,
                cb_kwargs={"extra": {
                    'pageid': str(pageid)
コード例 #54
0
 def compile_expressions(
         cls,
         expressions: Dict[str,
                           str]) -> Dict[str, path.parser.ParsedResult]:
     return {key: path.compile(value) for key, value in expressions.items()}
コード例 #55
0
ファイル: processors.py プロジェクト: imperator90/LGK-Hub
 def __init__(self, json_path):
     self.json_path = json_path
     import jmespath
     self.compiled_path = jmespath.compile(self.json_path)
コード例 #56
0
 def test_cloud_trail_resource(self):
     self.assertEqual(
         CloudWatchEvents.match(
             event_data('event-cloud-trail-s3.json')),
         {'source': 's3.amazonaws.com',
          'ids': jmespath.compile('detail.requestParameters.bucketName')})
コード例 #57
0
 def _format_response(self, response, stream):
     if self._args.query is not None:
         expression = jmespath.compile(self._args.query)
         response = expression.search(response)
     text.format_text(response, stream)
コード例 #58
0
ファイル: autoscaling.py プロジェクト: xu001186/skew
 def __init__(self, client, data, query=None):
     super(LaunchConfiguration, self).__init__(client, data, query)
     self._arn_query = jmespath.compile('LaunchConfigurationARN')
コード例 #59
0
 def __init__(self, endpoint, data):
     super(AutoScalingGroup, self).__init__(endpoint, data)
     self._arn_query = jmespath.compile('AutoScalingGroupARN')
コード例 #60
0
 def _initialise_mappings(self, config_object):
     return jmespath.compile(config_object)