def mediapackage_endpoint_speke_keyserver_ddb_items():
    """
    Identify and format MediaPackage origin endpoints to SPEKE keyservers for cache storage.
    """
    items = []
    # create an expression to find speke server urls
    jsonpath_expr = parse('$..SpekeKeyProvider.Url')
    try:
        # get SPEKE keyservers
        speke_keyservers_cached = cache.cached_by_service("speke-keyserver")
        # get MediaPackage origin endpoints
        mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
        # iterate over all distributions
        for keyserver in speke_keyservers_cached:
            keyserver_data = json.loads(keyserver["data"])
            keyserver_endpoint = keyserver_data["endpoint"]
            for mp_endpoint in mediapackage_ep_cached:
                mp_endpoint_data = json.loads(mp_endpoint["data"])
                for server_url in [match.value for match in jsonpath_expr.find(mp_endpoint_data)]:
                    if server_url == keyserver_endpoint:
                        config = {"from": mp_endpoint["arn"], "to": keyserver["arn"], "scheme": keyserver_data["scheme"]}
                        print(config)
                        items.append(connection_to_ddb_item(mp_endpoint["arn"], keyserver["arn"], "mediapackage-origin-endpoint-speke-keyserver", config))
    except ClientError as error:
        print(error)
    return items
Ejemplo n.º 2
0
    def execute(self, message: IRCMessage):
        if len(message.parameterList) < 2:
            return IRCResponse(ResponseType.Say,
                               'Not enough parameters, usage: {}'.format(self.help(None)),
                               message.replyTo)

        path = ' '.join(message.parameterList[1:])

        try:
            parser = jsonpath_ng.parse(path)
        except (jsonpath_ng.lexer.JsonPathLexerError, Exception) as e:
            # yep, jsonpath_ng uses generic exceptions, so this is the best we can do
            return IRCResponse(ResponseType.Say,
                               '[Jostle Error: {}]'.format(e),
                               message.replyTo)

        url = message.parameterList[0]
        if not re.match(r'^\w+://', url):
            url = 'http://{}'.format(url)

        if 'jostle' in message.metadata and url in message.metadata['jostle']:
            # use cached data if it exists
            j = message.metadata['jostle'][url]
        else:
            response = self.bot.moduleHandler.runActionUntilValue('fetch-url', url)
            if not response:
                return IRCResponse(ResponseType.Say,
                                   '[Jostle Error: problem fetching {}]'.format(url),
                                   message.replyTo)
            try:
                j = response.json()
            except JSONDecodeError:
                return IRCResponse(ResponseType.Say,
                                   '[Jostle Error: data at {} is not valid JSON]'.format(url),
                                   message.replyTo)

        m = parser.find(j)
        if not m:
            reply = '[Jostle Error: the jsonpath {!r} does not resolve a value from {!r}]'
            reply = reply.format(path, url)
            return IRCResponse(ResponseType.Say, reply, message.replyTo)

        value = m[0].value

        if not isinstance(value, str):
            value = ' '.join(value)

        # sanitize the value
        value = value.strip()
        value = re.sub(r'[\r\n]+', ' ', value)
        value = re.sub(r'\s+', ' ', value)

        return IRCResponse(ResponseType.Say, value, message.replyTo,
                           extraVars={'jostleURL': url},
                           metadata={'jostle': {url: j}})
def mediapackage_channels(region):
    """
    Return the MediaPackage channels for the given region.
    """
    service = boto3.client("mediapackage", region_name=region)
    jsonpath_expr = parse('$..Password')
    response = service.list_channels()
    items = response['Channels']
    while "NextToken" in response:
        response = service.list_channels(NextToken=response["NextToken"])
        items = items + response['Channels']
    jsonpath_expr.update(items, "XXXXXXXXXXXX")
    return items
def speke_server_ddb_items(region):
    """
    Find the SPEKE key servers based on MediaPackage endpoint configurations
    """
    items = []
    # create an expression to find speke server urls
    jsonpath_expr = parse('$..SpekeKeyProvider.Url')
    # get MediaPackage origin endpoints
    mediapackage_ep_cached = cache.cached_by_service_region("mediapackage-origin-endpoint", region)
    for endpoint in mediapackage_ep_cached:
        # decode the endpoint configuration
        endpoint_data = json.loads(endpoint["data"])
        for server_url in [match.value for match in jsonpath_expr.find(endpoint_data)]:
            parsed = urlparse(server_url)
            sha = hashlib.sha1()
            sha.update(server_url.encode('utf-8'))
            url_digest = sha.hexdigest()
            arn = "arn:oss:speke:::{}".format(url_digest)
            config = {"arn": arn, "endpoint": server_url, "scheme": parsed.scheme}
            service = "speke-keyserver"
            # print(config)
            items.append(node_to_ddb_item(arn, service, "global", config))
    return items
Ejemplo n.º 5
0
 def extract_path(data, path):
     r = jsonpath_ng.parse(path).find(data)
     if r:
         return r[0].value
     log.warn("Could not find %s in %s" % (path, data))
Ejemplo n.º 6
0
def json_path(txt):
    try:
        return jsonpath_ng.parse(txt)
    except Exception as e:
        raise SchemaError('Bad JsonPath format: %s' % txt)
Ejemplo n.º 7
0
def main(argv=None):
    itemData = item.get_data()
    datum = jsonpath_ng.parse("$..*").find(itemData)[0]
    _formatFullPath(path=str(datum.full_path), key=str(datum.path))
    _getDatumInformation(datum)
    JSONtoLongDataFrame(jsonObject=itemData, parseString="$..*")
Ejemplo n.º 8
0
    def process_etsy_request(self, conn_etsy, **kwargs):
        """Process Etsy request."""
        doc = kwargs.get('doc')
        doc_data = doc['data']
        url = kwargs.get('url')
        steps = kwargs.get('steps')
        method = kwargs.get('method')
        params = doc_data.get('params', None)
        vars_mapped = {}
        data = self._get_data(doc_data.get('get_data_on'))

        if doc_data.get('vars'):
            var_list = doc_data.get('vars').items()

            for k, v in var_list:
                jsonpath_expr = parse(v)
                vars_mapped[k] = [match.value for match in jsonpath_expr.find(data)][0]

            for k, v in vars_mapped.items():
                url = url.replace("$%s" % k, str(v))

        if method == 'get':
            if params:
                listings_generator = conn_etsy.iterate_pages(
                    'execute_authed',
                    url,
                    params=params
                )
            else:
                listings_generator = conn_etsy.iterate_pages(
                    'execute_authed',
                    url
                )

            for result in listings_generator:
                # In special cases save specific payloads into data, and
                # everything else are the results.
                try:
                    if result['type'] == 'ListingInventory':
                        self._put_data(result, doc_data.get('store_data_on'))

                    if result['type'] == 'Listing':
                        self._put_data(result['results'], doc_data.get('store_data_on'))
                except KeyError as err:
                    self._put_data(result['results'], doc_data.get('store_data_on'))

                self._run_steps(steps)

        if method == 'post':
            mapper = doc_data.get('export_mapper')
            _map = mapper['map']

            if isinstance(self.data, list):
                mapped_list = [form_doc(i, _map) for i in self.data]

                self.data = mapped_list
            else:
                self.data = form_doc(self.data, _map)

            if isinstance(self.data, list):
                for item in self.data:
                    conn_etsy.create_listings(payload=item)
            else:
                conn_etsy.create_listings(payload=self.data)
Ejemplo n.º 9
0
def jsonpath(path, obj, allow_none=False):
    r = [m.value for m in parse(path).find(obj)]
    return r[0] if not allow_none else (r[0] if r else None)
Ejemplo n.º 10
0
def parse_json_expression(json, expression):
    json_expression = parse(expression)
    json_thing = json_expression.find(json)

    return json_thing
Ejemplo n.º 11
0
import json

from jsonpath_ng import parse

with open("cloud.json", 'r') as json_file:
    json_data = json.load(json_file)
'''
print(json_data)
'''

jsonpath_expression = parse('$.ListBucketResult.Contents[*].Key')

for match in jsonpath_expression.find(json_data):
    print(f'Template name: {match.value}')
Ejemplo n.º 12
0
def resolve_path_multimatch(expression, match, context):
    pathmatches = jsonpath.parse(expression).find(context)
    if pathmatches:
        return unicode([m.value for m in pathmatches])
    return match.group(0)
Ejemplo n.º 13
0
def get_field_mappings(mapping_definition):
    mapping_obj = mapping_definition
    matches = parse('mapping[*]').find(mapping_obj)
    mappings = [match.value for match in matches]
    return mappings
Ejemplo n.º 14
0
def JSP_get_basic_fields(avro_obj):
    jsonpath_expr = parse('fields[*].name')
    return [match.value for match in jsonpath_expr.find(avro_obj)]
Ejemplo n.º 15
0
def custom(dataset) -> int:
    total_updated = 0
    running = True
    print("Custom data mutation mode.")
    print()
    print(
        "Enter selection JSONPath query, then enter mutation lambda to be applied to"
    )
    print("each. For mutation lambdas, '_' is the original value.")
    print("(type \q to quit, \c to cancel mutation)")
    while running:
        query = input("select> ")
        query = remove_ansi_escapes(query)
        if query == r'\q':
            running = False
            continue
        elif query == r'\c':
            continue

        try:
            expr = jsonpath_ng.parse(query)
        except JsonPathParserError as e:
            print(str(e))
            continue
        except AttributeError:
            print("Could not parse query, try again")
            continue

        matches = expr.find(dataset)
        if len(matches) < 1:
            print("(no results)")
            continue

        results = [match.value for match in matches]
        output = json.dumps(results, indent=2, sort_keys=True)
        print(output)

        mutie = input("MUTATE> ")
        mutie = remove_ansi_escapes(mutie)
        if query == r'\q':
            running = False
            continue
        elif query == r'\c':
            continue

        mutation_func_str = 'lambda _: ' + mutie
        mutation_func = eval(mutation_func_str)
        for m in matches:
            old_val = m.value
            new_val = mutation_func(old_val)
            print("NEW VAL: {!r}".format(new_val))
            dataset = m.full_path.update(dataset, new_val)

        # show user the result by selecting the updated data once more
        updated_matches = expr.find(dataset)
        if len(updated_matches) < 1:
            raise ValueError("updated_matches somehow resulted in no results")
        updated_results = [m.value for m in updated_matches]
        updated_output = json.dumps(updated_results, indent=2, sort_keys=True)
        print(output)

        print("{!r} row(s) updated with mutation lambda".format(len(matches)))
        total_updated += len(matches)
    return total_updated
Ejemplo n.º 16
0
 def get_base_url(self):
     page_url = parse('searchContent..pageMetadata.canonical')
     return [i.value for i in page_url.find(self.base)][0]
Ejemplo n.º 17
0
 def fake_parse(value):
     self.jsonpath_call_count += 1
     return jsonpath_ng.parse(value)
Ejemplo n.º 18
0
 def parse(cls, path):
     if not path in cls.finders: cls.finders[path] = jsonpath_ng.parse(path)
     return cls.finders[path]
Ejemplo n.º 19
0
def action_ibm_cf_invoke(context, event):
    class InvokeException(Exception):
        pass

    if not ibmcf_session:
        create_ibmcf_session()

    operator = context['operator'] if 'operator' in context else context

    cf_auth = operator['api_key'].split(':')
    cf_auth_handler = HTTPBasicAuth(cf_auth[0], cf_auth[1])

    url = operator['url']
    subject = context['subject'] if 'subject' in context else None

    triggerflow_meta = {'subject': subject, 'sink': operator['sink']}

    invoke_payloads = []
    if operator['iter_data']:
        keys = list(operator['iter_data'].keys())
        iterdata_keyword = keys.pop()
        for iterdata_value in operator['iter_data'][iterdata_keyword]:
            payload = operator['invoke_kwargs'].copy()
            payload[iterdata_keyword] = iterdata_value
            payload['__OW_TRIGGERFLOW'] = triggerflow_meta
            invoke_payloads.append(payload)
    else:
        for key, arg in operator['invoke_kwargs'].items():
            if isinstance(arg, str) and arg.startswith('$'):
                jsonpath_expr = jsonpath_ng.parse(arg)
                result_args = [
                    match.value
                    for match in jsonpath_expr.find(context['result'])
                ]
                operator['invoke_kwargs'][key] = result_args

        payload = operator['invoke_kwargs']
        payload['__OW_TRIGGERFLOW'] = triggerflow_meta
        invoke_payloads.append(payload)

    if 'max_retries' in context:
        max_retries = context['max_retries']
    else:
        max_retries = 5

    ################################################
    def invoke(call_id, payload):
        act_id = None
        retry = True
        retry_count = 0
        while retry:
            try:
                start_t = time.time()
                # response = requests.post(url, json=payload, auth=cf_auth_handler, timeout=10.0, verify=True)
                response = ibmcf_session.post(url,
                                              json=payload,
                                              auth=cf_auth_handler,
                                              timeout=10.0,
                                              verify=True)
                status_code = response.status_code
                res_json = response.json()
                et = round(time.time() - start_t, 3)
                if status_code in range(200, 300) and 'activationId' in res_json and \
                        res_json['activationId'] is not None:
                    retry = False
                    act_id = res_json['activationId']
                    logging.info(
                        '[{}][{}] Invocation success ({}s) - Activation ID: {}'
                        .format(context.workspace, call_id, et, act_id))
                elif status_code in range(
                        400, 500) and status_code not in [408, 409, 429]:
                    logging.error(
                        '[{}][{}] Invocation failed - Activation status code: {}'
                        .format(context.workspace, call_id, status_code))
                    raise InvokeException('Invocation failed')
            except requests.exceptions.RequestException as e:
                logging.error('[{}][{}] Error talking to OpenWhisk: {}'.format(
                    context.workspace, call_id, e))
                raise e
            except Exception as e:
                logging.error("[{}][{}] Exception - {}".format(
                    context.workspace, call_id, e))
            if retry:
                retry_count += 1
                if retry_count <= max_retries:
                    sleepy_time = pow(2, retry_count)
                    logging.info("[{}][{}] Retrying in {} second(s)".format(
                        context.workspace, call_id, sleepy_time))
                    time.sleep(sleepy_time)
                else:
                    logging.error(
                        "[{}][{}] Retrying failed after {} attempts".format(
                            context.workspace, call_id, max_retries))
                    raise InvokeException('Invocation failed')

        return call_id, act_id

    ################################################

    total_activations = len(invoke_payloads)
    logging.info("[{}] Firing trigger {} - Activations: {} ".format(
        context.workspace, subject, total_activations))
    futures = []
    responses = []
    if total_activations == 1:
        resp = invoke(0, invoke_payloads[0])
        responses.append(resp)
    else:
        with ThreadPoolExecutor(max_workers=128) as executor:
            for cid, payload in enumerate(invoke_payloads):
                res = executor.submit(invoke, cid, payload)
                futures.append(res)
        try:
            responses = [fut.result() for fut in futures]
        except InvokeException:
            pass

    activations_done = [call_id for call_id, _ in responses]
    activations_not_done = [
        call_id for call_id in range(total_activations)
        if call_id not in activations_done
    ]

    if subject in context.trigger_mapping:
        downstream_triggers = context.trigger_mapping[subject][
            'event.triggerflow.termination.success']
        for downstream_trigger in downstream_triggers:
            downstream_trigger_ctx = context.triggers[
                downstream_trigger].context
            if 'total_activations' in downstream_trigger_ctx:
                downstream_trigger_ctx[
                    'total_activations'] += total_activations
            else:
                downstream_trigger_ctx['total_activations'] = total_activations

            if 'dependencies' in downstream_trigger_ctx and subject in downstream_trigger_ctx[
                    'dependencies']:
                if downstream_trigger_ctx['dependencies'][subject]['join'] > 0:
                    downstream_trigger_ctx['dependencies'][subject][
                        'join'] += total_activations
                else:
                    downstream_trigger_ctx['dependencies'][subject][
                        'join'] = total_activations

    # All activations are unsuccessful
    if not activations_done:
        raise Exception('All invocations have failed')
    # At least one activation is successful
    else:
        # All activations are successful
        if len(activations_done) == total_activations:
            logging.info('[{}][{}] All invocations successful'.format(
                context.workspace, subject))
        # Only some activations are successful
        else:
            logging.info(
                "[{}][{}] Could not be completely triggered - {} activations pending"
                .format(context.workspace, subject, len(activations_not_done)))
Ejemplo n.º 20
0
from jsonpath_ng import jsonpath, parse
import influxdb
import json
import time
import datetime

#channel_id = "20002"
channels = ["20002", "20003", "20004", "20005", "20006", "20007", "20008", "20009", "20010", "20321", "18378"]
acc_key = "f90c94103d04abedaad2edebc38a3662"
olddate = datetime.datetime.strftime(datetime.datetime.utcnow()-datetime.timedelta(3), "%Y-%m-%d")
oldtime = datetime.datetime.strftime(datetime.datetime.utcnow()-datetime.timedelta(3), "%H:%M:%S")

for channel_id in channels:
  response = urllib.request.urlopen(f"https://api.ubibot.com/channels/{channel_id}/feeds.json?parameters&account_key={acc_key}&start={olddate}%20{oldtime}")
  d = json.loads(response.read())
  if parse('$.result').find(d) != 'success':
    print('result fail')
  
  expr = parse('$.feeds[*]')
  points = []
  for feed in [match.value for match in expr.find(d)]:
    if 'wifi' in feed:
      feed.pop('wifi')
    body = {}
    body['measurement'] = 'ubibot'
    body['tags'] = {'channel': channel_id}
    body['time'] = feed.pop('created_at')
    for bit in feed:
      feed[bit] = float(feed[bit])
    body['fields'] = feed
    if len(body['fields']) > 0:
Ejemplo n.º 21
0
def jsonpath_item_list(json_item, json_expression, items):
    jsonpath_expr = parse(json_expression)
    labels = jsonpath_expr.find(json_item)
    return len(labels)
Ejemplo n.º 22
0
def sync_records(qb, catalog_entry, state, counter, state_passed):
    chunked_bookmark = singer_utils.strptime_with_tz(qb.get_start_date(state, catalog_entry))
    stream = catalog_entry['stream']
    schema = catalog_entry['schema']
    stream_alias = catalog_entry.get('stream_alias')
    catalog_metadata = metadata.to_map(catalog_entry['metadata'])
    replication_key = catalog_metadata.get((), {}).get('replication-key')
    stream_version = get_stream_version(catalog_entry, state)
    activate_version_message = singer.ActivateVersionMessage(stream=(stream_alias or stream),
                                                             version=stream_version)

    start_time = singer_utils.now()

    LOGGER.info('Syncing Quickbooks data for stream %s', stream)

    previous_max_replication_key = None;

    query_func = qb.query
    if stream.endswith("Report"):
        query_func = qb.query_report

    for rec in query_func(catalog_entry, state, state_passed):

        counter.increment()
        with Transformer(pre_hook=transform_data_hook) as transformer:
            rec = transformer.transform(rec, schema)

        singer.write_message(
            singer.RecordMessage(
                stream=(
                        stream_alias or stream),
                record=rec,
                version=stream_version,
                time_extracted=start_time))

        if replication_key:
            jsonpath_expression = parse(f"$.{replication_key}")
            _rec = {'MetaData': json.loads(rec.get('MetaData', {}))}
            match = jsonpath_expression.find(_rec)
            original_replication_key_value = ""
            if replication_key and len(match) > 0:
                original_replication_key_value = match[0].value
                replication_key_value = singer_utils.strptime_with_tz(original_replication_key_value)

            # Before writing a bookmark, make sure Quickbooks has not given us a
            # record with one outside our range
            if previous_max_replication_key is None or (
                    replication_key_value and replication_key_value <= start_time and replication_key_value > previous_max_replication_key
            ):
                state = singer.write_bookmark(
                    state,
                    catalog_entry['tap_stream_id'],
                    replication_key,
                    original_replication_key_value)
                previous_max_replication_key = replication_key_value

            # Tables with no replication_key will send an
            # activate_version message for the next sync

    if not replication_key:
        singer.write_message(activate_version_message)
        state = singer.write_bookmark(
            state, catalog_entry['tap_stream_id'], 'version', None)
Ejemplo n.º 23
0
    def generate_clips_fact(self, json_object, json_path_func):
        """ Generic method that generates CLIPS fact 
            from given parsed JSON object

        Parameters
        ----------
        json_object : object
            Parsed JSON object
        json_path_func: function
            Function which determines which JSON path expression 
            will be used, possible functions: for single object & 
            for multitude of objects inside of array

        Details
        -------
        We use DSD model_id unique identifier for CLIPS fact name.
        CLIPS does not support BOOLEAN type, so we convert it to INTEGER.

        1. We loop through all fields of this data source definition.
        2. We locate DSD field in JSON object and read data from it.
        3. We translate BOOLEAN to INTEGER, if present.
        4. We add quotes to STRING type, if present.

        Returns
        -------
        clips_fact: str
            Generated CLIPS fact definition statement
        """

        clips_fact = "(" + str(self.dsd.model_id)
        clips_fields = []

        print("JSON")
        print(json.dumps(json_object))
        print("\n")

        for field in self.dsd.fields:
            jsonpath_expr = parse(str(json_path_func(field)))
            field_loc = [match.value for match in jsonpath_expr \
                                                  .find(json_object)]

            if not field_loc or len(field_loc) < 1:
                line, col = self.dsd._tx_parser \
                            .pos_to_linecol(field._tx_position)
                message = "Field '{0}' in DSD is not " \
                          "matched in provided JSON data." \
                          .format(field.field_name)
                raise AkashicError(message, line, col, ErrType.SEMANTIC)

            result = field_loc[0]

            # Resolve field value
            resolved_value = None
            if field.type == "INTEGER" or field.type == "FLOAT":
                resolved_value = result
            elif field.type == "BOOLEAN":
                if result == True:
                    resolved_value = 1
                else:
                    resolved_value = 0
            elif field.type == "STRING":
                resolved_value = "\"{0}\"" \
                                 .format(result)

            clips_fields.append("\t(" + str(field.field_name) + " " + \
                                str(resolved_value) + ")")

        clips_fact += "\n".join(clips_fields) + ")"
        return clips_fact
Ejemplo n.º 24
0
def _extract(document, jsonpath):
    p = jsonpath_ng.parse(jsonpath)
    matches = p.find(document)
    if matches:
        return matches[0].value
Ejemplo n.º 25
0
def find_matching_paths(json, jsonpexpr='$..id'):
    jsonpath_expr = parse(jsonpexpr)
    return [str(match.full_path) for match in jsonpath_expr.find(json)]
Ejemplo n.º 26
0
def properties_from_json(json, mapping, discovery_pattern=None, discovery_path=None):
    """Extract properties from a provider json result.

    :param json: the representation of a provider result as a json object
    :type json: dict
    :param mapping: a mapping between :class:`~eodag.api.product.EOProduct`'s metadata
                    keys and the location of the values of these properties in the json
                    representation, expressed as a
                    `jsonpath <http://goessner.net/articles/JsonPath/>`_
    :param discovery_pattern: regex pattern for metadata key discovery,
                                e.g. "^[a-zA-Z]+$"
    :type discovery_pattern: str
    :param discovery_path: str representation of jsonpath
    :type discovery_path: str
    :return: the metadata of the :class:`~eodag.api.product.EOProduct`
    :rtype: dict
    """
    properties = {}
    templates = {}
    used_jsonpaths = []
    for metadata, value in mapping.items():
        # Treat the case when the value is from a queryable metadata
        if isinstance(value, list):
            conversion_or_none, path_or_text = value[1]
        else:
            conversion_or_none, path_or_text = value
        if isinstance(path_or_text, str):
            if re.search(r"({[^{}]+})+", path_or_text):
                templates[metadata] = path_or_text
            else:
                properties[metadata] = path_or_text
        else:
            match = path_or_text.find(json)
            if len(match) == 1:
                extracted_value = match[0].value
                used_jsonpaths.append(match[0].path)
            else:
                extracted_value = NOT_AVAILABLE
            if extracted_value is None:
                properties[metadata] = None
            else:
                if conversion_or_none is None:
                    properties[metadata] = extracted_value
                else:
                    # reformat conversion_or_none as metadata#converter(args) or metadata#converter
                    if (
                        len(conversion_or_none) > 1
                        and isinstance(conversion_or_none, list)
                        and conversion_or_none[1] is not None
                    ):
                        conversion_or_none = "%s(%s)" % (
                            conversion_or_none[0],
                            conversion_or_none[1],
                        )
                    elif isinstance(conversion_or_none, list):
                        conversion_or_none = conversion_or_none[0]
                    properties[metadata] = format_metadata(
                        "{%s%s%s}" % (metadata, SEP, conversion_or_none),
                        **{metadata: extracted_value}
                    )
    # Resolve templates
    for metadata, template in templates.items():
        properties[metadata] = template.format(**properties)

    # adds missing discovered properties
    if discovery_pattern and discovery_path:
        discovered_properties = jsonpath.parse(discovery_path).find(json)
        for found_jsonpath in discovered_properties:
            found_key = found_jsonpath.path.fields[-1]
            if (
                re.compile(discovery_pattern).match(found_key)
                and found_key not in properties.keys()
                and found_jsonpath.path not in used_jsonpaths
            ):
                properties[found_key] = found_jsonpath.value

    return properties
Ejemplo n.º 27
0
import json
import bigjson
from jsonpath_ng import jsonpath, parse

jsonpath_expr = parse('features[*].properties.test')

with open('huge.json', 'rb') as file:
    obj = bigjson.load(file)
    match = jsonpath_expr.find(obj)

    for k in match:
        print(k.value)

    # for feature in obj['features']:
    #     print(feature['properties']['MAPBLKLOT'])
Ejemplo n.º 28
0
 def apply(self, item, path, value):
     jsonpath_expr = parse(path)
     jsonpath_expr.find(item)
     jsonpath_expr.update(item, value)
Ejemplo n.º 29
0
        Thread.__init__(self)
        self.stopped = event

    def run(self):
        while not self.stopped.wait(1.0):
            try:
                for key in cache:
                    expiry, value = cache[key]
                    if expiry < datetime.now():
                        del cache[key]
                        break
            except RuntimeError:
                pass


title = parse('$.data[*].title')
names = parse('$.data[*].subscriber.first_name')


@app.route('/events-with-subscriptions/<event_id>')
@cacheme
def events_with_subscriptions(event_id):
    headers = {
        'Accept': 'application/json',
        'Content-type': 'application/json',
        'Authorization': 'Token {}'.format(TOKEN)
    }
    url_evt = 'https://demo.calendar42.com/api/v2/events/{}'.format(event_id)
    url_subs = 'https://demo.calendar42.com/api/v2/event-subscriptions/'
    response = requests.get(url_evt, headers=headers)
    if response.status_code != 200:
Ejemplo n.º 30
0
if r.status_code != requests.codes.ok:
    print("error fetching golang versions")
    exit(1)

try:
    golangJson = r.json()
except:
    print("failed to parse json")
    exit(1)

if len(golangJson) != 2:
    # the script below assumes only two stable versions
    print("unexpected number of golang versions returned")
    exit(1)

fileExpr = parse('$.[*].files')
files = [match.value for match in fileExpr.find(golangJson)]
versionExpr = parse('$.[*].version')
versions = [match.value for match in versionExpr.find(golangJson)]
docker_images = []
for file in files:
    x = [f for f in file if (f['os'] == "linux" and f['arch'] == "amd64")][0]
    docker_images.append(x)

for docker_image in docker_images:
    generate_image(docker_image)

# write latest image
if docker_images[0]["version"].count('.') > 1:
    wildcard = re.sub("\.\d+$", ".x", docker_images[0]["version"])
else:
Ejemplo n.º 31
0
from jsonpath_ng import parse, Union, Child

from oasapi.common import OPERATIONS_LOWER, REFERENCE_SECTIONS

# list of JSPATH for different structures

JSPATH_INFO = parse(f"info")
JSPATH_ENDPOINTS = parse(f"paths.*")
JSPATH_OPERATIONS = Child(JSPATH_ENDPOINTS,
                          parse(f"({'|'.join(OPERATIONS_LOWER)})"))
JSPATH_OPERATION_RESPONSES = Child(JSPATH_OPERATIONS, parse("responses"))
JSPATH_OPERATIONID = Child(JSPATH_OPERATIONS, parse("operationId"))
JSPATH_OPERATION_TAGS = Child(JSPATH_OPERATIONS, parse(f"tags"))
JSPATH_SECURITY_OPERATION = Child(JSPATH_OPERATIONS, parse("security.[*].*"))
JSPATH_SECURITY_GLOBAL = parse("security.[*].*")
JSPATH_SECURITY = Union(JSPATH_SECURITY_GLOBAL, JSPATH_SECURITY_OPERATION)
JSPATH_PARAMETERS_GLOBAL = parse("parameters.[*]")
JSPATH_PARAMETERS_PATH = parse(f"paths.*.parameters.[*]")
JSPATH_PARAMETERS_OPERATION = Child(JSPATH_OPERATIONS, parse("parameters.[*]"))
JSPATH_PARAMETERS = Union(
    JSPATH_PARAMETERS_GLOBAL,
    Union(JSPATH_PARAMETERS_PATH, JSPATH_PARAMETERS_OPERATION))
JSPATH_PATHS_REFERENCES = parse("paths..'$ref'")
JSPATH_REFERENCES = parse("$..'$ref'")
JSPATH_COMPONENTS = parse(f"$.({'|'.join(REFERENCE_SECTIONS)}).*")
JSPATH_TAGS = parse("tags.[*].name")
Ejemplo n.º 32
0
def json_path_query(query: str, data: dict):
    path_expr = parse(query)
    return path_expr.find(data)
Ejemplo n.º 33
0
def get_task(ctx, cluster, sort_by, status, items, quiet, json_path, output):  # filter_val=None
    """
    \b
    # Show all running tasks.
    cmd::ecsctl get task

    \b
    # Show all running tasks with extra information.
    cmd::ecsctl get task --o wide

    \b
    # Show only ID of running tasks.
    cmd::ecsctl get task --quiet

    \b
    # Show task with status stopped and order by stopped time.
    cmd::ecsctl get task --status STOPPED --sort-by "stoppedAt"

    \b
    # Show last 10 task with status stopped and order by stopped time.
    cmd::ecsctl get task --status STOPPED --sort-by "stoppedAt" --items 10

    \b
    # Get all running task and create custom clumn: taks name and memory reservation.
    cmd::ecsctl get task --jsonpath "[*].containers[0].name" --jsonpath "[*].cpu" --jsonpath "[*].memory"
    """
    # TODO: ecsctl get po --filter "TASK_ID=~'.*'" --filter "STATUS=ACTIVE"
    #  =   : Select labels that are exactly equal to the provided string.
    #  !=  : Select labels that are not equal to the provided string.
    #  =~  : Select labels that regex-match the provided string.
    #  ~   : Select labels that do not regex-match the provided string.
    #  eg:
    #  environment=~"staging|testing|development"
    #  job=~".*"

    headers, out, instances = [], [], {}
    if not cluster:
        cluster = ctx.obj['cluster']
    bw = ctx.obj['bw']
    records = bw.get_tasks(cluster=cluster, status=status)
    if sort_by and records:
        records.sort(key=lambda r: parse(sort_by).find(r)[0].value)
    if json_path:
        for _jp in json_path:
            h = None
            jsonpath_expr = parse(_jp)
            for x, match in enumerate(jsonpath_expr.find(records)):
                try:
                    out[x].append(match.value)
                except IndexError:
                    out.append([match.value])
                h = str(match.path)
            headers.append(h)
        # if filter_val:
        #     raise ValueError
    else:
        now = datetime.datetime.now(pytz.utc)
        for inst in bw.get_container_instances(cluster=cluster):
            instances[inst['containerInstanceArn']] = inst
        for x, r in enumerate(records):
            status = r['lastStatus']
            created_at = r['createdAt']
            task_id = display.simple_task(r['taskArn'])
            task_def = display.simple_task_definition(r['taskDefinitionArn'])
            age = humanize.naturaltime(now - created_at)
            instance = instances[r['containerInstanceArn']]['ec2_data']['PrivateIpAddress']
            containers = ' | '.join([x['name'] for x in r['containers']])
            ports = []
            for c in r.get('containers', []):
                for port in c.get('networkBindings', []):
                    p = '{}{}->{}/{}'.format(
                        ':{}'.format(port.get('bindIP')) if not port.get('bindIP') == '0.0.0.0' else '0:',
                        port.get('hostPort'),
                        port.get('containerPort'),
                        port.get('protocol'))
                    ports.append(p)
            if output:
                info = bw.describe_task_definition(r['taskDefinitionArn'], cluster=cluster)
                c, l = [], []

                for td in info['containerDefinitions']:
                    c.append(td['name'])
                    log_config = td.get('logConfiguration', {})
                    if log_config.get('logDriver', {}) == 'awslogs':
                        options = log_config.get('options')
                        if options.get('awslogs-group') and \
                                options.get('awslogs-region') and \
                                options.get('awslogs-stream-prefix'):
                            l.append('awslogs')
                        else:
                            l.append('none')
                    else:
                        l.append('none')
                logs, containers = ' | '.join(l), ' | '.join(c)

            row = [task_id, status, containers, '\n'.join(ports), task_def, age, instance]
            if output:
                row.append(logs)
            # if filter_val:
            #     raise ValueError
            if quiet:
                row = [row[0]]
            out.append(row)

        if items:
            out = out[-items:]

        if not quiet:
            headers = ['TASK ID', 'STATUS', 'CONTAINERS', 'PORTS', 'TASK DEFINITION', 'AGE', 'EC2 PRIVATE IP', 'LOGS']
    output = tabulate.tabulate(out, headers=headers, tablefmt='plain')
    click.echo(output)
Ejemplo n.º 34
0
def lambda_handler(event, _):
    """
    Entry point for CloudWatch event receipt.
    """
    try:
        print(event)
        event["timestamp"] = int(
            datetime.datetime.strptime(event["time"],
                                       '%Y-%m-%dT%H:%M:%SZ').timestamp())
        event["expires"] = event["timestamp"] + int(os.environ["ITEM_TTL"])
        event["detail"]["time"] = event["time"]

        # catch all the various forms of ARN from the media services
        arn_expr = parse(
            '$..arn|aRN|resource-arn|channel_arn|multiplex_arn|flowArn|PlaybackConfigurationArn|resourceArn'
        )
        original_arns = [match.value for match in arn_expr.find(event)]
        arns = []
        # remove arn that is for userIdentity or inputSecurityGroup
        # note: can't remove an item from a list that's being iterated over so doing it this way
        for arn in original_arns:
            if "user" in arn or "role" in arn or "inputSecurityGroup" in arn:
                pass
            else:
                arns.append(arn)
        if arns:
            event["resource_arn"] = unquote(arns[0])
        # for certain events, the ARN is not labeled as an ARN but instead put in the resources list
        if not arns and event["resources"]:
            if "vod" not in event["resources"][0]:
                event["resource_arn"] = event["resources"][0]

        # handle alerts
        if "Alert" in event["detail-type"]:
            # medialive alerts
            if "MediaLive" in event["detail-type"]:
                event["alarm_id"] = event["detail"]["alarm_id"]
                event["alarm_state"] = event["detail"]["alarm_state"].lower()
                event["detail"]["pipeline_state"] = get_pipeline_state(event)

            # mediaconnect alerts
            elif "MediaConnect" in event["detail-type"]:
                event["alarm_id"] = event["detail"]["error-id"]
                if event["detail"]["errored"] == True:
                    event["alarm_state"] = "set"
                else:
                    event["alarm_state"] = "cleared"
                event["detail"]["alert_type"] = event["detail"]["error-code"]
                del event["detail"]["error-code"]
                event["detail"]["message"] = event["detail"]["error-message"]
                del event["detail"]["error-message"]
            #print(event)
            EVENTS_TABLE.put_item(Item=event)
            print(event["detail-type"] + " stored.")

        # set the rest of the information needed for storing as regular CWE
        # give timestamp a millisecond precision since it's sort key in CWE table
        event["timestamp"] = event["timestamp"] * 1000 + randint(1, 999)
        event["data"] = json.dumps(event["detail"])
        event["type"] = event["detail-type"]
        if "eventName" in event["detail"]:
            event["type"] = event["type"] + ": " + event["detail"]["eventName"]

        # handle specific cases depending on source
        if event["source"] == "aws.medialive":
            if "BatchUpdateSchedule" in event["type"]:
                print("Creating an ARN for BatchUpdateSchedule event.")
                event["resource_arn"] = "arn:aws:medialive:" + event['region'] + ":" + \
                    event['account'] + ":channel:" + \
                    event['detail']['requestParameters']['channelId']
        elif event["source"] == "aws.mediapackage":
            if "HarvestJob" in event["type"]:
                print(
                    "Asking MediaPackage for the ARN of endpoint in a HarvestJob event."
                )
                # to get the ARN, ask mediapackage to describe the origin endpoint
                # the ARN available through resources is the HarvestJob ARN, not the endpoint
                orig_id_expr = parse('$..origin_endpoint_id')
                orig_id = [match.value for match in orig_id_expr.find(event)]
                if orig_id:
                    emp_client = boto3.client('mediapackage')
                    response = emp_client.describe_origin_endpoint(
                        Id=orig_id[0])
                    event["resource_arn"] = response["Arn"]
                else:
                    print(
                        "Skipping this event. Origin ID not present in the HarvestJob event."
                        + event["type"])
        elif event["source"] == "aws.mediastore":
            # for object state change the resource is the object, not the container
            # so the captured arn needs to be fixed
            if "MediaStore Object State Change" in event["type"]:
                temp_arn = event["resource_arn"].split('/')
                event["resource_arn"] = temp_arn[0] + "/" + temp_arn[1]

        # if item has no resource arn, don't save in DB
        if "resource_arn" in event:
            #print(event)
            print("Storing media service event.")
            CLOUDWATCH_EVENTS_TABLE.put_item(Item=event)
        else:
            print("Skipping this event. " + event["type"])
    except ClientError as error:
        print(error)
    return True
Ejemplo n.º 35
0
def map_from_json(details, node, key, val):
    """construct pyspark map from jsonpaths"""
    keys = [x.value for x in jsonpath_ng.parse(node + key).find(details)]
    vals = [x.value for x in jsonpath_ng.parse(node + val).find(details)]
    flattened = itertools.chain.from_iterable(zip(keys, vals))
    return F.create_map(*map(F.lit, flattened)).cast('map<string,string>')