def mediapackage_endpoint_speke_keyserver_ddb_items():
"""
Identify and format MediaPackage origin endpoints to SPEKE keyservers for cache storage.
"""
items = []
# create an expression to find speke server urls
jsonpath_expr = parse('$..SpekeKeyProvider.Url')
try:
# get SPEKE keyservers
speke_keyservers_cached = cache.cached_by_service("speke-keyserver")
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service("mediapackage-origin-endpoint")
# iterate over all distributions
for keyserver in speke_keyservers_cached:
keyserver_data = json.loads(keyserver["data"])
keyserver_endpoint = keyserver_data["endpoint"]
for mp_endpoint in mediapackage_ep_cached:
mp_endpoint_data = json.loads(mp_endpoint["data"])
for server_url in [match.value for match in jsonpath_expr.find(mp_endpoint_data)]:
if server_url == keyserver_endpoint:
config = {"from": mp_endpoint["arn"], "to": keyserver["arn"], "scheme": keyserver_data["scheme"]}
print(config)
items.append(connection_to_ddb_item(mp_endpoint["arn"], keyserver["arn"], "mediapackage-origin-endpoint-speke-keyserver", config))
except ClientError as error:
print(error)
return items
def execute(self, message: IRCMessage):
if len(message.parameterList) < 2:
return IRCResponse(ResponseType.Say,
'Not enough parameters, usage: {}'.format(self.help(None)),
message.replyTo)
path = ' '.join(message.parameterList[1:])
try:
parser = jsonpath_ng.parse(path)
except (jsonpath_ng.lexer.JsonPathLexerError, Exception) as e:
# yep, jsonpath_ng uses generic exceptions, so this is the best we can do
return IRCResponse(ResponseType.Say,
'[Jostle Error: {}]'.format(e),
message.replyTo)
url = message.parameterList[0]
if not re.match(r'^\w+://', url):
url = 'http://{}'.format(url)
if 'jostle' in message.metadata and url in message.metadata['jostle']:
# use cached data if it exists
j = message.metadata['jostle'][url]
else:
response = self.bot.moduleHandler.runActionUntilValue('fetch-url', url)
if not response:
return IRCResponse(ResponseType.Say,
'[Jostle Error: problem fetching {}]'.format(url),
message.replyTo)
try:
j = response.json()
except JSONDecodeError:
return IRCResponse(ResponseType.Say,
'[Jostle Error: data at {} is not valid JSON]'.format(url),
message.replyTo)
m = parser.find(j)
if not m:
reply = '[Jostle Error: the jsonpath {!r} does not resolve a value from {!r}]'
reply = reply.format(path, url)
return IRCResponse(ResponseType.Say, reply, message.replyTo)
value = m[0].value
if not isinstance(value, str):
value = ' '.join(value)
# sanitize the value
value = value.strip()
value = re.sub(r'[\r\n]+', ' ', value)
value = re.sub(r'\s+', ' ', value)
return IRCResponse(ResponseType.Say, value, message.replyTo,
extraVars={'jostleURL': url},
metadata={'jostle': {url: j}})
def mediapackage_channels(region):
"""
Return the MediaPackage channels for the given region.
"""
service = boto3.client("mediapackage", region_name=region)
jsonpath_expr = parse('$..Password')
response = service.list_channels()
items = response['Channels']
while "NextToken" in response:
response = service.list_channels(NextToken=response["NextToken"])
items = items + response['Channels']
jsonpath_expr.update(items, "XXXXXXXXXXXX")
return items
def speke_server_ddb_items(region):
"""
Find the SPEKE key servers based on MediaPackage endpoint configurations
"""
items = []
# create an expression to find speke server urls
jsonpath_expr = parse('$..SpekeKeyProvider.Url')
# get MediaPackage origin endpoints
mediapackage_ep_cached = cache.cached_by_service_region("mediapackage-origin-endpoint", region)
for endpoint in mediapackage_ep_cached:
# decode the endpoint configuration
endpoint_data = json.loads(endpoint["data"])
for server_url in [match.value for match in jsonpath_expr.find(endpoint_data)]:
parsed = urlparse(server_url)
sha = hashlib.sha1()
sha.update(server_url.encode('utf-8'))
url_digest = sha.hexdigest()
arn = "arn:oss:speke:::{}".format(url_digest)
config = {"arn": arn, "endpoint": server_url, "scheme": parsed.scheme}
service = "speke-keyserver"
# print(config)
items.append(node_to_ddb_item(arn, service, "global", config))
return items
def extract_path(data, path):
r = jsonpath_ng.parse(path).find(data)
if r:
return r[0].value
log.warn("Could not find %s in %s" % (path, data))
def json_path(txt):
try:
return jsonpath_ng.parse(txt)
except Exception as e:
raise SchemaError('Bad JsonPath format: %s' % txt)
def main(argv=None):
itemData = item.get_data()
datum = jsonpath_ng.parse("$..*").find(itemData)[0]
_formatFullPath(path=str(datum.full_path), key=str(datum.path))
_getDatumInformation(datum)
JSONtoLongDataFrame(jsonObject=itemData, parseString="$..*")
def process_etsy_request(self, conn_etsy, **kwargs):
"""Process Etsy request."""
doc = kwargs.get('doc')
doc_data = doc['data']
url = kwargs.get('url')
steps = kwargs.get('steps')
method = kwargs.get('method')
params = doc_data.get('params', None)
vars_mapped = {}
data = self._get_data(doc_data.get('get_data_on'))
if doc_data.get('vars'):
var_list = doc_data.get('vars').items()
for k, v in var_list:
jsonpath_expr = parse(v)
vars_mapped[k] = [match.value for match in jsonpath_expr.find(data)][0]
for k, v in vars_mapped.items():
url = url.replace("$%s" % k, str(v))
if method == 'get':
if params:
listings_generator = conn_etsy.iterate_pages(
'execute_authed',
url,
params=params
)
else:
listings_generator = conn_etsy.iterate_pages(
'execute_authed',
url
)
for result in listings_generator:
# In special cases save specific payloads into data, and
# everything else are the results.
try:
if result['type'] == 'ListingInventory':
self._put_data(result, doc_data.get('store_data_on'))
if result['type'] == 'Listing':
self._put_data(result['results'], doc_data.get('store_data_on'))
except KeyError as err:
self._put_data(result['results'], doc_data.get('store_data_on'))
self._run_steps(steps)
if method == 'post':
mapper = doc_data.get('export_mapper')
_map = mapper['map']
if isinstance(self.data, list):
mapped_list = [form_doc(i, _map) for i in self.data]
self.data = mapped_list
else:
self.data = form_doc(self.data, _map)
if isinstance(self.data, list):
for item in self.data:
conn_etsy.create_listings(payload=item)
else:
conn_etsy.create_listings(payload=self.data)
def jsonpath(path, obj, allow_none=False):
r = [m.value for m in parse(path).find(obj)]
return r[0] if not allow_none else (r[0] if r else None)
def parse_json_expression(json, expression):
json_expression = parse(expression)
json_thing = json_expression.find(json)
return json_thing
import json
from jsonpath_ng import parse
with open("cloud.json", 'r') as json_file:
json_data = json.load(json_file)
'''
print(json_data)
'''
jsonpath_expression = parse('$.ListBucketResult.Contents[*].Key')
for match in jsonpath_expression.find(json_data):
print(f'Template name: {match.value}')
def resolve_path_multimatch(expression, match, context):
pathmatches = jsonpath.parse(expression).find(context)
if pathmatches:
return unicode([m.value for m in pathmatches])
return match.group(0)
def get_field_mappings(mapping_definition):
mapping_obj = mapping_definition
matches = parse('mapping[*]').find(mapping_obj)
mappings = [match.value for match in matches]
return mappings
def JSP_get_basic_fields(avro_obj):
jsonpath_expr = parse('fields[*].name')
return [match.value for match in jsonpath_expr.find(avro_obj)]
def custom(dataset) -> int:
total_updated = 0
running = True
print("Custom data mutation mode.")
print()
print(
"Enter selection JSONPath query, then enter mutation lambda to be applied to"
)
print("each. For mutation lambdas, '_' is the original value.")
print("(type \q to quit, \c to cancel mutation)")
while running:
query = input("select> ")
query = remove_ansi_escapes(query)
if query == r'\q':
running = False
continue
elif query == r'\c':
continue
try:
expr = jsonpath_ng.parse(query)
except JsonPathParserError as e:
print(str(e))
continue
except AttributeError:
print("Could not parse query, try again")
continue
matches = expr.find(dataset)
if len(matches) < 1:
print("(no results)")
continue
results = [match.value for match in matches]
output = json.dumps(results, indent=2, sort_keys=True)
print(output)
mutie = input("MUTATE> ")
mutie = remove_ansi_escapes(mutie)
if query == r'\q':
running = False
continue
elif query == r'\c':
continue
mutation_func_str = 'lambda _: ' + mutie
mutation_func = eval(mutation_func_str)
for m in matches:
old_val = m.value
new_val = mutation_func(old_val)
print("NEW VAL: {!r}".format(new_val))
dataset = m.full_path.update(dataset, new_val)
# show user the result by selecting the updated data once more
updated_matches = expr.find(dataset)
if len(updated_matches) < 1:
raise ValueError("updated_matches somehow resulted in no results")
updated_results = [m.value for m in updated_matches]
updated_output = json.dumps(updated_results, indent=2, sort_keys=True)
print(output)
print("{!r} row(s) updated with mutation lambda".format(len(matches)))
total_updated += len(matches)
return total_updated
def get_base_url(self):
page_url = parse('searchContent..pageMetadata.canonical')
return [i.value for i in page_url.find(self.base)][0]
def fake_parse(value):
self.jsonpath_call_count += 1
return jsonpath_ng.parse(value)
def parse(cls, path):
if not path in cls.finders: cls.finders[path] = jsonpath_ng.parse(path)
return cls.finders[path]
def action_ibm_cf_invoke(context, event):
class InvokeException(Exception):
pass
if not ibmcf_session:
create_ibmcf_session()
operator = context['operator'] if 'operator' in context else context
cf_auth = operator['api_key'].split(':')
cf_auth_handler = HTTPBasicAuth(cf_auth[0], cf_auth[1])
url = operator['url']
subject = context['subject'] if 'subject' in context else None
triggerflow_meta = {'subject': subject, 'sink': operator['sink']}
invoke_payloads = []
if operator['iter_data']:
keys = list(operator['iter_data'].keys())
iterdata_keyword = keys.pop()
for iterdata_value in operator['iter_data'][iterdata_keyword]:
payload = operator['invoke_kwargs'].copy()
payload[iterdata_keyword] = iterdata_value
payload['__OW_TRIGGERFLOW'] = triggerflow_meta
invoke_payloads.append(payload)
else:
for key, arg in operator['invoke_kwargs'].items():
if isinstance(arg, str) and arg.startswith('$'):
jsonpath_expr = jsonpath_ng.parse(arg)
result_args = [
match.value
for match in jsonpath_expr.find(context['result'])
]
operator['invoke_kwargs'][key] = result_args
payload = operator['invoke_kwargs']
payload['__OW_TRIGGERFLOW'] = triggerflow_meta
invoke_payloads.append(payload)
if 'max_retries' in context:
max_retries = context['max_retries']
else:
max_retries = 5
################################################
def invoke(call_id, payload):
act_id = None
retry = True
retry_count = 0
while retry:
try:
start_t = time.time()
# response = requests.post(url, json=payload, auth=cf_auth_handler, timeout=10.0, verify=True)
response = ibmcf_session.post(url,
json=payload,
auth=cf_auth_handler,
timeout=10.0,
verify=True)
status_code = response.status_code
res_json = response.json()
et = round(time.time() - start_t, 3)
if status_code in range(200, 300) and 'activationId' in res_json and \
res_json['activationId'] is not None:
retry = False
act_id = res_json['activationId']
logging.info(
'[{}][{}] Invocation success ({}s) - Activation ID: {}'
.format(context.workspace, call_id, et, act_id))
elif status_code in range(
400, 500) and status_code not in [408, 409, 429]:
logging.error(
'[{}][{}] Invocation failed - Activation status code: {}'
.format(context.workspace, call_id, status_code))
raise InvokeException('Invocation failed')
except requests.exceptions.RequestException as e:
logging.error('[{}][{}] Error talking to OpenWhisk: {}'.format(
context.workspace, call_id, e))
raise e
except Exception as e:
logging.error("[{}][{}] Exception - {}".format(
context.workspace, call_id, e))
if retry:
retry_count += 1
if retry_count <= max_retries:
sleepy_time = pow(2, retry_count)
logging.info("[{}][{}] Retrying in {} second(s)".format(
context.workspace, call_id, sleepy_time))
time.sleep(sleepy_time)
else:
logging.error(
"[{}][{}] Retrying failed after {} attempts".format(
context.workspace, call_id, max_retries))
raise InvokeException('Invocation failed')
return call_id, act_id
################################################
total_activations = len(invoke_payloads)
logging.info("[{}] Firing trigger {} - Activations: {} ".format(
context.workspace, subject, total_activations))
futures = []
responses = []
if total_activations == 1:
resp = invoke(0, invoke_payloads[0])
responses.append(resp)
else:
with ThreadPoolExecutor(max_workers=128) as executor:
for cid, payload in enumerate(invoke_payloads):
res = executor.submit(invoke, cid, payload)
futures.append(res)
try:
responses = [fut.result() for fut in futures]
except InvokeException:
pass
activations_done = [call_id for call_id, _ in responses]
activations_not_done = [
call_id for call_id in range(total_activations)
if call_id not in activations_done
]
if subject in context.trigger_mapping:
downstream_triggers = context.trigger_mapping[subject][
'event.triggerflow.termination.success']
for downstream_trigger in downstream_triggers:
downstream_trigger_ctx = context.triggers[
downstream_trigger].context
if 'total_activations' in downstream_trigger_ctx:
downstream_trigger_ctx[
'total_activations'] += total_activations
else:
downstream_trigger_ctx['total_activations'] = total_activations
if 'dependencies' in downstream_trigger_ctx and subject in downstream_trigger_ctx[
'dependencies']:
if downstream_trigger_ctx['dependencies'][subject]['join'] > 0:
downstream_trigger_ctx['dependencies'][subject][
'join'] += total_activations
else:
downstream_trigger_ctx['dependencies'][subject][
'join'] = total_activations
# All activations are unsuccessful
if not activations_done:
raise Exception('All invocations have failed')
# At least one activation is successful
else:
# All activations are successful
if len(activations_done) == total_activations:
logging.info('[{}][{}] All invocations successful'.format(
context.workspace, subject))
# Only some activations are successful
else:
logging.info(
"[{}][{}] Could not be completely triggered - {} activations pending"
.format(context.workspace, subject, len(activations_not_done)))
from jsonpath_ng import jsonpath, parse
import influxdb
import json
import time
import datetime
#channel_id = "20002"
channels = ["20002", "20003", "20004", "20005", "20006", "20007", "20008", "20009", "20010", "20321", "18378"]
acc_key = "f90c94103d04abedaad2edebc38a3662"
olddate = datetime.datetime.strftime(datetime.datetime.utcnow()-datetime.timedelta(3), "%Y-%m-%d")
oldtime = datetime.datetime.strftime(datetime.datetime.utcnow()-datetime.timedelta(3), "%H:%M:%S")
for channel_id in channels:
response = urllib.request.urlopen(f"https://api.ubibot.com/channels/{channel_id}/feeds.json?parameters&account_key={acc_key}&start={olddate}%20{oldtime}")
d = json.loads(response.read())
if parse('$.result').find(d) != 'success':
print('result fail')
expr = parse('$.feeds[*]')
points = []
for feed in [match.value for match in expr.find(d)]:
if 'wifi' in feed:
feed.pop('wifi')
body = {}
body['measurement'] = 'ubibot'
body['tags'] = {'channel': channel_id}
body['time'] = feed.pop('created_at')
for bit in feed:
feed[bit] = float(feed[bit])
body['fields'] = feed
if len(body['fields']) > 0:
def jsonpath_item_list(json_item, json_expression, items):
jsonpath_expr = parse(json_expression)
labels = jsonpath_expr.find(json_item)
return len(labels)
def sync_records(qb, catalog_entry, state, counter, state_passed):
chunked_bookmark = singer_utils.strptime_with_tz(qb.get_start_date(state, catalog_entry))
stream = catalog_entry['stream']
schema = catalog_entry['schema']
stream_alias = catalog_entry.get('stream_alias')
catalog_metadata = metadata.to_map(catalog_entry['metadata'])
replication_key = catalog_metadata.get((), {}).get('replication-key')
stream_version = get_stream_version(catalog_entry, state)
activate_version_message = singer.ActivateVersionMessage(stream=(stream_alias or stream),
version=stream_version)
start_time = singer_utils.now()
LOGGER.info('Syncing Quickbooks data for stream %s', stream)
previous_max_replication_key = None;
query_func = qb.query
if stream.endswith("Report"):
query_func = qb.query_report
for rec in query_func(catalog_entry, state, state_passed):
counter.increment()
with Transformer(pre_hook=transform_data_hook) as transformer:
rec = transformer.transform(rec, schema)
singer.write_message(
singer.RecordMessage(
stream=(
stream_alias or stream),
record=rec,
version=stream_version,
time_extracted=start_time))
if replication_key:
jsonpath_expression = parse(f"$.{replication_key}")
_rec = {'MetaData': json.loads(rec.get('MetaData', {}))}
match = jsonpath_expression.find(_rec)
original_replication_key_value = ""
if replication_key and len(match) > 0:
original_replication_key_value = match[0].value
replication_key_value = singer_utils.strptime_with_tz(original_replication_key_value)
# Before writing a bookmark, make sure Quickbooks has not given us a
# record with one outside our range
if previous_max_replication_key is None or (
replication_key_value and replication_key_value <= start_time and replication_key_value > previous_max_replication_key
):
state = singer.write_bookmark(
state,
catalog_entry['tap_stream_id'],
replication_key,
original_replication_key_value)
previous_max_replication_key = replication_key_value
# Tables with no replication_key will send an
# activate_version message for the next sync
if not replication_key:
singer.write_message(activate_version_message)
state = singer.write_bookmark(
state, catalog_entry['tap_stream_id'], 'version', None)
def generate_clips_fact(self, json_object, json_path_func):
""" Generic method that generates CLIPS fact
from given parsed JSON object
Parameters
----------
json_object : object
Parsed JSON object
json_path_func: function
Function which determines which JSON path expression
will be used, possible functions: for single object &
for multitude of objects inside of array
Details
-------
We use DSD model_id unique identifier for CLIPS fact name.
CLIPS does not support BOOLEAN type, so we convert it to INTEGER.
1. We loop through all fields of this data source definition.
2. We locate DSD field in JSON object and read data from it.
3. We translate BOOLEAN to INTEGER, if present.
4. We add quotes to STRING type, if present.
Returns
-------
clips_fact: str
Generated CLIPS fact definition statement
"""
clips_fact = "(" + str(self.dsd.model_id)
clips_fields = []
print("JSON")
print(json.dumps(json_object))
print("\n")
for field in self.dsd.fields:
jsonpath_expr = parse(str(json_path_func(field)))
field_loc = [match.value for match in jsonpath_expr \
.find(json_object)]
if not field_loc or len(field_loc) < 1:
line, col = self.dsd._tx_parser \
.pos_to_linecol(field._tx_position)
message = "Field '{0}' in DSD is not " \
"matched in provided JSON data." \
.format(field.field_name)
raise AkashicError(message, line, col, ErrType.SEMANTIC)
result = field_loc[0]
# Resolve field value
resolved_value = None
if field.type == "INTEGER" or field.type == "FLOAT":
resolved_value = result
elif field.type == "BOOLEAN":
if result == True:
resolved_value = 1
else:
resolved_value = 0
elif field.type == "STRING":
resolved_value = "\"{0}\"" \
.format(result)
clips_fields.append("\t(" + str(field.field_name) + " " + \
str(resolved_value) + ")")
clips_fact += "\n".join(clips_fields) + ")"
return clips_fact
def _extract(document, jsonpath):
p = jsonpath_ng.parse(jsonpath)
matches = p.find(document)
if matches:
return matches[0].value
def find_matching_paths(json, jsonpexpr='$..id'):
jsonpath_expr = parse(jsonpexpr)
return [str(match.full_path) for match in jsonpath_expr.find(json)]
def properties_from_json(json, mapping, discovery_pattern=None, discovery_path=None):
"""Extract properties from a provider json result.
:param json: the representation of a provider result as a json object
:type json: dict
:param mapping: a mapping between :class:`~eodag.api.product.EOProduct`'s metadata
keys and the location of the values of these properties in the json
representation, expressed as a
`jsonpath <http://goessner.net/articles/JsonPath/>`_
:param discovery_pattern: regex pattern for metadata key discovery,
e.g. "^[a-zA-Z]+$"
:type discovery_pattern: str
:param discovery_path: str representation of jsonpath
:type discovery_path: str
:return: the metadata of the :class:`~eodag.api.product.EOProduct`
:rtype: dict
"""
properties = {}
templates = {}
used_jsonpaths = []
for metadata, value in mapping.items():
# Treat the case when the value is from a queryable metadata
if isinstance(value, list):
conversion_or_none, path_or_text = value[1]
else:
conversion_or_none, path_or_text = value
if isinstance(path_or_text, str):
if re.search(r"({[^{}]+})+", path_or_text):
templates[metadata] = path_or_text
else:
properties[metadata] = path_or_text
else:
match = path_or_text.find(json)
if len(match) == 1:
extracted_value = match[0].value
used_jsonpaths.append(match[0].path)
else:
extracted_value = NOT_AVAILABLE
if extracted_value is None:
properties[metadata] = None
else:
if conversion_or_none is None:
properties[metadata] = extracted_value
else:
# reformat conversion_or_none as metadata#converter(args) or metadata#converter
if (
len(conversion_or_none) > 1
and isinstance(conversion_or_none, list)
and conversion_or_none[1] is not None
):
conversion_or_none = "%s(%s)" % (
conversion_or_none[0],
conversion_or_none[1],
)
elif isinstance(conversion_or_none, list):
conversion_or_none = conversion_or_none[0]
properties[metadata] = format_metadata(
"{%s%s%s}" % (metadata, SEP, conversion_or_none),
**{metadata: extracted_value}
)
# Resolve templates
for metadata, template in templates.items():
properties[metadata] = template.format(**properties)
# adds missing discovered properties
if discovery_pattern and discovery_path:
discovered_properties = jsonpath.parse(discovery_path).find(json)
for found_jsonpath in discovered_properties:
found_key = found_jsonpath.path.fields[-1]
if (
re.compile(discovery_pattern).match(found_key)
and found_key not in properties.keys()
and found_jsonpath.path not in used_jsonpaths
):
properties[found_key] = found_jsonpath.value
return properties
import json
import bigjson
from jsonpath_ng import jsonpath, parse
jsonpath_expr = parse('features[*].properties.test')
with open('huge.json', 'rb') as file:
obj = bigjson.load(file)
match = jsonpath_expr.find(obj)
for k in match:
print(k.value)
# for feature in obj['features']:
# print(feature['properties']['MAPBLKLOT'])
def apply(self, item, path, value):
jsonpath_expr = parse(path)
jsonpath_expr.find(item)
jsonpath_expr.update(item, value)
Thread.__init__(self)
self.stopped = event
def run(self):
while not self.stopped.wait(1.0):
try:
for key in cache:
expiry, value = cache[key]
if expiry < datetime.now():
del cache[key]
break
except RuntimeError:
pass
title = parse('$.data[*].title')
names = parse('$.data[*].subscriber.first_name')
@app.route('/events-with-subscriptions/<event_id>')
@cacheme
def events_with_subscriptions(event_id):
headers = {
'Accept': 'application/json',
'Content-type': 'application/json',
'Authorization': 'Token {}'.format(TOKEN)
}
url_evt = 'https://demo.calendar42.com/api/v2/events/{}'.format(event_id)
url_subs = 'https://demo.calendar42.com/api/v2/event-subscriptions/'
response = requests.get(url_evt, headers=headers)
if response.status_code != 200:
if r.status_code != requests.codes.ok:
print("error fetching golang versions")
exit(1)
try:
golangJson = r.json()
except:
print("failed to parse json")
exit(1)
if len(golangJson) != 2:
# the script below assumes only two stable versions
print("unexpected number of golang versions returned")
exit(1)
fileExpr = parse('$.[*].files')
files = [match.value for match in fileExpr.find(golangJson)]
versionExpr = parse('$.[*].version')
versions = [match.value for match in versionExpr.find(golangJson)]
docker_images = []
for file in files:
x = [f for f in file if (f['os'] == "linux" and f['arch'] == "amd64")][0]
docker_images.append(x)
for docker_image in docker_images:
generate_image(docker_image)
# write latest image
if docker_images[0]["version"].count('.') > 1:
wildcard = re.sub("\.\d+$", ".x", docker_images[0]["version"])
else:
from jsonpath_ng import parse, Union, Child
from oasapi.common import OPERATIONS_LOWER, REFERENCE_SECTIONS
# list of JSPATH for different structures
JSPATH_INFO = parse(f"info")
JSPATH_ENDPOINTS = parse(f"paths.*")
JSPATH_OPERATIONS = Child(JSPATH_ENDPOINTS,
parse(f"({'|'.join(OPERATIONS_LOWER)})"))
JSPATH_OPERATION_RESPONSES = Child(JSPATH_OPERATIONS, parse("responses"))
JSPATH_OPERATIONID = Child(JSPATH_OPERATIONS, parse("operationId"))
JSPATH_OPERATION_TAGS = Child(JSPATH_OPERATIONS, parse(f"tags"))
JSPATH_SECURITY_OPERATION = Child(JSPATH_OPERATIONS, parse("security.[*].*"))
JSPATH_SECURITY_GLOBAL = parse("security.[*].*")
JSPATH_SECURITY = Union(JSPATH_SECURITY_GLOBAL, JSPATH_SECURITY_OPERATION)
JSPATH_PARAMETERS_GLOBAL = parse("parameters.[*]")
JSPATH_PARAMETERS_PATH = parse(f"paths.*.parameters.[*]")
JSPATH_PARAMETERS_OPERATION = Child(JSPATH_OPERATIONS, parse("parameters.[*]"))
JSPATH_PARAMETERS = Union(
JSPATH_PARAMETERS_GLOBAL,
Union(JSPATH_PARAMETERS_PATH, JSPATH_PARAMETERS_OPERATION))
JSPATH_PATHS_REFERENCES = parse("paths..'$ref'")
JSPATH_REFERENCES = parse("$..'$ref'")
JSPATH_COMPONENTS = parse(f"$.({'|'.join(REFERENCE_SECTIONS)}).*")
JSPATH_TAGS = parse("tags.[*].name")
def json_path_query(query: str, data: dict):
path_expr = parse(query)
return path_expr.find(data)
def get_task(ctx, cluster, sort_by, status, items, quiet, json_path, output): # filter_val=None
"""
\b
# Show all running tasks.
cmd::ecsctl get task
\b
# Show all running tasks with extra information.
cmd::ecsctl get task --o wide
\b
# Show only ID of running tasks.
cmd::ecsctl get task --quiet
\b
# Show task with status stopped and order by stopped time.
cmd::ecsctl get task --status STOPPED --sort-by "stoppedAt"
\b
# Show last 10 task with status stopped and order by stopped time.
cmd::ecsctl get task --status STOPPED --sort-by "stoppedAt" --items 10
\b
# Get all running task and create custom clumn: taks name and memory reservation.
cmd::ecsctl get task --jsonpath "[*].containers[0].name" --jsonpath "[*].cpu" --jsonpath "[*].memory"
"""
# TODO: ecsctl get po --filter "TASK_ID=~'.*'" --filter "STATUS=ACTIVE"
# = : Select labels that are exactly equal to the provided string.
# != : Select labels that are not equal to the provided string.
# =~ : Select labels that regex-match the provided string.
# ~ : Select labels that do not regex-match the provided string.
# eg:
# environment=~"staging|testing|development"
# job=~".*"
headers, out, instances = [], [], {}
if not cluster:
cluster = ctx.obj['cluster']
bw = ctx.obj['bw']
records = bw.get_tasks(cluster=cluster, status=status)
if sort_by and records:
records.sort(key=lambda r: parse(sort_by).find(r)[0].value)
if json_path:
for _jp in json_path:
h = None
jsonpath_expr = parse(_jp)
for x, match in enumerate(jsonpath_expr.find(records)):
try:
out[x].append(match.value)
except IndexError:
out.append([match.value])
h = str(match.path)
headers.append(h)
# if filter_val:
# raise ValueError
else:
now = datetime.datetime.now(pytz.utc)
for inst in bw.get_container_instances(cluster=cluster):
instances[inst['containerInstanceArn']] = inst
for x, r in enumerate(records):
status = r['lastStatus']
created_at = r['createdAt']
task_id = display.simple_task(r['taskArn'])
task_def = display.simple_task_definition(r['taskDefinitionArn'])
age = humanize.naturaltime(now - created_at)
instance = instances[r['containerInstanceArn']]['ec2_data']['PrivateIpAddress']
containers = ' | '.join([x['name'] for x in r['containers']])
ports = []
for c in r.get('containers', []):
for port in c.get('networkBindings', []):
p = '{}{}->{}/{}'.format(
':{}'.format(port.get('bindIP')) if not port.get('bindIP') == '0.0.0.0' else '0:',
port.get('hostPort'),
port.get('containerPort'),
port.get('protocol'))
ports.append(p)
if output:
info = bw.describe_task_definition(r['taskDefinitionArn'], cluster=cluster)
c, l = [], []
for td in info['containerDefinitions']:
c.append(td['name'])
log_config = td.get('logConfiguration', {})
if log_config.get('logDriver', {}) == 'awslogs':
options = log_config.get('options')
if options.get('awslogs-group') and \
options.get('awslogs-region') and \
options.get('awslogs-stream-prefix'):
l.append('awslogs')
else:
l.append('none')
else:
l.append('none')
logs, containers = ' | '.join(l), ' | '.join(c)
row = [task_id, status, containers, '\n'.join(ports), task_def, age, instance]
if output:
row.append(logs)
# if filter_val:
# raise ValueError
if quiet:
row = [row[0]]
out.append(row)
if items:
out = out[-items:]
if not quiet:
headers = ['TASK ID', 'STATUS', 'CONTAINERS', 'PORTS', 'TASK DEFINITION', 'AGE', 'EC2 PRIVATE IP', 'LOGS']
output = tabulate.tabulate(out, headers=headers, tablefmt='plain')
click.echo(output)
def lambda_handler(event, _):
"""
Entry point for CloudWatch event receipt.
"""
try:
print(event)
event["timestamp"] = int(
datetime.datetime.strptime(event["time"],
'%Y-%m-%dT%H:%M:%SZ').timestamp())
event["expires"] = event["timestamp"] + int(os.environ["ITEM_TTL"])
event["detail"]["time"] = event["time"]
# catch all the various forms of ARN from the media services
arn_expr = parse(
'$..arn|aRN|resource-arn|channel_arn|multiplex_arn|flowArn|PlaybackConfigurationArn|resourceArn'
)
original_arns = [match.value for match in arn_expr.find(event)]
arns = []
# remove arn that is for userIdentity or inputSecurityGroup
# note: can't remove an item from a list that's being iterated over so doing it this way
for arn in original_arns:
if "user" in arn or "role" in arn or "inputSecurityGroup" in arn:
pass
else:
arns.append(arn)
if arns:
event["resource_arn"] = unquote(arns[0])
# for certain events, the ARN is not labeled as an ARN but instead put in the resources list
if not arns and event["resources"]:
if "vod" not in event["resources"][0]:
event["resource_arn"] = event["resources"][0]
# handle alerts
if "Alert" in event["detail-type"]:
# medialive alerts
if "MediaLive" in event["detail-type"]:
event["alarm_id"] = event["detail"]["alarm_id"]
event["alarm_state"] = event["detail"]["alarm_state"].lower()
event["detail"]["pipeline_state"] = get_pipeline_state(event)
# mediaconnect alerts
elif "MediaConnect" in event["detail-type"]:
event["alarm_id"] = event["detail"]["error-id"]
if event["detail"]["errored"] == True:
event["alarm_state"] = "set"
else:
event["alarm_state"] = "cleared"
event["detail"]["alert_type"] = event["detail"]["error-code"]
del event["detail"]["error-code"]
event["detail"]["message"] = event["detail"]["error-message"]
del event["detail"]["error-message"]
#print(event)
EVENTS_TABLE.put_item(Item=event)
print(event["detail-type"] + " stored.")
# set the rest of the information needed for storing as regular CWE
# give timestamp a millisecond precision since it's sort key in CWE table
event["timestamp"] = event["timestamp"] * 1000 + randint(1, 999)
event["data"] = json.dumps(event["detail"])
event["type"] = event["detail-type"]
if "eventName" in event["detail"]:
event["type"] = event["type"] + ": " + event["detail"]["eventName"]
# handle specific cases depending on source
if event["source"] == "aws.medialive":
if "BatchUpdateSchedule" in event["type"]:
print("Creating an ARN for BatchUpdateSchedule event.")
event["resource_arn"] = "arn:aws:medialive:" + event['region'] + ":" + \
event['account'] + ":channel:" + \
event['detail']['requestParameters']['channelId']
elif event["source"] == "aws.mediapackage":
if "HarvestJob" in event["type"]:
print(
"Asking MediaPackage for the ARN of endpoint in a HarvestJob event."
)
# to get the ARN, ask mediapackage to describe the origin endpoint
# the ARN available through resources is the HarvestJob ARN, not the endpoint
orig_id_expr = parse('$..origin_endpoint_id')
orig_id = [match.value for match in orig_id_expr.find(event)]
if orig_id:
emp_client = boto3.client('mediapackage')
response = emp_client.describe_origin_endpoint(
Id=orig_id[0])
event["resource_arn"] = response["Arn"]
else:
print(
"Skipping this event. Origin ID not present in the HarvestJob event."
+ event["type"])
elif event["source"] == "aws.mediastore":
# for object state change the resource is the object, not the container
# so the captured arn needs to be fixed
if "MediaStore Object State Change" in event["type"]:
temp_arn = event["resource_arn"].split('/')
event["resource_arn"] = temp_arn[0] + "/" + temp_arn[1]
# if item has no resource arn, don't save in DB
if "resource_arn" in event:
#print(event)
print("Storing media service event.")
CLOUDWATCH_EVENTS_TABLE.put_item(Item=event)
else:
print("Skipping this event. " + event["type"])
except ClientError as error:
print(error)
return True
def map_from_json(details, node, key, val):
"""construct pyspark map from jsonpaths"""
keys = [x.value for x in jsonpath_ng.parse(node + key).find(details)]
vals = [x.value for x in jsonpath_ng.parse(node + val).find(details)]
flattened = itertools.chain.from_iterable(zip(keys, vals))
return F.create_map(*map(F.lit, flattened)).cast('map<string,string>')