def parse_base64_feed(base64_feed):
decoded_feed = base64.b64decode(base64_feed)
feed = json.loads(decoded_feed)
if not isinstance(feed, dict) or 'feed' not in feed:
return False
errors = util.is_valid_json(feed['feed'], config.FEED_SCHEMA)
if len(errors) > 0:
raise Exception("Invalid json: {}".format(", ".join(errors)))
# get broadcast infos
params = {
'filters': {
'field': 'source',
'op': '=',
'value': feed['feed']['address']
},
'order_by': 'tx_index',
'order_dir': 'DESC',
'limit': 1
}
broadcasts = util.call_jsonrpc_api('get_broadcasts', params)['result']
if len(broadcasts) == 0:
raise Exception("invalid feed address")
complete_feed = {}
complete_feed['fee_fraction_int'] = broadcasts[0]['fee_fraction_int']
complete_feed['source'] = broadcasts[0]['source']
complete_feed['locked'] = broadcasts[0]['locked']
complete_feed['counters'] = get_feed_counters(broadcasts[0]['source'])
complete_feed['info_data'] = sanitize_json_data(feed['feed'])
feed['feed'] = complete_feed
return feed
def parse_base64_feed(base64_feed):
decoded_feed = base64.b64decode(base64_feed)
feed = json.loads(decoded_feed)
if not isinstance(feed, dict) or "feed" not in feed:
return False
errors = util.is_valid_json(feed["feed"], config.FEED_SCHEMA)
if len(errors) > 0:
raise Exception("Invalid json: {}".format(", ".join(errors)))
# get broadcast infos
params = {
"filters": {"field": "source", "op": "=", "value": feed["feed"]["address"]},
"order_by": "tx_index",
"order_dir": "DESC",
"limit": 1,
}
broadcasts = util.call_jsonrpc_api("get_broadcasts", params)["result"]
if len(broadcasts) == 0:
raise Exception("invalid feed address")
complete_feed = {}
complete_feed["fee_fraction_int"] = broadcasts[0]["fee_fraction_int"]
complete_feed["source"] = broadcasts[0]["source"]
complete_feed["locked"] = broadcasts[0]["locked"]
complete_feed["counters"] = get_feed_counters(broadcasts[0]["source"])
complete_feed["info_data"] = sanitize_json_data(feed["feed"])
feed["feed"] = complete_feed
return feed
def process_asset_info(asset, info_data):
def sanitize_json_data(data):
data['asset'] = util.sanitize_eliteness(data['asset'])
if 'description' in data: data['description'] = util.sanitize_eliteness(data['description'])
if 'website' in data: data['website'] = util.sanitize_eliteness(data['website'])
if 'pgpsig' in data: data['pgpsig'] = util.sanitize_eliteness(data['pgpsig'])
return data
# sanity check
assert asset['info_status'] == 'needfetch'
assert 'info_url' in asset
assert util.is_valid_url(asset['info_url'], allow_no_protocol=True) #already validated in the fetch
errors = util.is_valid_json(info_data, config.ASSET_SCHEMA)
if not isinstance(info_data, dict) or 'asset' not in info_data:
errors.append('Invalid data format')
elif asset['asset'] != info_data['asset']:
errors.append('asset field does not match asset name')
if len(errors) > 0:
inc_fetch_retry(asset, new_status='invalid', errors=errors)
return (False, errors)
asset['info_status'] = 'valid'
#fetch any associated images...
#TODO: parallelize this 2nd level asset image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(info_data['image'],
config.SUBDIR_ASSET_IMAGES, asset['asset'], fetch_timeout=5)
asset['info_data'] = sanitize_json_data(info_data)
config.mongo_db.asset_extended_info.save(asset)
return (True, None)
def parse_base64_feed(base64_feed):
decoded_feed = base64.b64decode(base64_feed)
feed = json.loads(decoded_feed)
if not isinstance(feed, dict) or 'feed' not in feed:
return False
errors = util.is_valid_json(feed['feed'], config.FEED_SCHEMA)
if len(errors) > 0:
raise Exception("Invalid json: {}".format(", ".join(errors)))
# get broadcast infos
params = {
'filters': {
'field': 'source',
'op': '=',
'value': feed['feed']['address']
},
'order_by': 'tx_index',
'order_dir': 'DESC',
'limit': 1
}
broadcasts = util.call_jsonrpc_api('get_broadcasts', params)['result']
if len(broadcasts) == 0:
raise Exception("invalid feed address")
complete_feed = {}
complete_feed['fee_fraction_int'] = broadcasts[0]['fee_fraction_int']
complete_feed['source'] = broadcasts[0]['source']
complete_feed['locked'] = broadcasts[0]['locked']
complete_feed['counters'] = get_feed_counters(broadcasts[0]['source'])
complete_feed['info_data'] = sanitize_json_data(feed['feed'])
feed['feed'] = complete_feed
return feed
def process_feed_info(feed, info_data):
# sanity check
assert feed['info_status'] == 'needfetch'
assert 'info_url' in feed
assert util.is_valid_url(
feed['info_url'],
allow_no_protocol=True) # already validated in the fetch
errors = util.is_valid_json(info_data, config.FEED_SCHEMA)
if not isinstance(info_data, dict) or 'address' not in info_data:
errors.append('Invalid data format')
elif feed['source'] != info_data['address']:
errors.append('Invalid address')
if len(errors) > 0:
inc_fetch_retry(feed, new_status='invalid', errors=errors)
return (False, errors)
feed['info_status'] = 'valid'
# fetch any associated images...
# TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(
info_data['image'],
config.SUBDIR_FEED_IMAGES,
feed['source'] + '_topic',
fetch_timeout=5)
if 'operator' in info_data and 'image' in info_data['operator']:
info_data['operator']['valid_image'] = util.fetch_image(
info_data['operator']['image'],
config.SUBDIR_FEED_IMAGES,
feed['source'] + '_owner',
fetch_timeout=5)
if 'targets' in info_data:
for i in range(len(info_data['targets'])):
if 'image' in info_data['targets'][i]:
image_name = feed['source'] + '_tv_' + str(
info_data['targets'][i]['value'])
info_data['targets'][i]['valid_image'] = util.fetch_image(
info_data['targets'][i]['image'],
config.SUBDIR_FEED_IMAGES,
image_name,
fetch_timeout=5)
feed['info_data'] = sanitize_json_data(info_data)
config.mongo_db.feeds.save(feed)
return (True, None)
def process_asset_info(asset, info_data):
def sanitize_json_data(data):
data['asset'] = util.sanitize_eliteness(data['asset'])
if 'description' in data:
data['description'] = util.sanitize_eliteness(data['description'])
if 'website' in data:
data['website'] = util.sanitize_eliteness(data['website'])
if 'pgpsig' in data:
data['pgpsig'] = util.sanitize_eliteness(data['pgpsig'])
return data
# sanity check
assert asset['info_status'] == 'needfetch'
assert 'info_url' in asset
assert util.is_valid_url(
asset['info_url'],
allow_no_protocol=True) # already validated in the fetch
errors = util.is_valid_json(info_data, config.ASSET_SCHEMA)
if not isinstance(info_data, dict) or 'asset' not in info_data:
errors.append('Invalid data format')
elif asset['asset'] != info_data['asset']:
errors.append('asset field does not match asset name')
if len(errors) > 0:
inc_fetch_retry(asset, new_status='invalid', errors=errors)
return (False, errors)
asset['info_status'] = 'valid'
# fetch any associated images...
# TODO: parallelize this 2nd level asset image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(info_data['image'],
config.SUBDIR_ASSET_IMAGES,
asset['asset'],
fetch_timeout=5)
asset['info_data'] = sanitize_json_data(info_data)
config.mongo_db.asset_extended_info.save(asset)
return (True, None)
def process_feed_info(feed, info_data):
# sanity check
assert feed["info_status"] == "needfetch"
assert "info_url" in feed
assert util.is_valid_url(feed["info_url"], allow_no_protocol=True) # already validated in the fetch
errors = util.is_valid_json(info_data, config.FEED_SCHEMA)
if not isinstance(info_data, dict) or "address" not in info_data:
errors.append("Invalid data format")
elif feed["source"] != info_data["address"]:
errors.append("Invalid address")
if len(errors) > 0:
inc_fetch_retry(feed, new_status="invalid", errors=errors)
return (False, errors)
feed["info_status"] = "valid"
# fetch any associated images...
# TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on)
if "image" in info_data:
info_data["valid_image"] = util.fetch_image(
info_data["image"], config.SUBDIR_FEED_IMAGES, feed["source"] + "_topic", fetch_timeout=5
)
if "operator" in info_data and "image" in info_data["operator"]:
info_data["operator"]["valid_image"] = util.fetch_image(
info_data["operator"]["image"], config.SUBDIR_FEED_IMAGES, feed["source"] + "_owner", fetch_timeout=5
)
if "targets" in info_data:
for i in range(len(info_data["targets"])):
if "image" in info_data["targets"][i]:
image_name = feed["source"] + "_tv_" + str(info_data["targets"][i]["value"])
info_data["targets"][i]["valid_image"] = util.fetch_image(
info_data["targets"][i]["image"], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5
)
feed["info_data"] = sanitize_json_data(info_data)
config.mongo_db.feeds.save(feed)
return (True, None)
def process_feed_info(feed, info_data):
# sanity check
assert feed['info_status'] == 'needfetch'
assert 'info_url' in feed
assert util.is_valid_url(feed['info_url'], allow_no_protocol=True) #already validated in the fetch
errors = util.is_valid_json(info_data, config.FEED_SCHEMA)
if not isinstance(info_data, dict) or 'address' not in info_data:
errors.append('Invalid data format')
elif feed['source'] != info_data['address']:
errors.append('Invalid address')
if len(errors) > 0:
inc_fetch_retry(feed, new_status='invalid', errors=errors)
return (False, errors)
feed['info_status'] = 'valid'
#fetch any associated images...
#TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(info_data['image'],
config.SUBDIR_FEED_IMAGES, feed['source'] + '_topic', fetch_timeout=5)
if 'operator' in info_data and 'image' in info_data['operator']:
info_data['operator']['valid_image'] = util.fetch_image(info_data['operator']['image'],
config.SUBDIR_FEED_IMAGES, feed['source'] + '_owner', fetch_timeout=5)
if 'targets' in info_data:
for i in range(len(info_data['targets'])):
if 'image' in info_data['targets'][i]:
image_name = feed['source'] + '_tv_' + str(info_data['targets'][i]['value'])
info_data['targets'][i]['valid_image'] = util.fetch_image(
info_data['targets'][i]['image'], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5)
feed['info_data'] = sanitize_json_data(info_data)
config.mongo_db.feeds.save(feed)
return (True, None)
