def process_feed_info(feed, info_data):
# sanity check
assert feed['info_status'] == 'needfetch'
assert 'info_url' in feed
assert util.is_valid_url(
feed['info_url'],
allow_no_protocol=True) # already validated in the fetch
errors = util.is_valid_json(info_data, config.FEED_SCHEMA)
if not isinstance(info_data, dict) or 'address' not in info_data:
errors.append('Invalid data format')
elif feed['source'] != info_data['address']:
errors.append('Invalid address')
if len(errors) > 0:
inc_fetch_retry(feed, new_status='invalid', errors=errors)
return (False, errors)
feed['info_status'] = 'valid'
# fetch any associated images...
# TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(
info_data['image'],
config.SUBDIR_FEED_IMAGES,
feed['source'] + '_topic',
fetch_timeout=5)
if 'operator' in info_data and 'image' in info_data['operator']:
info_data['operator']['valid_image'] = util.fetch_image(
info_data['operator']['image'],
config.SUBDIR_FEED_IMAGES,
feed['source'] + '_owner',
fetch_timeout=5)
if 'targets' in info_data:
for i in range(len(info_data['targets'])):
if 'image' in info_data['targets'][i]:
image_name = feed['source'] + '_tv_' + str(
info_data['targets'][i]['value'])
info_data['targets'][i]['valid_image'] = util.fetch_image(
info_data['targets'][i]['image'],
config.SUBDIR_FEED_IMAGES,
image_name,
fetch_timeout=5)
feed['info_data'] = sanitize_json_data(info_data)
config.mongo_db.feeds.save(feed)
return (True, None)
def process_asset_info(asset, info_data):
def sanitize_json_data(data):
data['asset'] = util.sanitize_eliteness(data['asset'])
if 'description' in data: data['description'] = util.sanitize_eliteness(data['description'])
if 'website' in data: data['website'] = util.sanitize_eliteness(data['website'])
if 'pgpsig' in data: data['pgpsig'] = util.sanitize_eliteness(data['pgpsig'])
return data
# sanity check
assert asset['info_status'] == 'needfetch'
assert 'info_url' in asset
assert util.is_valid_url(asset['info_url'], allow_no_protocol=True) #already validated in the fetch
errors = util.is_valid_json(info_data, config.ASSET_SCHEMA)
if not isinstance(info_data, dict) or 'asset' not in info_data:
errors.append('Invalid data format')
elif asset['asset'] != info_data['asset']:
errors.append('asset field does not match asset name')
if len(errors) > 0:
inc_fetch_retry(asset, new_status='invalid', errors=errors)
return (False, errors)
asset['info_status'] = 'valid'
#fetch any associated images...
#TODO: parallelize this 2nd level asset image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(info_data['image'],
config.SUBDIR_ASSET_IMAGES, asset['asset'], fetch_timeout=5)
asset['info_data'] = sanitize_json_data(info_data)
config.mongo_db.asset_extended_info.save(asset)
return (True, None)
def process_feed_info(feed, info_data):
# sanity check
assert feed["info_status"] == "needfetch"
assert "info_url" in feed
assert util.is_valid_url(feed["info_url"], allow_no_protocol=True) # already validated in the fetch
errors = util.is_valid_json(info_data, config.FEED_SCHEMA)
if not isinstance(info_data, dict) or "address" not in info_data:
errors.append("Invalid data format")
elif feed["source"] != info_data["address"]:
errors.append("Invalid address")
if len(errors) > 0:
inc_fetch_retry(feed, new_status="invalid", errors=errors)
return (False, errors)
feed["info_status"] = "valid"
# fetch any associated images...
# TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on)
if "image" in info_data:
info_data["valid_image"] = util.fetch_image(
info_data["image"], config.SUBDIR_FEED_IMAGES, feed["source"] + "_topic", fetch_timeout=5
)
if "operator" in info_data and "image" in info_data["operator"]:
info_data["operator"]["valid_image"] = util.fetch_image(
info_data["operator"]["image"], config.SUBDIR_FEED_IMAGES, feed["source"] + "_owner", fetch_timeout=5
)
if "targets" in info_data:
for i in range(len(info_data["targets"])):
if "image" in info_data["targets"][i]:
image_name = feed["source"] + "_tv_" + str(info_data["targets"][i]["value"])
info_data["targets"][i]["valid_image"] = util.fetch_image(
info_data["targets"][i]["image"], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5
)
feed["info_data"] = sanitize_json_data(info_data)
config.mongo_db.feeds.save(feed)
return (True, None)
def process_feed_info(feed, info_data):
# sanity check
assert feed['info_status'] == 'needfetch'
assert 'info_url' in feed
assert util.is_valid_url(feed['info_url'], allow_no_protocol=True) #already validated in the fetch
errors = util.is_valid_json(info_data, config.FEED_SCHEMA)
if not isinstance(info_data, dict) or 'address' not in info_data:
errors.append('Invalid data format')
elif feed['source'] != info_data['address']:
errors.append('Invalid address')
if len(errors) > 0:
inc_fetch_retry(feed, new_status='invalid', errors=errors)
return (False, errors)
feed['info_status'] = 'valid'
#fetch any associated images...
#TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(info_data['image'],
config.SUBDIR_FEED_IMAGES, feed['source'] + '_topic', fetch_timeout=5)
if 'operator' in info_data and 'image' in info_data['operator']:
info_data['operator']['valid_image'] = util.fetch_image(info_data['operator']['image'],
config.SUBDIR_FEED_IMAGES, feed['source'] + '_owner', fetch_timeout=5)
if 'targets' in info_data:
for i in range(len(info_data['targets'])):
if 'image' in info_data['targets'][i]:
image_name = feed['source'] + '_tv_' + str(info_data['targets'][i]['value'])
info_data['targets'][i]['valid_image'] = util.fetch_image(
info_data['targets'][i]['image'], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5)
feed['info_data'] = sanitize_json_data(info_data)
config.mongo_db.feeds.save(feed)
return (True, None)
def process_asset_info(asset, info_data):
def sanitize_json_data(data):
data['asset'] = util.sanitize_eliteness(data['asset'])
if 'description' in data:
data['description'] = util.sanitize_eliteness(data['description'])
if 'website' in data:
data['website'] = util.sanitize_eliteness(data['website'])
if 'pgpsig' in data:
data['pgpsig'] = util.sanitize_eliteness(data['pgpsig'])
return data
# sanity check
assert asset['info_status'] == 'needfetch'
assert 'info_url' in asset
assert util.is_valid_url(
asset['info_url'],
allow_no_protocol=True) # already validated in the fetch
errors = util.is_valid_json(info_data, config.ASSET_SCHEMA)
if not isinstance(info_data, dict) or 'asset' not in info_data:
errors.append('Invalid data format')
elif asset['asset'] != info_data['asset']:
errors.append('asset field does not match asset name')
if len(errors) > 0:
inc_fetch_retry(asset, new_status='invalid', errors=errors)
return (False, errors)
asset['info_status'] = 'valid'
# fetch any associated images...
# TODO: parallelize this 2nd level asset image fetching ... (e.g. just compose a list here, and process it in later on)
if 'image' in info_data:
info_data['valid_image'] = util.fetch_image(info_data['image'],
config.SUBDIR_ASSET_IMAGES,
asset['asset'],
fetch_timeout=5)
asset['info_data'] = sanitize_json_data(info_data)
config.mongo_db.asset_extended_info.save(asset)
return (True, None)
