def process_feed_info(db, feed, info_data): # sanity check assert feed['info_status'] == 'needfetch' assert 'info_url' in feed assert util.is_valid_url( feed['info_url'], allow_no_protocol=True) #already validated in the fetch errors = util.is_valid_json(info_data, config.FEED_SCHEMA) if not isinstance(info_data, dict) or 'address' not in info_data: errors.append('Invalid data format') elif feed['source'] != info_data['address']: errors.append('Invalid address') if len(errors) > 0: inc_fetch_retry(db, feed, new_status='invalid', errors=errors) return (False, errors) feed['info_status'] = 'valid' #fetch any associated images... #TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on) if 'image' in info_data: info_data['valid_image'] = util.fetch_image(info_data['image'], config.SUBDIR_FEED_IMAGES, feed['source'] + '_topic', fetch_timeout=5) if 'operator' in info_data and 'image' in info_data['operator']: info_data['operator']['valid_image'] = util.fetch_image( info_data['operator']['image'], config.SUBDIR_FEED_IMAGES, feed['source'] + '_owner', fetch_timeout=5) if 'targets' in info_data: for i in range(len(info_data['targets'])): if 'image' in info_data['targets'][i]: image_name = feed['source'] + '_tv_' + str( info_data['targets'][i]['value']) info_data['targets'][i]['valid_image'] = util.fetch_image( info_data['targets'][i]['image'], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5) feed['info_data'] = sanitize_json_data(info_data) db.feeds.save(feed) return (True, None)
def process_asset_info(db, asset, info_data): # sanity check assert asset['info_status'] == 'needfetch' assert 'info_url' in asset assert util.is_valid_url( asset['info_url'], allow_no_protocol=True) #already validated in the fetch errors = util.is_valid_json(info_data, config.ASSET_SCHEMA) if not isinstance(info_data, dict) or 'asset' not in info_data: errors.append('Invalid data format') elif asset['asset'] != info_data['asset']: errors.append('asset field does not match asset name') if len(errors) > 0: inc_fetch_retry(db, asset, new_status='invalid', errors=errors) return (False, errors) asset['info_status'] = 'valid' #fetch any associated images... #TODO: parallelize this 2nd level asset image fetching ... (e.g. just compose a list here, and process it in later on) if 'image' in info_data: info_data['valid_image'] = util.fetch_image(info_data['image'], config.SUBDIR_ASSET_IMAGES, asset['asset'], fetch_timeout=5) asset['info_data'] = sanitize_json_data(info_data) db.asset_extended_info.save(asset) return (True, None)
def process_asset_info(db, asset, info_data): # sanity check assert asset['info_status'] == 'needfetch' assert 'info_url' in asset assert util.is_valid_url(asset['info_url'], allow_no_protocol=True) #already validated in the fetch errors = util.is_valid_json(info_data, config.ASSET_SCHEMA) if not isinstance(info_data, dict) or 'asset' not in info_data: errors.append('Invalid data format') elif asset['asset'] != info_data['asset']: errors.append('asset field does not match asset name') if len(errors) > 0: inc_fetch_retry(db, asset, new_status='invalid', errors=errors) return (False, errors) asset['info_status'] = 'valid' #fetch any associated images... #TODO: parallelize this 2nd level asset image fetching ... (e.g. just compose a list here, and process it in later on) if 'image' in info_data: info_data['valid_image'] = util.fetch_image(info_data['image'], config.SUBDIR_ASSET_IMAGES, asset['asset']) asset['info_data'] = sanitize_json_data(info_data) db.asset_extended_info.save(asset) return (True, None)
def process_feed_info(db, feed, info_data): # sanity check assert feed["info_status"] == "needfetch" assert "info_url" in feed assert util.is_valid_url(feed["info_url"], allow_no_protocol=True) # already validated in the fetch errors = util.is_valid_json(info_data, config.FEED_SCHEMA) if not isinstance(info_data, dict) or "address" not in info_data: errors.append("Invalid data format") elif feed["source"] != info_data["address"]: errors.append("Invalid address") if len(errors) > 0: inc_fetch_retry(db, feed, new_status="invalid", errors=errors) return (False, errors) feed["info_status"] = "valid" # fetch any associated images... # TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on) if "image" in info_data: info_data["valid_image"] = util.fetch_image( info_data["image"], config.SUBDIR_FEED_IMAGES, feed["source"] + "_topic", fetch_timeout=5 ) if "operator" in info_data and "image" in info_data["operator"]: info_data["operator"]["valid_image"] = util.fetch_image( info_data["operator"]["image"], config.SUBDIR_FEED_IMAGES, feed["source"] + "_owner", fetch_timeout=5 ) if "targets" in info_data: for i in range(len(info_data["targets"])): if "image" in info_data["targets"][i]: image_name = feed["source"] + "_tv_" + str(info_data["targets"][i]["value"]) info_data["targets"][i]["valid_image"] = util.fetch_image( info_data["targets"][i]["image"], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5 ) feed["info_data"] = sanitize_json_data(info_data) db.feeds.save(feed) return (True, None)
def process_feed_info(db, feed, info_data): # sanity check assert feed['info_status'] == 'needfetch' assert 'info_url' in feed assert util.is_valid_url(feed['info_url'], allow_no_protocol=True) #already validated in the fetch errors = util.is_valid_json(info_data, config.FEED_SCHEMA) if not isinstance(info_data, dict) or 'address' not in info_data: errors.append('Invalid data format') elif feed['source'] != info_data['address']: errors.append('Invalid address') if len(errors) > 0: inc_fetch_retry(db, feed, new_status='invalid', errors=errors) return (False, errors) feed['info_status'] = 'valid' #fetch any associated images... #TODO: parallelize this 2nd level feed image fetching ... (e.g. just compose a list here, and process it in later on) if 'image' in info_data: info_data['valid_image'] = util.fetch_image(info_data['image'], config.SUBDIR_FEED_IMAGES, feed['source'] + '_topic', fetch_timeout=5) if 'operator' in info_data and 'image' in info_data['operator']: info_data['operator']['valid_image'] = util.fetch_image(info_data['operator']['image'], config.SUBDIR_FEED_IMAGES, feed['source'] + '_owner', fetch_timeout=5) if 'targets' in info_data: for i in range(len(info_data['targets'])): if 'image' in info_data['targets'][i]: image_name = feed['source'] + '_tv_' + str(info_data['targets'][i]['value']) info_data['targets'][i]['valid_image'] = util.fetch_image( info_data['targets'][i]['image'], config.SUBDIR_FEED_IMAGES, image_name, fetch_timeout=5) feed['info_data'] = sanitize_json_data(info_data) db.feeds.save(feed) return (True, None)