def test_distribute(self):
tiles_star = [
{
"imageURI": "data:image/png;base64,somedata",
"enhancedImageURI": "data:image/png;base64,somemoredata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
tiles_ca = [
{
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
data = ingest_links({"STAR/en-US": tiles_star}, self.channels[0].id)
distribute(data, self.channels[0].id, True)
# 6 files are uploaded, mirrors generate artifacts
assert_equal(6, self.key_mock.set_contents_from_string.call_count)
self.key_mock.set_contents_from_string = Mock()
data = ingest_links({
"STAR/en-US": tiles_star,
"CA/en-US": tiles_ca,
}, self.channels[0].id)
distribute(data, self.channels[0].id, True)
# includes two more upload: the locate data payload (for both versions)
assert_equal(8, self.key_mock.set_contents_from_string.call_count)
def test_ingest_dbpool(self):
"""
Test a ingestion of a large number of tiles that could use up connections to the db
"""
with open(self.get_fixture_path("2014-10-30.ja-pt.json"), 'r') as f:
tiles = json.load(f)
ingest_links(tiles, self.channels[0].id)
num_tiles = self.env.db.session.query(Tile).count()
assert(num_tiles > 30)
def test_ingest_no_duplicates(self):
"""
Test that there is no duplication when ingesting tiles
"""
with open(self.get_fixture_path("tiles_duplicates.json"), 'r') as f:
tiles = json.load(f)
num_tiles = self.env.db.session.query(Tile).count()
ingest_links(tiles, self.channels[0].id)
new_num_tiles = self.env.db.session.query(Tile).count()
assert_equal(num_tiles + 1, new_num_tiles)
def test_error_mid_ingestion(self):
"""
Test an error happening mid-ingestion
"""
tiles_star = [
{
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
},
{
"imageURI": "data:image/png;base64,someotherdata",
"url": "https://somewhereelse.com",
"title": "Some Other Title",
"type": "organic",
"bgColor": "#FFFFFF"
},
]
tile_count_before = self.env.db.session.query(Tile).count()
import splice.ingest
insert_function = splice.ingest.insert_tile
# put counts in a dict to get around python's
# non-local scope restrictions on variables
# for access in mock_ingest
counts = {
'call': 0,
'exception_at': 2,
}
def mock_ingest(*args, **kwargs):
counts['call'] += 1
if counts['call'] < counts['exception_at']:
return insert_function(*args, **kwargs)
else:
raise Exception('Boom')
function_mock = Mock(side_effect=mock_ingest)
splice.ingest.insert_tile = function_mock
ingest_links({"STAR/en-US": tiles_star}, self.channels[0].id)
tile_count_after = self.env.db.session.query(Tile).count()
# only one has been inserted out of two
assert_equal(1, tile_count_after - tile_count_before)
# put the module function back to what it was
splice.ingest.insert_tile = insert_function
def test_image_artifact_hash(self):
"""
Test that the correct number of image artifacts are produced
"""
with open(self.get_fixture_path("valid_tile.json"), 'r') as f:
fixture = json.load(f)
tile_1 = fixture["STAR/en-US"][0]
tile_2 = copy.deepcopy(tile_1)
tile_2['title'] = 'Some Other Title'
tile_3 = copy.deepcopy(tile_1)
tile_3['title'] = 'Yet Another Title'
tiles = {'STAR/en-US': [tile_1, tile_2, tile_3]}
data = ingest_links(tiles, self.channels[0].id)
artifacts = generate_artifacts(data, self.channels[0].name, True)
# even if there are 3 tiles, there should only be 2 images
image_count = 0
for a in artifacts:
mime = a.get('mime')
if mime and mime == 'image/png':
image_count += 1
assert_equal(2, image_count)
def test_distribute_suggested(self):
tiles_star = [
{
"imageURI": "data:image/png;base64,somedata",
"enhancedImageURI": "data:image/png;base64,somemoredata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"frecent_sites": ['http://xyz.com', 'http://abc.com']
}
]
tiles_ca = [
{
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Other Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
data = ingest_links({
"STAR/en-US": tiles_star,
"CA/en-US": tiles_ca,
}, self.channels[0].id)
distribute(data, self.channels[0].id, True)
# in this case, the 3rd element should be the mock of the s3 upload for the 'ag' index
frecents = json.loads(self.key_mock.set_contents_from_string.mock_calls[3][1][0])['suggested'][0]['frecent_sites']
assert_equal(frecents, ['http://abc.com', 'http://xyz.com'])
def test_frequency_caps(self):
"""
A simple test of frequency caps
"""
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"frequency_caps": {
"daily": 3,
"total": 10
}
}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
data = ingest_links({"US/en-US": [tile]}, self.channels[0].id)
assert_equal(1, len(data["US/en-US"]))
c = self.env.db.session.query(Adgroup).count()
assert_equal(31, c)
tile = self.env.db.session.query(Tile).filter(Tile.id == 31).one()
ag = self.env.db.session.query(Adgroup).filter(Adgroup.id == 31).one()
assert_equal(tile.adgroup_id, ag.id)
assert_equal(ag.frequency_cap_daily, 3)
assert_equal(ag.frequency_cap_total, 10)
def test_start_end_dates_optional(self):
"""
Ensure that start/end dates are optional
"""
tile_no_start = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {
"end": "2014-01-31T00:00:00.000"
}
}
tile_no_end = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhereelse.com",
"title": "Some Other Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {
"start": "2014-01-12T00:00:00.000",
}
}
tile_empty_limits = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://yetsomewhereelse.com",
"title": "Yet Some Other Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {}
}
dist = {"US/en-US": [tile_no_start, tile_no_end, tile_empty_limits]}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
data = ingest_links(dist, self.channels[0].id)
assert_equal(len(dist["US/en-US"]), len(data["US/en-US"]))
c = self.env.db.session.query(Adgroup).count()
assert_equal(30 + len(dist["US/en-US"]), c)
tile_tested = 0
for i, tile_def in enumerate(dist["US/en-US"]):
obj_id = 30 + 1 + i
tile = self.env.db.session.query(Tile).filter(Tile.id == obj_id).one()
ag = self.env.db.session.query(Adgroup).filter(Adgroup.id == obj_id).one()
assert_equal(tile.adgroup_id, ag.id)
if ag.start_date:
assert_equal(ag.start_date, tile_def['time_limits']['start'])
tile_tested += 1
if ag.end_date:
assert_equal(ag.end_date, tile_def['time_limits']['end'])
tile_tested += 1
# one tile not tested because it has neither start or end dates
assert_equal(len(dist["US/en-US"]) - 1, tile_tested)
def test_start_end_dates(self):
"""
a simple start/end date tile
"""
tile_no_tz = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {
"start": "2014-01-12T00:00:00.000",
"end": "2014-01-31T00:00:00.000"
}
}
dist = {"US/en-US": [tile_no_tz]}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
data = ingest_links(dist, self.channels[0].id)
assert_equal(1, len(data["US/en-US"]))
c = self.env.db.session.query(Adgroup).count()
assert_equal(31, c)
tile = self.env.db.session.query(Tile).filter(Tile.id == 31).one()
ag = self.env.db.session.query(Adgroup).filter(Adgroup.id == 31).one()
assert_equal(tile.adgroup_id, ag.id)
assert_equal(ag.start_date, dist["US/en-US"][0]['time_limits']['start'])
assert_equal(ag.end_date, dist["US/en-US"][0]['time_limits']['end'])
assert_equal(ag.start_date_dt, du_parse(dist["US/en-US"][0]['time_limits']['start']))
assert_equal(ag.end_date_dt, du_parse(dist["US/en-US"][0]['time_limits']['end']))
def ingest_tiles(in_file, out_path, console_out, deploy_flag, verbose, *args, **kwargs):
"""
Load a set of links for all country/locale combinations into data warehouse and optionally deploy
"""
if verbose:
logger = setup_command_logger(logging.DEBUG)
else:
logger = setup_command_logger(logging.INFO)
rawdata = None
with open(in_file, 'r') as f:
rawdata = json.load(f)
from splice.ingest import ingest_links, deploy, IngestError
try:
new_data = ingest_links(rawdata)
if console_out:
print json.dumps(new_data, sort_keys=True, indent=2)
if out_path:
directory, _ = os.path.split(out_path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(out_path, "w") as f:
json.dump(new_data, f, sort_keys=True, indent=2)
logger.info("wrote {0}".format(out_path))
if deploy_flag:
deploy(new_data)
except IngestError, e:
raise InvalidCommand(e.message)
def test_distribute_adgroup_explanation(self):
tile_en_us = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere_else.com",
"title": "Some Title US",
"type": "organic",
"bgColor": "#FFFFFF",
"adgroup_name": "Teçhnology".decode('utf-8'),
"explanation": "推荐 for %1$S fans who also like %2$S".decode('utf-8')
}
tiles_en_us_suggested = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title US Suggested",
"type": "organic",
"bgColor": "#FFFFFF",
"frecent_sites": ['http://xyz.com', 'http://abc.com'],
"adgroup_name": "Technology",
"explanation": "Suggested for %1$S fans who also like %2$S"
}
distribution = {
"US/en-US": [tile_en_us, tiles_en_us_suggested],
"GB/en-US": [tile_en_us],
}
data = ingest_links(distribution, self.channels[0].id)
distribute(data, self.channels[0].id, True)
# one image, 2 AG distributions, 2 legacy distributions, one index, one input distribution
assert_equal(7, self.key_mock.set_contents_from_string.call_count)
num_tiles_checked = 0
for i, key in enumerate(self.key_names):
ag = AG_DIST_PATHNAME.match(key)
leg = LEGACY_DIST_PATHNAME.match(key)
if ag:
country_locale, locale = ag.groups()
data = json.loads(self.key_contents[i])
for tile in data['directory']:
# index 0 expected, only for US/en-US
assert_equal(distribution[country_locale][0]['adgroup_name'], tile.get('adgroup_name'))
assert_equal(distribution[country_locale][0]['explanation'], tile.get('explanation'))
num_tiles_checked += 1
for tile in data['suggested']:
# index 1 expected, only for US/en-US
assert_equal(distribution[country_locale][1]['adgroup_name'], tile.get('adgroup_name'))
assert_equal(distribution[country_locale][1]['explanation'], tile.get('explanation'))
num_tiles_checked += 1
elif leg:
country_locale, locale = leg.groups()
data = json.loads(self.key_contents[i])
assert_equal(1, len(data[locale]))
tile = data[locale][0]
assert_equal(None, tile.get('adgroup_name'))
assert_equal(None, tile.get('explanation'))
num_tiles_checked += 1
assert_equal(5, num_tiles_checked)
def test_ingest_suggested_sites(self):
"""
Test that there is no duplication when ingesting tiles
"""
with open(self.get_fixture_path("tiles_suggested.json"), 'r') as f:
tiles = json.load(f)
num_tiles = self.env.db.session.query(Tile).count()
data = ingest_links(tiles, self.channels[0].id)
assert_equal(len(data['STAR/en-US']), 5)
new_num_tiles = self.env.db.session.query(Tile).count()
assert_equal(num_tiles + 4, new_num_tiles)
# ingesting the same thing a second time should be idempotent
data = ingest_links(tiles, self.channels[0].id)
assert_equal(len(data['STAR/en-US']), 5)
new_num_tiles = self.env.db.session.query(Tile).count()
assert_equal(num_tiles + 4, new_num_tiles)
def test_explanation_template_sanitization(self):
# test templates with html tags
tile = self._make_dist({
"adgroup_name": "<script>Technology</script>",
"explanation": "<br/>Suggested for %1$S, %2$S<br/>"})
ingest_links(tile, self.channels[0].id)
ag = self.env.db.session.query(Adgroup).filter(Adgroup.id == 31).one()
assert_equal(ag.name, "Technology")
assert_equal(ag.explanation, "Suggested for %1$S, %2$S")
# test templates with tags only and special characters
tile = self._make_dist({
"title": "Some Another Title",
"adgroup_name": "<script><script/>",
"explanation": "< Suggested for %1$S, %2$S >"})
ingest_links(tile, self.channels[0].id)
ag = self.env.db.session.query(Adgroup).filter(Adgroup.id == 32).one()
assert_equal(ag.name, None)
assert_equal(ag.explanation, "< Suggested for %1$S, %2$S >")
def test_adgroups_channel_id_uniqueness(self):
"""
Test that channel_ids in adgroups are part of what makes Tiles unique
"""
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
}
dist = {"US/en-US": [tile]}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
ingest_links(dist, self.channels[0].id)
ingest_links(dist, self.channels[1].id)
c = self.env.db.session.query(Adgroup).count()
assert_equal(32, c)
def test_start_end_dates_uniqueness(self):
"""
Test that start/end are part of what make tiles unique
"""
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {
"start": "2014-01-12T00:00:00.000",
"end": "2014-01-31T00:00:00.000"
}
}
tile_no_start = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {
"end": "2014-01-31T00:00:00.000"
}
}
tile_no_end = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {
"start": "2014-01-12T00:00:00.000",
}
}
tile_empty_limits = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Tile",
"type": "organic",
"bgColor": "#FFFFFF",
"time_limits": {}
}
dist = {"US/en-US": [tile, tile_no_start, tile_no_end, tile_empty_limits]}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
data = ingest_links(dist, self.channels[0].id)
assert_equal(len(dist["US/en-US"]), len(data["US/en-US"]))
c = self.env.db.session.query(Adgroup).count()
assert_equal(30 + len(dist["US/en-US"]), c)
def test_id_not_overwritten(self):
"""
Test an id is created for a valid tile
"""
tiles_star = [
{
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
data = ingest_links({"STAR/en-US": tiles_star}, self.channels[0].id)
directory_id = data["STAR/en-US"][0]["directoryId"]
assert_equal(31, directory_id)
data = ingest_links({"STAR/en-US": tiles_star}, self.channels[0].id)
directory_id = data["STAR/en-US"][0]["directoryId"]
assert_equal(31, directory_id)
def all_tiles():
try:
data = request.get_json(force=True)
new_data = ingest_links(data)
urls = deploy(new_data)
except ValidationError, e:
errors = []
error = {"path": e.path[0], "msg": e.message}
errors.append(error)
env.log("VALIDATION_ERROR path:{0} msg:{1}".format(e.path[0], e.message), level=logging.ERROR, name="client_error")
return jsonify({"err": errors}), 400
def test_generate_artifacts(self):
"""
Tests that the correct number of artifacts are generated
"""
with open(self.get_fixture_path("tiles_suggested.json"), 'r') as f:
fixture = json.load(f)
tile = fixture["STAR/en-US"][4]
data = ingest_links({"STAR/en-US": [tile]}, self.channels[0].id)
artifacts = generate_artifacts(data, self.channels[0].name, True)
# tile index, v2, v3 and 2 image files are generated
assert_equal(6, len(artifacts))
data = ingest_links({
"STAR/en-US": [tile],
"CA/en-US": [tile]
}, self.channels[0].id)
artifacts = generate_artifacts(data, self.channels[0].name, True)
# includes two more file: the locale data payload for each version
assert_equal(8, len(artifacts))
def test_image_content(self):
with open(self.get_fixture_path("valid_tile.json"), 'r') as f:
tiles = json.load(f)
data = ingest_links(tiles, self.channels[0].id)
artifacts = generate_artifacts(data, self.channels[0].name, True)
found_image = False
for file in artifacts:
if "mime" in file:
found_image = True
assert_equal(file["mime"], magic.from_buffer(file["data"], mime=True))
assert_true(found_image)
def test_deploy_always_generates_tile_index(self):
"""A tiles index file should always be generated"""
# this is a dict, because of a quirk in python's namespacing/scoping
# https://docs.python.org/2/tutorial/classes.html#python-scopes-and-namespaces
index_uploaded = {'count': 0}
def key_set_name(name):
if name == "{0}_tile_index.v3.json".format(self.channels[0].name):
index_uploaded['count'] += 1
name_mock = PropertyMock(side_effect=key_set_name)
type(self.key_mock).name = name_mock
with open(self.get_fixture_path("mozilla-tiles.fennec.json"), 'r') as f:
tiles = json.load(f)
data = ingest_links(tiles, self.channels[0].id)
distribute(data, self.channels[0].id, True)
assert_equal(1, index_uploaded['count'])
data = ingest_links(tiles, self.channels[0].id)
distribute(data, self.channels[0].id, True)
assert_equal(2, index_uploaded['count'])
def test_malformed_data_uri_meta(self):
"""
Tests that a malformed data uri declaration is rejected
"""
tiles_star = [
{
"imageURI": "data:image/somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
data = ingest_links({"STAR/en-US": tiles_star}, self.channels[0].id)
assert_raises(IngestError, generate_artifacts, data, self.channels[0].name, True)
def test_sorted_suggested_sites(self):
"""
ensure suggested sites are sorted
"""
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"frecent_sites": ["http://lmnop.org", "http://def.com", "http://abc.com", "http://def.com", "https://xyz.com"]
}
data = ingest_links({"CA/en-US": [tile]}, self.channels[0].id)
assert_equal(1, len(data["CA/en-US"]))
assert_equal(data["CA/en-US"][0]['frecent_sites'],
["http://abc.com", "http://def.com", "http://lmnop.org", "https://xyz.com"])
def test_unknown_mime_type(self):
"""
Tests that an unknown mime type is rejected
"""
tiles_star = [
{
"imageURI": "data:image/weirdimage;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
data = ingest_links({"STAR/en-US": tiles_star}, self.channels[0].id)
assert_raises(IngestError, generate_artifacts, data, self.channels[0].name, True)
def test_id_creation(self):
"""
Test an id is created for a valid tile
"""
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
data = ingest_links({"STAR/en-US": [tile]}, self.channels[0].id)
directory_id = data["STAR/en-US"][0]["directoryId"]
# the biggest ID is 30 - next one should be 31
assert_equal(31, directory_id)
def deploy_scheduled(console_out, deploy_flag, leniency, verbose, quiet, *args,
**kwargs):
"""
Find scheduled distributions and deploy
"""
if verbose:
logger = setup_command_logger(logging.DEBUG)
elif quiet:
logger = setup_command_logger(logging.ERROR)
else:
logger = setup_command_logger(logging.INFO)
from splice.queries import get_scheduled_distributions, unschedule_distribution
import requests
dt = datetime.utcnow()
distributions = get_scheduled_distributions(leniency, dt)
logger.info("{0} - found {1} distributions".format(dt, len(distributions)))
dist_data = []
for dist in distributions:
logger.info("fetching {0}".format(dist.url))
r = requests.get(dist.url)
if r.status_code == 200:
dist_data.append((r.json(), dist.channel_id, dist.id))
else:
logger.error("FETCH_ERROR status_code:{0} url:{1}".format(
r.status_code, dist.url))
from splice.ingest import ingest_links, distribute, IngestError
if deploy_flag:
for rawdata, channel_id, dist_id in dist_data:
try:
new_data = ingest_links(rawdata, channel_id)
if console_out:
print json.dumps(new_data, sort_keys=True, indent=2)
distribute(new_data, channel_id, deploy_flag)
unschedule_distribution(dist_id)
except IngestError, e:
raise InvalidCommand(e.message)
except:
def deploy_scheduled(console_out, deploy_flag, leniency, verbose, quiet, *args, **kwargs):
"""
Find scheduled distributions and deploy
"""
if verbose:
logger = setup_command_logger(logging.DEBUG)
elif quiet:
logger = setup_command_logger(logging.ERROR)
else:
logger = setup_command_logger(logging.INFO)
from splice.queries import get_scheduled_distributions, unschedule_distribution
import requests
dt = datetime.utcnow()
distributions = get_scheduled_distributions(leniency, dt)
logger.info("{0} - found {1} distributions".format(dt, len(distributions)))
dist_data = []
for dist in distributions:
logger.info("fetching {0}".format(dist.url))
r = requests.get(dist.url)
if r.status_code == 200:
dist_data.append((r.json(), dist.channel_id, dist.id))
else:
logger.error("FETCH_ERROR status_code:{0} url:{1}".format(r.status_code, dist.url))
from splice.ingest import ingest_links, distribute, IngestError
if deploy_flag:
for rawdata, channel_id, dist_id in dist_data:
try:
new_data = ingest_links(rawdata, channel_id)
if console_out:
print json.dumps(new_data, sort_keys=True, indent=2)
distribute(new_data, channel_id, deploy_flag)
unschedule_distribution(dist_id)
except IngestError, e:
raise InvalidCommand(e.message)
except:
def test_generate_artifacts_tile_count(self):
"""
Tests that the correct number of tiles are produced
"""
with open(self.get_fixture_path('mozilla-tiles.fennec.sg.json'), 'r') as f:
tiles = json.load(f)
data = ingest_links(tiles, self.channels[0].id)
artifacts = generate_artifacts(data, self.channels[0].name, True)
assertions_run = False
for a in artifacts:
m = DESKTOP_LOCALE_DISTRO_PATTERN.match(a['key'])
if m:
country_locale = m.groups()[0]
distro_data = json.loads(a['data'])
assert_equal(len(tiles[country_locale]) - 1, len(distro_data['directory']))
assert_equal(1, len(distro_data['suggested']))
assertions_run = True
assert(assertions_run)
def load_links(in_file, country_code, channel_id, out_path, console_out,
verbose, old_format, *args, **kwargs):
"""
Load a set of links in the data warehouse
"""
if verbose:
logger = setup_command_logger(logging.DEBUG)
else:
logger = setup_command_logger(logging.INFO)
rawdata = None
with open(in_file, 'r') as f:
rawdata = json.load(f)
from splice.ingest import ingest_links, IngestError
try:
locale = rawdata.keys()[0]
country_locale_str = "/".join([country_code, locale])
new_data = ingest_links({country_locale_str: rawdata[locale]},
channel_id)
if old_format:
new_data = new_data[new_data.keys()[0]]
if console_out:
print json.dumps(new_data, sort_keys=True, indent=2)
if out_path:
directory, _ = os.path.split(out_path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(out_path, "w") as f:
json.dump(new_data, f, sort_keys=True, indent=2)
logger.info("wrote {0}".format(out_path))
except IngestError, e:
raise InvalidCommand(e.message)
def test_id_not_duplicated(self):
"""
Test an id is created for a valid tile
"""
tiles_star = [
{
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
},
{
"imageURI": "data:image/png;base64,someotherdata",
"url": "https://somewhereelse.com",
"title": "Some Other Title",
"type": "organic",
"bgColor": "#FFFFFF"
},
]
tiles_ca = [
{
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF"
}
]
data = ingest_links({
"STAR/en-US": tiles_star,
"CA/en-US": tiles_ca,
}, self.channels[0].id)
directory_id_star = data["STAR/en-US"][0]["directoryId"]
directory_id_ca = data["CA/en-US"][0]["directoryId"]
assert_equal(31, directory_id_star)
assert_not_equal(data["STAR/en-US"][1]["directoryId"], directory_id_star)
assert_equal(directory_id_ca, directory_id_star)
def ingest_tiles(in_file, channel_id, out_path, console_out, deploy_flag,
verbose, *args, **kwargs):
"""
Load a set of links for all country/locale combinations into data warehouse and optionally deploy
"""
if verbose:
logger = setup_command_logger(logging.DEBUG)
else:
logger = setup_command_logger(logging.INFO)
rawdata = None
with open(in_file, 'r') as f:
rawdata = json.load(f)
from splice.ingest import ingest_links, distribute, IngestError
try:
new_data = ingest_links(rawdata, channel_id)
if console_out:
print json.dumps(new_data, sort_keys=True, indent=2)
if out_path:
directory, _ = os.path.split(out_path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(out_path, "w") as f:
json.dump(new_data, f, sort_keys=True, indent=2)
logger.info("wrote {0}".format(out_path))
if deploy_flag:
logger.info("Distributing AND Deploying data")
else:
logger.info("Distributing data (NO deploy)")
distribute(new_data, channel_id, deploy_flag)
except IngestError, e:
raise InvalidCommand(e.message)
def test_suggested_sites(self):
"""
just a simple suggested site tile
"""
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"frecent_sites": ["http://abc.com", "https://xyz.com"]
}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
c = self.env.db.session.query(AdgroupSite).count()
assert_equal(0, c)
data = ingest_links({"US/en-US": [tile]}, self.channels[0].id)
assert_equal(1, len(data["US/en-US"]))
c = self.env.db.session.query(Adgroup).count()
assert_equal(31, c)
c = self.env.db.session.query(AdgroupSite).count()
assert_equal(2, c)
def load_links(in_file, country_code, channel_id, out_path, console_out, verbose, old_format, *args, **kwargs):
"""
Load a set of links in the data warehouse
"""
if verbose:
logger = setup_command_logger(logging.DEBUG)
else:
logger = setup_command_logger(logging.INFO)
rawdata = None
with open(in_file, 'r') as f:
rawdata = json.load(f)
from splice.ingest import ingest_links, IngestError
try:
locale = rawdata.keys()[0]
country_locale_str = "/".join([country_code, locale])
new_data = ingest_links({country_locale_str: rawdata[locale]}, channel_id)
if old_format:
new_data = new_data[new_data.keys()[0]]
if console_out:
print json.dumps(new_data, sort_keys=True, indent=2)
if out_path:
directory, _ = os.path.split(out_path)
if not os.path.exists(directory):
os.makedirs(directory)
with open(out_path, "w") as f:
json.dump(new_data, f, sort_keys=True, indent=2)
logger.info("wrote {0}".format(out_path))
except IngestError, e:
raise InvalidCommand(e.message)
def test_explanation(self):
explanation = "Suggested for %1$S fans who visit site %2$S"
tile = {
"imageURI": "data:image/png;base64,somedata",
"url": "https://somewhere.com",
"title": "Some Title",
"type": "organic",
"bgColor": "#FFFFFF",
"adgroup_name": "Technology",
"explanation": explanation,
}
c = self.env.db.session.query(Adgroup).count()
assert_equal(30, c)
data = ingest_links({"US/en-US": [tile]}, self.channels[0].id)
assert_equal(1, len(data["US/en-US"]))
c = self.env.db.session.query(Adgroup).count()
assert_equal(31, c)
tile = self.env.db.session.query(Tile).filter(Tile.id == 31).one()
ag = self.env.db.session.query(Adgroup).filter(Adgroup.id == 31).one()
assert_equal(tile.adgroup_id, ag.id)
assert_equal(ag.name, "Technology")
assert_equal(ag.explanation, explanation)
def all_tiles():
deploy_flag = request.args.get('deploy')
channel_id = request.args.get('channelId')
scheduled_ts = request.args.get('scheduledTS')
scheduled_dt = None
deploy = deploy_flag == '1'
if channel_id is None:
msg = "channel_id not provided"
env.log("INGEST_ERROR msg: {0}".format(msg))
return jsonify({"err": [{"msg": msg}]}), 400
try:
data = request.get_json(force=True)
new_data = ingest_links(data, channel_id)
if scheduled_ts:
# scheduled_ts assumed to be in seconds
scheduled_dt = datetime.utcfromtimestamp(int(scheduled_ts))
urls = distribute(new_data, channel_id, deploy, scheduled_dt)
except NoResultFound, e:
msg = "channel_id {0} does not exist".format(channel_id)
env.log("INGEST_ERROR msg: {0}".format(msg))
return jsonify({"err": [{"msg": msg}]}), 404
