def test_links(self):
template = ExportTemplate()
template.fields = []
assert template.links == [], "should have no link"
template.fields = [{"value": "foo"}]
assert template.links == [], "should have no link"
template.fields = [{"value": "links['foo'] + bar"}]
assert template.links == ["foo"], "should extract the link"
template.fields = [{"value": "mainlinks + linksbar + mainlinks['bar']"}]
assert template.links == [], "should not be fooled by value including `links`"
template.fields = [
{"value": "links['foo']"},
{"value": "links['bar'] + links['foo']"},
]
assert template.links == ["bar", "foo"], "should dedupe links"
template.fields = [{"value": "links['bar'] + links['foo']"}]
tmp_links = template.links
template.fields = [{"value": "links['foo'] + links['bar']"}]
assert template.links == tmp_links, "order should stay the same"
def test_compile_template(self):
assert ExportTemplate(fields=[]).template == ""
assert (
ExportTemplate(fields=[{"value": "foo"}, {"value": "bar"}]).template
== "{{ (foo) | quote }}\t{{ (bar) | quote }}"
)
template = ExportTemplate(
fields=[{"value": "foo"}, {"value": "bar"}]
).compile_template()
assert (
template({"foo": "hello", "bar": "world"}) == "hello\tworld"
), "the compiled template should be a callable"
def test_generate(self, session):
platforms = [
Platform(name="Platform 1", type=PlatformType.TVOD, country="FR"),
Platform(name="Platform 2", type=PlatformType.SVOD, country="GB"),
]
group = PlatformGroup(name="Group Name", platforms=platforms)
session.add_all(platforms + [group])
session.commit()
scrap_session = Session()
template = ExportTemplate()
factory = ExportFactory(
template=template,
iterator=ExportFactoryIterator.PLATFORMS,
filters_template={"platform.id": "{{ platform.id }}"},
file_path_template=(
"{{ platform.type | upper }}_"
"{{ platform.country | lower }}_"
"{{ platform.name | pathify }}"
),
)
files = list(factory.generate(scrap_session=scrap_session, session=session))
assert len(files) == len(platforms)
assert all(file.factory == factory for file in files)
assert all(file.template == template for file in files)
assert all(file.session == scrap_session for file in files)
assert files[0].path == "TVOD_fr_platform_1"
assert files[1].path == "SVOD_gb_platform_2"
assert files[0].filters == {"platform.id": str(platforms[0].id)}
assert files[1].filters == {"platform.id": str(platforms[1].id)}
def test_count_links(self, session):
object_session = Session(name="test")
import_file = ImportFile(
sessions=[object_session],
status=ImportFileStatus.DONE,
filename="foo.csv",
fields={},
)
external_object = ExternalObject(type=ExternalObjectType.MOVIE)
empty_platform = Platform(name="empty", type=PlatformType.TVOD)
linked_platform = Platform(name="linked", type=PlatformType.TVOD)
link = ObjectLink(
external_object=external_object,
platform=linked_platform,
external_id="link",
imports=[import_file],
)
session.add_all(
[external_object, empty_platform, linked_platform, link, import_file]
)
session.commit()
template = ExportTemplate(
external_object_type=ExternalObjectType.MOVIE,
row_type=ExportRowType.OBJECT_LINK,
fields={},
)
assert (
ExportFile(
template=template,
session=object_session,
status=ExportFileStatus.DONE,
path="foo.csv",
filters={},
).count_links(session=session)
== 1
)
assert (
ExportFile(
template=template,
session=object_session,
status=ExportFileStatus.DONE,
path="foo.csv",
filters={"platform.id": str(linked_platform.id)},
).count_links(session=session)
== 1
)
assert (
ExportFile(
template=template,
session=object_session,
status=ExportFileStatus.DONE,
path="foo.csv",
filters={"platform.id": str(empty_platform.id)},
).count_links(session=session)
== 0
)
def test_header(self):
template = ExportTemplate()
template.fields = []
assert template.header == "", "header should be empty"
template.fields = [{"name": "foo"}, {"name": "bar"}]
assert template.header == "foo\tbar", "header should be correctly generated"
template.fields = [{}, {}, {}]
assert template.header == "\t\t", "empty column headers should stay as such"
template.fields = [{"name": "foo\tbar"}, {"name": '"baz"'}]
assert template.header == '"foo\tbar"\t"""baz"""', "column should be quoted"
def test_need(self):
template = ExportTemplate()
template.fields = []
assert template.needs == set(), "should have no need"
template.fields = [{"value": "'foo'"}, {"value": "3 + 2"}]
assert template.needs == set(), "should have no need"
template.fields = [{"value": "foo"}, {"value": "bar + baz"}]
assert template.needs == set(["foo", "bar", "baz"]), "should extract the needs"
template.fields = [{"value": "foo"}, {"value": "bar + foo"}, {"value": "bar"}]
assert template.needs == set(["foo", "bar"]), "should deduplicate the needs"
def test_valid_template(self):
assert ExportTemplate(fields=[]).valid_template
assert ExportTemplate(
fields=[{"value": "platform.name"}], row_type=ExportRowType.OBJECT_LINK
).valid_template
assert not ExportTemplate(
fields=[{"value": "platform.name"}], row_type=ExportRowType.EXTERNAL_OBJECT
).valid_template
assert ExportTemplate(
fields=[{"value": "external_object.id"}, {"value": "attributes.titles[0]"}],
row_type=ExportRowType.EXTERNAL_OBJECT,
).valid_template
assert not ExportTemplate(
fields=[{"value": "episodes_count"}],
external_object_type=ExternalObjectType.MOVIE,
).valid_template
assert ExportTemplate(
fields=[{"value": "episodes_count"}],
external_object_type=ExternalObjectType.SERIES,
).valid_template
def test_to_context(self):
platform = Platform()
external_object = ExternalObject()
object_link = ObjectLink(
external_object=external_object, platform=platform, external_id="foo"
)
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT, fields=[]
).to_context((external_object,))
assert context == {"external_object": external_object, "links": {}}
# OBJECT_LINK rows should have the `current` link
context = ExportTemplate(
row_type=ExportRowType.OBJECT_LINK, fields=[]
).to_context((object_link, external_object))
assert context == {
"external_object": external_object,
"links": {"current": "foo"},
}
# OBJECT_LINK rows should be able to request the platform
context = ExportTemplate(
row_type=ExportRowType.OBJECT_LINK, fields=[{"value": "platform.name"}]
).to_context((object_link, external_object))
assert context == {
"external_object": external_object,
"platform": platform,
"links": {"current": "foo"},
}
# EXTERNAL_OBJECT rows shall not ask for the platform
with raises(Exception):
context = ExportTemplate(
row_type=ExportRowType.PLATFORM, fields=[{"value": "platform.name"}]
).to_context((object_link, external_object))
fail(
"Platform can't be queried when the row_type is EXTERNAL_OBJECT"
)
# Requested links should be there
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT, fields=[{"value": "links['foo']"}]
).to_context((external_object, "bar"))
assert context == {"external_object": external_object, "links": {"foo": "bar"}}
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[{"value": "attributes.country | join(',')"}],
).to_context((external_object,))
assert "attributes" in context
assert (
context["attributes"].view != external_object.attributes
), "an empty AttributesView should be associated with the attributes"
assert (
context["attributes"].titles == []
), "attributes should fall back to empty lists"
with raises(AttributeError):
# Invalid attributes should fail
context["attributes"].foo
# Let's set some attributes
external_object.attributes = AttributesView(titles=["foo"])
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[{"value": "attributes.titles[0]"}],
).to_context((external_object,))
assert "attributes" in context
assert (
context["attributes"].view == external_object.attributes
), "the AttributesView should be associated"
assert context["attributes"].titles == [
"foo"
], "attributes that are set should work"
assert (
context["attributes"].genres == []
), "attributes that are not set should fall back to empty arrays"
with raises(AttributeError):
# Invalid attributes should fail
context["attributes"].foo
# Let's set some attributes
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT, fields=[{"value": "zones.EUROBS"}]
).to_context((external_object,))
assert "zones" in context, "the context should include zone informations"
assert (
"FR" in context["zones"].EUROBS
), "the zones should have some data in them"
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[{"value": "platform_countries"}],
).to_context((external_object, ["FR", "GB"]))
assert (
"platform_countries" in context
), "the context should include platform countries"
assert context["platform_countries"] == ["FR", "GB"]
context = ExportTemplate(
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[{"value": "platform_countries"}, {"value": "platform_names"}],
).to_context((external_object, ["FR"], ["Platform 1", "Platform 2"]))
assert (
"platform_countries" in context
), "the context should include platform countries"
assert "platform_names" in context, "the context should include platform names"
assert context["platform_countries"] == ["FR"]
assert context["platform_names"] == ["Platform 1", "Platform 2"]
def test_filtered_query(self, session):
platforms = [
Platform(name="TVOD FR", type=PlatformType.TVOD, country="FR"),
Platform(name="TVOD GB", type=PlatformType.TVOD, country="GB"),
Platform(name="SVOD FR", type=PlatformType.SVOD, country="FR"),
Platform(name="SVOD IT", type=PlatformType.SVOD, country="IT"),
Platform(name="SVOD DE", type=PlatformType.SVOD, country="DE"),
]
object_session = Session(name="test")
scrap = Scrap(platform=platforms[0], sessions=[object_session])
groups = [
PlatformGroup(name="Group TVOD", platforms=[platforms[0], platforms[1]]),
PlatformGroup(
name="Group SVOD", platforms=[platforms[2], platforms[3], platforms[4]]
),
]
objects = [
ExternalObject(
type=ExternalObjectType.MOVIE,
links=[
ObjectLink(
platform=platforms[0], external_id="tvod-fr-a", scraps=[scrap]
),
ObjectLink(
platform=platforms[1], external_id="tvod-gb-a", scraps=[scrap]
),
],
values=[
Value(
type=ValueType.TITLE,
text="A",
sources=[ValueSource(platform=platforms[0])],
),
Value(
type=ValueType.COUNTRY,
text="US",
sources=[
ValueSource(platform=platforms[0]),
ValueSource(platform=platforms[1]),
],
),
Value(
type=ValueType.COUNTRY,
text="GB",
sources=[ValueSource(platform=platforms[0])],
),
],
),
ExternalObject(
type=ExternalObjectType.MOVIE,
links=[
ObjectLink(
platform=platforms[0], external_id="tvod-fr-b", scraps=[scrap]
),
ObjectLink(
platform=platforms[3], external_id="svod-it-b", scraps=[scrap]
),
ObjectLink(
platform=platforms[4], external_id="svod-de-b", scraps=[scrap]
),
],
values=[
Value(
type=ValueType.TITLE,
text="B",
sources=[ValueSource(platform=platforms[0])],
)
],
),
]
# This is used to freeze the links orders
links = objects[0].links + objects[1].links
session.add_all(platforms + groups + objects + [scrap])
session.commit()
template = ExportTemplate(
fields=[{"value": 'links["{}"]'.format(platforms[0].slug)}],
external_object_type=ExternalObjectType.MOVIE,
)
file = ExportFile(
template=template,
session=object_session,
status=ExportFileStatus.SCHEDULED,
path="foo.csv",
)
template.row_type = ExportRowType.OBJECT_LINK
file.filters = {}
assert set(file.get_filtered_query(session=session)) == set(
[
(links[0], objects[0], "tvod-fr-a"),
(links[1], objects[0], "tvod-fr-a"),
(links[2], objects[1], "tvod-fr-b"),
(links[3], objects[1], "tvod-fr-b"),
(links[4], objects[1], "tvod-fr-b"),
]
)
template.row_type = ExportRowType.EXTERNAL_OBJECT
file.filters = {}
assert set(file.get_filtered_query(session=session)) == set(
[(objects[0], "tvod-fr-a"), (objects[1], "tvod-fr-b")]
)
template.row_type = ExportRowType.OBJECT_LINK
file.filters = {"platform.group_id": str(groups[0].id)}
assert set(file.get_filtered_query(session=session)) == set(
[
(links[0], objects[0], "tvod-fr-a"),
(links[1], objects[0], "tvod-fr-a"),
(links[2], objects[1], "tvod-fr-b"),
]
)
template.row_type = ExportRowType.EXTERNAL_OBJECT
file.filters = {"platform.group_id": str(groups[1].id)}
assert set(file.get_filtered_query(session=session)) == set(
[(objects[1], "tvod-fr-b")]
)
template.row_type = ExportRowType.OBJECT_LINK
file.filters = {"platform.country": "FR,IT"}
assert set(file.get_filtered_query(session=session)) == set(
[
(links[0], objects[0], "tvod-fr-a"),
(links[2], objects[1], "tvod-fr-b"),
(links[3], objects[1], "tvod-fr-b"),
]
)
template.row_type = ExportRowType.OBJECT_LINK
file.filters = {"platform.id": str(platforms[0].id)}
assert set(file.get_filtered_query(session=session)) == set(
[(links[0], objects[0], "tvod-fr-a"), (links[2], objects[1], "tvod-fr-b")]
)
def test_row_query(self, session):
platforms = [
Platform(slug="platform-" + str(i), name="Platform " + str(i))
for i in range(4)
]
platforms[0].country = "FR"
platforms[1].country = "FR"
platforms[2].country = "DE"
series = [ExternalObject(type=ExternalObjectType.SERIES) for _ in range(2)]
movies = [ExternalObject(type=ExternalObjectType.MOVIE) for _ in range(3)]
episodes = []
episodes_rel = []
for s in series:
for season in range(3):
for episode in range(10):
eo = ExternalObject(type=ExternalObjectType.EPISODE)
rel = Episode(
external_object=eo, series=s, episode=episode, season=season
)
episodes.append(eo)
episodes_rel.append(rel)
links = [
ObjectLink(
platform=p,
external_object=eo,
external_id="link-{}-{}-{}".format(p.slug, eo.type, index),
)
for (index, eo) in chain(
enumerate(series), enumerate(movies), enumerate(episodes)
)
for p in platforms
]
session.add_all(platforms + movies + series + links + episodes_rel + episodes)
session.commit()
# Start by checking the row count matches
assert ExportTemplate(
external_object_type=ExternalObjectType.MOVIE,
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[],
).get_row_query(session=session).count() == len(movies)
assert ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[],
).get_row_query(session=session).count() == len(series)
assert ExportTemplate(
external_object_type=ExternalObjectType.MOVIE,
row_type=ExportRowType.OBJECT_LINK,
fields=[],
).get_row_query(session=session).count() == len(movies) * len(platforms)
assert ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.OBJECT_LINK,
fields=[],
).get_row_query(session=session).count() == len(series) * len(platforms)
# Results are wrapped in sets because the order is not stable
assert set(
ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[
{"value": 'links["platform-1"]'},
{"value": 'links["platform-3"]'},
],
).get_row_query(session=session)
) == set(
[
(series[0], "link-platform-1-series-0", "link-platform-3-series-0"),
(series[1], "link-platform-1-series-1", "link-platform-3-series-1"),
]
)
# Export platform names and countries
rows = list(
sorted(
ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[
{"value": "platform_names"},
{"value": "platform_countries"},
],
).get_row_query(session=session),
key=lambda r: r[0].id,
)
)
assert rows[0][0] == series[0]
assert set(rows[0][1]) == set(["FR", "DE", None])
assert sorted(rows[0][2]) == sorted(p.name for p in platforms)
assert rows[1][0] == series[1]
assert set(rows[1][1]) == set(["FR", "DE", None])
assert sorted(rows[1][2]) == sorted(p.name for p in platforms)
# Export episodes and seasons count
assert set(
ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.EXTERNAL_OBJECT,
fields=[{"value": "seasons_count"}, {"value": "episodes_count"}],
).get_row_query(session=session)
) == set([(series[0], 3, 30), (series[1], 3, 30)])
# Results are wrapped in sets because the order is not stable
assert set(
ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.OBJECT_LINK,
fields=[
{"value": "attributes"},
{"value": "platform"},
{"value": 'links["platform-0"]'},
{"value": 'links["platform-2"]'},
],
).get_row_query(session=session)
) == set(
[
(
links[0],
series[0],
"link-platform-0-series-0",
"link-platform-2-series-0",
),
(
links[1],
series[0],
"link-platform-0-series-0",
"link-platform-2-series-0",
),
(
links[2],
series[0],
"link-platform-0-series-0",
"link-platform-2-series-0",
),
(
links[3],
series[0],
"link-platform-0-series-0",
"link-platform-2-series-0",
),
(
links[4],
series[1],
"link-platform-0-series-1",
"link-platform-2-series-1",
),
(
links[5],
series[1],
"link-platform-0-series-1",
"link-platform-2-series-1",
),
(
links[6],
series[1],
"link-platform-0-series-1",
"link-platform-2-series-1",
),
(
links[7],
series[1],
"link-platform-0-series-1",
"link-platform-2-series-1",
),
]
)
# Checking the generated contexts
template = ExportTemplate(
external_object_type=ExternalObjectType.SERIES,
row_type=ExportRowType.OBJECT_LINK,
fields=[{"value": "platform"}, {"value": 'links["platform-0"]'}],
)
# dicts are not hashable, we need to sort manually
def sort_key(d):
return d["links"]["current"]
rows = list(
template.to_context(row) for row in template.get_row_query(session=session)
)
expected = [
{
"external_object": series[0],
"platform": platforms[0],
"links": {
"current": links[0].external_id,
"platform-0": "link-platform-0-series-0",
},
},
{
"external_object": series[0],
"platform": platforms[1],
"links": {
"current": links[1].external_id,
"platform-0": "link-platform-0-series-0",
},
},
{
"external_object": series[0],
"platform": platforms[2],
"links": {
"current": links[2].external_id,
"platform-0": "link-platform-0-series-0",
},
},
{
"external_object": series[0],
"platform": platforms[3],
"links": {
"current": links[3].external_id,
"platform-0": "link-platform-0-series-0",
},
},
{
"external_object": series[1],
"platform": platforms[0],
"links": {
"current": links[4].external_id,
"platform-0": "link-platform-0-series-1",
},
},
{
"external_object": series[1],
"platform": platforms[1],
"links": {
"current": links[5].external_id,
"platform-0": "link-platform-0-series-1",
},
},
{
"external_object": series[1],
"platform": platforms[2],
"links": {
"current": links[6].external_id,
"platform-0": "link-platform-0-series-1",
},
},
{
"external_object": series[1],
"platform": platforms[3],
"links": {
"current": links[7].external_id,
"platform-0": "link-platform-0-series-1",
},
},
]
assert sorted(rows, key=sort_key) == sorted(expected, key=sort_key)