def test_update_publish_items_invalid_item(db, auth_header):
"""PUTting an item without object_key or link_to fails validation."""
publish_id = "11224567-e89b-12d3-a456-426614174000"
publish = Publish(id=uuid.UUID("{%s}" % publish_id),
env="test",
state="PENDING")
with TestClient(app) as client:
# ensure a publish object exists
db.add(publish)
db.commit()
# Try to add an item to it
r = client.put(
"/test/publish/%s" % publish_id,
json=[{
"web_uri": "/uri1"
}],
headers=auth_header(roles=["test-publisher"]),
)
expected_item = {
"web_uri": "/uri1",
"object_key": "",
"content_type": "",
"link_to": "",
}
# It should have failed with 400
assert r.status_code == 400
assert r.json() == {
"detail": ["No object key or link target: %s" % expected_item]
}
def test_update_publish_items_invalid_publish(db, auth_header):
"""PUTting items on a completed publish fails with code 409."""
publish_id = "11224567-e89b-12d3-a456-426614174000"
publish = Publish(id=uuid.UUID("{%s}" % publish_id),
env="test",
state="COMPLETE")
with TestClient(app) as client:
# ensure a publish object exists
db.add(publish)
db.commit()
# Try to add some items to it
r = client.put(
"/test/publish/%s" % publish_id,
json=[
{
"web_uri": "/uri1",
"object_key": "1" * 64,
},
],
headers=auth_header(roles=["test-publisher"]),
)
# It should have failed with 409
assert r.status_code == 409
assert r.json() == {
"detail": "Publish %s in unexpected state, 'COMPLETE'" % publish_id
}
def test_publish_task_before_update(db):
"""Changing object states updates timestamp."""
publish_id = "11224567-e89b-12d3-a456-426614174000"
publish = Publish(id=uuid.UUID(publish_id), env="test", state="PENDING")
task_id = "8d8a4692-c89b-4b57-840f-b3f0166148d2"
task = Task(
id=task_id,
publish_id=uuid.UUID(publish_id),
state="NOT_STARTED",
)
db.add(publish)
db.add(task)
db.commit()
# Updated should initially be null
assert publish.updated is None
assert task.updated is None
# Change state of publish and task
publish.state = "COMMITTING"
task.state = "IN_PROGRESS"
db.commit()
# Updated should now hold datetime objects
p_updated = publish.updated
assert isinstance(p_updated, datetime)
t_updated = task.updated
assert isinstance(t_updated, datetime)
# Change state of publish and task again
publish.state = "COMMITTED"
task.state = "COMPLETE"
db.commit()
# Updated should now hold different datetime objects
assert isinstance(publish.updated, datetime)
assert p_updated != publish.updated
assert isinstance(task.updated, datetime)
assert t_updated != task.updated
def make_publish(mode: str = None, db: Session = deps.db):
p = Publish(id=TEST_UUID, env="test", state="PENDING")
db.add(p)
if mode == "rollback":
db.rollback()
elif mode == "commit":
db.commit()
elif mode == "raise":
raise HTTPException(500)
def test_update_publish_items_path_normalization(db, auth_header):
"""URI and link target paths are normalized in PUT items."""
publish_id = "11224567-e89b-12d3-a456-426614174000"
publish = Publish(id=uuid.UUID("{%s}" % publish_id),
env="test",
state="PENDING")
with TestClient(app) as client:
# Ensure a publish object exists
db.add(publish)
db.commit()
# Add an item to it with some messy paths
r = client.put(
"/test/publish/%s" % publish_id,
json=[
{
"web_uri": "some/path",
"object_key": "1" * 64
},
{
"web_uri": "link/to/some/path",
"link_to": "/some/path"
},
],
headers=auth_header(roles=["test-publisher"]),
)
# It should have succeeded
assert r.ok
# Publish object should now have matching items
db.refresh(publish)
item_dicts = [{
"web_uri": item.web_uri,
"object_key": item.object_key,
"link_to": item.link_to,
} for item in publish.items]
# Should have stored normalized web_uri and link_to paths
assert item_dicts == [
{
"web_uri": "/some/path",
"object_key": "1" * 64,
"link_to": ""
},
{
"web_uri": "/link/to/some/path",
"object_key": "",
"link_to": "/some/path",
},
]
def test_cleanup_mixed(caplog, db):
"""Cleanup manipulates objects in the expected manner."""
logging.getLogger("exodus-gw").setLevel(logging.INFO)
# Note: datetimes in this test assume the default timeout values
# are used.
now = datetime.utcnow()
half_day_ago = now - timedelta(hours=12)
two_days_ago = now - timedelta(days=2)
eight_days_ago = now - timedelta(days=8)
twenty_days_ago = now - timedelta(days=30)
# Some objects with missing timestamps.
p1_missing_ts = Publish(id=uuid.uuid4(),
env="test",
state=PublishStates.failed,
updated=None)
p2_missing_ts = Publish(
id=uuid.uuid4(),
env="test2",
state=PublishStates.committed,
updated=None,
)
t1_missing_ts = Task(
id=uuid.uuid4(),
publish_id=p1_missing_ts.id,
state=TaskStates.failed,
updated=None,
)
# Some publishes which seem to be abandoned.
p1_abandoned = Publish(
id=uuid.uuid4(),
env="test",
state=PublishStates.pending,
updated=eight_days_ago,
items=[
Item(web_uri="/1", object_key="abc", link_to=""),
Item(web_uri="/2", object_key="aabbcc", link_to=""),
],
)
p2_abandoned = Publish(
id=uuid.uuid4(),
env="test",
state=PublishStates.committing,
updated=twenty_days_ago,
)
t1_abandoned = Task(
id=uuid.uuid4(),
publish_id=p1_abandoned.id,
state=TaskStates.in_progress,
updated=eight_days_ago,
)
# Some objects which are old enough to be cleaned up.
p1_old = Publish(
id=uuid.uuid4(),
env="test2",
state=PublishStates.committed,
updated=twenty_days_ago,
items=[
Item(web_uri="/1", object_key="abc", link_to=""),
Item(web_uri="/2", object_key="aabbcc", link_to=""),
],
)
p2_old = Publish(
id=uuid.uuid4(),
env="test3",
state=PublishStates.failed,
updated=twenty_days_ago,
)
t1_old = Task(
id=uuid.uuid4(),
publish_id=p1_old.id,
state=TaskStates.failed,
updated=twenty_days_ago,
)
# (Because these objects will be deleted, we need to keep their ids separately.)
p1_old_id = p1_old.id
p2_old_id = p2_old.id
t1_old_id = t1_old.id
# And finally some recent objects which should not be touched at all.
p1_recent = Publish(
id=uuid.uuid4(),
env="test3",
state=PublishStates.pending,
updated=half_day_ago,
)
t1_recent = Task(
id=uuid.uuid4(),
publish_id=p1_recent.id,
state=TaskStates.complete,
updated=two_days_ago,
)
# self.fix_abandoned_publishes()
# self.clean_old_data()
db.add_all([
p1_missing_ts,
p2_missing_ts,
t1_missing_ts,
p1_abandoned,
p2_abandoned,
t1_abandoned,
p1_old,
p2_old,
t1_old,
p1_recent,
t1_recent,
])
db.commit()
# Should run successfully
cleanup()
# Make sure we reload anything which has changed
db.expire_all()
# Missing timestamps should now be filled in (fuzzy comparison as exact
# time is not set)
assert (t1_missing_ts.updated - now) < timedelta(seconds=10)
assert (p1_missing_ts.updated - now) < timedelta(seconds=10)
assert (p2_missing_ts.updated - now) < timedelta(seconds=10)
# The abandoned objects should now be marked as failed
assert p1_abandoned.state == PublishStates.failed
assert p2_abandoned.state == PublishStates.failed
assert t1_abandoned.state == TaskStates.failed
# The old objects should no longer exist.
with pytest.raises(ObjectDeletedError):
p1_old.id
with pytest.raises(ObjectDeletedError):
p2_old.id
with pytest.raises(ObjectDeletedError):
t1_old.id
# Other objects should still exist as they were.
assert p1_recent.state == PublishStates.pending
assert t1_recent.state == TaskStates.complete
# It should have logged exactly what it did.
assert sorted(caplog.messages) == sorted([
####################################################
# Fixed timestamps
"Task %s: setting updated" % (t1_missing_ts.id, ),
"Publish %s: setting updated" % (p1_missing_ts.id, ),
"Publish %s: setting updated" % (p2_missing_ts.id, ),
####################################################
# Abandoned objects
"Task %s: marking as failed (last updated: %s)" %
(t1_abandoned.id, eight_days_ago),
"Publish %s: marking as failed (last updated: %s)" %
(p1_abandoned.id, eight_days_ago),
"Publish %s: marking as failed (last updated: %s)" %
(p2_abandoned.id, twenty_days_ago),
####################################################
# Deleted old stuff
"Task %s: cleaning old data (last updated: %s)" %
(t1_old_id, twenty_days_ago),
"Publish %s: cleaning old data (last updated: %s)" %
(p1_old_id, twenty_days_ago),
"Publish %s: cleaning old data (last updated: %s)" %
(p2_old_id, twenty_days_ago),
####################################################
# Completed cleanup
"Scheduled cleanup has completed",
])
def test_update_publish_items_typical(db, auth_header):
"""PUTting some items on a publish creates expected objects in DB."""
publish_id = "11224567-e89b-12d3-a456-426614174000"
publish = Publish(id=uuid.UUID("{%s}" % publish_id),
env="test",
state="PENDING")
with TestClient(app) as client:
# Ensure a publish object exists
db.add(publish)
db.commit()
# Try to add some items to it
r = client.put(
"/test/publish/%s" % publish_id,
json=[
{
"web_uri": "/uri1",
"object_key": "1" * 64,
"content_type": "application/octet-stream",
},
{
"web_uri": "/uri2",
"object_key": "2" * 64,
"content_type": "application/octet-stream",
},
{
"web_uri": "/uri3",
"link_to": "/uri1",
},
{
"web_uri": "/uri4",
"object_key": "absent",
},
],
headers=auth_header(roles=["test-publisher"]),
)
# It should have succeeded
assert r.ok
# Publish object should now have matching items
db.refresh(publish)
items = sorted(publish.items, key=lambda item: item.web_uri)
item_dicts = [{
"web_uri": item.web_uri,
"object_key": item.object_key,
"content_type": item.content_type,
"link_to": item.link_to,
} for item in items]
# Should have stored exactly what we asked for
assert item_dicts == [
{
"web_uri": "/uri1",
"object_key": "1" * 64,
"content_type": "application/octet-stream",
"link_to": "",
},
{
"web_uri": "/uri2",
"object_key": "2" * 64,
"content_type": "application/octet-stream",
"link_to": "",
},
{
"web_uri": "/uri3",
"object_key": "",
"content_type": "",
"link_to": "/uri1",
},
{
"web_uri": "/uri4",
"object_key": "absent",
"content_type": "",
"link_to": "",
},
]