def test_missing_crashes(self, boto_helper, monkeypatch):
"""Verify it finds a missing crash."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a raw and processed crash
crashid_1 = "000" + create_new_ooid()[3:]
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v2/raw_crash/%s/%s/%s" % (crashid_1[0:3], TODAY, crashid_1),
data=b"test",
)
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v1/processed_crash/%s" % crashid_1,
data=b"test",
)
# Create a raw crash
crashid_2 = "000" + create_new_ooid()[3:]
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v2/raw_crash/%s/%s/%s" % (crashid_2[0:3], TODAY, crashid_2),
data=b"test",
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == [crashid_2]
def test_missing_crashes(self, boto_helper):
"""Verify it finds a missing crash."""
boto_helper.get_or_create_bucket(BUCKET_NAME)
# Create a raw and processed crash
crashid_1 = create_new_ooid()
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key="/v2/raw_crash/%s/%s/%s" % (crashid_1[0:3], TODAY, crashid_1),
value="test",
)
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key="/v1/processed_crash/%s" % crashid_1,
value="test",
)
# Create a raw crash
crashid_2 = create_new_ooid()
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key="/v2/raw_crash/%s/%s/%s" % (crashid_2[0:3], TODAY, crashid_2),
value="test",
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == [crashid_2]
def test_no_missing_crashes(self, boto_helper):
"""Verify raw crashes with processed crashes result in no missing crashes."""
boto_helper.get_or_create_bucket(BUCKET_NAME)
# Create a couple raw and processed crashes
crashids = [
create_new_ooid(),
create_new_ooid(),
create_new_ooid(),
]
for crashid in crashids:
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key='/v2/raw_crash/%s/%s/%s' % (crashid[0:3], TODAY, crashid),
value='test'
)
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key='/v1/processed_crash/%s' % crashid,
value='test'
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_no_missing_crashes(self, boto_helper, monkeypatch):
"""Verify raw crashes with processed crashes result in no missing crashes."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a few raw and processed crashes
crashids = [
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
]
for crashid in crashids:
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v2/raw_crash/%s/%s/%s" % (crashid[0:3], TODAY, crashid),
data=b"test",
)
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v1/processed_crash/%s" % crashid,
data=b"test",
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_get_entropy(self):
cmd = Command()
entropy = list(sorted(cmd.get_entropy()))
# We don't want to assert the contents of the whole list, so let's
# just assert some basic facts and it's probably fine
assert len(entropy) == 4096
assert entropy[0] == "000"
assert entropy[-1] == "fff"
def test_no_crashes(self, boto_helper, monkeypatch):
"""Verify no crashes in bucket result in no missing crashes."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_handle_missing_some_missing(self, capsys, db):
crash_ids = [create_new_ooid(), create_new_ooid()]
crash_ids.sort()
cmd = Command()
cmd.handle_missing(TODAY, crash_ids)
captured = capsys.readouterr()
assert "Missing: %s" % crash_ids[0] in captured.out
assert "Missing: %s" % crash_ids[1] in captured.out
assert crash_ids == list(self.fetch_crashids())
def test_past_missing_still_missing(self, capsys, db):
# Create a MissingProcessedCrash row, but don't put the processed crash in the
# bucket. After check_past_missing() runs, the MissingProcessedCrash should
# still have is_processed=False.
crash_id = create_new_ooid()
mpe = MissingProcessedCrash(crash_id=crash_id, is_processed=False)
mpe.save()
cmd = Command()
cmd.check_past_missing()
mpe = MissingProcessedCrash.objects.get(crash_id=crash_id)
assert mpe.is_processed is False
def test_past_missing_no_longer_missing(self, capsys, db, boto_helper):
# Create a MissingProcessedCrash row and put the processed crash in the S3
# bucket. After check_past_missing() runs, the MissingProcessedCrash should
# have is_processed=True.
crash_id = create_new_ooid()
mpe = MissingProcessedCrash(crash_id=crash_id, is_processed=False)
mpe.save()
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v1/processed_crash/%s" % crash_id,
data=b"test",
)
cmd = Command()
cmd.check_past_missing()
mpe = MissingProcessedCrash.objects.get(crash_id=crash_id)
assert mpe.is_processed is True
def test_missing_crashes(self, boto_helper, es_conn, monkeypatch):
"""Verify it finds a missing crash."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a raw and processed crash
crash_id_1 = "000" + create_new_ooid()[3:]
self.create_raw_crash_in_s3(boto_helper, crash_id_1)
self.create_processed_crash_in_s3(boto_helper, crash_id_1)
self.create_processed_crash_in_es(es_conn, crash_id_1)
# Create a raw crash
crash_id_2 = "000" + create_new_ooid()[3:]
self.create_raw_crash_in_s3(boto_helper, crash_id_2)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == [crash_id_2]
def test_no_missing_crashes(self, boto_helper, es_conn, monkeypatch):
"""Verify raw crashes with processed crashes result in no missing crashes."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a few raw and processed crashes
crashids = [
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
]
for crash_id in crashids:
self.create_raw_crash_in_s3(boto_helper, crash_id)
self.create_processed_crash_in_s3(boto_helper, crash_id)
self.create_processed_crash_in_es(es_conn, crash_id)
es_conn.refresh()
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_handle_missing_none_missing(self, capsys):
cmd = Command()
cmd.handle_missing(TODAY, [])
captured = capsys.readouterr()
assert ("All crashes for %s were processed." % TODAY) in captured.out
def test_no_crashes(self, boto_helper):
"""Verify no crashes in bucket result in no missing crashes."""
boto_helper.get_or_create_bucket(BUCKET_NAME)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []