def test_missing_crashes(self, boto_helper):
"""Verify it finds a missing crash."""
boto_helper.get_or_create_bucket(BUCKET_NAME)
# Create a raw and processed crash
crashid_1 = create_new_ooid()
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key="/v2/raw_crash/%s/%s/%s" % (crashid_1[0:3], TODAY, crashid_1),
value="test",
)
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key="/v1/processed_crash/%s" % crashid_1,
value="test",
)
# Create a raw crash
crashid_2 = create_new_ooid()
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key="/v2/raw_crash/%s/%s/%s" % (crashid_2[0:3], TODAY, crashid_2),
value="test",
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == [crashid_2]
def test_missing_crashes(self, boto_helper, monkeypatch):
"""Verify it finds a missing crash."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a raw and processed crash
crashid_1 = "000" + create_new_ooid()[3:]
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v2/raw_crash/%s/%s/%s" % (crashid_1[0:3], TODAY, crashid_1),
data=b"test",
)
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v1/processed_crash/%s" % crashid_1,
data=b"test",
)
# Create a raw crash
crashid_2 = "000" + create_new_ooid()[3:]
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v2/raw_crash/%s/%s/%s" % (crashid_2[0:3], TODAY, crashid_2),
data=b"test",
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == [crashid_2]
def test_no_missing_crashes(self, boto_helper, monkeypatch):
"""Verify raw crashes with processed crashes result in no missing crashes."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a few raw and processed crashes
crashids = [
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
]
for crashid in crashids:
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v2/raw_crash/%s/%s/%s" % (crashid[0:3], TODAY, crashid),
data=b"test",
)
boto_helper.upload_fileobj(
bucket_name=BUCKET_NAME,
key="v1/processed_crash/%s" % crashid,
data=b"test",
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_no_missing_crashes(self, boto_helper):
"""Verify raw crashes with processed crashes result in no missing crashes."""
boto_helper.get_or_create_bucket(BUCKET_NAME)
# Create a couple raw and processed crashes
crashids = [
create_new_ooid(),
create_new_ooid(),
create_new_ooid(),
]
for crashid in crashids:
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key='/v2/raw_crash/%s/%s/%s' % (crashid[0:3], TODAY, crashid),
value='test'
)
boto_helper.set_contents_from_string(
bucket_name=BUCKET_NAME,
key='/v1/processed_crash/%s' % crashid,
value='test'
)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_no_crashes(self, boto_helper, monkeypatch):
"""Verify no crashes in bucket result in no missing crashes."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_missing_crashes(self, boto_helper, es_conn, monkeypatch):
"""Verify it finds a missing crash."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a raw and processed crash
crash_id_1 = "000" + create_new_ooid()[3:]
self.create_raw_crash_in_s3(boto_helper, crash_id_1)
self.create_processed_crash_in_s3(boto_helper, crash_id_1)
self.create_processed_crash_in_es(es_conn, crash_id_1)
# Create a raw crash
crash_id_2 = "000" + create_new_ooid()[3:]
self.create_raw_crash_in_s3(boto_helper, crash_id_2)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == [crash_id_2]
def test_no_missing_crashes(self, boto_helper, es_conn, monkeypatch):
"""Verify raw crashes with processed crashes result in no missing crashes."""
monkeypatch.setattr(Command, "get_entropy", get_small_entropy)
bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"]
boto_helper.create_bucket(bucket)
# Create a few raw and processed crashes
crashids = [
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
"000" + create_new_ooid()[3:],
]
for crash_id in crashids:
self.create_raw_crash_in_s3(boto_helper, crash_id)
self.create_processed_crash_in_s3(boto_helper, crash_id)
self.create_processed_crash_in_es(es_conn, crash_id)
es_conn.refresh()
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []
def test_no_crashes(self, boto_helper):
"""Verify no crashes in bucket result in no missing crashes."""
boto_helper.get_or_create_bucket(BUCKET_NAME)
cmd = Command()
missing = cmd.find_missing(num_workers=1, date=TODAY)
assert missing == []