def test_missing_crashes(self, boto_helper, monkeypatch): """Verify it finds a missing crash.""" monkeypatch.setattr(Command, "get_entropy", get_small_entropy) bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"] boto_helper.create_bucket(bucket) # Create a raw and processed crash crashid_1 = "000" + create_new_ooid()[3:] boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v2/raw_crash/%s/%s/%s" % (crashid_1[0:3], TODAY, crashid_1), data=b"test", ) boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v1/processed_crash/%s" % crashid_1, data=b"test", ) # Create a raw crash crashid_2 = "000" + create_new_ooid()[3:] boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v2/raw_crash/%s/%s/%s" % (crashid_2[0:3], TODAY, crashid_2), data=b"test", ) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == [crashid_2]
def test_missing_crashes(self, boto_helper): """Verify it finds a missing crash.""" boto_helper.get_or_create_bucket(BUCKET_NAME) # Create a raw and processed crash crashid_1 = create_new_ooid() boto_helper.set_contents_from_string( bucket_name=BUCKET_NAME, key="/v2/raw_crash/%s/%s/%s" % (crashid_1[0:3], TODAY, crashid_1), value="test", ) boto_helper.set_contents_from_string( bucket_name=BUCKET_NAME, key="/v1/processed_crash/%s" % crashid_1, value="test", ) # Create a raw crash crashid_2 = create_new_ooid() boto_helper.set_contents_from_string( bucket_name=BUCKET_NAME, key="/v2/raw_crash/%s/%s/%s" % (crashid_2[0:3], TODAY, crashid_2), value="test", ) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == [crashid_2]
def test_no_missing_crashes(self, boto_helper): """Verify raw crashes with processed crashes result in no missing crashes.""" boto_helper.get_or_create_bucket(BUCKET_NAME) # Create a couple raw and processed crashes crashids = [ create_new_ooid(), create_new_ooid(), create_new_ooid(), ] for crashid in crashids: boto_helper.set_contents_from_string( bucket_name=BUCKET_NAME, key='/v2/raw_crash/%s/%s/%s' % (crashid[0:3], TODAY, crashid), value='test' ) boto_helper.set_contents_from_string( bucket_name=BUCKET_NAME, key='/v1/processed_crash/%s' % crashid, value='test' ) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == []
def test_no_missing_crashes(self, boto_helper, monkeypatch): """Verify raw crashes with processed crashes result in no missing crashes.""" monkeypatch.setattr(Command, "get_entropy", get_small_entropy) bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"] boto_helper.create_bucket(bucket) # Create a few raw and processed crashes crashids = [ "000" + create_new_ooid()[3:], "000" + create_new_ooid()[3:], "000" + create_new_ooid()[3:], ] for crashid in crashids: boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v2/raw_crash/%s/%s/%s" % (crashid[0:3], TODAY, crashid), data=b"test", ) boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v1/processed_crash/%s" % crashid, data=b"test", ) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == []
def test_get_entropy(self): cmd = Command() entropy = list(sorted(cmd.get_entropy())) # We don't want to assert the contents of the whole list, so let's # just assert some basic facts and it's probably fine assert len(entropy) == 4096 assert entropy[0] == "000" assert entropy[-1] == "fff"
def test_no_crashes(self, boto_helper, monkeypatch): """Verify no crashes in bucket result in no missing crashes.""" monkeypatch.setattr(Command, "get_entropy", get_small_entropy) bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"] boto_helper.create_bucket(bucket) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == []
def test_handle_missing_some_missing(self, capsys, db): crash_ids = [create_new_ooid(), create_new_ooid()] crash_ids.sort() cmd = Command() cmd.handle_missing(TODAY, crash_ids) captured = capsys.readouterr() assert "Missing: %s" % crash_ids[0] in captured.out assert "Missing: %s" % crash_ids[1] in captured.out assert crash_ids == list(self.fetch_crashids())
def test_past_missing_still_missing(self, capsys, db): # Create a MissingProcessedCrash row, but don't put the processed crash in the # bucket. After check_past_missing() runs, the MissingProcessedCrash should # still have is_processed=False. crash_id = create_new_ooid() mpe = MissingProcessedCrash(crash_id=crash_id, is_processed=False) mpe.save() cmd = Command() cmd.check_past_missing() mpe = MissingProcessedCrash.objects.get(crash_id=crash_id) assert mpe.is_processed is False
def test_past_missing_no_longer_missing(self, capsys, db, boto_helper): # Create a MissingProcessedCrash row and put the processed crash in the S3 # bucket. After check_past_missing() runs, the MissingProcessedCrash should # have is_processed=True. crash_id = create_new_ooid() mpe = MissingProcessedCrash(crash_id=crash_id, is_processed=False) mpe.save() boto_helper.upload_fileobj( bucket_name=BUCKET_NAME, key="v1/processed_crash/%s" % crash_id, data=b"test", ) cmd = Command() cmd.check_past_missing() mpe = MissingProcessedCrash.objects.get(crash_id=crash_id) assert mpe.is_processed is True
def test_missing_crashes(self, boto_helper, es_conn, monkeypatch): """Verify it finds a missing crash.""" monkeypatch.setattr(Command, "get_entropy", get_small_entropy) bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"] boto_helper.create_bucket(bucket) # Create a raw and processed crash crash_id_1 = "000" + create_new_ooid()[3:] self.create_raw_crash_in_s3(boto_helper, crash_id_1) self.create_processed_crash_in_s3(boto_helper, crash_id_1) self.create_processed_crash_in_es(es_conn, crash_id_1) # Create a raw crash crash_id_2 = "000" + create_new_ooid()[3:] self.create_raw_crash_in_s3(boto_helper, crash_id_2) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == [crash_id_2]
def test_no_missing_crashes(self, boto_helper, es_conn, monkeypatch): """Verify raw crashes with processed crashes result in no missing crashes.""" monkeypatch.setattr(Command, "get_entropy", get_small_entropy) bucket = settings.SOCORRO_CONFIG["resource"]["boto"]["bucket_name"] boto_helper.create_bucket(bucket) # Create a few raw and processed crashes crashids = [ "000" + create_new_ooid()[3:], "000" + create_new_ooid()[3:], "000" + create_new_ooid()[3:], ] for crash_id in crashids: self.create_raw_crash_in_s3(boto_helper, crash_id) self.create_processed_crash_in_s3(boto_helper, crash_id) self.create_processed_crash_in_es(es_conn, crash_id) es_conn.refresh() cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == []
def test_handle_missing_none_missing(self, capsys): cmd = Command() cmd.handle_missing(TODAY, []) captured = capsys.readouterr() assert ("All crashes for %s were processed." % TODAY) in captured.out
def test_no_crashes(self, boto_helper): """Verify no crashes in bucket result in no missing crashes.""" boto_helper.get_or_create_bucket(BUCKET_NAME) cmd = Command() missing = cmd.find_missing(num_workers=1, date=TODAY) assert missing == []