def __init__(self):
big_query = BigQuery()
self.querier = SLIViewQuerier(big_query, QualityQuerySpecification())
self.streamer = SLIResultsStreamer(table_id="SLI_backup_quality")
self.table_newer_modification_predicate = SLITableNewerModificationPredicate(
big_query)
self.table_existence_predicate = SLITableExistsPredicate(
big_query, QualityQuerySpecification)
def __init__(self, x_days):
self.x_days = x_days
big_query = BigQuery()
self.streamer = SLIResultsStreamer(
table_id="SLI_backup_creation_latency"
)
self.table_existence_predicate = SLITableExistsPredicate(big_query, LatencyQuerySpecification)
self.table_recreation_predicate = SLITableRecreationPredicate(big_query)
self.table_emptiness_predicate = SLITableEmptinessPredicate(big_query)
self.table_has_any_backup_predicate = SLITableHasAnyBackupPredicate()
class LatencyViolationSliService(object):
def __init__(self, x_days):
self.x_days = x_days
big_query = BigQuery()
self.streamer = SLIResultsStreamer(
table_id="SLI_backup_creation_latency"
)
self.table_existence_predicate = SLITableExistsPredicate(big_query, LatencyQuerySpecification)
self.table_recreation_predicate = SLITableRecreationPredicate(big_query)
self.table_emptiness_predicate = SLITableEmptinessPredicate(big_query)
self.table_has_any_backup_predicate = SLITableHasAnyBackupPredicate()
def check_and_stream_violation(self, json_table):
if self.__should_stay_as_sli_violation(json_table):
filtered_table = [json_table]
self.streamer.stream(filtered_table)
def __should_stay_as_sli_violation(self, table):
try:
if not self.table_existence_predicate.exists(table):
return False
if self.table_recreation_predicate.is_recreated(table):
return False
if self.table_has_any_backup_predicate.has_any_backup(table) and \
self.table_emptiness_predicate.is_empty(table):
return False
return True
except Exception:
logging.exception("An error occurred while filtering table %s, "
"still it will be streamed", table)
return True
class QualityViolationSliService(object):
def __init__(self):
big_query = BigQuery()
self.querier = SLIViewQuerier(big_query, QualityQuerySpecification())
self.streamer = SLIResultsStreamer(table_id="SLI_backup_quality")
self.table_newer_modification_predicate = SLITableNewerModificationPredicate(
big_query)
self.table_existence_predicate = SLITableExistsPredicate(
big_query, QualityQuerySpecification)
def check_and_stream_violation(self, json_table):
if self.__should_stay_as_sli_violation(json_table):
filtered_table = [json_table]
self.streamer.stream(filtered_table)
def __should_stay_as_sli_violation(self, table):
try:
if not self.table_existence_predicate.exists(table):
return False
return not self.table_newer_modification_predicate.is_modified_since_last_census_snapshot(
table)
except Exception:
logging.exception(
"An error occurred while filtering table %s, "
"still it will be streamed", table)
return True
def test_should_return_false_when_there_is_no_schema(self):
# given
sli_table = self.__create_non_partitioned_sli_table()
# when
exists = SLITableExistsPredicate(BigQuery(), LatencyQuerySpecification).exists(sli_table)
# then
self.assertFalse(exists)
def test_should_return_true_for_existing_partition(self):
# given
sli_table = self.__create_partitioned_sli_table()
# when
exists = SLITableExistsPredicate(BigQuery(), LatencyQuerySpecification).exists(sli_table)
# then
self.assertTrue(exists)
def test_should_not_list_partitions_in_non_partitioned_table(self, list_table_partitions):
# given
sli_table = self.__create_non_partitioned_sli_table()
# when
exists = SLITableExistsPredicate(BigQuery(), LatencyQuerySpecification).exists(sli_table)
# then
self.assertTrue(exists)
list_table_partitions.assert_not_called()
def __init__(self, x_days):
self.x_days = x_days
big_query = BigQuery()
self.querier = SLIViewQuerier(big_query,
LatencyQuerySpecification(self.x_days))
self.streamer = SLIResultsStreamer(
table_id="SLI_backup_creation_latency")
self.table_existence_predicate = SLITableExistsPredicate(
big_query, LatencyQuerySpecification)
self.table_recreation_predicate = SLITableRecreationPredicate(
big_query)