def __init__(self, redis, cassandra, repositories=[], default_ttl_seconds=TTL_YEAR * 5, archive_ttl_seconds=TTL_WEEK * 8, async_processing=False, s3_credentials=None):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
if archive_ttl_seconds is not None and archive_ttl_seconds < 2 * self.TTL_WEEK:
raise ValueError('Archive TTL must be at least 2 weeks')
self.default_ttl_seconds = default_ttl_seconds
self.archive_ttl_seconds = archive_ttl_seconds or default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
with self.cassandra:
self.cassandra.create_table(self.HealthTable)
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.failure_context = FailureContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [self.ci_context, self.failure_context, self.suite_context, self.test_context]:
self.upload_context.register_upload_callback(context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.archive_ttl_seconds,
s3_credentials=s3_credentials,
)
def __init__(self,
redis,
cassandra,
repositories=[],
default_ttl_seconds=TTL_YEAR * 5,
async_processing=False):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
self.default_ttl_seconds = default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [
self.suite_context, self.test_context, self.ci_context
]:
self.upload_context.register_upload_callback(
context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
class Model(object):
TTL_DAY = 60 * 60 * 24
TTL_WEEK = 7 * TTL_DAY
TTL_YEAR = 365 * TTL_DAY
class HealthTable(CassandraModel):
__table_name__ = 'health'
key = columns.Text(partition_key=True, required=True)
value = columns.Text(required=True)
def __init__(self,
redis,
cassandra,
repositories=[],
default_ttl_seconds=TTL_YEAR * 5,
archive_ttl_seconds=TTL_WEEK * 8,
async_processing=False,
s3_credentials=None):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
if archive_ttl_seconds is not None and archive_ttl_seconds < 2 * self.TTL_WEEK:
raise ValueError('Archive TTL must be at least 2 weeks')
self.default_ttl_seconds = default_ttl_seconds
self.archive_ttl_seconds = archive_ttl_seconds or default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
with self.cassandra:
self.cassandra.create_table(self.HealthTable)
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.failure_context = FailureContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [
self.ci_context, self.failure_context, self.suite_context,
self.test_context, self.commit_context
]:
self.upload_context.register_upload_callback(
context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.archive_ttl_seconds,
s3_credentials=s3_credentials,
)
def healthy(self, writable=True):
if writable:
self.redis.set('health-check', b'healthy', ex=5 * 60)
value = self.redis.get('health-check')
if value is not None and value != b'healthy':
return False
with self.cassandra:
if writable:
self.cassandra.insert_row(
self.HealthTable.__table_name__,
key='health-check',
value='healthy',
ttl=5 * 60,
)
values = [
element.value for element in self.cassandra.select_from_table(
self.HealthTable.__table_name__,
limit=100,
key='health-check',
)
]
if values and values != ['healthy']:
return False
return True
def do_work(self):
if not self._async_processing:
raise RuntimeError(
'No work to be done, asynchronous processing disabled')
try:
return self.upload_context.do_processing_work()
except Exception as e:
sys.stderr.write(f'{traceback.format_exc()}\n')
sys.stderr.write(f'{e}\n')
return False
def test_zipping(self):
zipped_bytes = UploadContext.to_zip('somestring' * 100)
self.assertNotEqual(zipped_bytes, 'somestring' * 100)
self.assertEqual(UploadContext.from_zip(zipped_bytes),
'somestring' * 100)
class Model(object):
TTL_DAY = 60 * 60 * 24
TTL_WEEK = 7 * TTL_DAY
TTL_YEAR = 365 * TTL_DAY
def __init__(self,
redis,
cassandra,
repositories=[],
default_ttl_seconds=TTL_YEAR * 5,
async_processing=False):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
self.default_ttl_seconds = default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [
self.suite_context, self.test_context, self.ci_context
]:
self.upload_context.register_upload_callback(
context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
def do_work(self):
if not self._async_processing:
raise RuntimeError(
'No work to be done, asynchronous processing disabled')
try:
self.upload_context.do_processing_work()
except Exception as e:
sys.stderr.write(f'{traceback.format_exc()}\n')
sys.stderr.write(f'{e}\n')
