def __create__random_result(self):
def rnd():
return randint(1, 1000)
suffix = rnd()
env = Env(name=f'Test Env {suffix}', short_name=f'TE{suffix}')
component = Component(name=f'Test Component {suffix}')
item = Item(name=f'Test Item {suffix}', args=f'args {suffix}')
status = Status(test_status='Passed', priority=suffix)
platform = Platform(name=f'Test Platform {suffix}')
os = Os(name=f'Test Os {suffix}')
run = Run(name=f'Test Run {suffix}', session=f'Test Session {suffix}')
return Result(
validation=Validation(
name=f'Test Validation {suffix}',
env=env,
platform=platform,
os=os,
),
env=env,
platform=platform,
os=os,
component=component,
item=item,
status=status,
run=run,
)
def heroes_create():
hero_json = request.get_json()
status = Status.query.filter_by(status=hero_json['status']).first()
if not status:
status = Status(status=hero_json['status'])
db.session.add(status)
db.session.commit()
# https://docs.sqlalchemy.org/en/latest/orm/tutorial.html#common-filter-operators
hero = db.session.query(SuperHero).filter(or_(SuperHero.superhero_alias == hero_json.get('superhero_alias'),\
SuperHero.email_address == hero_json.get('email_address')))\
.first()
if not hero:
new_hero = SuperHero(superhero_alias=hero_json.get('superhero_alias'),
email_address=hero_json.get('email_address'),
first_name=hero_json.get('first_name'),
last_name=hero_json.get('last_name'),
started_on=hero_json.get('started_on',
datetime.utcnow()),
finished_on=hero_json.get('finished_on'),
income=hero_json.get('income'),
status=status)
db.session.add(new_hero)
db.session.commit()
return 'KER-SPLOOSH! New hero added to the data trust.', 201
else:
# https://httpstatuses.com/409
return 'ZOWIE! A hero with the same superhero_alias or email_address already exists.', 409
def get_oldest_object_to_retry(self):
"""Grab the oldest report or report slice object to retry.
returns: object to retry or None.
"""
status_info = Status()
current_time = datetime.now(pytz.utc)
objects_count = self.calculate_queued_objects(current_time,
status_info)
if self.object_class == Report:
QUEUED_REPORTS.set(objects_count)
else:
QUEUED_REPORT_SLICES.set(objects_count)
LOG.info(
format_message(
self.prefix, 'Number of %s waiting to be processed: %s' %
(self.object_prefix.lower() + 's', objects_count)))
# first we have to query for all objects with commit retries
commit_retry_query = self.object_class.objects.filter(
retry_type=self.object_class.GIT_COMMIT)
# then we grab the oldest object from the query
oldest_commit_object = self.return_queryset_object(
queryset=commit_retry_query)
if oldest_commit_object:
same_commit = oldest_commit_object.git_commit == status_info.git_commit
if not same_commit:
return oldest_commit_object
# If the above doesn't return, we should query for all time retries
time_retry_query = self.object_class.objects.filter(
retry_type=self.object_class.TIME)
oldest_time_object = self.return_queryset_object(
queryset=time_retry_query)
if oldest_time_object:
minutes_passed = int(
(current_time -
oldest_time_object.last_update_time).total_seconds() / 60)
if minutes_passed >= RETRY_TIME:
return oldest_time_object
# if we haven't returned a retry object, return None
return None
def setUpClass(cls):
"""Test Class setup."""
# remove filters on logging
logging.disable(logging.NOTSET)
cls.status_info = Status()
def update_slice_state(self, options,
report_slice): # noqa: C901 (too-complex)
"""
Update the report processor state and save.
:param options: <dict> dictionary potentially containing the following:
report_slice: <ReportSlice> the report slice to update
state: <str> the state to update to
retry: <enum> Retry.clear=clear count, RETRY.increment=increase count
retry_type: <str> either time=retry after time,
git_commit=retry after new commit
report_json: <dict> dictionary containing the report json
report_platform_id: <str> string containing report_platform_id
candidate_hosts: <dict> dictionary containing hosts that were
successfully verified and uploaded
failed_hosts: <dict> dictionary containing hosts that failed
verification or upload
ready_to_archive: <bool> boolean on whether or not to archive
"""
try:
state = options.get('state')
retry_type = options.get('retry_type')
retry = options.get('retry', RETRY.clear)
candidate_hosts = options.get('candidate_hosts')
failed_hosts = options.get('failed_hosts')
ready_to_archive = options.get('ready_to_archive')
status_info = Status()
report_slice.last_update_time = datetime.now(pytz.utc)
report_slice.state = state
report_slice.git_commit = status_info.git_commit
report_slice_data = {
'last_update_time': datetime.now(pytz.utc),
'state': state,
'git_commit': status_info.git_commit
}
if not retry_type:
retry_type = ReportSlice.TIME
if retry == RETRY.clear:
# After a successful transaction when we have reached the update
# point, we want to set the retry count back to 0 because
# any future failures should be unrelated
report_slice_data['retry_count'] = 0
report_slice_data['retry_type'] = ReportSlice.TIME
elif retry == RETRY.increment:
current_count = report_slice.retry_count
report_slice_data['retry_count'] = current_count + 1
report_slice_data['retry_type'] = ReportSlice.TIME
# the other choice for retry is RETRY.keep_same in which case we don't
# want to do anything to the retry count bc we want to preserve as is
if candidate_hosts is not None:
# candidate_hosts will get smaller and smaller until it hopefully
# is empty because we have taken care of all ofthe candidates so
# we rewrite this each time
report_slice_data['candidate_hosts'] = json.dumps(
candidate_hosts)
if failed_hosts:
# for failed hosts this list can keep growing, so we add the
# newly failed hosts to the previous value
failed = json.loads(report_slice.failed_hosts)
for host in failed_hosts:
failed.append(host)
report_slice_data['failed_hosts'] = json.dumps(failed)
if ready_to_archive:
report_slice_data['ready_to_archive'] = ready_to_archive
state_info = json.loads(report_slice.state_info)
state_info.append(state)
report_slice_data['state_info'] = json.dumps(state_info)
serializer = ReportSliceSerializer(instance=report_slice,
data=report_slice_data,
partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
LOG.info(
format_message(self.prefix,
'Successfully updated report slice %s' %
report_slice.report_slice_id,
account_number=self.account_number,
report_platform_id=self.report_platform_id))
except Exception as error: # pylint: disable=broad-except
DB_ERRORS.inc()
self.should_run = False
LOG.error(
format_message(
self.prefix,
'Could not update report slice record due to the following error %s.'
% str(error),
account_number=self.account_number,
report_platform_id=self.report_platform_id))
print_error_loop_event()
def update_object_state(self, options): # noqa: C901 (too-complex)
"""
Update the report processor state and save.
:param options: <dict> potentially containing the following:
retry: <enum> Retry.clear=clear count, RETRY.increment=increase count
retry_type: <str> either time=retry after time,
git_commit=retry after new commit
report_json: <dict> dictionary containing the report json
report_platform_id: <str> string containing report_platform_id
candidate_hosts: <dict> dictionary containing hosts that were
successfully verified and uploaded
failed_hosts: <dict> dictionary containing hosts that failed
verification or upload
status: <str> either success or failure based on the report
host_inventory_api_version: <str> the inventory api version
source: <str> containing either qpc or satellite
source_metadata: <dict> containing metadata info about the source
ready_to_archive: <bool> bool regarding archive
"""
try:
status_info = Status()
self.state = self.next_state
# grab all of the potential options
retry_type = options.get('retry_type', self.object_class.TIME)
retry = options.get('retry', RETRY.clear)
report_json = options.get('report_json')
report_platform_id = options.get('report_platform_id')
candidate_hosts = options.get('candidate_hosts')
failed_hosts = options.get('failed_hosts')
status = options.get('status')
host_inventory_api_version = options.get('host_inventory_api_version')
source = options.get('source')
source_metadata = options.get('source_metadata')
ready_to_archive = options.get('ready_to_archive')
start_processing = options.get('start_processing')
update_data = {
'last_update_time': datetime.now(pytz.utc),
'state': self.next_state,
'git_commit': status_info.git_commit
}
# if this is the start of the processing, update the processing
# start time
if start_processing:
update_data['processing_start_time'] = datetime.now(pytz.utc)
if retry == RETRY.clear:
# After a successful transaction when we have reached the update
# point, we want to set the Retry count back to 0 because
# any future failures should be unrelated
update_data['retry_count'] = 0
update_data['retry_type'] = self.object_class.TIME
elif retry == RETRY.increment:
retry_count = self.report_or_slice.retry_count
update_data['retry_count'] = retry_count + 1
update_data['retry_type'] = retry_type
# the other choice for retry is RETRY.keep_same in which case we don't
# want to do anything to the retry count bc we want to preserve as is
if report_json:
update_data['report_json'] = json.dumps(report_json)
if report_platform_id:
update_data['report_platform_id'] = report_platform_id
if candidate_hosts is not None:
# candidate_hosts will get smaller and smaller until it hopefully
# is empty because we have taken care of all ofthe candidates so
# we rewrite this each time
update_data['candidate_hosts'] = json.dumps(candidate_hosts)
if failed_hosts:
# for failed hosts this list can keep growing, so we add the
# newly failed hosts to the previous value
failed = json.loads(self.report_or_slice.failed_hosts)
for host in failed_hosts:
failed.append(host)
update_data['failed_hosts'] = json.dumps(failed)
if status:
update_data['upload_ack_status'] = status
if host_inventory_api_version:
update_data['host_inventory_api_version'] = \
host_inventory_api_version
if source:
update_data['source'] = source
if source_metadata:
update_data['source_metadata'] = json.dumps(source_metadata)
if ready_to_archive:
update_data['ready_to_archive'] = ready_to_archive
state_info = json.loads(self.report_or_slice.state_info)
state_info.append(self.next_state)
update_data['state_info'] = json.dumps(state_info)
serializer = self.object_serializer(
instance=self.report_or_slice,
data=update_data,
partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
except Exception as error:
DB_ERRORS.inc()
self.should_run = False
LOG.error(format_message(
self.prefix,
'Could not update %s record due to the following error %s.' % (
self.object_prefix.lower(), str(error)),
account_number=self.account_number, report_platform_id=self.report_platform_id))
print_error_loop_event()
def test_calculating_queued_reports(self):
"""Test the calculate_queued_reports method."""
status_info = Status()
current_time = datetime.now(pytz.utc)
self.report_record.state = Report.NEW
self.report_record.save()
reports_to_process = self.processor.calculate_queued_objects(
current_time, status_info)
self.assertEqual(reports_to_process, 1)
min_old_time = current_time - timedelta(hours=8)
older_report = Report(upload_srv_kafka_msg=json.dumps(self.msg),
account='4321',
report_platform_id=self.uuid2,
state=Report.STARTED,
state_info=json.dumps([Report.NEW]),
last_update_time=min_old_time,
retry_count=1,
retry_type=Report.TIME)
older_report.save()
retry_commit_report = Report(upload_srv_kafka_msg=json.dumps(self.msg),
account='4321',
report_platform_id=self.uuid2,
state=Report.DOWNLOADED,
state_info=json.dumps([Report.NEW]),
last_update_time=min_old_time,
git_commit='3948384729',
retry_type=Report.GIT_COMMIT,
retry_count=1)
retry_commit_report.save()
# create some reports that should not be counted
not_old_enough = current_time - timedelta(hours=1)
too_young_report = Report(upload_srv_kafka_msg=json.dumps(self.msg),
account='4321',
report_platform_id=self.uuid2,
state=Report.DOWNLOADED,
state_info=json.dumps([Report.NEW]),
last_update_time=not_old_enough,
git_commit='3948384729',
retry_type=Report.TIME,
retry_count=1)
too_young_report.save()
same_commit_report = Report(upload_srv_kafka_msg=json.dumps(self.msg),
account='4321',
report_platform_id=self.uuid2,
state=Report.DOWNLOADED,
state_info=json.dumps([Report.NEW]),
last_update_time=min_old_time,
git_commit=status_info.git_commit,
retry_type=Report.GIT_COMMIT,
retry_count=1)
same_commit_report.save()
reports_to_process = self.processor.calculate_queued_objects(
current_time, status_info)
self.assertEqual(reports_to_process, 3)
# delete the older report object
Report.objects.get(id=older_report.id).delete()
Report.objects.get(id=retry_commit_report.id).delete()
Report.objects.get(id=too_young_report.id).delete()
Report.objects.get(id=same_commit_report.id).delete()
def update_slice_state(self, options,
report_slice): # noqa: C901 (too-complex)
"""
Update the report processor state and save.
:param options: <dict> dictionary potentially containing the following:
report_slice: <ReportSlice> the report slice to update
state: <str> the state to update to
retry: <enum> Retry.clear=clear count, RETRY.increment=increase count
retry_type: <str> either time=retry after time,
git_commit=retry after new commit
report_json: <dict> dictionary containing the report json
report_platform_id: <str> string containing report_platform_id
ready_to_archive: <bool> boolean on whether or not to archive
"""
try:
state = options.get('state')
retry_type = options.get('retry_type')
retry = options.get('retry', RETRY.clear)
ready_to_archive = options.get('ready_to_archive')
status_info = Status()
report_slice.last_update_time = datetime.now(pytz.utc)
report_slice.state = state
report_slice.git_commit = status_info.git_commit
report_slice_data = {
'last_update_time': datetime.now(pytz.utc),
'state': state,
'git_commit': status_info.git_commit
}
if not retry_type:
retry_type = ReportSlice.TIME
if retry == RETRY.clear:
# After a successful transaction when we have reached the update
# point, we want to set the retry count back to 0 because
# any future failures should be unrelated
report_slice_data['retry_count'] = 0
report_slice_data['retry_type'] = ReportSlice.TIME
elif retry == RETRY.increment:
current_count = report_slice.retry_count
report_slice_data['retry_count'] = current_count + 1
report_slice_data['retry_type'] = ReportSlice.TIME
# the other choice for retry is RETRY.keep_same in which case we don't
# want to do anything to the retry count bc we want to preserve as is
if ready_to_archive:
report_slice_data['ready_to_archive'] = ready_to_archive
state_info = json.loads(report_slice.state_info)
state_info.append(state)
report_slice_data['state_info'] = json.dumps(state_info)
serializer = ReportSliceSerializer(instance=report_slice,
data=report_slice_data,
partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
LOG.info(
format_message(self.prefix,
'Successfully updated report slice %s' %
report_slice.report_slice_id,
account_number=self.account_number,
report_platform_id=self.report_platform_id))
except Exception as error: # pylint: disable=broad-except
DB_ERRORS.inc()
self.should_run = False
LOG.error(
format_message(
self.prefix,
'Could not update report slice record due to the following error %s.'
% str(error),
account_number=self.account_number,
report_platform_id=self.report_platform_id))
stop_all_event_loops()
def update_object_state(self, options): # noqa: C901 (too-complex)
"""
Update the report processor state and save.
:param options: <dict> potentially containing the following:
retry: <enum> Retry.clear=clear count, RETRY.increment=increase count
retry_type: <str> either time=retry after time,
git_commit=retry after new commit
report_json: <dict> dictionary containing the report json
report_platform_id: <str> string containing report_platform_id
status: <str> either success or failure based on the report
source: <str> containing OpenShift cluster ID
source_metadata: <dict> containing metadata info about the source
ready_to_archive: <bool> bool regarding archive
"""
try:
status_info = Status()
self.state = self.next_state
# grab all of the potential options
retry_type = options.get('retry_type', self.object_class.TIME)
retry = options.get('retry', RETRY.clear)
report_json = options.get('report_json')
report_platform_id = options.get('report_platform_id')
status = options.get('status')
source = options.get('source')
source_metadata = options.get('source_metadata')
ready_to_archive = options.get('ready_to_archive')
start_processing = options.get('start_processing')
update_data = {
'last_update_time': datetime.now(pytz.utc),
'state': self.next_state,
'git_commit': status_info.git_commit
}
# if this is the start of the processing, update the processing
# start time
if start_processing:
update_data['processing_start_time'] = datetime.now(pytz.utc)
if retry == RETRY.clear:
# After a successful transaction when we have reached the update
# point, we want to set the Retry count back to 0 because
# any future failures should be unrelated
update_data['retry_count'] = 0
update_data['retry_type'] = self.object_class.TIME
elif retry == RETRY.increment:
retry_count = self.report_or_slice.retry_count
update_data['retry_count'] = retry_count + 1
update_data['retry_type'] = retry_type
# the other choice for retry is RETRY.keep_same in which case we don't
# want to do anything to the retry count bc we want to preserve as is
if report_json:
update_data['report_json'] = json.dumps(report_json)
if report_platform_id:
update_data['report_platform_id'] = report_platform_id
if status:
update_data['upload_ack_status'] = status
if source:
update_data['source'] = source
if source_metadata:
update_data['source_metadata'] = json.dumps(source_metadata)
if ready_to_archive:
update_data['ready_to_archive'] = ready_to_archive
state_info = json.loads(self.report_or_slice.state_info)
state_info.append(self.next_state)
update_data['state_info'] = json.dumps(state_info)
serializer = self.object_serializer(instance=self.report_or_slice,
data=update_data,
partial=True)
serializer.is_valid(raise_exception=True)
serializer.save()
except Exception as error:
DB_ERRORS.inc()
self.should_run = False
LOG.error(
format_message(
self.prefix,
'Could not update %s record due to the following error %s.'
% (self.object_prefix.lower(), str(error)),
account_number=self.account_number,
report_platform_id=self.report_platform_id))
stop_all_event_loops()
