def create_service(self, policy):
"""
Create job based on the policy.
The payload is tailored for the AxOps POST /v1/services. This might get improved in the future.
"""
logger.info(
"Start triggering job based on cron schedule. Policy info: \n%s",
AxPrettyPrinter().pformat(policy))
service_template = self.axops_client.get_templates(
policy['repo'], policy['branch'], name=policy['template'])[0]
commit_res = self.axops_client.get_commit_info(repo=policy['repo'],
branch=policy['branch'],
limit=1)
if not commit_res or len(commit_res) != 1:
logger.error(
"Error retrieving latest commit info for cron job, commit_info: %s. Return",
commit_res)
return
commit_json = commit_schema(commit_res[0])
notification_info = policy['notifications']
commit_info = {
'revision': commit_json['revision'],
'repo': commit_json['repo'],
'branch': commit_json['branch'],
'author': commit_json['author'],
'committer': commit_json['committer'],
'description': commit_json['description'],
'date': commit_json['date']
}
parameters = copy.deepcopy(policy['arguments'])
parameters['session.commit'] = commit_json['revision']
parameters['session.branch'] = commit_json['branch']
parameters['session.repo'] = commit_json['repo']
service = {
'template_id': service_template['id'],
'arguments': parameters,
'policy_id': policy['id'],
'commit': commit_info,
}
if notification_info:
service['notifications'] = notification_info
logger.info("Creating new service with the following payload ...\n%s",
AxPrettyPrinter().pformat(service))
service = self.axops_client.create_service(service)
logger.info('Successfully created service (id: %s)', service['id'])
def refresh_scheduler(self):
"""
Refresh the job scheduler.
The major functionality of this service. Read all the cron policies from AxOps, then
load the schedules into the job scheduler.
"""
if self._schedule_lock.acquire(
timeout=2): # Try to acquire lock for 2 seconds
try:
scheduler = BackgroundScheduler()
logger.info("Start refreshing the scheduler.")
for policy in self.axops_client.get_policy(enabled=True):
self.add_policy(policy, scheduler)
# Scheduler swap
self.stop_scheduler()
self.scheduler = scheduler
self.scheduler.start()
logger.info(
"Successfully finish refreshing the scheduler. \n%s",
AxPrettyPrinter().pformat(self.get_schedules()))
return {}
finally:
self._schedule_lock.release()
else:
with self._schedule_lock:
logger.info(
"Some other thread is refreshing the scheduler. Instant return."
)
return {'Details': 'Instant return'}
def events(self, request):
"""Create a DevOps event.
:param request:
:return:
"""
payload, headers = request.data, request.META
try:
logger.info('Translating SCM event ...')
events = EventTranslator.translate(payload, headers)
except Exception as e:
logger.error('Failed to translate event: %s', e)
# Todo Tianhe Issue: #330 comment out for now because it is distracting
# event_notification_client.send_message_to_notification_center(CODE_JOB_CI_EVENT_TRANSLATE_FAILURE,
# detail={
# 'payload': payload,
# 'error': str(e)
# })
raise AXApiInternalError('Failed to translate event',
detail=str(e))
else:
logger.info('Successfully translated event')
kafka_client = ProducerClient()
successful_events = []
for event in events:
if event['type'] == AxEventTypes.PING:
logger.info(
'Received a PING event, skipping service creation ...')
continue
else:
try:
logger.info('Creating AX event ...\n%s',
AxPrettyPrinter().pformat(event))
key = '{}_{}_{}'.format(event['repo'], event['branch'],
event['commit'])
kafka_client.send(AxSettings.TOPIC_DEVOPS_CI_EVENT,
key=key,
value=event,
timeout=120)
except Exception as e:
event_notification_client.send_message_to_notification_center(
CODE_JOB_CI_EVENT_CREATION_FAILURE,
detail={
'event_type': event.get('type', 'UNKNOWN'),
'error': str(e)
})
logger.warning('Failed to create AX event: %s', e)
else:
logger.info('Successfully created AX event')
successful_events.append(event)
kafka_client.close()
return Response(successful_events)
def test_single_workflow(self):
"""Test workflow with a single subtask.
:return:
"""
logger.info('Creating a service request with a single subtask ... ')
service_request = self._create_single_workflow()
logger.info('Successfully created a service request: \n%s',
AxPrettyPrinter().pformat(service_request))
logger.info('Start executing service ...')
service = self.axops_client.create_service(service_request)
result = self._wait_for_result(service['id'], self.timeout)
self.assertTrue(result,
'Failed to execute service ({})'.format(service['id']))
logger.info('Successfully executed service (%s)', service['id'])
def add_policy(self, policy, scheduler):
"""
Add a schedule into scheduler based on policy.
Ignore exceptions (for now).
"""
try:
policy_json = policy_schema(policy)
policy_id = policy_json['id']
event_list = policy_json['when']
logger.info("Processing policy, %s", policy_id)
for event in event_list:
if event.get('event', None) != 'on_cron':
continue
event_json = schedule_schema(event)
cron_str = event_json['schedule'].strip().split(
' ') # Parse the cron string
assert len(cron_str) == 5, "Invalid cron schedule format"
logger.info("Adding cron event, \n %s",
AxPrettyPrinter().pformat(event_json))
scheduler.add_job(
self.create_service,
'cron', # Add cron job into scheduler
id='{}-{}'.format(policy_id, cron_str),
args=[policy_json],
minute=cron_str[0],
hour=cron_str[1],
day=cron_str[2],
month=cron_str[3],
day_of_week=cron_str[4],
timezone=event_json['timezone'])
except MultipleInvalid as e:
logger.exception("Invalid cron policy format, \n%s. Details: %s",
AxPrettyPrinter().pformat(policy), str(e))
try:
if 'when' in policy:
policy['when'] = json.dumps(policy['when'])
self.event_notification_client.send_message_to_notification_center(
CODE_JOB_SCHEDULER_INVALID_POLICY_DEFINITION,
detail=policy)
except Exception:
logger.exception(
"Failed to send out alert to notification center.")
except AssertionError as e:
logger.exception(
"Invalid cron policy format, \n%s, cron string. Details: %s",
AxPrettyPrinter().pformat(policy), str(e))
try:
if 'when' in policy:
policy['when'] = json.dumps(policy['when'])
self.event_notification_client.send_message_to_notification_center(
CODE_JOB_SCHEDULER_INVALID_CRON_EXPRESSION, detail=policy)
except Exception:
logger.exception(
"Failed to send out alert to notification center.")
except Exception as e:
logger.exception(
"Failed to add event, \n%s into scheduler. Details: %s",
AxPrettyPrinter().pformat(policy), str(e))
try:
if 'when' in policy:
policy['when'] = json.dumps(policy['when'])
self.event_notification_client.send_message_to_notification_center(
CODE_JOB_SCHEDULER_CANNOT_ADD_POLICY, detail=policy)
except Exception:
logger.exception(
"Failed to send out alert to notification center.")