def _handle_msg(self, event_source, event):
'''Handle message from initiator.'''
logging.debug('InitiatorConnection: from %s: %r', self.initiator_name,
event.msg)
try:
if event.msg['type'] == 'build-request':
if (event.msg.get('protocol_version') !=
distbuild.protocol.VERSION):
msg = distbuild.message('build-failed',
id=event.msg['id'],
reason=('Protocol version mismatch between server & '
'initiator: distbuild network uses distbuild '
'protocol version %i, but client uses version'
' %i.', distbuild.protocol.VERSION,
event.msg.get('protocol_version')))
self.jm.send(msg)
self._log_send(msg)
return
new_id = self._idgen.next()
self.our_ids.add(new_id)
self._route_map.add(event.msg['id'], new_id)
event.msg['id'] = new_id
build_controller = distbuild.BuildController(
self, event.msg, self.artifact_cache_server,
self.morph_instance)
self.mainloop.add_state_machine(build_controller)
except (KeyError, ValueError) as ex:
logging.error('Invalid message from initiator: %s: exception %s',
event.msg, ex)
def _maybe_cancel(self, event_source, build_cancel):
if build_cancel.id not in self._job.initiators:
return # event not relevant
logging.debug('WC: BuildController %r requested a cancel',
event_source)
if (len(self._job.initiators) == 1):
logging.debug('WC: Cancelling running job %s '
'with job id %s running on %s',
self._job.artifact.basename(),
self._job.id,
self.name())
msg = distbuild.message('exec-cancel', id=self._job.id)
self._jm.send(msg)
self.mainloop.queue_event(self, _BuildCancelled())
else:
logging.debug('WC: Not cancelling running job %s with job id %s, '
'other initiators want it done: %s',
self._job.artifact.basename(),
self._job.id,
[i for i in self._job.initiators
if i != build_cancel.id])
self._job.initiators.remove(build_cancel.id)
def _start_graphing(self, event_source, event):
distbuild.crash_point()
logging.info('Start constructing build graph')
self._artifact_data = distbuild.StringBuffer()
self._artifact_error = distbuild.StringBuffer()
argv = [
self._morph_instance,
'serialise-artifact',
'--quiet',
self._request['repo'],
self._request['ref'],
self._request['morphology'],
]
if 'original_ref' in self._request:
argv.append(self._request['original_ref'])
msg = distbuild.message('exec-request',
id=self._idgen.next(),
argv=argv,
stdin_contents='')
self._helper_id = msg['id']
req = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, req)
progress = BuildProgress(self._request['id'], 'Computing build graph')
self.mainloop.queue_event(BuildController, progress)
def _send_build_progress_message(self, event_source, event):
if event.id in self.our_ids:
msg = distbuild.message('build-progress',
id=self._route_map.get_incoming_id(event.id),
message=event.message_text)
self.jm.send(msg)
self._log_send(msg)
def _start_build(self, event_source, event):
distbuild.crash_point()
self._job = event.job
self._helper_id = None
self._exec_response_msg = None
logging.debug('WC: starting build: %s for %s' %
(self._job.artifact.name, self._job.initiators))
argv = [
self._morph_instance,
'worker-build',
'--build-log-on-stdout',
self._job.artifact.name,
]
msg = distbuild.message('exec-request',
id=self._job.id,
argv=argv,
stdin_contents=distbuild.serialise_artifact(self._job.artifact),
)
self._jm.send(msg)
if self._debug_json:
logging.debug('WC: sent to worker %s: %r'
% (self._worker_name, msg))
started = WorkerBuildStepStarted(self._job.initiators,
self._job.artifact.source.cache_key, self.name())
self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job))
self.mainloop.queue_event(WorkerConnection, started)
def _start_annotating(self, event_source, event):
distbuild.crash_point()
self._artifact = event.artifact
self._helper_id = self._idgen.next()
artifact_names = []
def set_state_and_append(artifact):
artifact.state = UNKNOWN
artifact_names.append(artifact.basename())
map_build_graph(self._artifact, set_state_and_append)
url = urlparse.urljoin(self._artifact_cache_server, '/1.0/artifacts')
msg = distbuild.message('http-request',
id=self._helper_id,
url=url,
headers={'Content-type': 'application/json'},
body=json.dumps(artifact_names),
method='POST')
request = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, request)
logging.debug('Made cache request for state of artifacts '
'(helper id: %s)' % self._helper_id)
def _start_annotating(self, event_source, event):
distbuild.crash_point()
self._artifact = event.artifact
self._helper_id = self._idgen.next()
artifact_names = []
def set_state_and_append(artifact):
artifact.state = UNKNOWN
artifact_names.append(artifact.basename())
map_build_graph(self._artifact, set_state_and_append)
url = urlparse.urljoin(self._artifact_cache_server, '/1.0/artifacts')
msg = distbuild.message('http-request',
id=self._helper_id,
url=url,
headers={'Content-type': 'application/json'},
body=json.dumps(artifact_names),
method='POST')
request = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, request)
logging.debug('Made cache request for state of artifacts '
'(helper id: %s)' % self._helper_id)
def setup(self):
distbuild.crash_point()
self._jm = distbuild.JsonMachine(self._conn)
self.mainloop.add_state_machine(self._jm)
logging.debug('initiator: _jm=%s' % repr(self._jm))
spec = [
# state, source, event_class, new_state, callback
('waiting', self._jm, distbuild.JsonEof, None, self._terminate),
('waiting', self._jm, distbuild.JsonNewMessage, 'waiting',
self._handle_json_message),
('waiting', self, _Finished, None, self._succeed),
('waiting', self, _Failed, None, self._fail),
]
self.add_transitions(spec)
random_id = random.randint(0, 2**32-1)
self._app.status(
msg='Requesting build of %(repo)s %(ref)s %(morph)s',
repo=self._repo_name,
ref=self._ref,
morph=self._morphology)
msg = distbuild.message('build-request',
id=random_id,
repo=self._repo_name,
ref=self._ref,
morphology=self._morphology,
original_ref=self._original_ref,
protocol_version=distbuild.protocol.VERSION
)
self._jm.send(msg)
logging.debug('Initiator: sent to controller: %s', repr(msg))
def setup(self):
distbuild.crash_point()
self._jm = distbuild.JsonMachine(self._conn)
self.mainloop.add_state_machine(self._jm)
logging.debug('initiator: _jm=%s' % repr(self._jm))
spec = [
# state, source, event_class, new_state, callback
('waiting', self._jm, distbuild.JsonEof, None, self._terminate),
('waiting', self._jm, distbuild.JsonNewMessage, 'waiting',
self._handle_json_message),
('waiting', self, _Finished, None, self._succeed),
('waiting', self, _Failed, None, self._fail),
]
self.add_transitions(spec)
random_id = random.randint(0, 2**32 - 1)
self._app.status(msg='Requesting build of %(repo)s %(ref)s %(morph)s',
repo=self._repo_name,
ref=self._ref,
morph=self._morphology)
msg = distbuild.message('build-request',
id=random_id,
repo=self._repo_name,
ref=self._ref,
morphology=self._morphology,
original_ref=self._original_ref)
self._jm.send(msg)
logging.debug('Initiator: sent to controller: %s', repr(msg))
def _start_graphing(self, event_source, event):
distbuild.crash_point()
logging.info('Start constructing build graph')
self._artifact_data = distbuild.StringBuffer()
self._artifact_error = distbuild.StringBuffer()
argv = [
self._morph_instance,
'serialise-artifact',
'--quiet',
self._request['repo'],
self._request['ref'],
self._request['morphology'],
]
if 'original_ref' in self._request:
argv.append(self._request['original_ref'])
msg = distbuild.message('exec-request',
id=self._idgen.next(),
argv=argv,
stdin_contents='')
self._helper_id = msg['id']
req = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, req)
progress = BuildProgress(self._request['id'], 'Computing build graph')
self.mainloop.queue_event(BuildController, progress)
def _send_build_step_failed_message(self, event_source, event):
if event.id in self.our_ids:
msg = distbuild.message('step-failed',
id=self._route_map.get_incoming_id(event.id),
step_name=event.step_name)
self.jm.send(msg)
self._log_send(msg)
def _start_build(self, event_source, event):
distbuild.crash_point()
self._job = event.job
self._helper_id = None
self._exec_response_msg = None
logging.debug('WC: starting build: %s for %s' %
(self._job.artifact.name, self._job.initiators))
argv = [
self._morph_instance,
'worker-build',
'--build-log-on-stdout',
self._job.artifact.name,
]
msg = distbuild.message(
'exec-request',
id=self._job.id,
argv=argv,
stdin_contents=distbuild.serialise_artifact(self._job.artifact),
)
self._jm.send(msg)
if self._debug_json:
logging.debug('WC: sent to worker %s: %r' %
(self._worker_name, msg))
started = WorkerBuildStepStarted(self._job.initiators,
self._job.artifact.source.cache_key,
self.name())
self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job))
self.mainloop.queue_event(WorkerConnection, started)
def _maybe_cancel(self, event_source, build_cancel):
if build_cancel.id not in self._job.initiators:
return # event not relevant
logging.debug('WC: BuildController %r requested a cancel',
event_source)
if (len(self._job.initiators) == 1):
logging.debug(
'WC: Cancelling running job %s '
'with job id %s running on %s', self._job.artifact.basename(),
self._job.id, self.name())
msg = distbuild.message('exec-cancel', id=self._job.id)
self._jm.send(msg)
self.mainloop.queue_event(self, _BuildCancelled())
else:
logging.debug(
'WC: Not cancelling running job %s with job id %s, '
'other initiators want it done: %s',
self._job.artifact.basename(), self._job.id,
[i for i in self._job.initiators if i != build_cancel.id])
self._job.initiators.remove(build_cancel.id)
def _send_build_step_failed_message(self, event_source, event):
if event.id in self.our_ids:
msg = distbuild.message('step-failed',
id=self._route_map.get_incoming_id(
event.id),
step_name=event.step_name)
self.jm.send(msg)
self._log_send(msg)
def _send_build_progress_message(self, event_source, event):
if event.id in self.our_ids:
msg = distbuild.message('build-progress',
id=self._route_map.get_incoming_id(
event.id),
message=event.message_text)
self.jm.send(msg)
self._log_send(msg)
def _send_build_step_finished_message(self, event_source, event):
logging.debug('heard built step finished: event.id: %s our_ids: %s'
% (str(event.id), str(self.our_ids)))
if event.id in self.our_ids:
msg = distbuild.message('step-finished',
id=self._route_map.get_incoming_id(event.id),
step_name=event.step_name)
self.jm.send(msg)
self._log_send(msg)
def _send_build_failed_message(self, event_source, event):
if event.id in self.our_ids:
msg = distbuild.message('build-failed',
id=self._route_map.get_incoming_id(event.id),
reason=event.reason)
self._route_map.remove(event.id)
self.our_ids.remove(event.id)
self.jm.send(msg)
self._log_send(msg)
def _send_build_failed_message(self, event_source, event):
if event.id in self.our_ids:
msg = distbuild.message('build-failed',
id=self._route_map.get_incoming_id(
event.id),
reason=event.reason)
self._route_map.remove(event.id)
self.our_ids.remove(event.id)
self.jm.send(msg)
self._log_send(msg)
def _send_build_step_finished_message(self, event_source, event):
logging.debug('heard built step finished: event.id: %s our_ids: %s' %
(str(event.id), str(self.our_ids)))
if event.id in self.our_ids:
msg = distbuild.message('step-finished',
id=self._route_map.get_incoming_id(
event.id),
step_name=event.step_name)
self.jm.send(msg)
self._log_send(msg)
def _send_build_step_started_message(self, event_source, event):
logging.debug('InitiatorConnection: build_step_started: '
'id=%s step_name=%s worker_name=%s' %
(event.id, event.step_name, event.worker_name))
if event.id in self.our_ids:
msg = distbuild.message('step-started',
id=self._route_map.get_incoming_id(event.id),
step_name=event.step_name,
worker_name=event.worker_name)
self.jm.send(msg)
self._log_send(msg)
def _request_command_execution(self, argv, request_id):
'''Tell the controller's distbuild-helper to run a command.'''
if self.mainloop.n_state_machines_of_type(distbuild.HelperRouter) == 0:
self.fail('No distbuild-helper process running on controller!')
msg = distbuild.message('exec-request',
id=request_id,
argv=argv,
stdin_contents='')
req = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, req)
def _send_build_output_message(self, event_source, event):
logging.debug('InitiatorConnection: build_output: '
'id=%s stdout=%s stderr=%s' %
(repr(event.id), repr(event.stdout), repr(event.stderr)))
if event.id in self.our_ids:
msg = distbuild.message('step-output',
id=self._route_map.get_incoming_id(event.id),
step_name=event.step_name,
stdout=event.stdout,
stderr=event.stderr)
self.jm.send(msg)
self._log_send(msg)
def _send_build_step_started_message(self, event_source, event):
logging.debug('InitiatorConnection: build_step_started: '
'id=%s step_name=%s worker_name=%s' %
(event.id, event.step_name, event.worker_name))
if event.id in self.our_ids:
msg = distbuild.message('step-started',
id=self._route_map.get_incoming_id(
event.id),
step_name=event.step_name,
worker_name=event.worker_name)
self.jm.send(msg)
self._log_send(msg)
def _send_build_output_message(self, event_source, event):
logging.debug('InitiatorConnection: build_output: '
'id=%s stdout=%s stderr=%s' %
(repr(event.id), repr(event.stdout), repr(event.stderr)))
if event.id in self.our_ids:
msg = distbuild.message('step-output',
id=self._route_map.get_incoming_id(
event.id),
step_name=event.step_name,
stdout=event.stdout,
stderr=event.stderr)
self.jm.send(msg)
self._log_send(msg)
def _request_caching(self, event_source, event):
# This code should be moved into the morphlib.remoteartifactcache
# module. It would be good to share it with morphlib.buildcommand,
# which also wants to fetch artifacts from a remote cache.
distbuild.crash_point()
logging.debug('Requesting shared artifact cache to get artifacts')
kind = self._job.artifact.source.morphology['kind']
if kind == 'chunk':
source_artifacts = self._job.artifact.source.artifacts
suffixes = ['%s.%s' % (kind, name) for name in source_artifacts]
suffixes.append('build-log')
else:
filename = '%s.%s' % (kind, self._job.artifact.name)
suffixes = [filename]
if kind == 'stratum':
suffixes.append(filename + '.meta')
suffixes = [urllib.quote(x) for x in suffixes]
suffixes = ','.join(suffixes)
worker_host = self._conn.getpeername()[0]
url = urlparse.urljoin(
self._writeable_cache_server,
'/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' %
(urllib.quote(worker_host), self._worker_cache_server_port,
urllib.quote(self._job.artifact.source.cache_key), suffixes))
msg = distbuild.message('http-request',
id=self._request_ids.next(),
url=url,
method='GET',
body=None,
headers=None)
self._helper_id = msg['id']
req = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, req)
progress = WorkerBuildCaching(self._job.initiators,
self._job.artifact.source.cache_key)
self.mainloop.queue_event(WorkerConnection, progress)
def _request_caching(self, event_source, event):
# This code should be moved into the morphlib.remoteartifactcache
# module. It would be good to share it with morphlib.buildcommand,
# which also wants to fetch artifacts from a remote cache.
distbuild.crash_point()
logging.debug('Requesting shared artifact cache to get artifacts')
kind = self._job.artifact.source.morphology['kind']
if kind == 'chunk':
source_artifacts = self._job.artifact.source.artifacts
suffixes = ['%s.%s' % (kind, name) for name in source_artifacts]
suffixes.append('build-log')
else:
filename = '%s.%s' % (kind, self._job.artifact.name)
suffixes = [filename]
if kind == 'stratum':
suffixes.append(filename + '.meta')
suffixes = [urllib.quote(x) for x in suffixes]
suffixes = ','.join(suffixes)
worker_host = self._conn.getpeername()[0]
url = urlparse.urljoin(
self._writeable_cache_server,
'/1.0/fetch?host=%s:%d&cacheid=%s&artifacts=%s' %
(urllib.quote(worker_host),
self._worker_cache_server_port,
urllib.quote(self._job.artifact.source.cache_key),
suffixes))
msg = distbuild.message(
'http-request', id=self._request_ids.next(), url=url,
method='GET', body=None, headers=None)
self._helper_id = msg['id']
req = distbuild.HelperRequest(msg)
self.mainloop.queue_event(distbuild.HelperRouter, req)
progress = WorkerBuildCaching(self._job.initiators,
self._job.artifact.source.cache_key)
self.mainloop.queue_event(WorkerConnection, progress)
def _send_build_steps_message(self, event_source, event):
def make_step_dict(artifact):
return {
'name': distbuild.build_step_name(artifact),
'build-depends': [
distbuild.build_step_name(x)
for x in artifact.source.dependencies
]
}
if event.id in self.our_ids:
step_names = distbuild.map_build_graph(
event.artifact, make_step_dict)
msg = distbuild.message('build-steps',
id=self._route_map.get_incoming_id(event.id),
steps=step_names)
self.jm.send(msg)
self._log_send(msg)
def _send_build_steps_message(self, event_source, event):
def make_step_dict(artifact):
return {
'name':
distbuild.build_step_name(artifact),
'build-depends': [
distbuild.build_step_name(x)
for x in artifact.source.dependencies
]
}
if event.id in self.our_ids:
step_names = distbuild.map_build_graph(event.artifact,
make_step_dict)
msg = distbuild.message('build-steps',
id=self._route_map.get_incoming_id(
event.id),
steps=step_names)
self.jm.send(msg)
self._log_send(msg)
