def _client_request_received(self, server, msg):
try:
request = json.loads(msg[1])
except json.JSONDecodeError as e:
# we ignore invalid requests
log.info('Received invalid JSON request from client: %s (%s)', msg,
str(e))
return
try:
reply = self._worker.process_client_message(request)
except Exception as e:
reply = json_compact_dump(
{'error': 'Internal Error: {}'.format(e)}, as_bytes=True)
# an empty result means we shouldn't send a message back
if not reply:
return
# ensure we have the bytes of a JSON string
# (workers are permitted to return e.g. True or UTF-8 strings)
if type(reply) is str:
reply = reply.encode('utf-8')
elif type(reply) is not bytes:
reply = json_compact_dump(reply, as_bytes=True)
reply_msg = [msg[0], reply]
log.debug('Sending %s', reply_msg)
server.send_multipart(reply_msg)
def sign_json(json_object, signature_name, signing_key):
'''
Sign the JSON object. Stores the signature in json_object['signatures'].
Args:
json_object (dict): The JSON object to sign.
signature_name (str): The name of the signing entity.
signing_key (syutil.crypto.SigningKey): The key to sign the JSON with.
Returns:
The modified, signed JSON object.
'''
signatures = json_object.pop('signatures', {})
unsigned = json_object.pop('unsigned', None)
message = json_compact_dump(json_object, as_bytes=True)
signed = signing_key.sign(message)
signature_base64 = encode_base64(signed.signature)
key_id = '%s:%s' % (signing_key.alg, signing_key.version)
signatures.setdefault(signature_name, {})[key_id] = signature_base64
json_object['signatures'] = signatures
if unsigned is not None:
json_object['unsigned'] = unsigned
return json_object
def submit_event_message(socket, sender, tag, data, key):
'''
Create a new event message, sign it and send it via the specified socket.
'''
if not socket:
return # don't send the message if we do not have a valid socket
msg = create_event_message(sender, tag, data, key)
socket.send_string(json_compact_dump(msg))
def _emit_event(self, subject, data):
if not self._event_pub_queue:
return # do nothing if event publishing is disabled
tag = create_message_tag('jobs', subject)
msg = {'tag': tag,
'uuid': str(uuid.uuid1()),
'format': '1.0',
'time': datetime.now().isoformat(),
'data': data}
self._event_pub_queue.put([bytes(tag, 'utf-8'),
bytes(json_compact_dump(msg), 'utf-8')])
def _publish_event(self, event):
# sign outgoing trusted message with our key
# anything that is in this queue has already been
# checked and is trusted
event = self._sign_message(event)
new_data = json_compact_dump(event)
# create message
msg = [bytes(event['tag'], 'utf-8'),
bytes(new_data, 'utf-8')]
# send message
for socket in self._sockets:
try:
socket.send_multipart(msg)
except Exception as e:
log.warning('Unable to publish event: {} (data was: {})'.format(str(e), str(msg)))
def verify_signed_json(json_object, signature_name, verify_key):
'''
Check a signature on a signed JSON object.
Args:
json_object (dict): The signed JSON object to check.
signature_name (str): The name of the signature to check.
verify_key (syutil.crypto.VerifyKey): The key to verify the signature.
Raises:
InvalidSignature: If the signature isn't valid
'''
try:
signatures = json_object['signatures']
except KeyError:
raise SignatureVerifyException('No signatures on this object')
key_id = '%s:%s' % (verify_key.alg, verify_key.version)
try:
signature_b64 = signatures[signature_name][key_id]
except Exception:
raise SignatureVerifyException('Missing signature for {}, {}'.format(
signature_name, key_id))
try:
signature = decode_base64(signature_b64)
except Exception:
raise SignatureVerifyException(
'Invalid signature base64 for {}, {}'.format(
signature_name, key_id))
json_object_copy = dict(json_object)
del json_object_copy['signatures']
json_object_copy.pop('unsigned', None)
message = json_compact_dump(json_object_copy, as_bytes=True)
try:
verify_key.verify(message, signature)
except Exception:
log.exception('Error verifying signature')
raise SignatureVerifyException(
'Unable to verify signature for {}'.format(signature_name))
def _error_reply(self, message):
return json_compact_dump({'error': message})
def _process_job_request(self, session, req_data):
'''
Read job request and return a job matching the request or
null in case we couldn't find any job.
'''
client_name = req_data.get('machine_name')
client_id = req_data.get('machine_id')
architectures = req_data.get('architectures', [])
# update information about this client
worker = session.query(SparkWorker) \
.filter(SparkWorker.uuid == client_id).one_or_none()
# we might have a new machine, so set the ID again to create an empty new worker
if not worker:
worker = SparkWorker()
# this may throw an exception which is caought and sent back to the worker
# (the worker then has the oportunity to fix its UUID)
try:
worker.uuid = uuid.UUID(client_id)
except TypeError as e:
return self._error_reply(
'Failed to parse client UUID: {}'.format(str(e)))
worker.name = client_name
worker.enabled = True
session.add(worker)
worker.last_ping = datetime.utcnow()
accepted_kinds = req_data.get('accepts', [])
if type(accepted_kinds) is list:
worker.accepts = accepted_kinds
else:
worker.accepts = [str(accepted_kinds)]
session.commit()
job_data = None
job_assigned = False
for accepted_kind in worker.accepts:
for arch_name in architectures:
job = None
if arch_name == self._arch_indep_affinity:
# we can maybe assign an arch:all job to this machine
job = self._assign_suitable_job(session, accepted_kind,
'all', worker.uuid)
# use the first job with a matching architecture/kind if we didn't find an arch:all job previously
if not job:
job = self._assign_suitable_job(session, accepted_kind,
arch_name, worker.uuid)
if job:
job_data = self._get_job_details(session, job)
job_assigned = True
event_data = {
'job_id': job_data['uuid'],
'client_name': client_name,
'client_id': client_id,
'job_module': job_data['module'],
'job_kind': job_data['kind'],
'job_version': job_data['version'],
'job_architecture': job_data['architecture']
}
self._emit_event('job-assigned', event_data)
break
if job_assigned:
break
return json_compact_dump(job_data)