def delete(self, key, **kw):
key = self.prefixify(key)
result = yield From(self.delete_backend(Bucket=self.bucket, Key=key))
if result.get('ResponseMetadata').get('HTTPStatusCode') == 204:
raise Return("Success! Deleted: %s" % key)
else:
raise Return(result)
def get_unicode_from_response(r):
"""Returns the requested content back in unicode.
:param r: Response object to get unicode content from.
Tried:
1. charset from content-type
2. fall back and replace all unicode characters
"""
tried_encodings = []
# Try charset from content-type
encoding = get_encoding_from_headers(r.headers)
if encoding:
try:
rc = yield From(r.content)
raise Return(str(rc, encoding))
except UnicodeError:
tried_encodings.append(encoding)
# Fall back:
rc = yield From(r.content)
try:
raise Return(str(rc, encoding, errors='replace'))
except TypeError:
raise Return(rc)
def json(self, **kwargs):
"""Returns the json-encoded content of a response, if any.
:param \*\*kwargs: Optional arguments that ``json.loads`` takes.
"""
_content = yield From(self.content)
if not self.encoding and len(_content) > 3:
# No encoding set. JSON RFC 4627 section 3 states we should expect
# UTF-8, -16 or -32. Detect which one to use; If the detection or
# decoding fails, fall back to `self.text` (using chardet to make
# a best guess).
encoding = guess_json_utf(_content)
if encoding is not None:
try:
raise Return(
json.loads(_content.decode(encoding), **kwargs))
except UnicodeDecodeError:
# Wrong UTF codec detected; usually because it's not UTF-8
# but some other 8-bit codec. This is an RFC violation,
# and the server didn't bother to tell us what codec *was*
# used.
pass
_text = yield From(self.text)
raise Return(json.loads(_text, **kwargs))
def _on_ready(self, token, version):
logger.debug('On ready called (token "%s")', token)
self._worker_version = version
if not version in SUPPORTED_WORKER_VERSIONS:
logger.warning(
'Build component (token "%s") is running an out-of-date version: %s',
token, version)
raise Return(False)
if self._component_status != ComponentStatus.WAITING:
logger.warning('Build component (token "%s") is already connected',
self.expected_token)
raise Return(False)
if token != self.expected_token:
logger.warning(
'Builder token mismatch. Expected: "%s". Found: "%s"',
self.expected_token, token)
raise Return(False)
yield From(self._set_status(ComponentStatus.RUNNING))
# Start the heartbeat check and updating loop.
loop = trollius.get_event_loop()
loop.create_task(self._heartbeat())
logger.debug('Build worker %s is connected and ready',
self.builder_realm)
raise Return(True)
def _get_packet_from_stream(self,
stream,
existing_data,
psml_structure=None):
"""
A coroutine which returns a single packet if it can be read from the given StreamReader.
:return a tuple of (packet, remaining_data). The packet will be None if there was not enough XML data to create
a packet. remaining_data is the leftover data which was not enough to create a packet from.
:raises EOFError if EOF was reached.
"""
#yield each packet in existing_data
#Maybe there is already a packet in our buffer
packet, existing_data = self._extract_tag_from_data(existing_data)
if packet:
packet = packet_from_xml_packet(packet,
psml_structure=psml_structure)
raise Return(packet, existing_data)
new_data = yield From(stream.read(self.DEFAULT_BATCH_SIZE))
existing_data += new_data
if not new_data:
# Reached EOF
raise EOFError()
raise Return(None, existing_data)
def sock_connect(self, sock, address):
assert sock.gettimeout() == 0.0
try:
result = sock.connect(address)
raise Return()
except socket.error as e:
if e.args[0] != errno.EINPROGRESS:
raise e
pass # ignore, as we need to wait
# Nope, now we have to wait.
event = asyncio.Event()
try:
self.add_writer(sock.fileno(), event.set)
self.add_exception(sock.fileno(), event.set)
yield From(event.wait())
finally:
self.remove_writer(sock.fileno())
self.remove_exception(sock.fileno())
error = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if error != 0:
# TODO jpieper: This doesn't seem to result in a usable
# error message.
raise socket.error(error)
raise Return()
def put(self, key, value, **kw):
if value is None:
raise Return("Error! No value provided.")
if kw.get('prefixify', True):
key = self.prefixify(key)
is_file = False
is_binary = False # TODO: --binary
mode = 'rb'
# TODO --encode flag to encode value
if os.path.isfile(value):
kwargs = dict(mode=mode)
if PY3:
kwargs = dict(mode=mode, encoding=ENCODING)
value = codecs.open(os.path.expandvars(os.path.expanduser(value)),
**kwargs).read().rstrip('\n')
is_file = True
if PY3:
value = value.encode(ENCODING)
# boto expects bytestrings
data = self.vault.encrypt(value, is_file=is_file, is_binary=is_binary)
data['name'] = key
result = yield From(
self.put_backend(Bucket=self.bucket,
Key=key,
Body=json.dumps(data)))
if result.get('ResponseMetadata').get('HTTPStatusCode') == 200:
raise Return("Success! Wrote: %s" % key)
else:
raise Return(result)
def get(self, key, **kw):
if key is None:
raise Return("Error! No key provided.")
key = self.prefixify(key)
extra = {}
if kw.get('version'):
extra['VersionId'] = kw.get('version')
try:
result = yield From(
self.get_backend(Bucket=self.bucket, Key=key, **extra))
except Exception as e:
if kw.get('in_group_check', True):
# check for grouped keys
key = key.rstrip('/') + '/'
kw['in_group_check'] = True
result = yield From(self.config(key=key, **kw))
if not result:
result = "Error! The specified key does not exist."
raise Return(result)
raise
body = json.loads(result['Body'].read().decode(ENCODING))
data = self.vault.decrypt(body)
is_binary = body.get('is_binary', False)
if not is_binary:
data = data.decode(ENCODING)
raise Return(data)
def handle_401(self, r, **kwargs):
"""Takes the given response and tries digest-auth, if needed."""
#if self.pos is not None:
# Rewind the file position indicator of the body to where
# it was to resend the request.
# r.request.body.seek(self.pos)
num_401_calls = getattr(self, 'num_401_calls', 1)
s_auth = r.headers.get('www-authenticate', '')
if 'digest' in s_auth.lower() and num_401_calls < 2:
setattr(self, 'num_401_calls', num_401_calls + 1)
pat = re.compile(r'digest ', flags=re.IGNORECASE)
self.chal = parse_dict_header(pat.sub('', s_auth, count=1))
# Consume content and release the original connection
# to allow our new request to reuse the same one.
yield From(r.content)
r.raw.release_conn() # redundant
prep = r.request.copy()
extract_cookies_to_jar(prep._cookies, r.request, r.raw)
prep.prepare_cookies(prep._cookies)
prep.headers['Authorization'] = self.build_digest_header(
prep.method, prep.url)
_r = yield From(r.connection.send(prep, **kwargs))
_r.history.append(r)
_r.request = prep
raise Return(_r)
setattr(self, 'num_401_calls', 1)
raise Return(r)
def _cancel_callback(self, key_change):
if key_change.event not in (KeyEvent.CREATE, KeyEvent.SET):
raise Return()
build_uuid = key_change.value
build_info = self._build_uuid_to_info.get(build_uuid, None)
if build_info is None:
logger.debug('No build info for "%s" job %s', key_change.event,
build_uuid)
raise Return(False)
lock_key = slash_join(self._canceled_lock_prefix, build_uuid,
build_info.execution_id)
lock_acquired = yield From(self._orchestrator.lock(lock_key))
if lock_acquired:
builder_realm = build_info.component.builder_realm
yield From(self.kill_builder_executor(build_uuid))
yield From(
self._orchestrator.delete_key(self._realm_key(builder_realm)))
yield From(
self._orchestrator.delete_key(self._metric_key(builder_realm)))
yield From(
self._orchestrator.delete_key(
slash_join(self._job_prefix, build_uuid)))
# This is outside the lock so we can un-register the component wherever it is registered to.
yield From(build_info.component.cancel_build())
def analyze_sdf(self, sdf, max_attempts=5):
"""
Coroutine that returns with a (collisions, bounding box) tuple,
assuming analysis succeeds.
:param max_attempts:
:param sdf:
:type sdf: SDF
:return:
"""
msg = None
rh = self.request_handler
for _ in range(max_attempts):
future = yield From(rh.do_gazebo_request("analyze_body", str(sdf)))
yield From(future)
response = future.result()
if response.response == "success":
msg = BodyAnalysisResponse()
msg.ParseFromString(response.serialized_data)
break
if not msg:
# Error return
raise Return(None)
if msg.HasField("boundingBox"):
bbox = msg.boundingBox
else:
bbox = None
internal_collisions = len(msg.contact)
raise Return(internal_collisions, bbox)
def attempt_mate(self, ra, rb):
"""
Attempts mating between two robots.
:param ra:
:param rb:
:return:
"""
logger.debug("Attempting mating between `%s` and `%s`..." % (ra.name, rb.name))
# Attempt to create a child through crossover
success, child = self.crossover.crossover(ra.tree, rb.tree)
if not success:
logger.debug("Crossover failed.")
raise Return(False)
# Apply mutation
logger.debug("Crossover succeeded, applying mutation...")
self.mutator.mutate(child, in_place=True)
if self.conf.enforce_planarity:
make_planar(child.root)
_, outputs, _ = child.root.io_count(recursive=True)
if not outputs:
logger.debug("Evolution resulted in child without motors.")
raise Return(False)
# Check if the robot body is valid
ret = yield From(self.analyze_tree(child))
if ret is None or ret[0]:
logger.debug("Intersecting body parts: Miscarriage.")
raise Return(False)
logger.debug("Viable child created.")
raise Return(child, ret[1])
def data(self):
# For backwords-compat with earlier urllib3 0.4 and earlier.
if self._body:
raise Return(self._body)
if self._fp:
_d = yield From(self.read(cache_content=True))
raise Return(_d)
def voltage(self, servo):
try:
data = yield From(self.ram_read(servo, 54, 1))
value = data.data[0]
raise Return(0.074 * value)
except asyncio.TimeoutError:
raise Return(None)
def perform_request(self, dua, request):
message_id = int(request.getComponentByName('messageID'))
dict_req = BaseStrategy.decode_request(request)
if message_id not in dua.pending:
dua.pending[message_id] = dict_req
if dict_req['type'] == 'bindRequest':
if NATIVE_ASYNCIO:
response, response_type = yield from do_bind_operation(
dua, message_id, dict_req)
else:
response, response_type = yield From(
do_bind_operation(dua, message_id, dict_req))
elif dict_req['type'] == 'unbindRequest':
if NATIVE_ASYNCIO:
yield from do_unbind_operation(dua, message_id)
else:
yield From(do_unbind_operation(dua, message_id))
dua.writer.close()
if NATIVE_ASYNCIO:
return
else:
raise Return()
elif dict_req['type'] == 'extendedReq':
response, response_type = do_extended_operation(
dua, message_id, dict_req)
if response[
'responseName'] == '1.3.6.1.4.1.1466.20037' and response[
'result'] == RESULT_SUCCESS: # issue start_tls
ldap_message = build_ldap_message(message_id,
response_type, response,
None)
dua.send(ldap_message)
dua.start_tls()
response = None
else:
dua.abort(diagnostic_message='unknown operation')
if NATIVE_ASYNCIO:
return
else:
raise Return()
del dua.pending[message_id]
if not response: # notice of disconnection sent while doing operation
if NATIVE_ASYNCIO:
return
else:
raise Return()
controls = None # TODO
ldap_message = build_ldap_message(message_id, response_type,
response, controls)
else: # pending message with same id of previous message
dua.abort(diagnostic_message='duplicate message ID')
if NATIVE_ASYNCIO:
return
else:
raise Return()
dua.send(ldap_message)
def _job_callback(self, key_change):
"""
This is the callback invoked when keys related to jobs are changed.
It ignores all events related to the creation of new jobs.
Deletes or expirations cause checks to ensure they've been properly marked as completed.
:param key_change: the event and value produced by a key changing in the orchestrator
:type key_change: :class:`KeyChange`
"""
if key_change.event in (KeyEvent.CREATE, KeyEvent.SET):
raise Return()
elif key_change.event in (KeyEvent.DELETE, KeyEvent.EXPIRE):
# Handle the expiration/deletion.
job_metadata = json.loads(key_change.value)
build_job = BuildJob(AttrDict(job_metadata["job_queue_item"]))
logger.debug('Got "%s" of job %s', key_change.event, build_job.build_uuid)
# Get the build info.
build_info = self._build_uuid_to_info.get(build_job.build_uuid, None)
if build_info is None:
logger.debug(
'No build info for "%s" job %s (%s); probably already deleted by this manager',
key_change.event,
build_job.build_uuid,
job_metadata,
)
raise Return()
if key_change.event != KeyEvent.EXPIRE:
# If the etcd action was not an expiration, then it was already deleted by some manager and
# the execution was therefore already shutdown. All that's left is to remove the build info.
self._build_uuid_to_info.pop(build_job.build_uuid, None)
raise Return()
logger.debug(
"got expiration for job %s with metadata: %s", build_job.build_uuid, job_metadata
)
if not job_metadata.get("had_heartbeat", False):
# If we have not yet received a heartbeat, then the node failed to boot in some way.
# We mark the job as incomplete here.
yield From(self._mark_job_incomplete(build_job, build_info))
# Finally, we terminate the build execution for the job. We don't do this under a lock as
# terminating a node is an atomic operation; better to make sure it is terminated than not.
logger.info(
"Terminating expired build executor for job %s with execution id %s",
build_job.build_uuid,
build_info.execution_id,
)
yield From(self.kill_builder_executor(build_job.build_uuid))
else:
logger.warning(
"Unexpected KeyEvent (%s) on job key: %s", key_change.event, key_change.key
)
def schedule(self, build_job):
""" Schedules a build for an Enterprise Registry. """
if self.shutting_down or not self.ready_components:
raise Return(False, RETRY_TIMEOUT)
component = self.ready_components.pop()
yield From(component.start_build(build_job))
raise Return(True, None)
def _get_tshark_process(self, packet_count=None):
if self._current_tshark:
raise Return(self._current_tshark)
proc = yield From(super(InMemCapture, self)._get_tshark_process(packet_count=packet_count, stdin=subprocess.PIPE))
self._current_tshark = proc
# Create PCAP header
header = struct.pack("IHHIIII", 0xa1b2c3d4, 2, 4, 0, 0, 0x7fff, self._current_linktype)
proc.stdin.write(header)
raise Return(proc)
def position(self, servo):
try:
# NOTE: The datasheet appears to be off here.
data = yield From(self.ram_read(servo, 60, 2))
value = data.data
if value is None:
raise Return(None)
raise Return(value[1] << 8 | value[0])
except asyncio.TimeoutError:
raise Return(None)
def temperature_C(self, servo):
try:
data = yield From(self.ram_read(servo, 55, 1))
value = data.data[0]
# Note, this formula was derived from the Dongbu lookup table,
# and becomes terribly inaccurate below -20C.
raise Return((value - 40) * 0.5125 - 19.38)
except asyncio.TimeoutError:
raise Return(None)
def config(self, key=None, **kw):
contents = yield From(self.list_backend(prefix=self.prefixify(key)))
contents = contents.get('Contents', [])
kw.setdefault('in_group_check', True)
tasks = [self.get(key=obj['Key'], **kw) for obj in contents]
try:
results = yield From(asyncio.gather(*tasks))
except ValueError as e:
raise Return({})
keys = [k['Key'].split('/')[-1] for k in contents]
raise Return(OrderedDict(zip(keys, results)))
def birth(self, tree, bbox, parents):
"""
Birth process, picks a robot position and inserts
the robot into the world.
Robots are currently placed at a random position within the circular
birth clinic. In this process, 5 attempts are made to place the robot
at the minimum drop distance from other robots. If this fails however
the last generated position is used anyway.
:param tree:
:param bbox:
:param parents:
:return:
"""
s = len(tree)
if (s + self.total_size()) > self.conf.part_limit:
print("Not enough parts in pool to create robot of size %d." % s)
raise Return(False)
# Pick a random radius and angle within the birth clinic
pos = Vector3()
pos.z = -bbox.min.z + self.conf.drop_height
done = False
min_drop = self.conf.min_drop_distance
for i in range(5):
angle = random.random() * 2 * math.pi
radius = self.conf.birth_clinic_diameter * 0.5 * random.random()
pos.x = radius * math.cos(angle)
pos.y = radius * math.sin(angle)
done = True
for robot in self.robots.values():
if robot.distance_to(pos, planar=True) < min_drop:
done = False
break
if done:
break
if not done:
logger.warning("Warning: could not satisfy minimum drop distance.")
# Note that we register the reproduction only if
# the child is actually born, i.e. there were enough parts
# left in the world to satisfy the request.
if parents:
ra, rb = parents
ra.did_mate_with(rb)
rb.did_mate_with(ra)
self.births += 1
fut = yield From(self.insert_robot(tree, Pose(position=pos), parents=parents))
raise Return(fut)
def process(self, url_item):
scheme = url_item.url_info.scheme
if scheme in ('http', 'https'):
raise Return((yield From(self.web_processor.process(url_item))))
elif scheme == 'ftp':
raise Return((yield From(self.ftp_processor.process(url_item))))
else:
_logger.warning(__(
_('No processor available to handle {scheme} scheme.'),
scheme=repr(scheme)
))
def set_phase(self, phase, extra_data=None):
if phase == self._current_phase:
raise Return(False)
self._current_phase = phase
yield From(
self._append_log_message(phase, self._build_logs.PHASE,
extra_data))
# Update the repository build with the new phase
raise Return(
self._build_model.update_phase_then_close(self._uuid, phase))
def getData(self):
if self.first:
self.first = False
dat = yield From(self.fetch())
raise Return(dat)
else:
if self.repeat > 0:
yield From(asyncio.sleep(self.repeat))
dat = yield From(self.fetch())
raise Return(dat)
else:
raise Return(None)
def job_completed(self, build_job, job_status, build_component):
logger.debug("Calling job_completed for job %s with status: %s",
build_job.build_uuid, job_status)
yield From(
self._write_duration_metric(build_duration,
build_component.builder_realm,
job_status=job_status))
# Mark the job as completed. Since this is being invoked from the component, we don't need
# to ask for the phase to be updated as well.
build_info = self._build_uuid_to_info.get(build_job.build_uuid, None)
executor_name = build_info.executor_name if build_info else None
yield From(
self.job_complete_callback(build_job,
job_status,
executor_name,
update_phase=False))
# Kill the ephemeral builder.
yield From(self.kill_builder_executor(build_job.build_uuid))
# Delete the build job from the orchestrator.
try:
job_key = self._job_key(build_job)
yield From(self._orchestrator.delete_key(job_key))
except KeyError:
logger.debug(
"Builder is asking for job to be removed, but work already completed"
)
except OrchestratorConnectionError:
logger.exception(
"Could not remove job key as orchestrator is not available")
yield From(sleep(ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION))
raise Return()
# Delete the metric from the orchestrator.
try:
metric_key = self._metric_key(build_component.builder_realm)
yield From(self._orchestrator.delete_key(metric_key))
except KeyError:
logger.debug(
"Builder is asking for metric to be removed, but key not found"
)
except OrchestratorConnectionError:
logger.exception(
"Could not remove metric key as orchestrator is not available")
yield From(sleep(ORCHESTRATOR_UNAVAILABLE_SLEEP_DURATION))
raise Return()
logger.debug("job_completed for job %s with status: %s",
build_job.build_uuid, job_status)
def produce_individual(self):
"""
:return:
"""
p1, p2 = self.select_parents()
for j in xrange(self.conf.max_mating_attempts):
pair = yield From(self.attempt_mate(p1, p2))
if pair:
raise Return((pair[0], pair[1], (p1, p2)))
raise Return(False)
def _heartbeat(self):
"""
Coroutine that runs every HEARTBEAT_TIMEOUT seconds, both checking the worker's heartbeat
and updating the heartbeat in the build status dictionary (if applicable).
This allows the build system to catch crashes from either end.
"""
yield From(trollius.sleep(INITIAL_TIMEOUT))
while True:
# If the component is no longer running or actively building, nothing more to do.
if (
self._component_status != ComponentStatus.RUNNING
and self._component_status != ComponentStatus.BUILDING
):
raise Return()
# If there is an active build, write the heartbeat to its status.
if self._build_status is not None:
with self._build_status as status_dict:
status_dict["heartbeat"] = int(time.time())
# Mark the build item.
current_job = self._current_job
if current_job is not None:
yield From(self.parent_manager.job_heartbeat(current_job))
# Check the heartbeat from the worker.
logger.debug("Checking heartbeat on realm %s", self.builder_realm)
if (
self._last_heartbeat
and self._last_heartbeat < datetime.datetime.utcnow() - HEARTBEAT_DELTA
):
logger.debug(
"Heartbeat on realm %s has expired: %s",
self.builder_realm,
self._last_heartbeat,
)
yield From(self._timeout())
raise Return()
logger.debug(
"Heartbeat on realm %s is valid: %s (%s).",
self.builder_realm,
self._last_heartbeat,
self._component_status,
)
yield From(trollius.sleep(HEARTBEAT_TIMEOUT))
def get(self, key, **kw):
try:
data = yield From(self._get(key, **kw))
except Exception as e:
if kw.get('in_group_check', True):
# check for grouped keys
key = key.rstrip('/') + '/'
kw['in_group_check'] = True
result = yield From(self.config(key=key, **kw))
if not result:
result = "Error! The specified key does not exist."
raise Return(result)
raise
raise Return(data["data"])
def _transition_to_idle(self):
# Command 0 speed, wait for the command to be active, then
# wait for at least 1 phase, then wait for all legs to be in
# stance.
self.gait.set_command(ripple.Command())
yield From(self._run_while(
lambda: (self.gait.is_command_pending() and
not self.next_command),
allow_new=False))
if self.next_command:
raise Return(False)
class PhaseChecker(object):
def __init__(self, parent):
self.phase_delta = 0.0
self.parent = parent
self.old_phase = self.parent.state.phase
def check(self):
self.phase_delta += _wrap_phase(
self.parent.state.phase - self.old_phase)
self.old_phase = self.parent.state.phase
return (self.phase_delta < 0.95 and
not self.parent.next_command)
checker = PhaseChecker(self)
yield From(self._run_while(checker.check, allow_new=False))
if self.next_command:
raise Return(False)
def any_non_stance():
if self.next_command:
return False
for leg in self.state.legs.values():
if leg.mode != ripple.STANCE:
return True
return False
yield From(self._run_while(any_non_stance, allow_new=False))
if self.next_command:
raise Return(False)
raise Return(True)