def pset_field(item_type, optional=False):
"""
Create checked ``PSet`` field.
:param item_type: The required type for the items in the set.
:param bool optional: If true, ``None`` can be used as a value for
this field.
:return: A ``field`` containing a ``CheckedPSet`` of the given type.
"""
class TheSet(CheckedPSet):
__type__ = item_type
TheSet.__name__ = item_type.__name__.capitalize() + "PSet"
if optional:
def factory(argument):
if argument is None:
return None
else:
return TheSet(argument)
else:
factory = TheSet
return field(type=optional_type(TheSet) if optional else TheSet,
factory=factory, mandatory=True,
initial=TheSet())
def pmap_field(key_type, value_type, optional=False, invariant=_valid):
"""
Create a checked ``PMap`` field.
:param key: The required type for the keys of the map.
:param value: The required type for the values of the map.
:param bool optional: If true, ``None`` can be used as a value for
this field.
:param invariant: Pass-through to ``field``.
:return: A ``field`` containing a ``CheckedPMap``.
"""
class TheMap(CheckedPMap):
__key_type__ = key_type
__value_type__ = value_type
TheMap.__name__ = (key_type.__name__.capitalize() +
value_type.__name__.capitalize() + "PMap")
if optional:
def factory(argument):
if argument is None:
return None
else:
return TheMap(argument)
else:
factory = TheMap
return field(mandatory=True, initial=TheMap(),
type=optional_type(TheMap) if optional else TheMap,
factory=factory, invariant=invariant)
def _sequence_field(checked_class, suffix, item_type, optional, initial):
"""
Create checked field for either ``PSet`` or ``PVector``.
:param checked_class: ``CheckedPSet`` or ``CheckedPVector``.
:param suffix: Suffix for new type name.
:param item_type: The required type for the items in the set.
:param bool optional: If true, ``None`` can be used as a value for
this field.
:param initial: Initial value to pass to factory.
:return: A ``field`` containing a checked class.
"""
class TheType(checked_class):
__type__ = item_type
TheType.__name__ = item_type.__name__.capitalize() + suffix
if optional:
def factory(argument):
if argument is None:
return None
else:
return TheType(argument)
else:
factory = TheType
return field(type=optional_type(TheType) if optional else TheType,
factory=factory, mandatory=True,
initial=factory(initial))
def _volume():
"""
Create and return a ``PRecord`` ``field`` to hold a ``BlockDeviceVolume``.
"""
return field(
type=BlockDeviceVolume, mandatory=True,
# Disable the automatic PRecord.create factory. Callers can just
# supply the right type, we don't need the magic coercion behavior
# supplied by default.
factory=lambda x: x
)
def pmap_field(
key_type, value_type, optional=False, invariant=_valid,
initial=_UNDEFINED, factory=None
):
"""
Create a checked ``PMap`` field.
:param key: The required type for the keys of the map.
:param value: The required type for the values of the map.
:param bool optional: If true, ``None`` can be used as a value for this
field.
:param invariant: Pass-through to ``field``.
:param initial: An initial value for the field. This will first be coerced
using the field's factory. If not given, the initial value is an empty
map.
:param factory: A factory used to convert input arguments to the stored
value whenever it is set. Note that this will be composed with the
constructor for the ``CheckedPMap`` class constructed for this field.
:return: A ``field`` containing a ``CheckedPMap``.
"""
input_factory = factory
class TheMap(CheckedPMap):
__key_type__ = key_type
__value_type__ = value_type
TheMap.__name__ = (key_type.__name__.capitalize() +
value_type.__name__.capitalize() + "PMap")
if optional:
def mapping_factory(argument):
if argument is None:
return None
else:
return TheMap(argument)
else:
mapping_factory = TheMap
if input_factory:
factory = lambda x: mapping_factory(input_factory(x))
else:
factory = mapping_factory
if initial is _UNDEFINED:
initial = TheMap()
else:
initial = factory(initial)
return field(mandatory=True, initial=initial,
type=optional_type(TheMap) if optional else TheMap,
factory=factory, invariant=invariant)
def _make_intent_from_args(args):
"""
Create an intent type for a given set of arguments.
:param args: a dict with keys as the names of arguments and values as
:class:`argument`s.
:returns: A new type that can hold all of the data to call a function that
has the given arguments.
"""
class _Intent(PClass):
pass
for name, arg in iteritems(args):
setattr(_Intent, name, field(type=arg.type))
_PIntent = add_metaclass(PClassMeta)(_Intent)
return _PIntent
def field_factory(self, *args, **kwargs):
"""
This function does two things: it forwards args to parser.add_arguments, and other args to field()
There is one keyword collision for add_argument and field, which is 'type'. If 'type' exists, we will
use it for field
:param long_name: The "--long-arg" name
:param short_name:
:param cli_help:
:param parser:
:param kwargs:
:return:
"""
short_name = None
if len(args) == 2:
long_name = args[1]
short_name = args[0]
elif len(args) == 1:
long_name = args[0]
if short_name is not None and short_name in self.short_names:
log.warning("{0} already used. Not setting {0} for {1}".format(short_name, long_name))
else:
self.short_names.add(short_name)
if "short_name" in kwargs:
kwargs.pop("short_name")
parser_kwargs = {}
for x in ["required", "nargs", "choices", "default", "help", "dest"]:
if x in kwargs:
parser_kwargs[x] = kwargs.pop(x)
field_kwargs = kwargs
if short_name is not None:
self.parser.add_argument(short_name, long_name, **parser_kwargs)
else:
self.parser.add_argument(long_name, **parser_kwargs)
# print "Option {}:".format(long_name), field_kwargs
return field(**field_kwargs)
def interface_field(interfaces, **field_kwargs):
"""
A ``PClass`` field which checks that the assigned value provides all the
``interfaces``.
:param tuple interfaces: The ``Interface`` that a value must provide.
"""
if not isinstance(interfaces, tuple):
raise TypeError(
"The ``interfaces`` argument must be a tuple. "
"Got: {!r}".format(interfaces)
)
original_invariant = field_kwargs.pop("invariant", None)
def invariant(value):
error_messages = []
if original_invariant is not None:
(original_invariant_result,
_original_invariant_message) = original_invariant(value)
if original_invariant_result:
error_messages.append(original_invariant_result)
missing_interfaces = []
for interface in interfaces:
if not interface.providedBy(value):
missing_interfaces.append(interface.getName())
if missing_interfaces:
error_messages.append(
"The value {!r} "
"did not provide these required interfaces: {}".format(
value,
", ".join(missing_interfaces)
)
)
if error_messages:
return (False, "\n".join(error_messages))
else:
return (True, "")
field_kwargs["invariant"] = invariant
return field(**field_kwargs)
def pmap_field(
key_type, value_type, optional=False, invariant=_valid,
initial=_UNDEFINED
):
"""
Create a checked ``PMap`` field.
:param key: The required type for the keys of the map.
:param value: The required type for the values of the map.
:param bool optional: If true, ``None`` can be used as a value for this
field.
:param invariant: Pass-through to ``field``.
:param initial: An initial value for the field. This will first be coerced
using the field's factory. If not given, the initial value is an empty
map.
:return: A ``field`` containing a ``CheckedPMap``.
"""
class TheMap(CheckedPMap):
__key_type__ = key_type
__value_type__ = value_type
TheMap.__name__ = (key_type.__name__.capitalize() +
value_type.__name__.capitalize() + "PMap")
if optional:
def factory(argument):
if argument is None:
return None
else:
return TheMap(argument)
else:
factory = TheMap
if initial is _UNDEFINED:
initial = TheMap()
else:
initial = factory(initial)
return field(mandatory=True, initial=initial,
type=optional_type(TheMap) if optional else TheMap,
factory=factory, invariant=invariant)
class CRecord(PRecord):
x = field(serializer=1)
def any_field(**kw):
return pyrsistent.field(mandatory=True, **kw)
def new_record_type(name, headings):
return type(
name,
(PClass,),
{heading: field(type=unicode, mandatory=True) for heading in headings}
)
class BRecord(PRecord):
__invariant__ = lambda r: (r.x % r.y == 0, 'modulo')
x = field()
y = field()
class BRecord(PRecord):
x = field(type=int, initial='foo')
class BRecord(PRecord):
x = field(initial=1)
y = field(initial=2)
class BRecord(PRecord):
x = field(invariant=lambda x: (x > 1, 'x too small'))
y = field(mandatory=True)
class A(PRecord):
x = field(type=int)
class MyRecord(PRecord):
a = field(int, initial=lambda: 2)
class Foo(PRecord):
foo = field(type=str)
class BRecord(ARecord):
z = field()
class Node(PRecord):
applications = field(type=ApplicationVector)
class Application(PRecord):
name = field(type=(six.text_type, ) + six.string_types)
image = field(type=(six.text_type, ) + six.string_types)
def fieldm():
return field(mandatory=True)
class BRecord(PRecord):
d = field(serializer=lambda format, d: format)
class WrittenAction(PClass):
"""
An Action that has been logged.
This class is intended to provide a definition within Eliot of what an
action actually is, and a means of constructing actions that are known to
be valid.
@ivar WrittenMessage start_message: A start message whose task UUID and
level match this action, or C{None} if it is not yet set on the
action.
@ivar WrittenMessage end_message: An end message hose task UUID and
level match this action. Can be C{None} if the action is
unfinished.
@ivar TaskLevel task_level: The action's task level, e.g. if start
message has level C{[2, 3, 1]} it will be
C{TaskLevel(level=[2, 3])}.
@ivar UUID task_uuid: The UUID of the task to which this action belongs.
@ivar _children: A L{pmap} from L{TaskLevel} to the L{WrittenAction} and
L{WrittenMessage} objects that make up this action.
"""
start_message = field(type=optional(WrittenMessage), mandatory=True,
initial=None)
end_message = field(type=optional(WrittenMessage), mandatory=True,
initial=None)
task_level = field(type=TaskLevel, mandatory=True)
task_uuid = field(type=unicode, mandatory=True, factory=unicode)
# Pyrsistent doesn't support pmap_field with recursive types.
_children = pmap_field(TaskLevel, object)
@classmethod
def from_messages(cls, start_message=None, children=pvector(),
end_message=None):
"""
Create a C{WrittenAction} from C{WrittenMessage}s and other
C{WrittenAction}s.
@param WrittenMessage start_message: A message that has
C{ACTION_STATUS_FIELD}, C{ACTION_TYPE_FIELD}, and a C{task_level}
that ends in C{1}, or C{None} if unavailable.
@param children: An iterable of C{WrittenMessage} and C{WrittenAction}
@param WrittenMessage end_message: A message that has the same
C{action_type} as this action.
@raise WrongTask: If C{end_message} has a C{task_uuid} that differs
from C{start_message.task_uuid}.
@raise WrongTaskLevel: If any child message or C{end_message} has a
C{task_level} that means it is not a direct child.
@raise WrongActionType: If C{end_message} has an C{ACTION_TYPE_FIELD}
that differs from the C{ACTION_TYPE_FIELD} of C{start_message}.
@raise InvalidStatus: If C{end_message} doesn't have an
C{action_status}, or has one that is not C{SUCCEEDED_STATUS} or
C{FAILED_STATUS}.
@raise InvalidStartMessage: If C{start_message} does not have a
C{ACTION_STATUS_FIELD} of C{STARTED_STATUS}, or if it has a
C{task_level} indicating that it is not the first message of an
action.
@return: A new C{WrittenAction}.
"""
actual_message = [message for message in
[start_message, end_message] + list(children)
if message][0]
action = cls(
task_level=actual_message.task_level.parent(),
task_uuid=actual_message.task_uuid,
)
if start_message:
action = action._start(start_message)
for child in children:
if action._children.get(child.task_level, child) != child:
raise DuplicateChild(action, child)
action = action._add_child(child)
if end_message:
action = action._end(end_message)
return action
@property
def action_type(self):
"""
The type of this action, e.g. C{"yourapp:subsystem:dosomething"}.
"""
if self.start_message:
return self.start_message.contents[ACTION_TYPE_FIELD]
elif self.end_message:
return self.end_message.contents[ACTION_TYPE_FIELD]
else:
return None
@property
def status(self):
"""
One of C{STARTED_STATUS}, C{SUCCEEDED_STATUS}, C{FAILED_STATUS} or
C{None}.
"""
message = self.end_message if self.end_message else self.start_message
if message:
return message.contents[ACTION_STATUS_FIELD]
else:
return None
@property
def start_time(self):
"""
The Unix timestamp of when the action started, or C{None} if there has
been no start message added so far.
"""
if self.start_message:
return self.start_message.timestamp
@property
def end_time(self):
"""
The Unix timestamp of when the action ended, or C{None} if there has been
no end message.
"""
if self.end_message:
return self.end_message.timestamp
@property
def exception(self):
"""
If the action failed, the name of the exception that was raised to cause
it to fail. If the action succeeded, or hasn't finished yet, then
C{None}.
"""
if self.end_message:
return self.end_message.contents.get(EXCEPTION_FIELD, None)
@property
def reason(self):
"""
The reason the action failed. If the action succeeded, or hasn't finished
yet, then C{None}.
"""
if self.end_message:
return self.end_message.contents.get(REASON_FIELD, None)
@property
def children(self):
"""
The list of child messages and actions sorted by task level, excluding the
start and end messages.
"""
return pvector(sorted(self._children.values(), key=lambda m: m.task_level))
def _validate_message(self, message):
"""
Is C{message} a valid direct child of this action?
@param message: Either a C{WrittenAction} or a C{WrittenMessage}.
@raise WrongTask: If C{message} has a C{task_uuid} that differs from the
action's C{task_uuid}.
@raise WrongTaskLevel: If C{message} has a C{task_level} that means
it's not a direct child.
"""
if message.task_uuid != self.task_uuid:
raise WrongTask(self, message)
if not message.task_level.parent() == self.task_level:
raise WrongTaskLevel(self, message)
def _add_child(self, message):
"""
Return a new action with C{message} added as a child.
Assumes C{message} is not an end message.
@param message: Either a C{WrittenAction} or a C{WrittenMessage}.
@raise WrongTask: If C{message} has a C{task_uuid} that differs from the
action's C{task_uuid}.
@raise WrongTaskLevel: If C{message} has a C{task_level} that means
it's not a direct child.
@return: A new C{WrittenAction}.
"""
self._validate_message(message)
level = message.task_level
return self.transform(('_children', level), message)
def _start(self, start_message):
"""
Start this action given its start message.
@param WrittenMessage start_message: A start message that has the
same level as this action.
@raise InvalidStartMessage: If C{start_message} does not have a
C{ACTION_STATUS_FIELD} of C{STARTED_STATUS}, or if it has a
C{task_level} indicating that it is not the first message of an
action.
"""
if start_message.contents.get(
ACTION_STATUS_FIELD, None) != STARTED_STATUS:
raise InvalidStartMessage.wrong_status(start_message)
if start_message.task_level.level[-1] != 1:
raise InvalidStartMessage.wrong_task_level(start_message)
return self.set(start_message=start_message)
def _end(self, end_message):
"""
End this action with C{end_message}.
Assumes that the action has not already been ended.
@param WrittenMessage end_message: An end message that has the
same level as this action.
@raise WrongTask: If C{end_message} has a C{task_uuid} that differs
from the action's C{task_uuid}.
@raise WrongTaskLevel: If C{end_message} has a C{task_level} that means
it's not a direct child.
@raise InvalidStatus: If C{end_message} doesn't have an
C{action_status}, or has one that is not C{SUCCEEDED_STATUS} or
C{FAILED_STATUS}.
@return: A new, completed C{WrittenAction}.
"""
action_type = end_message.contents.get(ACTION_TYPE_FIELD, None)
if self.action_type not in (None, action_type):
raise WrongActionType(self, end_message)
self._validate_message(end_message)
status = end_message.contents.get(ACTION_STATUS_FIELD, None)
if status not in (FAILED_STATUS, SUCCEEDED_STATUS):
raise InvalidStatus(self, end_message)
return self.set(end_message=end_message)
class BRecord(PRecord):
__invariant__ = lambda r: (r.x <= r.y, 'y smaller than x')
x = field()
y = field()
class AWSProvisioner(PClass):
"""
A provisioner that creates nodes on AWS.
:ivar boto.ec2.connection.EC2Connection _connection: The boto connection to
use.
:ivar bytes _keyname: The name of an existing ssh public key configured
with the cloud provider.
:ivar _security_groups: List of security groups to put the instances
created by this provisioner in.
:type _security_groups: `list` of `bytes`
:param bytes _zone: The AWS availability zone to put instances created by
this provisioner in.
"""
_connection = field(mandatory=True)
_keyname = field(type=bytes, mandatory=True)
_security_groups = field(mandatory=True)
_zone = field(type=bytes, mandatory=True)
_default_size = field(type=bytes, mandatory=True)
def get_ssh_key(self):
"""
Return the public key associated with the provided keyname.
:return Key: The ssh public key or ``None`` if it can't be determined.
"""
# EC2 only provides the SSH2 fingerprint (for uploaded keys)
# or the SHA-1 hash of the private key (for EC2 generated keys)
# https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_KeyPairInfo.html
return None
def create_node(self,
name,
distribution,
size=None,
disk_size=8,
metadata={}):
if size is None:
size = self._default_size
with start_action(
action_type=u"flocker:provision:aws:create_node",
name=name,
distribution=distribution,
image_size=size,
disk_size=disk_size,
metadata=metadata,
):
metadata = metadata.copy()
metadata['Name'] = name
disk1 = EBSBlockDeviceType()
disk1.size = disk_size
disk1.delete_on_termination = True
diskmap = BlockDeviceMapping()
diskmap['/dev/sda1'] = disk1
images = self._connection.get_all_images(
filters={'name': IMAGE_NAMES[distribution]}, )
with start_action(
action_type=
u"flocker:provision:aws:create_node:run_instances",
) as context:
reservation = self._connection.run_instances(
images[0].id,
key_name=self._keyname,
instance_type=size,
security_groups=self._security_groups,
block_device_map=diskmap,
placement=self._zone,
# On some operating systems, a tty is requried for sudo.
# Since AWS systems have a non-root user as the login,
# disable this, so we can use sudo with conch.
user_data=dedent("""\
#!/bin/sh
sed -i '/Defaults *requiretty/d' /etc/sudoers
"""),
)
instance = reservation.instances[0]
context.add_success_fields(instance_id=instance.id)
self._connection.create_tags([instance.id], metadata)
# Display state as instance starts up, to keep user informed that
# things are happening.
_wait_until_running(instance)
return AWSNode(
name=name,
_provisioner=self,
_instance=instance,
distribution=distribution,
)
class BRecord(PRecord):
x = field(type=1)
class AWSNode(PClass):
_provisioner = field(mandatory=True)
_instance = field(mandatory=True)
distribution = field(mandatory=True)
name = field(mandatory=True)
@property
def address(self):
return self._instance.ip_address.encode('ascii')
@property
def private_address(self):
return self._instance.private_ip_address.encode('ascii')
def destroy(self):
with start_action(
action_type=u"flocker:provision:aws:destroy",
instance_id=self._instance.id,
):
self._instance.terminate()
def get_default_username(self):
"""
Return the username available by default on a system.
"""
return _usernames[self.distribution]
def provision(self, package_source, variants=()):
"""
Provision flocker on this node.
:param LibcloudNode node: Node to provision.
:param PackageSource package_source: See func:`task_install_flocker`
:param set variants: The set of variant configurations to use when
provisioning
"""
username = self.get_default_username()
commands = []
# cloud-init may not have allowed sudo without tty yet, so try SSH key
# installation for a few more seconds:
start = []
def for_thirty_seconds(*args, **kwargs):
if not start:
start.append(time())
return Effect(Constant((time() - start[0]) < 30))
commands.append(
run_remotely(
username=username,
address=self.address,
commands=retry(task_install_ssh_key(), for_thirty_seconds),
))
commands.append(
run_remotely(
username='******',
address=self.address,
commands=provision(
package_source=package_source,
distribution=self.distribution,
variants=variants,
),
))
return sequence(commands)
def reboot(self):
"""
Reboot the node.
:return Effect:
"""
def do_reboot(_):
with start_action(
action_type=u"flocker:provision:aws:reboot",
instance_id=self._instance.id,
):
self._instance.reboot()
_wait_until_running(self._instance)
return run_remotely(username="******",
address=self.address,
commands=run_from_args(["sync"
])).on(success=do_reboot)
class BRecord(PRecord):
x = field(invariant='foo')
class PendingTrade(PRecord):
buyers_price = field(type=float)
class BRecord(PRecord):
x = field(type=int, factory=1)
class AlgorithmicModel(PRecord):
pending_trades = pvector_field(PendingTrade)
selling_threshold = field(type=float)
cut_losses_threshold = field(type=float)
class BRecord(PRecord):
x = field()
y = field()
def test_enum():
f = field(type=TestEnum)
assert TestEnum in f.type
assert len(f.type) == 1
class BRecord(PRecord):
x = field(factory=int, invariant=lambda x: (x >= 0, 'x negative'))
y = field(mandatory=True)
class CRecord(PRecord):
b = field()
class CRecord(PRecord):
a = field(invariant=lambda x: (x != 0, 'a zero'))
b = field(type=BRecord)
class _LoggingNovaVolumeManager(PRecord):
_nova_volumes = field(mandatory=True)
class _LoggingNovaServerManager(PRecord):
_nova_servers = field(mandatory=True)
class ARecord(PRecord):
x = field(type=(int, float))
y = field()
class _LoggingCinderVolumeManager(PRecord):
_cinder_volumes = field(mandatory=True)
class BRecord(PRecord):
d = field(
type=datetime.date,
factory=lambda d: datetime.datetime.strptime(d, "%d%m%Y").date(),
serializer=lambda format, d: d.strftime('%Y-%m-%d')
if format == 'ISO' else d.strftime('%d%m%Y'))
