def pset_field(item_type, optional=False): """ Create checked ``PSet`` field. :param item_type: The required type for the items in the set. :param bool optional: If true, ``None`` can be used as a value for this field. :return: A ``field`` containing a ``CheckedPSet`` of the given type. """ class TheSet(CheckedPSet): __type__ = item_type TheSet.__name__ = item_type.__name__.capitalize() + "PSet" if optional: def factory(argument): if argument is None: return None else: return TheSet(argument) else: factory = TheSet return field(type=optional_type(TheSet) if optional else TheSet, factory=factory, mandatory=True, initial=TheSet())
def pmap_field(key_type, value_type, optional=False, invariant=_valid): """ Create a checked ``PMap`` field. :param key: The required type for the keys of the map. :param value: The required type for the values of the map. :param bool optional: If true, ``None`` can be used as a value for this field. :param invariant: Pass-through to ``field``. :return: A ``field`` containing a ``CheckedPMap``. """ class TheMap(CheckedPMap): __key_type__ = key_type __value_type__ = value_type TheMap.__name__ = (key_type.__name__.capitalize() + value_type.__name__.capitalize() + "PMap") if optional: def factory(argument): if argument is None: return None else: return TheMap(argument) else: factory = TheMap return field(mandatory=True, initial=TheMap(), type=optional_type(TheMap) if optional else TheMap, factory=factory, invariant=invariant)
def _sequence_field(checked_class, suffix, item_type, optional, initial): """ Create checked field for either ``PSet`` or ``PVector``. :param checked_class: ``CheckedPSet`` or ``CheckedPVector``. :param suffix: Suffix for new type name. :param item_type: The required type for the items in the set. :param bool optional: If true, ``None`` can be used as a value for this field. :param initial: Initial value to pass to factory. :return: A ``field`` containing a checked class. """ class TheType(checked_class): __type__ = item_type TheType.__name__ = item_type.__name__.capitalize() + suffix if optional: def factory(argument): if argument is None: return None else: return TheType(argument) else: factory = TheType return field(type=optional_type(TheType) if optional else TheType, factory=factory, mandatory=True, initial=factory(initial))
def _volume(): """ Create and return a ``PRecord`` ``field`` to hold a ``BlockDeviceVolume``. """ return field( type=BlockDeviceVolume, mandatory=True, # Disable the automatic PRecord.create factory. Callers can just # supply the right type, we don't need the magic coercion behavior # supplied by default. factory=lambda x: x )
def pmap_field( key_type, value_type, optional=False, invariant=_valid, initial=_UNDEFINED, factory=None ): """ Create a checked ``PMap`` field. :param key: The required type for the keys of the map. :param value: The required type for the values of the map. :param bool optional: If true, ``None`` can be used as a value for this field. :param invariant: Pass-through to ``field``. :param initial: An initial value for the field. This will first be coerced using the field's factory. If not given, the initial value is an empty map. :param factory: A factory used to convert input arguments to the stored value whenever it is set. Note that this will be composed with the constructor for the ``CheckedPMap`` class constructed for this field. :return: A ``field`` containing a ``CheckedPMap``. """ input_factory = factory class TheMap(CheckedPMap): __key_type__ = key_type __value_type__ = value_type TheMap.__name__ = (key_type.__name__.capitalize() + value_type.__name__.capitalize() + "PMap") if optional: def mapping_factory(argument): if argument is None: return None else: return TheMap(argument) else: mapping_factory = TheMap if input_factory: factory = lambda x: mapping_factory(input_factory(x)) else: factory = mapping_factory if initial is _UNDEFINED: initial = TheMap() else: initial = factory(initial) return field(mandatory=True, initial=initial, type=optional_type(TheMap) if optional else TheMap, factory=factory, invariant=invariant)
def _make_intent_from_args(args): """ Create an intent type for a given set of arguments. :param args: a dict with keys as the names of arguments and values as :class:`argument`s. :returns: A new type that can hold all of the data to call a function that has the given arguments. """ class _Intent(PClass): pass for name, arg in iteritems(args): setattr(_Intent, name, field(type=arg.type)) _PIntent = add_metaclass(PClassMeta)(_Intent) return _PIntent
def field_factory(self, *args, **kwargs): """ This function does two things: it forwards args to parser.add_arguments, and other args to field() There is one keyword collision for add_argument and field, which is 'type'. If 'type' exists, we will use it for field :param long_name: The "--long-arg" name :param short_name: :param cli_help: :param parser: :param kwargs: :return: """ short_name = None if len(args) == 2: long_name = args[1] short_name = args[0] elif len(args) == 1: long_name = args[0] if short_name is not None and short_name in self.short_names: log.warning("{0} already used. Not setting {0} for {1}".format(short_name, long_name)) else: self.short_names.add(short_name) if "short_name" in kwargs: kwargs.pop("short_name") parser_kwargs = {} for x in ["required", "nargs", "choices", "default", "help", "dest"]: if x in kwargs: parser_kwargs[x] = kwargs.pop(x) field_kwargs = kwargs if short_name is not None: self.parser.add_argument(short_name, long_name, **parser_kwargs) else: self.parser.add_argument(long_name, **parser_kwargs) # print "Option {}:".format(long_name), field_kwargs return field(**field_kwargs)
def interface_field(interfaces, **field_kwargs): """ A ``PClass`` field which checks that the assigned value provides all the ``interfaces``. :param tuple interfaces: The ``Interface`` that a value must provide. """ if not isinstance(interfaces, tuple): raise TypeError( "The ``interfaces`` argument must be a tuple. " "Got: {!r}".format(interfaces) ) original_invariant = field_kwargs.pop("invariant", None) def invariant(value): error_messages = [] if original_invariant is not None: (original_invariant_result, _original_invariant_message) = original_invariant(value) if original_invariant_result: error_messages.append(original_invariant_result) missing_interfaces = [] for interface in interfaces: if not interface.providedBy(value): missing_interfaces.append(interface.getName()) if missing_interfaces: error_messages.append( "The value {!r} " "did not provide these required interfaces: {}".format( value, ", ".join(missing_interfaces) ) ) if error_messages: return (False, "\n".join(error_messages)) else: return (True, "") field_kwargs["invariant"] = invariant return field(**field_kwargs)
def pmap_field( key_type, value_type, optional=False, invariant=_valid, initial=_UNDEFINED ): """ Create a checked ``PMap`` field. :param key: The required type for the keys of the map. :param value: The required type for the values of the map. :param bool optional: If true, ``None`` can be used as a value for this field. :param invariant: Pass-through to ``field``. :param initial: An initial value for the field. This will first be coerced using the field's factory. If not given, the initial value is an empty map. :return: A ``field`` containing a ``CheckedPMap``. """ class TheMap(CheckedPMap): __key_type__ = key_type __value_type__ = value_type TheMap.__name__ = (key_type.__name__.capitalize() + value_type.__name__.capitalize() + "PMap") if optional: def factory(argument): if argument is None: return None else: return TheMap(argument) else: factory = TheMap if initial is _UNDEFINED: initial = TheMap() else: initial = factory(initial) return field(mandatory=True, initial=initial, type=optional_type(TheMap) if optional else TheMap, factory=factory, invariant=invariant)
class CRecord(PRecord): x = field(serializer=1)
def any_field(**kw): return pyrsistent.field(mandatory=True, **kw)
def new_record_type(name, headings): return type( name, (PClass,), {heading: field(type=unicode, mandatory=True) for heading in headings} )
class BRecord(PRecord): __invariant__ = lambda r: (r.x % r.y == 0, 'modulo') x = field() y = field()
class BRecord(PRecord): x = field(type=int, initial='foo')
class BRecord(PRecord): x = field(initial=1) y = field(initial=2)
class BRecord(PRecord): x = field(invariant=lambda x: (x > 1, 'x too small')) y = field(mandatory=True)
class A(PRecord): x = field(type=int)
class MyRecord(PRecord): a = field(int, initial=lambda: 2)
class Foo(PRecord): foo = field(type=str)
class BRecord(ARecord): z = field()
class Node(PRecord): applications = field(type=ApplicationVector)
class Application(PRecord): name = field(type=(six.text_type, ) + six.string_types) image = field(type=(six.text_type, ) + six.string_types)
def fieldm(): return field(mandatory=True)
class BRecord(PRecord): d = field(serializer=lambda format, d: format)
class WrittenAction(PClass): """ An Action that has been logged. This class is intended to provide a definition within Eliot of what an action actually is, and a means of constructing actions that are known to be valid. @ivar WrittenMessage start_message: A start message whose task UUID and level match this action, or C{None} if it is not yet set on the action. @ivar WrittenMessage end_message: An end message hose task UUID and level match this action. Can be C{None} if the action is unfinished. @ivar TaskLevel task_level: The action's task level, e.g. if start message has level C{[2, 3, 1]} it will be C{TaskLevel(level=[2, 3])}. @ivar UUID task_uuid: The UUID of the task to which this action belongs. @ivar _children: A L{pmap} from L{TaskLevel} to the L{WrittenAction} and L{WrittenMessage} objects that make up this action. """ start_message = field(type=optional(WrittenMessage), mandatory=True, initial=None) end_message = field(type=optional(WrittenMessage), mandatory=True, initial=None) task_level = field(type=TaskLevel, mandatory=True) task_uuid = field(type=unicode, mandatory=True, factory=unicode) # Pyrsistent doesn't support pmap_field with recursive types. _children = pmap_field(TaskLevel, object) @classmethod def from_messages(cls, start_message=None, children=pvector(), end_message=None): """ Create a C{WrittenAction} from C{WrittenMessage}s and other C{WrittenAction}s. @param WrittenMessage start_message: A message that has C{ACTION_STATUS_FIELD}, C{ACTION_TYPE_FIELD}, and a C{task_level} that ends in C{1}, or C{None} if unavailable. @param children: An iterable of C{WrittenMessage} and C{WrittenAction} @param WrittenMessage end_message: A message that has the same C{action_type} as this action. @raise WrongTask: If C{end_message} has a C{task_uuid} that differs from C{start_message.task_uuid}. @raise WrongTaskLevel: If any child message or C{end_message} has a C{task_level} that means it is not a direct child. @raise WrongActionType: If C{end_message} has an C{ACTION_TYPE_FIELD} that differs from the C{ACTION_TYPE_FIELD} of C{start_message}. @raise InvalidStatus: If C{end_message} doesn't have an C{action_status}, or has one that is not C{SUCCEEDED_STATUS} or C{FAILED_STATUS}. @raise InvalidStartMessage: If C{start_message} does not have a C{ACTION_STATUS_FIELD} of C{STARTED_STATUS}, or if it has a C{task_level} indicating that it is not the first message of an action. @return: A new C{WrittenAction}. """ actual_message = [message for message in [start_message, end_message] + list(children) if message][0] action = cls( task_level=actual_message.task_level.parent(), task_uuid=actual_message.task_uuid, ) if start_message: action = action._start(start_message) for child in children: if action._children.get(child.task_level, child) != child: raise DuplicateChild(action, child) action = action._add_child(child) if end_message: action = action._end(end_message) return action @property def action_type(self): """ The type of this action, e.g. C{"yourapp:subsystem:dosomething"}. """ if self.start_message: return self.start_message.contents[ACTION_TYPE_FIELD] elif self.end_message: return self.end_message.contents[ACTION_TYPE_FIELD] else: return None @property def status(self): """ One of C{STARTED_STATUS}, C{SUCCEEDED_STATUS}, C{FAILED_STATUS} or C{None}. """ message = self.end_message if self.end_message else self.start_message if message: return message.contents[ACTION_STATUS_FIELD] else: return None @property def start_time(self): """ The Unix timestamp of when the action started, or C{None} if there has been no start message added so far. """ if self.start_message: return self.start_message.timestamp @property def end_time(self): """ The Unix timestamp of when the action ended, or C{None} if there has been no end message. """ if self.end_message: return self.end_message.timestamp @property def exception(self): """ If the action failed, the name of the exception that was raised to cause it to fail. If the action succeeded, or hasn't finished yet, then C{None}. """ if self.end_message: return self.end_message.contents.get(EXCEPTION_FIELD, None) @property def reason(self): """ The reason the action failed. If the action succeeded, or hasn't finished yet, then C{None}. """ if self.end_message: return self.end_message.contents.get(REASON_FIELD, None) @property def children(self): """ The list of child messages and actions sorted by task level, excluding the start and end messages. """ return pvector(sorted(self._children.values(), key=lambda m: m.task_level)) def _validate_message(self, message): """ Is C{message} a valid direct child of this action? @param message: Either a C{WrittenAction} or a C{WrittenMessage}. @raise WrongTask: If C{message} has a C{task_uuid} that differs from the action's C{task_uuid}. @raise WrongTaskLevel: If C{message} has a C{task_level} that means it's not a direct child. """ if message.task_uuid != self.task_uuid: raise WrongTask(self, message) if not message.task_level.parent() == self.task_level: raise WrongTaskLevel(self, message) def _add_child(self, message): """ Return a new action with C{message} added as a child. Assumes C{message} is not an end message. @param message: Either a C{WrittenAction} or a C{WrittenMessage}. @raise WrongTask: If C{message} has a C{task_uuid} that differs from the action's C{task_uuid}. @raise WrongTaskLevel: If C{message} has a C{task_level} that means it's not a direct child. @return: A new C{WrittenAction}. """ self._validate_message(message) level = message.task_level return self.transform(('_children', level), message) def _start(self, start_message): """ Start this action given its start message. @param WrittenMessage start_message: A start message that has the same level as this action. @raise InvalidStartMessage: If C{start_message} does not have a C{ACTION_STATUS_FIELD} of C{STARTED_STATUS}, or if it has a C{task_level} indicating that it is not the first message of an action. """ if start_message.contents.get( ACTION_STATUS_FIELD, None) != STARTED_STATUS: raise InvalidStartMessage.wrong_status(start_message) if start_message.task_level.level[-1] != 1: raise InvalidStartMessage.wrong_task_level(start_message) return self.set(start_message=start_message) def _end(self, end_message): """ End this action with C{end_message}. Assumes that the action has not already been ended. @param WrittenMessage end_message: An end message that has the same level as this action. @raise WrongTask: If C{end_message} has a C{task_uuid} that differs from the action's C{task_uuid}. @raise WrongTaskLevel: If C{end_message} has a C{task_level} that means it's not a direct child. @raise InvalidStatus: If C{end_message} doesn't have an C{action_status}, or has one that is not C{SUCCEEDED_STATUS} or C{FAILED_STATUS}. @return: A new, completed C{WrittenAction}. """ action_type = end_message.contents.get(ACTION_TYPE_FIELD, None) if self.action_type not in (None, action_type): raise WrongActionType(self, end_message) self._validate_message(end_message) status = end_message.contents.get(ACTION_STATUS_FIELD, None) if status not in (FAILED_STATUS, SUCCEEDED_STATUS): raise InvalidStatus(self, end_message) return self.set(end_message=end_message)
class BRecord(PRecord): __invariant__ = lambda r: (r.x <= r.y, 'y smaller than x') x = field() y = field()
class AWSProvisioner(PClass): """ A provisioner that creates nodes on AWS. :ivar boto.ec2.connection.EC2Connection _connection: The boto connection to use. :ivar bytes _keyname: The name of an existing ssh public key configured with the cloud provider. :ivar _security_groups: List of security groups to put the instances created by this provisioner in. :type _security_groups: `list` of `bytes` :param bytes _zone: The AWS availability zone to put instances created by this provisioner in. """ _connection = field(mandatory=True) _keyname = field(type=bytes, mandatory=True) _security_groups = field(mandatory=True) _zone = field(type=bytes, mandatory=True) _default_size = field(type=bytes, mandatory=True) def get_ssh_key(self): """ Return the public key associated with the provided keyname. :return Key: The ssh public key or ``None`` if it can't be determined. """ # EC2 only provides the SSH2 fingerprint (for uploaded keys) # or the SHA-1 hash of the private key (for EC2 generated keys) # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_KeyPairInfo.html return None def create_node(self, name, distribution, size=None, disk_size=8, metadata={}): if size is None: size = self._default_size with start_action( action_type=u"flocker:provision:aws:create_node", name=name, distribution=distribution, image_size=size, disk_size=disk_size, metadata=metadata, ): metadata = metadata.copy() metadata['Name'] = name disk1 = EBSBlockDeviceType() disk1.size = disk_size disk1.delete_on_termination = True diskmap = BlockDeviceMapping() diskmap['/dev/sda1'] = disk1 images = self._connection.get_all_images( filters={'name': IMAGE_NAMES[distribution]}, ) with start_action( action_type= u"flocker:provision:aws:create_node:run_instances", ) as context: reservation = self._connection.run_instances( images[0].id, key_name=self._keyname, instance_type=size, security_groups=self._security_groups, block_device_map=diskmap, placement=self._zone, # On some operating systems, a tty is requried for sudo. # Since AWS systems have a non-root user as the login, # disable this, so we can use sudo with conch. user_data=dedent("""\ #!/bin/sh sed -i '/Defaults *requiretty/d' /etc/sudoers """), ) instance = reservation.instances[0] context.add_success_fields(instance_id=instance.id) self._connection.create_tags([instance.id], metadata) # Display state as instance starts up, to keep user informed that # things are happening. _wait_until_running(instance) return AWSNode( name=name, _provisioner=self, _instance=instance, distribution=distribution, )
class BRecord(PRecord): x = field(type=1)
class AWSNode(PClass): _provisioner = field(mandatory=True) _instance = field(mandatory=True) distribution = field(mandatory=True) name = field(mandatory=True) @property def address(self): return self._instance.ip_address.encode('ascii') @property def private_address(self): return self._instance.private_ip_address.encode('ascii') def destroy(self): with start_action( action_type=u"flocker:provision:aws:destroy", instance_id=self._instance.id, ): self._instance.terminate() def get_default_username(self): """ Return the username available by default on a system. """ return _usernames[self.distribution] def provision(self, package_source, variants=()): """ Provision flocker on this node. :param LibcloudNode node: Node to provision. :param PackageSource package_source: See func:`task_install_flocker` :param set variants: The set of variant configurations to use when provisioning """ username = self.get_default_username() commands = [] # cloud-init may not have allowed sudo without tty yet, so try SSH key # installation for a few more seconds: start = [] def for_thirty_seconds(*args, **kwargs): if not start: start.append(time()) return Effect(Constant((time() - start[0]) < 30)) commands.append( run_remotely( username=username, address=self.address, commands=retry(task_install_ssh_key(), for_thirty_seconds), )) commands.append( run_remotely( username='******', address=self.address, commands=provision( package_source=package_source, distribution=self.distribution, variants=variants, ), )) return sequence(commands) def reboot(self): """ Reboot the node. :return Effect: """ def do_reboot(_): with start_action( action_type=u"flocker:provision:aws:reboot", instance_id=self._instance.id, ): self._instance.reboot() _wait_until_running(self._instance) return run_remotely(username="******", address=self.address, commands=run_from_args(["sync" ])).on(success=do_reboot)
class BRecord(PRecord): x = field(invariant='foo')
class PendingTrade(PRecord): buyers_price = field(type=float)
class BRecord(PRecord): x = field(type=int, factory=1)
class AlgorithmicModel(PRecord): pending_trades = pvector_field(PendingTrade) selling_threshold = field(type=float) cut_losses_threshold = field(type=float)
class BRecord(PRecord): x = field() y = field()
def test_enum(): f = field(type=TestEnum) assert TestEnum in f.type assert len(f.type) == 1
class BRecord(PRecord): x = field(factory=int, invariant=lambda x: (x >= 0, 'x negative')) y = field(mandatory=True)
class CRecord(PRecord): b = field()
class CRecord(PRecord): a = field(invariant=lambda x: (x != 0, 'a zero')) b = field(type=BRecord)
class _LoggingNovaVolumeManager(PRecord): _nova_volumes = field(mandatory=True)
class _LoggingNovaServerManager(PRecord): _nova_servers = field(mandatory=True)
class ARecord(PRecord): x = field(type=(int, float)) y = field()
class _LoggingCinderVolumeManager(PRecord): _cinder_volumes = field(mandatory=True)
class BRecord(PRecord): d = field( type=datetime.date, factory=lambda d: datetime.datetime.strptime(d, "%d%m%Y").date(), serializer=lambda format, d: d.strftime('%Y-%m-%d') if format == 'ISO' else d.strftime('%d%m%Y'))