from eliot import Field, ActionType __all__ = [ "JSON_REQUEST", "REQUEST", ] LOG_SYSTEM = u"api" METHOD = Field(u"method", lambda method: method, u"The HTTP method of the request.") REQUEST_PATH = Field( u"request_path", lambda path: path, u"The absolute path of the resource to which the request was issued.") JSON = Field.forTypes( u"json", [unicode, bytes, dict, list, None, bool, float], u"The JSON request body.") RESPONSE_CODE = Field.forTypes( u"code", [int], u"The response code for the request.") # It would be nice if RESPONSE_CODE was in REQUEST instead of # JSON_REQUEST; see FLOC-1586. REQUEST = ActionType( LOG_SYSTEM + u":request", [REQUEST_PATH, METHOD], [], u"A request was received on the public HTTP interface.") # NB we deliberately do not log the entire JSON response body because the
LOG_START_SERVICE = ActionType( u'fusion_index:service:start', [_SERVICE_DESCRIPTION], [], u'Indexing service is starting') LOG_STOP_SERVICE = ActionType( u'fusion_index:service:stop', [], [], u'Indexing service is stopping') LOG_LOOKUP_GET = ActionType( u'fusion_index:lookup:get', fields(environment=unicode, indexType=unicode, key=unicode), [Field.for_types('value', [bytes, None], u'Value in the index, if any')], u'Retrieving a value from the lookup index') LOG_LOOKUP_PUT = ActionType( u'fusion_index:lookup:put', fields(environment=unicode, indexType=unicode, key=unicode), fields(value=bytes), u'Storing a value in the lookup index') _SEARCH_TYPE = Field.for_types( 'searchType', [unicode, None], u'The search type') LOG_SEARCH_GET = ActionType( u'fusion_index:search:get', fields(
import attr from eliot import ( ActionType, Field, ) from eliot.testing import capture_logging from twisted.internet.defer import ( maybeDeferred, ) _NAME = Field.for_types( u"name", [unicode], u"The name of the test.", ) RUN_TEST = ActionType( u"run-test", [_NAME], [], u"A test is run.", ) def eliot_logged_test(f): """ Decorate a test method to run in a dedicated Eliot action context.
from twisted.internet.defer import succeed, fail from twisted.internet.threads import deferToThread from twisted.web.http import NOT_FOUND, INTERNAL_SERVER_ERROR from ..common import ( poll_until, retry_if, decorate_methods, with_retry, get_default_retry_steps, ) from ..control._model import ( RestartNever, RestartAlways, RestartOnFailure, pset_field, pvector_field) LOG_CACHED_IMAGE = MessageType( u"flocker:node:docker:image_from_cache", [Field.for_types(u"image", [unicode], "The image ID.")], "An image was retrieved from the cache." ) class AlreadyExists(Exception): """A unit with the given name already exists.""" @with_cmp(["address", "apierror"]) class AddressInUse(Exception): """ The listen address for an exposed port was in use and could not be bound. """ def __init__(self, address, apierror): """
if not isinstance(v, t): raise ValidationError("{} not an instance of {}".format(v, t)) return validator def validateSetMembership(s): """ Return an Eliot validator that requires values to be elements of ``s``. """ def validator(v): if v not in s: raise ValidationError("{} not in {}".format(v, s)) return validator RELPATH = Field.for_types( u"relpath", [unicode], u"The relative path of a file in a magic-folder.", ) VERSION = Field.for_types( u"version", [int, long], u"The version of the file.", ) LAST_UPLOADED_URI = Field.for_types( u"last_uploaded_uri", [unicode, bytes, None], u"The filecap to which this version of this file was uploaded.", )
from twisted.trial.unittest import SynchronousTestCase from eliot.testing import ( assertHasMessage, capture_logging ) from eliot import Field, MessageType from zope.interface import Interface, implementer from .. import interface_decorator # Eliot structures for testing ``interface_decorator``. METHOD = Field.for_types( u"method", [unicode], u"The name of the decorated method.") TEST_MESSAGE = MessageType(u"flocker:common:test:interface:message", [METHOD]) TEST_EXCEPTION = MessageType(u"flocker:common:test:interface:exception", [METHOD]) class IDummy(Interface): """ Dummy interface with two test methods. """ def return_method(): """ Return something. """
from characteristic import attributes from eliot import MessageType, ActionType, Field from eliot.twisted import DeferredContext from twisted.python.failure import Failure from twisted.internet.error import ProcessTerminated, ProcessDone from twisted.internet.defer import Deferred from twisted.internet.protocol import ProcessProtocol from twisted.protocols.basic import LineOnlyReceiver RUN_ACTION = ActionType( action_type="flocker.common.runner:run", startFields=[ Field.for_types(u"command", [list], u"The command.") ], successFields=[], description="Run a command.", ) RUN_OUTPUT_MESSAGE = MessageType( message_type="flocker.common.runner:run:stdout", fields=[ Field.for_types(u"line", [bytes], u"The output."), ], description=u"A line of command output.", ) RUN_ERROR_MESSAGE = MessageType( message_type="flocker.common.runner:run:stderr", fields=[ Field.for_types(u"line", [bytes], u"The error."),
# Copyright ClusterHQ Ltd. See LICENSE file for details. from eliot import Field, MessageType CINDER_VOLUME = MessageType( u"flocker:functional:cinder:cinder_volume:created", [Field.for_types( u"id", [bytes, unicode], u"The Cinder-assigned unique identifier for the volume that was " u"created.", )], )
IN_MOVE_SELF, IN_UNMOUNT, IN_Q_OVERFLOW, IN_IGNORED, IN_ONLYDIR, IN_DONT_FOLLOW, \ IN_MASK_ADD, IN_ISDIR, IN_ONESHOT, IN_CLOSE, IN_MOVED, IN_CHANGED, \ _FLAG_TO_HUMAN from . import _watchdog_541 _watchdog_541.patch() NOT_STARTED = "NOT_STARTED" STARTED = "STARTED" STOPPING = "STOPPING" STOPPED = "STOPPED" _PATH = Field.for_types( u"path", [bytes, unicode], u"The path an inotify event concerns.", ) _EVENT = Field( u"event", lambda e: e.__class__.__name__, u"The watchdog event that has taken place.", validateInstanceOf(FileSystemEvent), ) ANY_INOTIFY_EVENT = ActionType( u"watchdog:inotify:any-event", [_PATH, _EVENT], [], u"An inotify event is being dispatched.",
def validate_ipv4_address(value): if not isinstance(value, IPv4Address): raise ValidationError( value, u"Field %s requires type to be IPv4Address (not %s)" % (u"target_ip", type(value))) def serialize_ipv4_address(address): return unicode(address) TARGET_IP = Field( key=u"target_ip", serializer=serialize_ipv4_address, extraValidator=validate_ipv4_address, description=u"The IP address which is the target of a proxy.") TARGET_PORT = Field.forTypes( u"target_port", [int], u"The port number which is the target of a proxy.") ARGV = Field.forTypes(u"argv", [list], u"The argument list of a child process being executed.") IPTABLES = ActionType( _system(u"iptables"), [ARGV], [], u"An iptables command which Flocker is executing against the system.") CREATE_PROXY_TO = ActionType(_system(u"create_proxy_to"), [TARGET_IP, TARGET_PORT], [],
# Copyright Hybrid Logic Ltd. See LICENSE file for details. """ This module defines the Eliot log events emitted by the API implementation. """ __all__ = [ "REQUEST_PATH", "REQUEST", ] from eliot import Field, ActionType LOG_SYSTEM = u"api" REQUEST_PATH = Field.forTypes( u"request_path", [unicode], u"The absolute path of the resource to which the request was issued.") REQUEST = ActionType( LOG_SYSTEM, [REQUEST_PATH], [], u"A request was received on the public HTTP interface.")
CALLBACK, validateInstanceOf, ) from . import _watchdog_541 _watchdog_541.patch() NOT_STARTED = "NOT_STARTED" STARTED = "STARTED" STOPPING = "STOPPING" STOPPED = "STOPPED" _PATH = Field.for_types( u"path", [bytes, unicode], u"The path an inotify event concerns.", ) _EVENT = Field( u"event", lambda e: e.__class__.__name__, u"The watchdog event that has taken place.", validateInstanceOf(FileSystemEvent), ) ANY_INOTIFY_EVENT = ActionType( u"watchdog:inotify:any-event", [_PATH, _EVENT], [], u"An inotify event is being dispatched.",
u"target_ip", type(value))) def serialize_ipv4_address(address): return unicode(address) TARGET_IP = Field( key=u"target_ip", serializer=serialize_ipv4_address, extraValidator=validate_ipv4_address, description=u"The IP address which is the target of a proxy.") TARGET_PORT = Field.forTypes( u"target_port", [int], u"The port number which is the target of a proxy.") ARGV = Field.forTypes( u"argv", [list], u"The argument list of a child process being executed.") IPTABLES = ActionType( _system(u"iptables"), [ARGV], [], u"An iptables command which Flocker is executing against the system.")
class AlreadyAttachedVolume(VolumeException): """ A failed attempt to attach a block device that is already attached. """ class UnattachedVolume(VolumeException): """ An attempt was made to operate on an unattached volume but the operation requires the volume to be attached. """ DATASET = Field( u"dataset", lambda dataset: dataset.dataset_id, u"The unique identifier of a dataset." ) VOLUME = Field( u"volume", lambda volume: volume.blockdevice_id, u"The unique identifier of a volume." ) DATASET_ID = Field( u"dataset_id", lambda dataset_id: unicode(dataset_id), u"The unique identifier of a dataset." )
NOT_FOUND, PRECONDITION_FAILED, ) from twisted.internet.utils import getProcessOutput from twisted.internet.task import deferLater from treq import json_content, content from ..ca import treq_with_authentication from ..control import Leases as LeasesModel, LeaseError, DockerImage from ..common import retry_failure from .. import __version__ _LOG_HTTP_REQUEST = ActionType("flocker:apiclient:http_request", [ Field.forTypes("url", [bytes, unicode], "Request URL."), Field.forTypes("method", [bytes, unicode], "Request method."), Field("request_body", lambda o: o, "Request JSON body.") ], [ Field.forTypes("response_code", [int], "Response code."), Field("response_body", lambda o: o, "JSON response body.") ], "A HTTP request.") _LOG_CONDITIONAL_CREATE = ActionType(u"flocker:apiclient:conditional_create", [], [], u"Conditionally create a dataset.") NoneType = type(None) class ServerResponseMissingElementError(Exception):
The goal here is to mostly focus on performance of serialization, in a vaguely realistic manner. That is, mesages are logged in context of a message with a small number of fields. """ from __future__ import unicode_literals import time from eliot import Logger, MessageType, Field, ActionType def _ascii(s): return s.decode("ascii") F1 = Field.forTypes("integer", [int], "") F2 = Field("string", _ascii, "") F3 = Field("string2", _ascii, "") F4 = Field.forTypes("list", [list], "list of integers") M = MessageType("system:message", [F1, F2, F3, F4], "description") A = ActionType("action", [], [], [], "desc") log = Logger() N = 100000 def run(): start = time.time() with A(log): for i in xrange(N):
from eliot import ActionType, Field, fields _SEARCH_CLASS = Field(u'searchClass', lambda c: c.value, u'The search class') LOG_LOOKUP_GET = ActionType( u'fusion_index:lookup:get', fields(environment=unicode, indexType=unicode, key=unicode), [Field.for_types('value', [bytes, None], u'Value in the index, if any')], u'Retrieving a value from the lookup index') LOG_LOOKUP_PUT = ActionType( u'fusion_index:lookup:put', fields(environment=unicode, indexType=unicode, key=unicode), fields(value=bytes), u'Storing a value in the lookup index') _SEARCH_TYPE = Field.for_types('searchType', [unicode, None], u'The search type') LOG_SEARCH_GET = ActionType( u'fusion_index:search:get', fields(_SEARCH_CLASS, _SEARCH_TYPE, environment=unicode, indexType=unicode, searchValue=unicode), fields(results=list), u'Searching the search index') LOG_SEARCH_PUT = ActionType( u'fusion_index:search:put', fields(_SEARCH_CLASS, environment=unicode, indexType=unicode,
from twisted.internet.error import ConnectionDone from twisted.protocols.basic import LineOnlyReceiver from twisted.python.filepath import FilePath import os from ...common import loop_until, timeout from ._model import ( Run, Sudo, Put, Comment, RunRemotely, perform_comment, perform_put, perform_sudo) from .._effect import dispatcher as base_dispatcher RUN_OUTPUT_MESSAGE = MessageType( message_type="flocker.provision.ssh:run:output", fields=[ Field.for_types(u"line", [bytes], u"The output."), ], description=u"A line of command output.", ) def extReceived(self, type, data): from twisted.conch.ssh.connection import EXTENDED_DATA_STDERR if type == EXTENDED_DATA_STDERR: self.dataReceived(data) @attributes([ "deferred", "context", ])
__all__ = [ "JSON_REQUEST", "REQUEST", ] from eliot import Field, ActionType LOG_SYSTEM = u"api" METHOD = Field(u"method", lambda method: method, u"The HTTP method of the request.") REQUEST_PATH = Field( u"request_path", lambda path: path, u"The absolute path of the resource to which the request was issued.") JSON = Field.forTypes( u"json", [unicode, bytes, dict, list, None, bool, float], u"JSON, either request or response depending on context.") RESPONSE_CODE = Field.forTypes( u"code", [int], u"The response code for the request.") # It would be nice if RESPONSE_CODE was in REQUEST instead of # JSON_REQUEST; see FLOC-1586. REQUEST = ActionType( LOG_SYSTEM + u":request", [REQUEST_PATH, METHOD], [], u"A request was received on the public HTTP interface.") JSON_REQUEST = ActionType( LOG_SYSTEM + u":json_request",
from twisted.internet.defer import succeed, fail from twisted.python.filepath import FilePath from twisted.web.http import CREATED, OK, CONFLICT, NOT_FOUND from twisted.internet.utils import getProcessOutput from treq import json_content, content from ..ca import treq_with_authentication from ..control import Leases as LeasesModel, LeaseError from ..common import retry_failure from .. import __version__ _LOG_HTTP_REQUEST = ActionType( "flocker:apiclient:http_request", [Field.forTypes("url", [bytes, unicode], "Request URL."), Field.forTypes("method", [bytes, unicode], "Request method."), Field("request_body", lambda o: o, "Request JSON body.")], [Field.forTypes("response_code", [int], "Response code."), Field("response_body", lambda o: o, "JSON response body.")], "A HTTP request.") NoneType = type(None) class Dataset(PClass): """ A dataset in the configuration. :attr UUID primary: The node where the dataset should manifest.
OpenStack-related tools. """ # After _interface_decorator is public, move this and auto_openstack_logging # into (or at least nearer) flocker/node/agents/cinder.py. from eliot import Field, MessageType from novaclient.exceptions import ClientException as NovaClientException from keystoneclient.openstack.common.apiclient.exceptions import ( HttpError as KeystoneHttpError, ) from ._thread import _interface_decorator CODE = Field.for_types("code", [int], u"The HTTP response code.") MESSAGE = Field.for_types( "message", [bytes, unicode], u"A human-readable error message given by the response.", ) DETAILS = Field.for_types("details", [dict], u"Extra details about the error.") REQUEST_ID = Field.for_types( "request_id", [bytes, unicode], u"The unique identifier assigned by the server for this request.", ) URL = Field.for_types("url", [bytes, unicode], u"The request URL.") METHOD = Field.for_types("method", [bytes, unicode], u"The request method.") NOVA_CLIENT_EXCEPTION = MessageType( u"openstack:nova_client_exception", [ CODE,
from zope.interface import Attribute, Interface, implementer, provider from zope.interface.exceptions import DoesNotImplement from eliot import Field, ActionType, Logger from twisted.python.util import FancyStrMixin, FancyEqMixin from twisted.python.components import proxyForInterface from twisted.internet.defer import succeed def _system(suffix): return u":".join((u"fsm", suffix)) FSM_IDENTIFIER = Field.forTypes( u"fsm_identifier", [unicode], u"An unique identifier for the FSM to which the event pertains.") FSM_STATE = Field.forTypes( u"fsm_state", [unicode], u"The state of the FSM prior to the transition.") FSM_RICH_INPUT = Field.forTypes( u"fsm_rich_input", [unicode], u"The string representation of the rich input delivered to the FSM.") FSM_INPUT = Field.forTypes( u"fsm_input", [unicode], u"The string representation of the input symbol delivered to the FSM.") FSM_NEXT_STATE = Field.forTypes( u"fsm_next_state", [unicode], u"The string representation of the state of the FSM after the transition.") FSM_OUTPUT = Field.forTypes( u"fsm_output", [list], # of unicode u"A list of the string representations of the outputs produced by the "
""" Helper module to provide macros for logging support for storage drivers (AWS, Cinder). See https://clusterhq.atlassian.net/browse/FLOC-2053 for consolidation opportunities. """ from eliot import Field, ActionType, MessageType # Begin: Common structures used by all (AWS, OpenStack) # storage drivers. # An OPERATION is a list of: # IBlockDeviceAPI name, positional arguments, keyword arguments. OPERATION = Field.for_types( u"operation", [list], u"The IBlockDeviceAPI operation being executed," u"along with positional and keyword arguments.") # End: Common structures used by all storage drivers. # Begin: Helper datastructures to log IBlockDeviceAPI calls # from AWS storage driver using Eliot. # ActionType used by AWS storage driver. AWS_ACTION = ActionType( u"flocker:node:agents:blockdevice:aws", [OPERATION], [], u"An IBlockDeviceAPI operation is executing using AWS storage driver.") # Three fields to gather from EC2 response to Boto.
dictionary = dictionary.copy() dictionary.pop(_CLASS_MARKER) return _CONFIG_CLASS_MAP[class_name].create(dictionary) else: return dictionary return loads(data, object_hook=decode) _DEPLOYMENT_FIELD = Field(u"configuration", repr) _LOG_STARTUP = MessageType(u"flocker-control:persistence:startup", [_DEPLOYMENT_FIELD]) _LOG_SAVE = ActionType(u"flocker-control:persistence:save", [_DEPLOYMENT_FIELD], []) _UPGRADE_SOURCE_FIELD = Field.for_types( u"source_version", [int], u"Configuration version to upgrade from.") _UPGRADE_TARGET_FIELD = Field.for_types( u"target_version", [int], u"Configuration version to upgrade to.") _LOG_UPGRADE = ActionType(u"flocker-control:persistence:migrate_configuration", [_DEPLOYMENT_FIELD, _UPGRADE_SOURCE_FIELD, _UPGRADE_TARGET_FIELD, ], []) _LOG_EXPIRE = MessageType( u"flocker-control:persistence:lease-expired", [Field(u"dataset_id", unicode), Field(u"node_id", unicode)], u"A lease for a dataset has expired.") _LOG_UNCHANGED_DEPLOYMENT_NOT_SAVED = MessageType( u"flocker-control:persistence:unchanged-deployment-not-saved", [], u"The persistence service was told to save a deployment which is the same " u"as the already-saved deployment. It has optimized this away."
lambda v: None if v is None else { "size": v.size, "mtime_ns": v.mtime_ns, "ctime_ns": v.ctime_ns, "version": v.version, "last_uploaded_uri": v.last_uploaded_uri, "last_downloaded_uri": v.last_downloaded_uri, "last_downloaded_timestamp": v.last_downloaded_timestamp, }, u"The local database state of a file.", validateInstanceOf((type(None), PathEntry)), ) _INSERT_OR_UPDATE = Field.for_types( u"insert_or_update", [unicode], u"An indication of whether the record for this upload was new or an update to a previous entry.", validateSetMembership({u"insert", u"update"}), ) UPDATE_ENTRY = ActionType( u"magic-folder-db:update-entry", [RELPATH, VERSION, LAST_UPLOADED_URI, LAST_DOWNLOADED_URI, LAST_DOWNLOADED_TIMESTAMP, PATHINFO], [_INSERT_OR_UPDATE], u"Record some metadata about a relative path in the magic-folder.", ) # magic-folder db schema version 1 SCHEMA_v1 = """ CREATE TABLE version (
:param arguments: A ``list`` of ``bytes``, command-line arguments to ``zfs``. :return: A :class:`Deferred` firing with the bytes of the result (on exit code 0), or errbacking with :class:`CommandFailed` or :class:`BadArguments` depending on the exit code (1 or 2). """ endpoint = ProcessEndpoint(reactor, b"zfs", [b"zfs"] + arguments, os.environ) d = connectProtocol(endpoint, _AccumulatingProtocol()) d.addCallback(lambda protocol: protocol._result) return d _ZFS_COMMAND = Field.forTypes( "zfs_command", [bytes], u"The command which was run.") _OUTPUT = Field.forTypes( "output", [bytes], u"The output generated by the command.") _STATUS = Field.forTypes( "status", [int], u"The exit status of the command") ZFS_ERROR = MessageType( "filesystem:zfs:error", [_ZFS_COMMAND, _OUTPUT, _STATUS], u"The zfs command signaled an error.") def _sync_command_error_squashed(arguments, logger): """ Synchronously run a command-line tool with the given arguments.
u'txacme:jws:http:get', fields(), fields(), u'A JWSClient GET request') LOG_JWS_POST = ActionType( u'txacme:jws:http:post', fields(), fields(), u'A JWSClient POST request') LOG_JWS_REQUEST = ActionType( u'txacme:jws:http:request', fields(url=unicode), fields(Field.for_types(u'content_type', [unicode, None], u'Content-Type header field'), code=int), u'A JWSClient request') LOG_JWS_CHECK_RESPONSE = ActionType( u'txacme:jws:http:check-response', fields(Field.for_types(u'response_content_type', [unicode, None], u'Content-Type header field'), expected_content_type=unicode), fields(), u'Checking a JWSClient response') LOG_JWS_GET_NONCE = ActionType( u'txacme:jws:nonce:get',