class EndDeviceGroup(ComplexModel):
"""Abstraction for management of group communications within a two-way AMR
system or the data for a group of related end devices. Commands can be
issued to all of the end devices that belong to the group using a
defined group address and the underlying AMR communication
infrastructure. A DERGroup and a PANDeviceGroup is an
EndDeviceGroup."""
_type_info = [
('mRID', Uuid.customize(max_occurs=1, min_occurs=0)),
('Names', Name.customize(max_occurs='unbounded', min_occurs=0)),
]
def __init__(self, mRID=None, names=None, **kwargs):
super().__init__(mRID=mRID, Names=names, **kwargs)
self.mRID = mRID
if names is None:
self.names = []
else:
self.names = names
def test_uuid_deserialize(self):
value = uuid.UUID('12345678123456781234567812345678')
assert ProtocolBase().from_unicode(Uuid,
'12345678-1234-5678-1234-567812345678') == value
assert ProtocolBase().from_unicode(Uuid(serialize_as='hex'),
'12345678123456781234567812345678') == value
assert ProtocolBase().from_unicode(Uuid(serialize_as='urn'),
'urn:uuid:12345678-1234-5678-1234-567812345678') == value
assert ProtocolBase().from_string(Uuid(serialize_as='bytes'),
b'\x124Vx\x124Vx\x124Vx\x124Vx') == value
assert ProtocolBase().from_string(Uuid(serialize_as='bytes_le'),
b'xV4\x124\x12xV\x124Vx\x124Vx') == value
assert ProtocolBase().from_unicode(Uuid(serialize_as='fields'),
(305419896, 4660, 22136, 18, 52, 95073701484152)) == value
assert ProtocolBase().from_unicode(Uuid(serialize_as='int'),
24197857161011715162171839636988778104) == value
def test_uuid_serialize(self):
value = uuid.UUID('12345678123456781234567812345678')
assert ProtocolBase().to_bytes(Uuid, value) == \
'12345678-1234-5678-1234-567812345678'
assert ProtocolBase().to_bytes(Uuid(serialize_as='hex'), value) == \
'12345678123456781234567812345678'
assert ProtocolBase().to_bytes(Uuid(serialize_as='urn'), value) == \
'urn:uuid:12345678-1234-5678-1234-567812345678'
assert ProtocolBase().to_bytes(Uuid(serialize_as='bytes'), value) == \
b'\x124Vx\x124Vx\x124Vx\x124Vx'
assert ProtocolBase().to_bytes(Uuid(serialize_as='bytes_le'), value) == \
b'xV4\x124\x12xV\x124Vx\x124Vx'
assert ProtocolBase().to_bytes(Uuid(serialize_as='fields'), value) == \
(305419896, 4660, 22136, 18, 52, 95073701484152)
assert ProtocolBase().to_bytes(Uuid(serialize_as='int'), value) == \
24197857161011715162171839636988778104
def test_uuid_validate(self):
assert Uuid.validate_string(Uuid,
'12345678-1234-5678-1234-567812345678')
assert Uuid.validate_native(Uuid,
uuid.UUID('12345678-1234-5678-1234-567812345678'))
class Daemon(ComplexModel):
"""A couple of neurons."""
LOGGING_DEVEL_FORMAT = "%(module)-15s | %(message)s"
LOGGING_PROD_FORMAT = "%(asctime)s | %(module)-8s | %(message)s"
_type_info = [
('uuid',
Uuid(no_cli=True,
help="Daemon uuid. Regenerated every time a new "
"config file is written. It could come in handy.")),
('secret',
ByteArray(no_cli=True,
help="Secret key for signing cookies "
"and other stuff.")),
('daemonize',
Boolean(default=False, help="Daemonizes before everything else.")),
('uid',
Unicode(help="The daemon user. You need to start the server as"
" a priviledged user for this to work.")),
('gid',
Unicode(help="The daemon group. You need to start the server as"
" a priviledged user for this to work.")),
('pid_file',
String(help="The path to a text file that contains the pid"
"of the daemonized process.")),
('logger_dest',
String(
help="The path to the log file. The server won't"
" daemonize without this. Converted to an absolute path if not.")
),
('log_rpc', Boolean(help="Log raw rpc data.")),
('log_queries', Boolean(help="Log sql queries.")),
('log_results',
Boolean(help="Log query results in addition to queries"
"themselves.")),
('main_store',
Unicode(help="The name of the store for binding "
"neurons.TableModel's metadata to.")),
('bootstrap',
Boolean(help="Bootstrap the application. Create schema, "
"insert initial data, etc.",
no_config=True)),
('_services', Array(Service, sub_name='services')),
('_stores', Array(StorageInfo, sub_name='stores')),
('_loggers', Array(Logger, sub_name='loggers')),
]
# FIXME: we need this atrocity with custom constructor and properties
# because spyne doesn't support custom containers
def __init__(self, *args, **kwargs):
super(Daemon, self).__init__(*args, **kwargs)
services = kwargs.get('services', None)
if services is not None:
self.services = services
if not hasattr(self, 'services') or self.services is None:
self.services = _Twrdict('name')()
stores = kwargs.get('stores', None)
if stores is not None:
self.stores = stores
if not hasattr(self, 'stores') or self.stores is None:
self.stores = _wdict()
loggers = kwargs.get('loggers', None)
if loggers is not None:
self.loggers = loggers
if not hasattr(self, 'loggers') or self.loggers is None:
self.loggers = _wdict()
@property
def _services(self):
if self.services is not None:
for k, v in self.services.items():
v.name = k
return self.services.values()
self.services = _Twrdict('name')()
return []
@_services.setter
def _services(self, what):
self.services = what
if what is not None:
self.services = _Twrdict('name')([(s.name, s) for s in what])
@property
def _stores(self):
if self.stores is not None:
for k, v in self.stores.items():
v.name = k
return self.stores.values()
self.stores = _wdict()
return []
@_stores.setter
def _stores(self, what):
self.stores = what
if what is not None:
self.stores = _wdict([(s.name, s) for s in what])
@property
def _loggers(self):
if self.loggers is not None:
for k, v in self.loggers.items():
v.name = k
return self.loggers.values()
self.loggers = _wdict()
return []
@_loggers.setter
def _loggers(self, what):
self.loggers = what
if what is not None:
self.loggers = _wdict([(s.path, s) for s in what])
@classmethod
def get_default(cls, daemon_name):
return cls(
uuid=uuid1(),
secret=os.urandom(64),
_stores=[
Relational(
name="sql_main",
backend="sqlalchemy",
pool_size=10,
pool_recycle=3600,
pool_timeout=30,
max_overflow=3,
conn_str='postgres://postgres:@localhost:5432/%s_%s' %
(daemon_name, getpass.getuser()),
sync_pool=True,
async_pool=True,
),
],
main_store='sql_main',
_loggers=[
Logger(path='.',
level='DEBUG',
format=cls.LOGGING_DEVEL_FORMAT),
],
)
def apply_logging(self):
# We're using twisted logging only for IO.
from twisted.python.logger import FileLogObserver
from twisted.python.logger import Logger, LogLevel, globalLogPublisher
LOGLEVEL_TWISTED_MAP = {
logging.DEBUG: LogLevel.debug,
logging.INFO: LogLevel.info,
logging.WARN: LogLevel.warn,
logging.ERROR: LogLevel.error,
logging.CRITICAL: LogLevel.critical,
}
class TwistedHandler(logging.Handler):
def emit(self, record):
assert isinstance(record, logging.LogRecord)
Logger(record.name).emit(LOGLEVEL_TWISTED_MAP[record.levelno],
log_text=self.format(record))
if self.logger_dest is not None:
from twisted.python.logfile import DailyLogFile
self.logger_dest = abspath(self.logger_dest)
if access(dirname(self.logger_dest), os.R_OK | os.W_OK):
log_dest = DailyLogFile.fromFullPath(self.logger_dest)
else:
Logger().warn("%r is not accessible. We need rwx on it to "
"rotate logs." % dirname(self.logger_dest))
log_dest = open(self.logger_dest, 'wb+')
formatter = logging.Formatter(self.LOGGING_PROD_FORMAT)
else:
formatter = logging.Formatter(self.LOGGING_DEVEL_FORMAT)
log_dest = open('/dev/stdout', 'wb+')
try:
import colorama
colorama.init()
logger.debug("colorama loaded.")
except Exception as e:
logger.debug("coloarama not loaded: %r" % e)
def record_as_string(record):
if 'log_text' in record:
return record['log_text'] + "\n"
if 'message' in record:
return record['message'] + "\n"
if 'log_failure' in record:
failure = record['log_failure']
return "%s: %s" % (failure.type, pformat(vars(failure.value)))
return pformat(record)
observer = FileLogObserver(log_dest, record_as_string)
globalLogPublisher.addObserver(observer)
handler = TwistedHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
for l in self._loggers or []:
l.apply()
if self.log_rpc or self.log_queries or self.log_results:
logging.getLogger().setLevel(logging.DEBUG)
if self.log_rpc:
logging.getLogger('spyne.protocol').setLevel(logging.DEBUG)
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
logging.getLogger('spyne.protocol.dictdoc').setLevel(logging.DEBUG)
if self.log_queries:
logging.getLogger('sqlalchemy').setLevel(logging.INFO)
if self.log_results:
logging.getLogger('sqlalchemy').setLevel(logging.DEBUG)
def sanitize(self):
if self.logger_dest is not None:
self.logger_dest = abspath(self.logger_dest)
if self.pid_file is not None:
self.pid_file = abspath(self.pid_file)
def apply(self, for_testing=False):
"""Daemonizes the process if requested, then sets up logging and data
stores.
"""
# FIXME: apply_storage could return a deferred due to txpool init.
# Daemonization won't work if twisted is imported before fork().
# It's best to know this in advance or you'll have to deal with daemons
# that work perfectly well in development environments but won't boot
# in production ones, solely because of fork()ingw.
assert for_testing or not ('twisted' in sys.modules), \
"Twisted is already imported!"
self.sanitize()
if self.daemonize:
assert self.logger_dest, "Refusing to start without any log output."
daemonize()
self.apply_logging()
if self.pid_file is not None:
pid = os.getpid()
with open(self.pid_file, 'w') as f:
f.write(str(pid))
logger.debug("Pid file is at: %r", self.pid_file)
self.apply_storage()
def apply_storage(self):
for store in self._stores or []:
try:
store.apply()
except Exception as e:
logger.exception(e)
raise
if self.main_store == store.name:
engine = store.itself.engine
import neurons
neurons.TableModel.Attributes.sqla_metadata.bind = engine
@classmethod
def parse_config(cls, daemon_name, argv=None):
_apply_custom_attributes(cls)
retval = cls.get_default(daemon_name)
file_name = abspath('%s.yaml' % daemon_name)
argv_parser = spyne_to_argparse(cls)
cli = {}
if argv is not None and len(argv) > 1:
cli = dict(argv_parser.parse_args(argv[1:]).__dict__.items())
if cli['config_file'] is not None:
file_name = abspath(cli['config_file'])
del cli['config_file']
exists = isfile(file_name) and os.access(file_name, os.R_OK)
if exists:
retval = yaml_loads(open(file_name).read(),
cls,
validator='soft',
polymorphic=True)
else:
if not access(dirname(file_name), os.R_OK | os.W_OK):
raise Exception("File %r can't be created in %r" %
(file_name, dirname(file_name)))
for k, v in cli.items():
if not v in (None, False):
setattr(retval, k, v)
retval.config_file = file_name
return retval
def write_config(self):
open(self.config_file, 'wb').write(
get_object_as_yaml(self, self.__class__, polymorphic=True))
