class SomeService(Service):
@rpc(Unicode(sub_name="fileName"),
ByteArray(sub_name='binaryData'),
ByteArray(sub_name="hash"),
_returns=Unicode)
def documentRequest(ctx, file_name, file_data, data_hash):
assert file_name == FILE_NAME
assert file_data == (PAYLOAD, )
return file_name
class DaemonConfig(ComplexModel):
SECTION_NAME = 'basic'
daemonize = Boolean(default=False)
"""Fork the process to the background."""
log_file = AbsolutePath
"""Log file."""
pid_file = AbsolutePath
"""File that will contain the pid of the daemon process."""
config_file = AbsolutePath
"""Alternative configuration file.."""
uid = SystemUser
"""Daemon will drop privileges and switch to this uid when specified"""
gid = SystemGroup
"""Daemon will drop privileges and switch to this gid when specified"""
log_level = Unicode(values=['DEBUG', 'INFO'], default='DEBUG')
"""Logging level"""
show_rpc = Boolean(default=False)
"""Log raw request and response data."""
secret = ByteArray(default_factory=lambda: [os.urandom(64)],
no_cmdline=True)
"""Cookie encryption key. Keep secret."""
thread_min = UnsignedInteger(default=3)
"""Min number of threads in the thread pool"""
thread_max = UnsignedInteger(default=10)
"""Max number of threads in the thread pool"""
listeners = Array(ListenerConfig)
class Certificate(ComplexModel):
__namespace__ = ''
ID = Integer.customize(max_occurs=1, min_occurs=0)
GUID = Unicode(128, pattern='[^@]+@[^@]+')
SendDateTime = DateTime.customize(max_occurs=1, min_occurs=0)
Number = String(max_len=100).customize(max_occurs=1, min_occurs=0)
Date = DateTime.customize(max_occurs=1, min_occurs=0)
Blanc = String(max_len=100).customize(max_occurs=1, min_occurs=0)
DepartureCountry = String(max_len=100).customize(max_occurs=1,
min_occurs=0)
DepartureCountryCode = String(max_len=2).customize(max_occurs=1,
min_occurs=0)
DestinationCountry = String(max_len=100).customize(max_occurs=1,
min_occurs=0)
DestinationCountryCode = String(max_len=2).customize(max_occurs=1,
min_occurs=0)
EntryCheckpoint = String(max_len=255).customize(max_occurs=1, min_occurs=0)
EntryCheckpointCode = Integer.customize(max_occurs=1, min_occurs=0)
consignor = Consignor
consignee = Consignee
transport = Transport
disinfection = Disinfection
productdescription = ProductDescription
GeneralMarking = String(max_len=512).customize(max_occurs=1, min_occurs=0)
GeneralQuarantineCondition = String(max_len=512).customize(max_occurs=1,
min_occurs=0)
GeneralBaseDocument = String(max_len=512).customize(max_occurs=1,
min_occurs=0)
GeneralAdditionalDeclaration = String(max_len=512).customize(max_occurs=1,
min_occurs=0)
GeneralMandatoryActions = String(max_len=512).customize(max_occurs=1,
min_occurs=0)
AdditionalInfo = String(max_len=2000).customize(max_occurs=1, min_occurs=0)
Inspector = String(max_len=255).customize(max_occurs=1, min_occurs=0)
AnnexDoc = String(max_len=100000).customize(max_occurs=1, min_occurs=0)
AnnexText = String(max_len=100000).customize(max_occurs=1, min_occurs=0)
PDF = ByteArray.customize(max_occurs=1, min_occurs=0)
class Daemon(ComplexModel):
"""A couple of neurons."""
LOGGING_DEVEL_FORMAT = "%(module)-15s | %(message)s"
LOGGING_PROD_FORMAT = "%(asctime)s | %(module)-8s | %(message)s"
_type_info = [
('uuid',
Uuid(no_cli=True,
help="Daemon uuid. Regenerated every time a new "
"config file is written. It could come in handy.")),
('secret',
ByteArray(no_cli=True,
help="Secret key for signing cookies "
"and other stuff.")),
('daemonize',
Boolean(default=False, help="Daemonizes before everything else.")),
('uid',
Unicode(help="The daemon user. You need to start the server as"
" a priviledged user for this to work.")),
('gid',
Unicode(help="The daemon group. You need to start the server as"
" a priviledged user for this to work.")),
('pid_file',
String(help="The path to a text file that contains the pid"
"of the daemonized process.")),
('logger_dest',
String(
help="The path to the log file. The server won't"
" daemonize without this. Converted to an absolute path if not.")
),
('log_rpc', Boolean(help="Log raw rpc data.")),
('log_queries', Boolean(help="Log sql queries.")),
('log_results',
Boolean(help="Log query results in addition to queries"
"themselves.")),
('main_store',
Unicode(help="The name of the store for binding "
"neurons.TableModel's metadata to.")),
('bootstrap',
Boolean(help="Bootstrap the application. Create schema, "
"insert initial data, etc.",
no_config=True)),
('_services', Array(Service, sub_name='services')),
('_stores', Array(StorageInfo, sub_name='stores')),
('_loggers', Array(Logger, sub_name='loggers')),
]
# FIXME: we need this atrocity with custom constructor and properties
# because spyne doesn't support custom containers
def __init__(self, *args, **kwargs):
super(Daemon, self).__init__(*args, **kwargs)
services = kwargs.get('services', None)
if services is not None:
self.services = services
if not hasattr(self, 'services') or self.services is None:
self.services = _Twrdict('name')()
stores = kwargs.get('stores', None)
if stores is not None:
self.stores = stores
if not hasattr(self, 'stores') or self.stores is None:
self.stores = _wdict()
loggers = kwargs.get('loggers', None)
if loggers is not None:
self.loggers = loggers
if not hasattr(self, 'loggers') or self.loggers is None:
self.loggers = _wdict()
@property
def _services(self):
if self.services is not None:
for k, v in self.services.items():
v.name = k
return self.services.values()
self.services = _Twrdict('name')()
return []
@_services.setter
def _services(self, what):
self.services = what
if what is not None:
self.services = _Twrdict('name')([(s.name, s) for s in what])
@property
def _stores(self):
if self.stores is not None:
for k, v in self.stores.items():
v.name = k
return self.stores.values()
self.stores = _wdict()
return []
@_stores.setter
def _stores(self, what):
self.stores = what
if what is not None:
self.stores = _wdict([(s.name, s) for s in what])
@property
def _loggers(self):
if self.loggers is not None:
for k, v in self.loggers.items():
v.name = k
return self.loggers.values()
self.loggers = _wdict()
return []
@_loggers.setter
def _loggers(self, what):
self.loggers = what
if what is not None:
self.loggers = _wdict([(s.path, s) for s in what])
@classmethod
def get_default(cls, daemon_name):
return cls(
uuid=uuid1(),
secret=os.urandom(64),
_stores=[
Relational(
name="sql_main",
backend="sqlalchemy",
pool_size=10,
pool_recycle=3600,
pool_timeout=30,
max_overflow=3,
conn_str='postgres://postgres:@localhost:5432/%s_%s' %
(daemon_name, getpass.getuser()),
sync_pool=True,
async_pool=True,
),
],
main_store='sql_main',
_loggers=[
Logger(path='.',
level='DEBUG',
format=cls.LOGGING_DEVEL_FORMAT),
],
)
def apply_logging(self):
# We're using twisted logging only for IO.
from twisted.python.logger import FileLogObserver
from twisted.python.logger import Logger, LogLevel, globalLogPublisher
LOGLEVEL_TWISTED_MAP = {
logging.DEBUG: LogLevel.debug,
logging.INFO: LogLevel.info,
logging.WARN: LogLevel.warn,
logging.ERROR: LogLevel.error,
logging.CRITICAL: LogLevel.critical,
}
class TwistedHandler(logging.Handler):
def emit(self, record):
assert isinstance(record, logging.LogRecord)
Logger(record.name).emit(LOGLEVEL_TWISTED_MAP[record.levelno],
log_text=self.format(record))
if self.logger_dest is not None:
from twisted.python.logfile import DailyLogFile
self.logger_dest = abspath(self.logger_dest)
if access(dirname(self.logger_dest), os.R_OK | os.W_OK):
log_dest = DailyLogFile.fromFullPath(self.logger_dest)
else:
Logger().warn("%r is not accessible. We need rwx on it to "
"rotate logs." % dirname(self.logger_dest))
log_dest = open(self.logger_dest, 'wb+')
formatter = logging.Formatter(self.LOGGING_PROD_FORMAT)
else:
formatter = logging.Formatter(self.LOGGING_DEVEL_FORMAT)
log_dest = open('/dev/stdout', 'wb+')
try:
import colorama
colorama.init()
logger.debug("colorama loaded.")
except Exception as e:
logger.debug("coloarama not loaded: %r" % e)
def record_as_string(record):
if 'log_text' in record:
return record['log_text'] + "\n"
if 'message' in record:
return record['message'] + "\n"
if 'log_failure' in record:
failure = record['log_failure']
return "%s: %s" % (failure.type, pformat(vars(failure.value)))
return pformat(record)
observer = FileLogObserver(log_dest, record_as_string)
globalLogPublisher.addObserver(observer)
handler = TwistedHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
for l in self._loggers or []:
l.apply()
if self.log_rpc or self.log_queries or self.log_results:
logging.getLogger().setLevel(logging.DEBUG)
if self.log_rpc:
logging.getLogger('spyne.protocol').setLevel(logging.DEBUG)
logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
logging.getLogger('spyne.protocol.dictdoc').setLevel(logging.DEBUG)
if self.log_queries:
logging.getLogger('sqlalchemy').setLevel(logging.INFO)
if self.log_results:
logging.getLogger('sqlalchemy').setLevel(logging.DEBUG)
def sanitize(self):
if self.logger_dest is not None:
self.logger_dest = abspath(self.logger_dest)
if self.pid_file is not None:
self.pid_file = abspath(self.pid_file)
def apply(self, for_testing=False):
"""Daemonizes the process if requested, then sets up logging and data
stores.
"""
# FIXME: apply_storage could return a deferred due to txpool init.
# Daemonization won't work if twisted is imported before fork().
# It's best to know this in advance or you'll have to deal with daemons
# that work perfectly well in development environments but won't boot
# in production ones, solely because of fork()ingw.
assert for_testing or not ('twisted' in sys.modules), \
"Twisted is already imported!"
self.sanitize()
if self.daemonize:
assert self.logger_dest, "Refusing to start without any log output."
daemonize()
self.apply_logging()
if self.pid_file is not None:
pid = os.getpid()
with open(self.pid_file, 'w') as f:
f.write(str(pid))
logger.debug("Pid file is at: %r", self.pid_file)
self.apply_storage()
def apply_storage(self):
for store in self._stores or []:
try:
store.apply()
except Exception as e:
logger.exception(e)
raise
if self.main_store == store.name:
engine = store.itself.engine
import neurons
neurons.TableModel.Attributes.sqla_metadata.bind = engine
@classmethod
def parse_config(cls, daemon_name, argv=None):
_apply_custom_attributes(cls)
retval = cls.get_default(daemon_name)
file_name = abspath('%s.yaml' % daemon_name)
argv_parser = spyne_to_argparse(cls)
cli = {}
if argv is not None and len(argv) > 1:
cli = dict(argv_parser.parse_args(argv[1:]).__dict__.items())
if cli['config_file'] is not None:
file_name = abspath(cli['config_file'])
del cli['config_file']
exists = isfile(file_name) and os.access(file_name, os.R_OK)
if exists:
retval = yaml_loads(open(file_name).read(),
cls,
validator='soft',
polymorphic=True)
else:
if not access(dirname(file_name), os.R_OK | os.W_OK):
raise Exception("File %r can't be created in %r" %
(file_name, dirname(file_name)))
for k, v in cli.items():
if not v in (None, False):
setattr(retval, k, v)
retval.config_file = file_name
return retval
def write_config(self):
open(self.config_file, 'wb').write(
get_object_as_yaml(self, self.__class__, polymorphic=True))
class TaskQueue(TableModel):
__tablename__ = 'task_queue'
id = Integer32(primary_key=True)
data = ByteArray(nullable=False)