class Sandbox(object): def __init__(self): self._events = dict() self._logger = Logger() def invoke(self, event_name, request, stream): """ Connect worker and decorator """ event_closure = self._events.get(event_name, None) if event_closure is not None: event_handler = event_closure() event_handler.invoke(request, stream) else: self._logger.warn("There is no handler for event %s" % event_name) def on(self, event_name, event_handler): try: # Try to construct handler. closure = event_handler() except Exception: # If this callable object is not our wrapper - may raise Exception closure = default(event_handler)() if hasattr(closure, "_wrapped"): event_handler = default(event_handler) else: if not hasattr(closure, "_wrapped"): event_handler = default(event_handler) self._events[event_name] = event_handler
def __init__(self, storage, key, namespace): self.storage = SecureStorage(storage) self.key = key self.logger = Logger() self.namespace = namespace_prefix + namespace self.dbnamespace = namespace_prefix + "apps" self.lognamespace = namespace_prefix + "logs" self.logger.info("UserDB has been initialized. Use namespace %s" % self.namespace)
def __init__(self, app_name, addresses=None, attempts=3, delay=0.1, max_delay=60.0, delay_exp=2.0, connect_timeout=None, timeout=None, logger=None): self.delay = delay self.max_delay = max_delay self.delay_exp = delay_exp self.connect_timeout = connect_timeout self.timeout = timeout self.attempts = attempts self.logger = logger or Logger() self._reset() addresses = addresses or ReconnectableService.DEFAULT_ADDRESS pairs = [] for address in addresses.split(','): address_parts = address.split(':') host = address_parts[0] port = (len(address_parts) > 1 and int(address_parts[1]) or ReconnectableService.DEFAULT_PORT) pairs.append((host, port)) self.addresses = itertools.cycle(pairs) self.app_name = app_name self.upstream = None
def __init__(self, init_args=sys.argv): self._logger = Logger() self._init_endpoint(init_args) self.sessions = dict() self.sandbox = Sandbox() self.service = ev.Service() self.disown_timer = ev.Timer(self.on_disown, 2, self.service) self.heartbeat_timer = ev.Timer(self.on_heartbeat, 5, self.service) self.disown_timer.start() self.heartbeat_timer.start() self.pipe = Pipe(self.endpoint) self.service.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.service, self.pipe) self.r_stream = ReadableStream(self.service, self.pipe) self.r_stream.bind(self.decoder.decode) self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) self._send_handshake()
def __init__(self, init_args=sys.argv, disown_timeout=2, heartbeat_timeout=20): self._logger = Logger() self._init_endpoint(init_args) self.sessions = dict() self.sandbox = Sandbox() self.loop = ev.Loop() self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop) self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop) self.disown_timer.start() self.heartbeat_timer.start() self.pipe = Pipe(self.endpoint) self.loop.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) # Send both messages - to run timers properly. This messages will be sent # only after all initialization, so they have same purpose. self._send_handshake() self._send_heartbeat()
def __init__(self): self._logger = Logger() self.cache = list() self._clbk = None # Callback - on chunk self._errbk = None # Errorback - translate error to handler self._errmsg = None # Store message self._state = 1 # Status of stream (close/open)
def __init__(self, **config): self.logger = Logger() self.place = None self.tablename = '' try: unix_socket = config.get('MysqlSocket', "/var/run/mysqld/mysqld.sock") self.dbname = config.get('local_db_name', 'COMBAINE') self.user = config.get('user', 'root') self.password = config.get('password', "") self.db = MySQLdb.connect(unix_socket=unix_socket, user=self.user, passwd=self.password) self.cursor = self.db.cursor() self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname) self.db.commit() self.db.select_db(self.dbname) except Exception as err: self.logger.error('Error in init MySQLdb %s' % err) raise Exception
def __init__(self, **config): self.logger = Logger() self.place = None self.tablename = '' try: # port = config.get('local_db_port', 3306) unix_socket = config.get('MysqlSocket', "/var/run/mysqld/mysqld.sock") self.dbname = config.get('local_db_name', 'COMBAINE') self.db = MySQLdb.connect(unix_socket=unix_socket, user='******', ) self.cursor = self.db.cursor() self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname) self.db.commit() self.db.select_db(self.dbname) except Exception as err: self.logger.error('Error in init MySQLdb %s' % err) raise Exception
def __init__(self): self._events = dict() self._logger = Logger()
class MySqlDG(object): def __init__(self, **config): self.logger = Logger() self.place = None self.tablename = '' try: unix_socket = config.get('MysqlSocket', "/var/run/mysqld/mysqld.sock") self.dbname = config.get('local_db_name', 'COMBAINE') self.user = config.get('user', 'root') self.password = config.get('password', "") self.db = MySQLdb.connect(unix_socket=unix_socket, user=self.user, passwd=self.password) self.cursor = self.db.cursor() self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname) self.db.commit() self.db.select_db(self.dbname) except Exception as err: self.logger.error('Error in init MySQLdb %s' % err) raise Exception def putData(self, data, tablename): try: tablename = tablename.replace('.', '_').replace('-', '_').replace('+', '_') line = None fname = '/dev/shm/%s-%i' % ('COMBAINE', random.randint(0, 65535)) with open(fname, 'w') as table_file: for line in data: table_file.write('GOPA'.join([str(x) for _, x in line]) + '\n') table_file.close() if not line: self.logger.info("Data for mysql is missed") os.remove(table_file.name) return False self.logger.debug( 'Data has been written to a temporary file %s, size: %d bytes' % (table_file.name, os.lstat(table_file.name).st_size)) if not self._preparePlace(line): self.logger.error( 'Unsupported field types. Look at preparePlace()') return False self.cursor.execute('DROP TABLE IF EXISTS %s' % tablename) query = "CREATE TABLE IF NOT EXISTS `%(tablename)s` %(struct)s ENGINE = MEMORY DATA DIRECTORY='/dev/shm/'" % { 'tablename': tablename, 'struct': self.place } self.cursor.execute(query) self.db.commit() query = "LOAD DATA INFILE '%(filename)s' INTO TABLE `%(tablename)s` FIELDS TERMINATED BY 'GOPA'" % { 'filename': table_file.name, 'tablename': tablename } self.cursor.execute(query) self.db.commit() if os.path.isfile(table_file.name): os.remove(table_file.name) except Exception as err: self.logger.error('Error in putData %s' % err) if os.path.isfile(table_file.name): os.remove(table_file.name) return False else: self.tablename = tablename return True def _preparePlace(self, example): ftypes = { types.IntType: "BIGINT", types.UnicodeType: "VARCHAR(200)", types.StringType: "VARCHAR(200)", types.FloatType: "DOUBLE" } try: self.place = '( %s )' % ','.join([ " `%s` %s" % (field_name, ftypes[type(field_type)]) for field_name, field_type in example ]) except Exception as err: self.logger.error('Error in preparePlace() %s' % err) self.place = None return False else: return True def perfomCustomQuery(self, query_string): self.logger.debug("Execute query: %s" % query_string) self.cursor.execute(query_string) _ret = self.cursor.fetchall() self.db.commit() return _ret
class UserDB(object): def __init__(self, storage, key, namespace): self.storage = SecureStorage(storage) self.key = key self.logger = Logger() self.namespace = namespace_prefix + namespace self.dbnamespace = namespace_prefix + "apps" self.lognamespace = namespace_prefix + "logs" self.logger.info("UserDB has been initialized. Use namespace %s" % self.namespace) @asynchronous def exists(self, name): try: yield self.storage.read(self.namespace, name) except Exception as err: self.logger.error(str(err)) yield False else: yield True @asynchronous def get(self, name): yield self.storage.read(self.namespace, name) @asynchronous def create(self, info): user_info = dict() uid = uuid.uuid4().hex name = info['name'] password = info['password'] exists = yield self.exists(name) if exists: raise Exception("Already exists") # Store user uid user_info['uid'] = uid # Store username user_info['name'] = name # Crypt user passwd h = HMAC.new(uid) h.update(password) user_info['hash'] = h.hexdigest() try: yield self.storage.write(self.namespace, name, user_info, USER_TAG) except Exception as err: self.logger.error(str(err)) yield False else: yield True @asynchronous def remove(self, name): try: self.logger.info("Remove user %s" % name) yield self.storage.remove(self.namespace, name) except Exception as err: self.logger.error(repr(err)) try: self.logger.info("Remove user %s application info" % name) yield self.storage.remove(self.dbnamespace, name) except Exception as err: self.logger.error(repr(err)) try: self.logger.info("Remove user %s upload logs" % name) tags = LOG_TAG + [name] logkeys = yield self.storage.find(self.lognamespace, tags) self.logger.debug("Uploadlogs keys %s" % logkeys) for key in logkeys: yield self.storage.remove(self.lognamespace, key) except Exception as err: self.logger.error(repr(err)) @asynchronous def login(self, name, password): user_info = yield self.get(name) self.logger.info(str(user_info)) h = HMAC.new(user_info['uid']) h.update(password) self.logger.error("%s %s" % (h.hexdigest(), user_info['hash'])) if (h.hexdigest() == user_info['hash']): user_info.pop('uid') yield user_info else: raise Exception("Invalid pair of login/password") @asynchronous def users(self): yield self.storage.find(self.namespace, USER_TAG) @asynchronous def user_apps(self, user): apps = list() try: raw_apps = yield self.storage.read(self.dbnamespace, user) apps = msgpack.unpackb(raw_apps) except Exception as err: self.logger.error(repr(err)) finally: yield apps @asynchronous def write_app_info(self, user, name): def handler(data): apps = list() if data is None: apps = list() else: apps = msgpack.unpackb(data) if name in apps: self.logger.error("App %s already exists" % name) return None apps.append(name) return msgpack.packb(apps) reader = partial(self.storage.read, self.dbnamespace, user) writer = lambda result: self.storage.write(self.dbnamespace, user, result, USER_TAG) yield self.quasi_atomic_write(reader, writer, handler) @asynchronous def write_buildlog(self, user, key, logdata): tags = LOG_TAG + [user] yield self.storage.write(self.lognamespace, key, logdata, tags) @asynchronous def read_buildlog(self, key): yield self.storage.read(self.lognamespace, key) @asynchronous def list_buildlog(self, user): tags = [user] if user else LOG_TAG yield self.storage.find(self.lognamespace, tags) @asynchronous def quasi_atomic_write(self, reader, writer, handler): while True: data = None summ = "" try: data = yield reader() summ = hashlib.md5(data).hexdigest() except Exception as err: self.logger.error(repr(err)) result = handler(data) if result is None: break try: data = yield reader() except Exception as err: self.logger.error(repr(err)) if data is None or summ == hashlib.md5(data).hexdigest(): self.logger.info("MD5 is still valid. Do write") yield writer(result) break self.logger.info("MD5 mismatchs. Continue")
#!/usr/bin/env python import msgpack from cocaine.worker import Worker from cocaine.logging import Logger __author__ = 'EvgenySafronov <*****@*****.**>' log = Logger() def chunker(request, response): chunks = yield request.read() try: chunks = int(msgpack.loads(chunks)) except ValueError: chunks = int(chunks) for num in xrange(chunks): response.write(msgpack.dumps('{0:-<1024}'.format(num))) response.write(msgpack.dumps('Done')) response.close() W = Worker() W.run({'chunkMe': chunker})
import logging from cocaine.logging import Logger _cocaine_logger = Logger() class CocaineHandler(logging.Handler): def __init__(self, level=logging.NOTSET): # do not work in python 2.6.5 - logging.Handler is an old-style class # super(CocaineHandler, self).__init__(level=level) logging.Handler.__init__(self, level=level) def emit(self, record): try: levelname = record.levelname.lower() # due to bug in cocaine.Logger if levelname == 'warning': levelname = 'warn' log = getattr(_cocaine_logger, levelname) except AttributeError: _cocaine_logger.warn('No appropriate method for log records ' 'of level "{0}"'.format(record.levelname)) log = _cocaine_logger.info log(self.format(record)) root_logger = logging.getLogger('mm') tornado_logger = logging.getLogger('tornado')
import os import sys import imp import logging from time import time import msgpack # pylint: disable=import-error from cocaine.worker import Worker from cocaine.logging import Logger from cocaine.logging.hanlders import CocaineHandler # pylint: enable=import-error Logger().info("Initialize custom.py") PATH = os.environ.get('PLUGINS_PATH', '/usr/lib/yandex/combaine/custom') sys.path.insert(0, PATH) EXTS = [ext for (ext, _, _) in imp.get_suffixes()] def _is_plugin(candidate): name, extension = os.path.splitext(candidate) return name != "__init__" and extension in EXTS def _is_candidate(name): return not name.startswith("_") and name[0].isupper() class Custom(object):
class MySqlDG(object): def __init__(self, **config): self.logger = Logger() self.place = None self.tablename = '' try: # port = config.get('local_db_port', 3306) unix_socket = config.get('MysqlSocket', "/var/run/mysqld/mysqld.sock") self.dbname = config.get('local_db_name', 'COMBAINE') self.db = MySQLdb.connect(unix_socket=unix_socket, user='******', ) self.cursor = self.db.cursor() self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname) self.db.commit() self.db.select_db(self.dbname) except Exception as err: self.logger.error('Error in init MySQLdb %s' % err) raise Exception def putData(self, data, tablename): try: tablename = tablename.replace('.', '_').replace('-', '_').replace('+', '_') line = None fname = '/dev/shm/%s-%i' % ('COMBAINE', random.randint(0, 65535)) with open(fname, 'w') as table_file: for line in data: table_file.write('GOPA'.join([str(x) for x in line.values()]) + '\n') table_file.close() if not line: self.logger.info("Data for mysql is missed") os.remove(table_file.name) return False self.logger.debug('Data written to a temporary file %s, size: %d bytes' % (table_file.name, os.lstat(table_file.name).st_size)) if not self._preparePlace(line): self.logger.error('Unsupported field types. Look at preparePlace()') return False self.cursor.execute('DROP TABLE IF EXISTS %s' % tablename) query = "CREATE TABLE IF NOT EXISTS %(tablename)s %(struct)s ENGINE = MEMORY DATA DIRECTORY='/dev/shm/'" % {'tablename': tablename, 'struct': self.place} self.cursor.execute(query) self.db.commit() query = "LOAD DATA INFILE '%(filename)s' INTO TABLE %(tablename)s FIELDS TERMINATED BY 'GOPA'" % {'filename': table_file.name, 'tablename': tablename} self.cursor.execute(query) self.db.commit() if os.path.isfile(table_file.name): os.remove(table_file.name) except Exception as err: self.logger.error('Error in putData %s' % err) if os.path.isfile(table_file.name): os.remove(table_file.name) return False else: self.tablename = tablename return True def _preparePlace(self, example): ftypes = {types.IntType: "INT", types.UnicodeType: "VARCHAR(200)", types.StringType: "VARCHAR(200)", types.FloatType: "FLOAT"} try: self.place = '( %s )' % ','.join([" %s %s" % (field_name, ftypes[type(field_type)]) for field_name, field_type in example.items()]) except Exception as err: self.logger.error('Error in preparePlace() %s' % err) self.place = None return False else: return True def perfomCustomQuery(self, query_string): self.logger.debug("Execute query: %s" % query_string) self.cursor.execute(query_string) _ret = self.cursor.fetchall() self.db.commit() return _ret def __del__(self): if self.db: self.cursor.close() self.db.commit() self.db.close()
class Worker(object): def __init__(self, init_args=sys.argv, disown_timeout=2, heartbeat_timeout=20): self._logger = Logger() self._init_endpoint(init_args) self.sessions = dict() self.sandbox = Sandbox() self.loop = ev.Loop() self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop) self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop) self.disown_timer.start() self.heartbeat_timer.start() self.pipe = Pipe(self.endpoint) self.loop.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) # Send both messages - to run timers properly. This messages will be sent # only after all initialization, so they have same purpose. self._send_handshake() self._send_heartbeat() def _init_endpoint(self, init_args): try: self.id = init_args[init_args.index("--uuid") + 1] app_name = init_args[init_args.index("--app") + 1] self.endpoint = init_args[init_args.index("--endpoint") + 1] except Exception as err: self._logger.error("Wrong cmdline arguments: %s " % err) raise RuntimeError("Wrong cmdline arguments") def run(self, binds=None): if not binds: binds = {} for event, name in binds.iteritems(): self.on(event, name) self.loop.run() def terminate(self, reason, msg): self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack()) self.loop.stop() # Event machine def on(self, event, callback): self.sandbox.on(event, callback) # Events def on_heartbeat(self): self._send_heartbeat() def on_message(self, args): msg = Message.initialize(args) if msg is None: return elif msg.id == message.RPC_INVOKE: try: _request = Request() _stream = Stream(msg.session, self) self.sandbox.invoke(msg.event, _request, _stream) self.sessions[msg.session] = _request except Exception as err: self._logger.error("On invoke error: %s" % err) traceback.print_stack() elif msg.id == message.RPC_CHUNK: self._logger.debug("Receive chunk: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: try: _session.push(msg.data) except Exception as err: self._logger.error("On push error: %s" % str(err)) elif msg.id == message.RPC_CHOKE: self._logger.debug("Receive choke: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: _session.close() self.sessions.pop(msg.session) elif msg.id == message.RPC_HEARTBEAT: self._logger.debug("Receive heartbeat. Stop disown timer") self.disown_timer.stop() elif msg.id == message.RPC_TERMINATE: self._logger.debug("Receive terminate. Reason: %s, message: %s " % (msg.reason, msg.message)) self.terminate(msg.reason, msg.message) elif msg.id == message.RPC_ERROR: _session = self.sessions.get(msg.session, None) if _session is not None: _session.error(RequestError(msg.message)) def on_disown(self): try: self._logger.error("Disowned") finally: self.loop.stop() # Private: def _send_handshake(self): self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack()) def _send_heartbeat(self): self.disown_timer.start() self._logger.debug("Send heartbeat. Start disown timer") self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack()) def send_choke(self, session): self.w_stream.write(Message(message.RPC_CHOKE, session).pack()) def send_chunk(self, session, data): self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack()) def send_error(self, session, code, msg): self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
#! /usr/bin/env python from hashlib import sha512 from cocaine.worker import Worker from cocaine.logging import Logger from cocaine.services import Service from cocaine.exceptions import * from cocaine.decorators import http, fs L = Logger() urlfetcher_service = Service("urlfetch") storage_service = Service("storage") import sys def example(request, response): L.info("INITIALIZE FUNCTION") try: ls = yield storage_service.list() L.info("From storage: %s" % str(ls)) except ServiceError as err: L.error("S: %s" % err) ls = yield storage_service.list("manifests") L.info("From storage: %s" % str(ls)) chunk_from_cloud = yield request.read() L.info("from dealer: %s" % chunk_from_cloud) try:
class Request(Future): def __init__(self): self._logger = Logger() self.cache = list() self._clbk = None # Callback - on chunk self._errbk = None # Errorback - translate error to handler self._errmsg = None # Store message self._state = 1 # Status of stream (close/open) def push(self, chunk): if self._clbk is None: # If there is no attachment object, put chunk in the cache self._logger.debug("Cache chunk") self.cache.append(chunk) else: # Copy callback to temp, clear current callback and perform temp # Do it so because self._clbk may change, # while performing callback function. # Avoid double chunk sending to the task self._logger.debug("Send chunk to application") temp = self._clbk self._clbk = None temp(chunk) def error(self, errormsg): self._errmsg = errormsg def close(self): self._logger.debug("Close request") self._state = None if len(self.cache) == 0 and self._clbk is not None: self._logger.warn("Chunks are over,\ but the application requests them") if self._errbk is not None: self._logger.error("Throw error") self._errbk(RequestError("No chunks are available")) else: self._logger.error("No errorback. Can't throw error") def read(self): return chain.ChainFactory().then(lambda : self) def default_errorback(self, err): self._logger.error("No errorback.\ Can't throw error: %s" % str(self._errmsg)) def bind(self, callback, errorback=None, on_done=None): #self._logger.debug("Bind request") if len(self.cache) > 0: callback(self.cache.pop(0)) elif self._errmsg is not None: if errorback is not None: errorback(self._errmsg) # translate error into worker else: self.default_errorback(self._errmsg) elif self._state is not None: self._clbk = callback self._errbk = errorback or self.default_errorback else: # Stream closed by choke # Raise exception here because no chunks # from cocaine-runtime are available self._logger.warn("Chunks are over,\ but the application requests them") if errorback: errorback(RequestError("No chunks are available"))
class UrlFetcher(): def __init__(self, io_loop): self.io_loop = io_loop self.http_client = AsyncHTTPClient() self.logger = Logger() @chain.source def perform_request(self, request, response, method): try: constants = request_consts[method] url = request[constants.URL] timeout = request[constants.TIMEOUT] http_request = HTTPRequest(url=url, method=method) http_request.request_timeout = float(timeout)/1000 if method == 'POST': http_request.body = request[constants.BODY] #adds cookies to request params_num = len(request) if constants.COOKIES <= params_num - 1: cookies = request[constants.COOKIES] if len(cookies) > 0: list_of_cookies = list('{0}={1}'.format(cookie, value) for cookie, value in cookies.iteritems()) cookies_str = '; '.join(list_of_cookies) http_request.headers.add('Cookie', cookies_str) #adds headers to request if constants.HEADERS <= params_num - 1: for name, values_list in request[constants.HEADERS].iteritems(): for value in values_list: http_request.headers.add(name, value) self.logger.info("Downloading {0}, headers {1}, method {2}".format(url, http_request.headers, method)) http_response = yield self.http_client.fetch(http_request) response_headers = self._get_headers_from_response(http_response) response.write((True, http_response.body, http_response.code, response_headers,)) response.close() self.logger.info("{0} has been successfuly downloaded".format(url)) except HTTPError as e: self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url)) if e.response is not None: http_response = e.response response_headers = self._get_headers_from_response(http_response) response.write((False, http_response.body, http_response.code, response_headers,)) else: response.write((False, '', e.code, {},)) response.close() except socket.gaierror as e: self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url)) response.write((False, '', e.errno, {},)) response.close() except Exception as e: self.logger.error("Unhandled error ({0}) occured in perform_request, report about this problem " "to httpclient service developers. Method is {1}, stacktrace is: {2}".format( e.message, method, traceback.format_exc())) response.write((False, '', 0, {},)) response.close() @chain.source def on_get_request(self, request, response): try: request_data_packed = yield request.read() request_data = msgpack.unpackb(request_data_packed) yield self.perform_request(request_data, response, 'GET') except StopIteration: pass except Exception as e: self.logger.error("Unhandled error ({0}) occured in on_get_request, report about this problem " "to httpclient service developers. Stacktrace is: {1}".format(e.message, traceback.format_exc())) response.write((False, '', 0, {},)) response.close() @chain.source def on_post_request(self, request, response): try: request_data_packed = yield request.read() request_data = msgpack.unpackb(request_data_packed) yield self.perform_request(request_data, response, 'POST') except StopIteration: pass except Exception as e: self.logger.error("Unhandled error ({0}) occured in on_post_request, report about this problem " "to httpclient service developers. Stacktrace is: {1}".format(e.message, traceback.format_exc())) response.write((False, '', 0, {},)) response.close() def _get_headers_from_response(self, http_response): response_headers = {} for header_tuple in http_response.headers.items(): name = header_tuple[0] value = header_tuple[1] if not name in response_headers: response_headers[name] = [] response_headers[name].append(value) return response_headers
from functools import partial from Crypto.Hash import HMAC import msgpack from cocaine.asio.engine import asynchronous from cocaine.logging import Logger namespace_prefix = "flow-users@" USER_TAG = ["FLOW_USER"] LOG_TAG = ["FLOW_UPLOAD_LOG"] encoder = msgpack.packb decoder = msgpack.unpackb logger = Logger() class SecureStorage(object): def __init__(self, storage): self.storage = storage @asynchronous def write(self, namespace, key, blob, tags): yield self.storage.write(namespace, key, encoder(blob), tags) @asynchronous def read(self, namespace, key): res = yield self.storage.read(namespace, key) yield decoder(res)
def __init__(self, io_loop): self.io_loop = io_loop self.http_client = AsyncHTTPClient() self.logger = Logger()
# (at your option) any later version. # # Combaine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import logging from cocaine.logging import Logger from cocaine.logging.hanlders import CocaineHandler Log = Logger() Log.error("INITIALIZE") l = logging.getLogger("combaine") l.setLevel(logging.DEBUG) ch = CocaineHandler() formatter = logging.Formatter("%(tid)s %(message)s") ch.setFormatter(formatter) ch.setLevel(logging.DEBUG) l.addHandler(ch) def get_logger_adapter(tid): return logging.LoggerAdapter(l, {"tid": tid})
class Worker(object): def __init__(self, init_args=sys.argv): self._logger = Logger() self._init_endpoint(init_args) self.sessions = dict() self.sandbox = Sandbox() self.service = ev.Service() self.disown_timer = ev.Timer(self.on_disown, 2, self.service) self.heartbeat_timer = ev.Timer(self.on_heartbeat, 5, self.service) self.disown_timer.start() self.heartbeat_timer.start() self.pipe = Pipe(self.endpoint) self.service.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.service, self.pipe) self.r_stream = ReadableStream(self.service, self.pipe) self.r_stream.bind(self.decoder.decode) self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) self._send_handshake() def _init_endpoint(self, init_args): try: self.id = init_args[init_args.index("--uuid") + 1] app_name = init_args[init_args.index("--app") + 1] self.endpoint = init_args[init_args.index("--endpoint") + 1] except Exception as err: self._logger.error("Wrong cmdline argumensts: %s " % err) raise RuntimeError("Wrong cmdline arguments") def run(self, binds={}): for event, name in binds.iteritems(): self.on(event, name) self.service.run() def terminate(self, reason, msg): self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack()) self.service.stop() # Event machine def on(self, event, callback): self.sandbox.on(event, callback) # Events def on_heartbeat(self): self._send_heartbeat() def on_message(self, args): msg = Message.initialize(args) if msg is None: return elif msg.id == message.RPC_INVOKE: #PROTOCOL_LIST.index("rpc::invoke"): try: _request = Request() _stream = Stream(msg.session, self) self.sandbox.invoke(msg.event, _request, _stream) self.sessions[msg.session] = _request except Exception as err: self._logger.error("On invoke error: %s" % err) elif msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"): self._logger.debug("Receive chunk: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: try: _session.push(msg.data) except Exception as err: self._logger.error("On push error: %s" % str(err)) elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"): self._logger.debug("Receive choke: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: _session.close() self.sessions.pop(msg.session) elif msg.id == message.RPC_HEARTBEAT: #PROTOCOL_LIST.index("rpc::heartbeat"): self.disown_timer.stop() elif msg.id == message.RPC_TERMINATE: #PROTOCOL_LIST.index("rpc::terminate"): self._logger.debug("Receive terminate. Reason: %s, message: %s " % (msg.reason, msg.message)) self.terminate(msg.reason, msg.message) elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"): _session = self.sessions.get(msg.session, None) if _session is not None: _session.error(RequestError(msg.message)) def on_disown(self): self._logger.error("Disowned") self.service.stop() # Private: def _send_handshake(self): self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack()) def _send_heartbeat(self): self.disown_timer.start() self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack()) def send_choke(self, session): self.w_stream.write(Message(message.RPC_CHOKE, session).pack()) def send_chunk(self, session, data): self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())