class Request(object): def __init__(self): self._logger = Logger() self.cache = list() self._clbk = None # Callback - on chunk self._errbk = None # Errorback - translate error to handler self._errmsg = None # Store message self._state = 1 # Status of stream (close/open) def push(self, chunk): if self._clbk is None: # If there is no attachment object, put chunk in the cache self._logger.debug("Cache chunk") self.cache.append(chunk) else: # Copy callback to temp, clear current callback and perform temp # Do it so because self._clbk may change, while perfoming callback function. # Avoid double chunk sending to the task self._logger.debug("Send chunk to application") temp = self._clbk self._clbk = None temp(chunk) def error(self, errormsg): self._errmsg = errormsg def close(self): self._logger.debug("Close request") self._state = None if len(self.cache) == 0 and self._clbk is not None: self._logger.warn("Chunks are over, but the application requests them") if self._errbk is not None: self._logger.error("Throw error") self._errbk(RequestError("No chunks are available")) else: self._logger.error("No errorback. Can't throw error") def read(self): def wrapper(clbk, errorback=None): self._read(clbk, errorback) return wrapper def _read(self, callback, errorback): if len(self.cache) > 0: callback(self.cache.pop(0)) elif self._errmsg is not None: errorback(self._errmsg) #traslate error into worker elif self._state is not None: self._clbk = callback self._errbk = errorback else: #Stream closed by choke #Raise exception here because no chunks from cocaine-runtime are availaible self._logger.warn("Chunks are over, but the application requests them") errorback(RequestError("No chunks are available"))
class UserDB(object): def __init__(self, storage, key, namespace): self.storage = SecureStorage(storage) self.key = key self.logger = Logger() self.namespace = namespace_prefix + namespace self.dbnamespace = namespace_prefix + "apps" self.lognamespace = namespace_prefix + "logs" self.logger.info("UserDB has been initialized. Use namespace %s" % self.namespace) @asynchronous def exists(self, name): try: yield self.storage.read(self.namespace, name) except Exception as err: self.logger.error(str(err)) yield False else: yield True @asynchronous def get(self, name): yield self.storage.read(self.namespace, name) @asynchronous def create(self, info): user_info = dict() uid = uuid.uuid4().hex name = info['name'] password = info['password'] exists = yield self.exists(name) if exists: raise Exception("Already exists") # Store user uid user_info['uid'] = uid # Store username user_info['name'] = name # Crypt user passwd h = HMAC.new(uid) h.update(password) user_info['hash'] = h.hexdigest() try: yield self.storage.write(self.namespace, name, user_info, USER_TAG) except Exception as err: self.logger.error(str(err)) yield False else: yield True @asynchronous def remove(self, name): try: self.logger.info("Remove user %s" % name) yield self.storage.remove(self.namespace, name) except Exception as err: self.logger.error(repr(err)) try: self.logger.info("Remove user %s application info" % name) yield self.storage.remove(self.dbnamespace, name) except Exception as err: self.logger.error(repr(err)) try: self.logger.info("Remove user %s upload logs" % name) tags = LOG_TAG + [name] logkeys = yield self.storage.find(self.lognamespace, tags) self.logger.debug("Uploadlogs keys %s" % logkeys) for key in logkeys: yield self.storage.remove(self.lognamespace, key) except Exception as err: self.logger.error(repr(err)) @asynchronous def login(self, name, password): user_info = yield self.get(name) self.logger.info(str(user_info)) h = HMAC.new(user_info['uid']) h.update(password) self.logger.error("%s %s" % (h.hexdigest(), user_info['hash'])) if (h.hexdigest() == user_info['hash']): user_info.pop('uid') yield user_info else: raise Exception("Invalid pair of login/password") @asynchronous def users(self): yield self.storage.find(self.namespace, USER_TAG) @asynchronous def user_apps(self, user): apps = list() try: raw_apps = yield self.storage.read(self.dbnamespace, user) apps = msgpack.unpackb(raw_apps) except Exception as err: self.logger.error(repr(err)) finally: yield apps @asynchronous def write_app_info(self, user, name): def handler(data): apps = list() if data is None: apps = list() else: apps = msgpack.unpackb(data) if name in apps: self.logger.error("App %s already exists" % name) return None apps.append(name) return msgpack.packb(apps) reader = partial(self.storage.read, self.dbnamespace, user) writer = lambda result: self.storage.write(self.dbnamespace, user, result, USER_TAG) yield self.quasi_atomic_write(reader, writer, handler) @asynchronous def write_buildlog(self, user, key, logdata): tags = LOG_TAG + [user] yield self.storage.write(self.lognamespace, key, logdata, tags) @asynchronous def read_buildlog(self, key): yield self.storage.read(self.lognamespace, key) @asynchronous def list_buildlog(self, user): tags = [user] if user else LOG_TAG yield self.storage.find(self.lognamespace, tags) @asynchronous def quasi_atomic_write(self, reader, writer, handler): while True: data = None summ = "" try: data = yield reader() summ = hashlib.md5(data).hexdigest() except Exception as err: self.logger.error(repr(err)) result = handler(data) if result is None: break try: data = yield reader() except Exception as err: self.logger.error(repr(err)) if data is None or summ == hashlib.md5(data).hexdigest(): self.logger.info("MD5 is still valid. Do write") yield writer(result) break self.logger.info("MD5 mismatchs. Continue")
class MySqlDG(object): def __init__(self, **config): self.logger = Logger() self.place = None self.tablename = '' try: unix_socket = config.get('MysqlSocket', "/var/run/mysqld/mysqld.sock") self.dbname = config.get('local_db_name', 'COMBAINE') self.user = config.get('user', 'root') self.password = config.get('password', "") self.db = MySQLdb.connect(unix_socket=unix_socket, user=self.user, passwd=self.password) self.cursor = self.db.cursor() self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname) self.db.commit() self.db.select_db(self.dbname) except Exception as err: self.logger.error('Error in init MySQLdb %s' % err) raise Exception def putData(self, data, tablename): try: tablename = tablename.replace('.', '_').replace('-', '_').replace('+', '_') line = None fname = '/dev/shm/%s-%i' % ('COMBAINE', random.randint(0, 65535)) with open(fname, 'w') as table_file: for line in data: table_file.write('GOPA'.join([str(x) for _, x in line]) + '\n') table_file.close() if not line: self.logger.info("Data for mysql is missed") os.remove(table_file.name) return False self.logger.debug( 'Data has been written to a temporary file %s, size: %d bytes' % (table_file.name, os.lstat(table_file.name).st_size)) if not self._preparePlace(line): self.logger.error( 'Unsupported field types. Look at preparePlace()') return False self.cursor.execute('DROP TABLE IF EXISTS %s' % tablename) query = "CREATE TABLE IF NOT EXISTS `%(tablename)s` %(struct)s ENGINE = MEMORY DATA DIRECTORY='/dev/shm/'" % { 'tablename': tablename, 'struct': self.place } self.cursor.execute(query) self.db.commit() query = "LOAD DATA INFILE '%(filename)s' INTO TABLE `%(tablename)s` FIELDS TERMINATED BY 'GOPA'" % { 'filename': table_file.name, 'tablename': tablename } self.cursor.execute(query) self.db.commit() if os.path.isfile(table_file.name): os.remove(table_file.name) except Exception as err: self.logger.error('Error in putData %s' % err) if os.path.isfile(table_file.name): os.remove(table_file.name) return False else: self.tablename = tablename return True def _preparePlace(self, example): ftypes = { types.IntType: "BIGINT", types.UnicodeType: "VARCHAR(200)", types.StringType: "VARCHAR(200)", types.FloatType: "DOUBLE" } try: self.place = '( %s )' % ','.join([ " `%s` %s" % (field_name, ftypes[type(field_type)]) for field_name, field_type in example ]) except Exception as err: self.logger.error('Error in preparePlace() %s' % err) self.place = None return False else: return True def perfomCustomQuery(self, query_string): self.logger.debug("Execute query: %s" % query_string) self.cursor.execute(query_string) _ret = self.cursor.fetchall() self.db.commit() return _ret
class UrlFetcher(): def __init__(self, io_loop): self.io_loop = io_loop self.http_client = AsyncHTTPClient() self.logger = Logger() @chain.source def perform_request(self, request, response, method): try: constants = request_consts[method] url = request[constants.URL] timeout = request[constants.TIMEOUT] http_request = HTTPRequest(url=url, method=method) http_request.request_timeout = float(timeout)/1000 if method == 'POST': http_request.body = request[constants.BODY] #adds cookies to request params_num = len(request) if constants.COOKIES <= params_num - 1: cookies = request[constants.COOKIES] if len(cookies) > 0: list_of_cookies = list('{0}={1}'.format(cookie, value) for cookie, value in cookies.iteritems()) cookies_str = '; '.join(list_of_cookies) http_request.headers.add('Cookie', cookies_str) #adds headers to request if constants.HEADERS <= params_num - 1: for name, values_list in request[constants.HEADERS].iteritems(): for value in values_list: http_request.headers.add(name, value) self.logger.info("Downloading {0}, headers {1}, method {2}".format(url, http_request.headers, method)) http_response = yield self.http_client.fetch(http_request) response_headers = self._get_headers_from_response(http_response) response.write((True, http_response.body, http_response.code, response_headers,)) response.close() self.logger.info("{0} has been successfuly downloaded".format(url)) except HTTPError as e: self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url)) if e.response is not None: http_response = e.response response_headers = self._get_headers_from_response(http_response) response.write((False, http_response.body, http_response.code, response_headers,)) else: response.write((False, '', e.code, {},)) response.close() except socket.gaierror as e: self.logger.info("Error ({0}) occured while downloading {1}".format(e.message, url)) response.write((False, '', e.errno, {},)) response.close() except Exception as e: self.logger.error("Unhandled error ({0}) occured in perform_request, report about this problem " "to httpclient service developers. Method is {1}, stacktrace is: {2}".format( e.message, method, traceback.format_exc())) response.write((False, '', 0, {},)) response.close() @chain.source def on_get_request(self, request, response): try: request_data_packed = yield request.read() request_data = msgpack.unpackb(request_data_packed) yield self.perform_request(request_data, response, 'GET') except StopIteration: pass except Exception as e: self.logger.error("Unhandled error ({0}) occured in on_get_request, report about this problem " "to httpclient service developers. Stacktrace is: {1}".format(e.message, traceback.format_exc())) response.write((False, '', 0, {},)) response.close() @chain.source def on_post_request(self, request, response): try: request_data_packed = yield request.read() request_data = msgpack.unpackb(request_data_packed) yield self.perform_request(request_data, response, 'POST') except StopIteration: pass except Exception as e: self.logger.error("Unhandled error ({0}) occured in on_post_request, report about this problem " "to httpclient service developers. Stacktrace is: {1}".format(e.message, traceback.format_exc())) response.write((False, '', 0, {},)) response.close() def _get_headers_from_response(self, http_response): response_headers = {} for header_tuple in http_response.headers.items(): name = header_tuple[0] value = header_tuple[1] if not name in response_headers: response_headers[name] = [] response_headers[name].append(value) return response_headers
class MySqlDG(object): def __init__(self, **config): self.logger = Logger() self.place = None self.tablename = '' try: # port = config.get('local_db_port', 3306) unix_socket = config.get('MysqlSocket', "/var/run/mysqld/mysqld.sock") self.dbname = config.get('local_db_name', 'COMBAINE') self.db = MySQLdb.connect(unix_socket=unix_socket, user='******', ) self.cursor = self.db.cursor() self.cursor.execute('CREATE DATABASE IF NOT EXISTS %s' % self.dbname) self.db.commit() self.db.select_db(self.dbname) except Exception as err: self.logger.error('Error in init MySQLdb %s' % err) raise Exception def putData(self, data, tablename): try: tablename = tablename.replace('.', '_').replace('-', '_').replace('+', '_') line = None fname = '/dev/shm/%s-%i' % ('COMBAINE', random.randint(0, 65535)) with open(fname, 'w') as table_file: for line in data: table_file.write('GOPA'.join([str(x) for x in line.values()]) + '\n') table_file.close() if not line: self.logger.info("Data for mysql is missed") os.remove(table_file.name) return False self.logger.debug('Data written to a temporary file %s, size: %d bytes' % (table_file.name, os.lstat(table_file.name).st_size)) if not self._preparePlace(line): self.logger.error('Unsupported field types. Look at preparePlace()') return False self.cursor.execute('DROP TABLE IF EXISTS %s' % tablename) query = "CREATE TABLE IF NOT EXISTS %(tablename)s %(struct)s ENGINE = MEMORY DATA DIRECTORY='/dev/shm/'" % {'tablename': tablename, 'struct': self.place} self.cursor.execute(query) self.db.commit() query = "LOAD DATA INFILE '%(filename)s' INTO TABLE %(tablename)s FIELDS TERMINATED BY 'GOPA'" % {'filename': table_file.name, 'tablename': tablename} self.cursor.execute(query) self.db.commit() if os.path.isfile(table_file.name): os.remove(table_file.name) except Exception as err: self.logger.error('Error in putData %s' % err) if os.path.isfile(table_file.name): os.remove(table_file.name) return False else: self.tablename = tablename return True def _preparePlace(self, example): ftypes = {types.IntType: "INT", types.UnicodeType: "VARCHAR(200)", types.StringType: "VARCHAR(200)", types.FloatType: "FLOAT"} try: self.place = '( %s )' % ','.join([" %s %s" % (field_name, ftypes[type(field_type)]) for field_name, field_type in example.items()]) except Exception as err: self.logger.error('Error in preparePlace() %s' % err) self.place = None return False else: return True def perfomCustomQuery(self, query_string): self.logger.debug("Execute query: %s" % query_string) self.cursor.execute(query_string) _ret = self.cursor.fetchall() self.db.commit() return _ret def __del__(self): if self.db: self.cursor.close() self.db.commit() self.db.close()
class Worker(object): def __init__(self, init_args=sys.argv, disown_timeout=2, heartbeat_timeout=20): self._logger = Logger() self._init_endpoint(init_args) self.sessions = dict() self.sandbox = Sandbox() self.loop = ev.Loop() self.disown_timer = ev.Timer(self.on_disown, disown_timeout, self.loop) self.heartbeat_timer = ev.Timer(self.on_heartbeat, heartbeat_timeout, self.loop) self.disown_timer.start() self.heartbeat_timer.start() self.pipe = Pipe(self.endpoint) self.loop.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.loop, self.pipe) self.r_stream = ReadableStream(self.loop, self.pipe) self.r_stream.bind(self.decoder.decode) self.loop.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) # Send both messages - to run timers properly. This messages will be sent # only after all initialization, so they have same purpose. self._send_handshake() self._send_heartbeat() def _init_endpoint(self, init_args): try: self.id = init_args[init_args.index("--uuid") + 1] app_name = init_args[init_args.index("--app") + 1] self.endpoint = init_args[init_args.index("--endpoint") + 1] except Exception as err: self._logger.error("Wrong cmdline arguments: %s " % err) raise RuntimeError("Wrong cmdline arguments") def run(self, binds=None): if not binds: binds = {} for event, name in binds.iteritems(): self.on(event, name) self.loop.run() def terminate(self, reason, msg): self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack()) self.loop.stop() # Event machine def on(self, event, callback): self.sandbox.on(event, callback) # Events def on_heartbeat(self): self._send_heartbeat() def on_message(self, args): msg = Message.initialize(args) if msg is None: return elif msg.id == message.RPC_INVOKE: try: _request = Request() _stream = Stream(msg.session, self) self.sandbox.invoke(msg.event, _request, _stream) self.sessions[msg.session] = _request except Exception as err: self._logger.error("On invoke error: %s" % err) traceback.print_stack() elif msg.id == message.RPC_CHUNK: self._logger.debug("Receive chunk: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: try: _session.push(msg.data) except Exception as err: self._logger.error("On push error: %s" % str(err)) elif msg.id == message.RPC_CHOKE: self._logger.debug("Receive choke: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: _session.close() self.sessions.pop(msg.session) elif msg.id == message.RPC_HEARTBEAT: self._logger.debug("Receive heartbeat. Stop disown timer") self.disown_timer.stop() elif msg.id == message.RPC_TERMINATE: self._logger.debug("Receive terminate. Reason: %s, message: %s " % (msg.reason, msg.message)) self.terminate(msg.reason, msg.message) elif msg.id == message.RPC_ERROR: _session = self.sessions.get(msg.session, None) if _session is not None: _session.error(RequestError(msg.message)) def on_disown(self): try: self._logger.error("Disowned") finally: self.loop.stop() # Private: def _send_handshake(self): self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack()) def _send_heartbeat(self): self.disown_timer.start() self._logger.debug("Send heartbeat. Start disown timer") self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack()) def send_choke(self, session): self.w_stream.write(Message(message.RPC_CHOKE, session).pack()) def send_chunk(self, session, data): self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack()) def send_error(self, session, code, msg): self.w_stream.write(Message(message.RPC_ERROR, session, code, msg).pack())
class Worker(object): def __init__(self, init_args=sys.argv): self._logger = Logger() self._init_endpoint(init_args) self.sessions = dict() self.sandbox = Sandbox() self.service = ev.Service() self.disown_timer = ev.Timer(self.on_disown, 2, self.service) self.heartbeat_timer = ev.Timer(self.on_heartbeat, 5, self.service) self.disown_timer.start() self.heartbeat_timer.start() self.pipe = Pipe(self.endpoint) self.service.bind_on_fd(self.pipe.fileno()) self.decoder = Decoder() self.decoder.bind(self.on_message) self.w_stream = WritableStream(self.service, self.pipe) self.r_stream = ReadableStream(self.service, self.pipe) self.r_stream.bind(self.decoder.decode) self.service.register_read_event(self.r_stream._on_event, self.pipe.fileno()) self._logger.debug("Worker with %s send handshake" % self.id) self._send_handshake() def _init_endpoint(self, init_args): try: self.id = init_args[init_args.index("--uuid") + 1] app_name = init_args[init_args.index("--app") + 1] self.endpoint = init_args[init_args.index("--endpoint") + 1] except Exception as err: self._logger.error("Wrong cmdline argumensts: %s " % err) raise RuntimeError("Wrong cmdline arguments") def run(self, binds={}): for event, name in binds.iteritems(): self.on(event, name) self.service.run() def terminate(self, reason, msg): self.w_stream.write(Message(message.RPC_TERMINATE, 0, reason, msg).pack()) self.service.stop() # Event machine def on(self, event, callback): self.sandbox.on(event, callback) # Events def on_heartbeat(self): self._send_heartbeat() def on_message(self, args): msg = Message.initialize(args) if msg is None: return elif msg.id == message.RPC_INVOKE: #PROTOCOL_LIST.index("rpc::invoke"): try: _request = Request() _stream = Stream(msg.session, self) self.sandbox.invoke(msg.event, _request, _stream) self.sessions[msg.session] = _request except Exception as err: self._logger.error("On invoke error: %s" % err) elif msg.id == message.RPC_CHUNK: #PROTOCOL_LIST.index("rpc::chunk"): self._logger.debug("Receive chunk: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: try: _session.push(msg.data) except Exception as err: self._logger.error("On push error: %s" % str(err)) elif msg.id == message.RPC_CHOKE: #PROTOCOL_LIST.index("rpc::choke"): self._logger.debug("Receive choke: %d" % msg.session) _session = self.sessions.get(msg.session, None) if _session is not None: _session.close() self.sessions.pop(msg.session) elif msg.id == message.RPC_HEARTBEAT: #PROTOCOL_LIST.index("rpc::heartbeat"): self.disown_timer.stop() elif msg.id == message.RPC_TERMINATE: #PROTOCOL_LIST.index("rpc::terminate"): self._logger.debug("Receive terminate. Reason: %s, message: %s " % (msg.reason, msg.message)) self.terminate(msg.reason, msg.message) elif msg.id == message.RPC_ERROR: #PROTOCOL_LIST.index("rpc::error"): _session = self.sessions.get(msg.session, None) if _session is not None: _session.error(RequestError(msg.message)) def on_disown(self): self._logger.error("Disowned") self.service.stop() # Private: def _send_handshake(self): self.w_stream.write(Message(message.RPC_HANDSHAKE, 0, self.id).pack()) def _send_heartbeat(self): self.disown_timer.start() self.w_stream.write(Message(message.RPC_HEARTBEAT, 0).pack()) def send_choke(self, session): self.w_stream.write(Message(message.RPC_CHOKE, session).pack()) def send_chunk(self, session, data): self.w_stream.write(Message(message.RPC_CHUNK, session, data).pack())
# (at your option) any later version. # # Combaine is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # import logging from cocaine.logging import Logger from cocaine.logging.hanlders import CocaineHandler Log = Logger() Log.error("INITIALIZE") l = logging.getLogger("combaine") l.setLevel(logging.DEBUG) ch = CocaineHandler() formatter = logging.Formatter("%(tid)s %(message)s") ch.setFormatter(formatter) ch.setLevel(logging.DEBUG) l.addHandler(ch) def get_logger_adapter(tid): return logging.LoggerAdapter(l, {"tid": tid})