def mail_send(self, subject, message, sender=get_config('NOODLES_ERROR_SENDER'), recipient=get_config('NOODLES_ERROR_RECIPIENT'), conn=None, with_hostname=False, attachments=None, is_html=True): if with_hostname: sender = socket.gethostname() + '_' + sender assert type(recipient) == list if len(recipient) == 0: return if 'example.com' in ''.join(recipient): return if conn is None: conn = self.obtain_conn() try: body = self.construct(subject, message, sender, recipient, with_hostname=with_hostname, attachments=attachments, is_html=is_html) conn.sendmail(sender, recipient, body) finally: conn.quit()
def __init__(self, server=get_config('MAIL_SERVER'), port=get_config('MAIL_PORT'), login=get_config('MAIL_LOGIN'), password=get_config('MAIL_PASSWORD'), recipient=None): fnd = False if recipient: for rcp in recipient: for sndk, snd in get_config('SECONDARY_EMAIL_SERVERS').items(): if rcp.endswith(sndk): assert fnd!=True and \ hasattr(self,'server') and \ getattr(self,'server')!=snd['server'],\ "allow only one override (%s != %s)"%(getattr(self,'server'),snd['server']) self.server = snd['server'] self.port = snd['port'] self.login = snd['login'] self.password = snd['password'] fnd = True if not fnd: self.server = server self.port = port self.login = login self.password = password self.recipient = recipient #this is used to choose a connection
def send_email(subject, message, sender=get_config('NOODLES_ERROR_SENDER'), recipient=get_config('NOODLES_ERROR_RECIPIENT'), conn=None, with_hostname=False, attachments=None, is_html=True): mailman = MailMan(recipient=recipient) mailman.mail_send(subject, message, sender, recipient, conn, with_hostname, attachments, is_html)
def startapp(port=PORT, host=None, start_time=None): global SERVER_INSTANCE if get_config('DONT_USE_GEVENT'): from paste import httpserver SERVER_INSTANCE = httpserver.serve(noodlesapp, host=host, port=port) else: startgeventapp(port, host, start_time)
def pre_set_up(self): if not self.is_tracked(): return data = { 'job': get_config('JOB', ''), 'suite_id': self.suite_id, 'test_id': self.test_id, 'testname': self.testname, 'method': self._testMethodName, 'class': str(self.__class__), 'timestamp': time.ctime(), } track_file_name = get_config('TRACK_FILE_NAME', 'api_debug.json') with open(track_file_name, 'w') as f: f.write(json.dumps(data))
def report_exception(exc, env=None, extra=None): if extra is None: extra = {} formatted = format_exception(exc, env, extra) if isinstance(exc, tuple): 'pick an exception from the tuple (type, value, traceback)' exc = exc[1] if not get_config('NOODLES_SEND_FULL_MAIL'): send_email(repr(exc), formatted, with_hostname=True, is_html=False) return subject = repr(exc) report = '' try: subject = repr(exc) report = _collect(exc) report = HTMLFormatter().format(report, maxdepth=4) except exceptions.MakoException: error = exceptions.text_error_template().render() formatted += '\nTemplate rendering error ' + error log.error(error) attachments = None if report: attachments = { 'report.html': report, } send_email(subject, formatted, with_hostname=True, attachments=attachments, is_html=False)
def startbackdoor(host=None, port=8998): if get_config('DONT_USE_GEVENT'): raise('Backdoor is not available') if host is None: host=HOST from gevent.backdoor import BackdoorServer print('Backdoor is on %s:%s' % (host, port)) bs = BackdoorServer((host, port), locals()) bs.start()
def noodlesapp(env, start_response): """ :rtype : noodles.http.Response :param env: :param start_response: :return: :rtype: :raise: """ # Get request object if get_config('ENCODE_SEMICOLON_IN_REQUEST') is True: env['QUERY_STRING'] = re.sub('[;]', '%3b', env['QUERY_STRING']) request = Request(env) if "HTTP_X_FORWARDED_FOR" in env: x_forwarded_for = env["HTTP_X_FORWARDED_FOR"].split(',')[:1] if x_forwarded_for: request.remote_addr = x_forwarded_for[0] #print("Try to handle url_path '%s'" % request.path) # Get callable object with routine method to handle request producer = dispatcher.get_callable(request) if not producer: # May be here an error,raise exception raise Exception('Can\'t find callable for this url path') # Callable function must return Response object try: response = middlewares.run_chain(producer, request) if not hasattr(response, 'is_noodles_response'): response = producer() # Capture traceback here and send it if debug mode except Exception as e: f = Formatter() if EXCEPTION_FLAVOR=='html': traceback = '<pre>%s\n\n%s</pre>' \ % (json.dumps(env, indent=2, default=datahandler), xml_escape(f.formatException(sys.exc_info()))) else: traceback = '%s\n%s' \ % (json.dumps(env, indent=2, default=datahandler), f.formatException(sys.exc_info())) extra = {'request': request} log.error(format_exception(e, None), extra=extra) report_exception(e, extra=extra) if DEBUG: response = Error500(e, traceback) else: response = Error500() finally: middlewares.end_chain(lambda x: x, request) return response(env, start_response)
""" pid = os.getpid() _proc_fname = '/proc/%d/status' % pid status = None mb = 1024 * 1024 * 1024 try: status = open(_proc_fname, mode='r') while True: status.seek(0) content = status.read() for key in ['VmSize', 'VmRSS', 'VmStk']: i = content.index(key) portion = content[i:].split(None, 3) metric = portion[2] data = float(portion[1]) if metric in ['kB', 'KB']: data /= 1024.0 metric = 'mb' riemann_key = 'internal.%d.%s_%s' % (pid, key, metric) RIEMANN_QUEUE.put((riemann_key, data)) gevent.sleep(10) except Exception as e: logging.error(e) finally: if status: status.close() if get_config('RIEMANN_USE'): gevent.spawn(infinite_control)
def get_partial_digest(self, login): pw = self.storage.get(login, {}) return self._generate_partial_digest(login, pw, get_config('DIGEST_REALM'))
def __init__(self, realm=None, timeout=None): self.realm = realm or get_config('DIGEST_REALM') self._account_storage = SimpleDigestAuth() self.timeout = timeout or get_config('DIGEST_TIMEOUT_IN_SECONDS') self.secret_key = get_config('SECRET_KEY')
# -*- coding: utf-8 -*- """ Machinery for launching the wsgi server """ from noodles.utils.helpers import get_config if not get_config('DONT_USE_GEVENT'): from gevent import monkey monkey.patch_all() from logging import Formatter import os import re import sys import threading import time import json from mako.filters import xml_escape from config import (URL_RESOLVER, CONTROLLERS, MIDDLEWARES, DEBUG, AUTO_STOP, HOST, PORT, SERVER_LOGTYPE, EXCEPTION_FLAVOR) from noodles.dispatcher import Dispatcher from noodles.http import Request, Error500 from noodles.middleware.middleware import AppMiddlewares from noodles.websockserver import server from noodles.utils.datahandler import datahandler from noodles.utils.logger import log from noodles.utils.mailer import report_exception, format_exception resolver = __import__(URL_RESOLVER, globals(), locals())
def pubsub_key(*args): """ use to create redis pub/sub channel name """ return mkey(*args + (get_config('RDB', ''), ))
# -*- coding: utf-8 -*- """ Machinery for launching the wsgi server """ from noodles.utils.helpers import get_config if not get_config('DONT_USE_GEVENT'): from gevent import monkey monkey.patch_all() from logging import Formatter import os import re import sys import threading import time import json from mako.filters import xml_escape from config import (URL_RESOLVER, CONTROLLERS, MIDDLEWARES, DEBUG, AUTO_RELOAD, HOST, PORT, SERVER_LOGTYPE, EXCEPTION_FLAVOR) from noodles.dispatcher import Dispatcher from noodles.http import Request, Error500 from noodles.middleware.middleware import AppMiddlewares from noodles.websockserver import server from noodles.utils.datahandler import datahandler from noodles.utils.logger import log from noodles.utils.mailer import report_exception, format_exception resolver = __import__(URL_RESOLVER, globals(), locals())
# -*- coding: utf-8 -*- import sys import os import logging from logging.handlers import WatchedFileHandler from pythonjsonlogger import jsonlogger from noodles.utils.helpers import get_config LOG_DIR = get_config('LOG_DIR',ignore_import_errors=True) LOG_LEVEL = get_config('LOG_LEVEL',ignore_import_errors=True) formatter = logging.Formatter('%(asctime)s.%(msecs)d::' '%(levelname)s::' '%(filename)s L[%(lineno)d] ' '%(module)s::' '%(message)s', '%Y-%m-%d %H:%M:%S') json_formatter = jsonlogger.JsonFormatter('%(asctime)s ' '%(created)f ' '%(filename)s ' '%(funcName)s ' '%(levelname)s ' '%(lineno)d ' '%(module)s ' '%(message)s ' '%(name)s ' '%(pathname)s ') is_error = lambda level: level >= logging.ERROR log = logging.getLogger('GameServer') if LOG_LEVEL: log.setLevel(LOG_LEVEL)
# -*- coding: utf-8 -*- """ filedesc: stores traceback on db connect @author: Yura A """ from gevent import getcurrent from functools import partial from traceback import extract_stack, format_list from noodles.utils.helpers import get_config CONNECTIONS_INFO_HOLDER = {} CONNECTIONS_COUNTER = {} POOL_SIZES = {} RAW_TRACEBACK = get_config('DEBUG_GREENLET_RAW_TRACEBACK', False) def exhaust(shard_id): result = POOL_SIZES[shard_id] <= len(CONNECTIONS_INFO_HOLDER[shard_id]) + 2 return result def print_holder(req_greenlet_id, shard_id): slot = CONNECTIONS_INFO_HOLDER[shard_id] i = 1 print('requested by greenlet_id', req_greenlet_id) for k, v in slot.items(): (greenlet_id, tb, parallel) = v print( i, ',',
# -*- coding: utf-8 -*- """ Redis connection wrapper which gives a soft error in case that noodles is run on a machine without redis """ import redis from redis.client import Lock from redis.exceptions import LockError from noodles.utils.helpers import get_config from functools import wraps pool = redis.ConnectionPool(**get_config('REDIS_BASE_CONN_DATA')) RedisConn = redis.Redis(connection_pool=pool) lock_timeout = get_config('REDIS_LOCK_TIMEOUT', 600) def single_instance_task(func): @wraps(func) def wrapper(*args, **kwargs): key = "celery-single-instance-%s" % func.__name__ suffix = kwargs.get('sit_suffix') if suffix: key = "%s-%s" % (key, suffix) lock = RedisConn.lock(key, timeout=lock_timeout) acquire_lock = lambda: lock.acquire(blocking=False) force = kwargs.get('force', False) if acquire_lock() or force: try:
def is_tracked(self): result = get_config('TRACK_TESTS_EXECUTION', False) return result
# -*- coding: utf-8 -*- """ filedesc: stores traceback on db connect @author: Yura A """ from gevent import getcurrent from functools import partial from traceback import extract_stack, format_list from noodles.utils.helpers import get_config CONNECTIONS_INFO_HOLDER = {} CONNECTIONS_COUNTER = {} POOL_SIZES = {} RAW_TRACEBACK = get_config('DEBUG_GREENLET_RAW_TRACEBACK', False) def exhaust(shard_id): result = POOL_SIZES[shard_id] <= len(CONNECTIONS_INFO_HOLDER[shard_id]) + 2 return result def print_holder(req_greenlet_id, shard_id): slot = CONNECTIONS_INFO_HOLDER[shard_id] i = 1 print('requested by greenlet_id', req_greenlet_id) for k, v in slot.items(): (greenlet_id, tb, parallel) = v print(i, ',', 'shard#', shard_id, ' greenlet#', greenlet_id,) if parallel:
# -*- coding: utf-8 -*- """ Redis connection wrapper which gives a soft error in case that noodles is run on a machine without redis """ import redis from redis.client import Lock from redis.exceptions import LockError from noodles.utils.helpers import get_config from functools import wraps pool = redis.ConnectionPool(**get_config('REDIS_BASE_CONN_DATA')) RedisConn = redis.Redis(connection_pool=pool) lock_timeout = get_config('REDIS_LOCK_TIMEOUT', 600) def single_instance_task(func): @wraps(func) def wrapper(*args, **kwargs): key = "celery-single-instance-%s" % func.__name__ suffix = kwargs.get('sit_suffix') if suffix: key = "%s-%s" % (key, suffix) lock = RedisConn.lock(key, timeout=lock_timeout) acquire_lock = lambda: lock.acquire(blocking=False) force = kwargs.get('force', False) if acquire_lock() or force: try: result = func(*args, **kwargs)
def pubsub_key(*args): """ use to create redis pub/sub channel name """ return mkey(*args + (get_config('RDB', ''),))