def mail_send(self,
subject,
message,
sender=get_config('NOODLES_ERROR_SENDER'),
recipient=get_config('NOODLES_ERROR_RECIPIENT'),
conn=None,
with_hostname=False,
attachments=None,
is_html=True):
if with_hostname:
sender = socket.gethostname() + '_' + sender
assert type(recipient) == list
if len(recipient) == 0:
return
if 'example.com' in ''.join(recipient):
return
if conn is None:
conn = self.obtain_conn()
try:
body = self.construct(subject,
message,
sender,
recipient,
with_hostname=with_hostname,
attachments=attachments,
is_html=is_html)
conn.sendmail(sender, recipient, body)
finally:
conn.quit()
def __init__(self,
server=get_config('MAIL_SERVER'),
port=get_config('MAIL_PORT'),
login=get_config('MAIL_LOGIN'),
password=get_config('MAIL_PASSWORD'),
recipient=None):
fnd = False
if recipient:
for rcp in recipient:
for sndk, snd in get_config('SECONDARY_EMAIL_SERVERS').items():
if rcp.endswith(sndk):
assert fnd!=True and \
hasattr(self,'server') and \
getattr(self,'server')!=snd['server'],\
"allow only one override (%s != %s)"%(getattr(self,'server'),snd['server'])
self.server = snd['server']
self.port = snd['port']
self.login = snd['login']
self.password = snd['password']
fnd = True
if not fnd:
self.server = server
self.port = port
self.login = login
self.password = password
self.recipient = recipient #this is used to choose a connection
def send_email(subject,
message,
sender=get_config('NOODLES_ERROR_SENDER'),
recipient=get_config('NOODLES_ERROR_RECIPIENT'),
conn=None,
with_hostname=False,
attachments=None,
is_html=True):
mailman = MailMan(recipient=recipient)
mailman.mail_send(subject, message, sender, recipient, conn, with_hostname,
attachments, is_html)
def startapp(port=PORT, host=None, start_time=None):
global SERVER_INSTANCE
if get_config('DONT_USE_GEVENT'):
from paste import httpserver
SERVER_INSTANCE = httpserver.serve(noodlesapp, host=host, port=port)
else:
startgeventapp(port, host, start_time)
def startapp(port=PORT, host=None, start_time=None):
global SERVER_INSTANCE
if get_config('DONT_USE_GEVENT'):
from paste import httpserver
SERVER_INSTANCE = httpserver.serve(noodlesapp, host=host, port=port)
else:
startgeventapp(port, host, start_time)
def pre_set_up(self):
if not self.is_tracked():
return
data = {
'job': get_config('JOB', ''),
'suite_id': self.suite_id,
'test_id': self.test_id,
'testname': self.testname,
'method': self._testMethodName,
'class': str(self.__class__),
'timestamp': time.ctime(),
}
track_file_name = get_config('TRACK_FILE_NAME', 'api_debug.json')
with open(track_file_name, 'w') as f:
f.write(json.dumps(data))
def report_exception(exc, env=None, extra=None):
if extra is None:
extra = {}
formatted = format_exception(exc, env, extra)
if isinstance(exc, tuple):
'pick an exception from the tuple (type, value, traceback)'
exc = exc[1]
if not get_config('NOODLES_SEND_FULL_MAIL'):
send_email(repr(exc), formatted, with_hostname=True, is_html=False)
return
subject = repr(exc)
report = ''
try:
subject = repr(exc)
report = _collect(exc)
report = HTMLFormatter().format(report, maxdepth=4)
except exceptions.MakoException:
error = exceptions.text_error_template().render()
formatted += '\nTemplate rendering error ' + error
log.error(error)
attachments = None
if report:
attachments = {
'report.html': report,
}
send_email(subject,
formatted,
with_hostname=True,
attachments=attachments,
is_html=False)
def startbackdoor(host=None, port=8998):
if get_config('DONT_USE_GEVENT'):
raise('Backdoor is not available')
if host is None:
host=HOST
from gevent.backdoor import BackdoorServer
print('Backdoor is on %s:%s' % (host, port))
bs = BackdoorServer((host, port), locals())
bs.start()
def startbackdoor(host=None, port=8998):
if get_config('DONT_USE_GEVENT'):
raise('Backdoor is not available')
if host is None:
host=HOST
from gevent.backdoor import BackdoorServer
print('Backdoor is on %s:%s' % (host, port))
bs = BackdoorServer((host, port), locals())
bs.start()
def noodlesapp(env, start_response):
"""
:rtype : noodles.http.Response
:param env:
:param start_response:
:return: :rtype: :raise:
"""
# Get request object
if get_config('ENCODE_SEMICOLON_IN_REQUEST') is True:
env['QUERY_STRING'] = re.sub('[;]', '%3b', env['QUERY_STRING'])
request = Request(env)
if "HTTP_X_FORWARDED_FOR" in env:
x_forwarded_for = env["HTTP_X_FORWARDED_FOR"].split(',')[:1]
if x_forwarded_for:
request.remote_addr = x_forwarded_for[0]
#print("Try to handle url_path '%s'" % request.path)
# Get callable object with routine method to handle request
producer = dispatcher.get_callable(request)
if not producer:
# May be here an error,raise exception
raise Exception('Can\'t find callable for this url path')
# Callable function must return Response object
try:
response = middlewares.run_chain(producer, request)
if not hasattr(response, 'is_noodles_response'):
response = producer()
# Capture traceback here and send it if debug mode
except Exception as e:
f = Formatter()
if EXCEPTION_FLAVOR=='html':
traceback = '<pre>%s\n\n%s</pre>' \
% (json.dumps(env, indent=2, default=datahandler),
xml_escape(f.formatException(sys.exc_info())))
else:
traceback = '%s\n%s' \
% (json.dumps(env, indent=2, default=datahandler),
f.formatException(sys.exc_info()))
extra = {'request': request}
log.error(format_exception(e, None), extra=extra)
report_exception(e, extra=extra)
if DEBUG:
response = Error500(e, traceback)
else:
response = Error500()
finally:
middlewares.end_chain(lambda x: x, request)
return response(env, start_response)
def noodlesapp(env, start_response):
"""
:rtype : noodles.http.Response
:param env:
:param start_response:
:return: :rtype: :raise:
"""
# Get request object
if get_config('ENCODE_SEMICOLON_IN_REQUEST') is True:
env['QUERY_STRING'] = re.sub('[;]', '%3b', env['QUERY_STRING'])
request = Request(env)
if "HTTP_X_FORWARDED_FOR" in env:
x_forwarded_for = env["HTTP_X_FORWARDED_FOR"].split(',')[:1]
if x_forwarded_for:
request.remote_addr = x_forwarded_for[0]
#print("Try to handle url_path '%s'" % request.path)
# Get callable object with routine method to handle request
producer = dispatcher.get_callable(request)
if not producer:
# May be here an error,raise exception
raise Exception('Can\'t find callable for this url path')
# Callable function must return Response object
try:
response = middlewares.run_chain(producer, request)
if not hasattr(response, 'is_noodles_response'):
response = producer()
# Capture traceback here and send it if debug mode
except Exception as e:
f = Formatter()
if EXCEPTION_FLAVOR=='html':
traceback = '<pre>%s\n\n%s</pre>' \
% (json.dumps(env, indent=2, default=datahandler),
xml_escape(f.formatException(sys.exc_info())))
else:
traceback = '%s\n%s' \
% (json.dumps(env, indent=2, default=datahandler),
f.formatException(sys.exc_info()))
extra = {'request': request}
log.error(format_exception(e, None), extra=extra)
report_exception(e, extra=extra)
if DEBUG:
response = Error500(e, traceback)
else:
response = Error500()
finally:
middlewares.end_chain(lambda x: x, request)
return response(env, start_response)
"""
pid = os.getpid()
_proc_fname = '/proc/%d/status' % pid
status = None
mb = 1024 * 1024 * 1024
try:
status = open(_proc_fname, mode='r')
while True:
status.seek(0)
content = status.read()
for key in ['VmSize', 'VmRSS', 'VmStk']:
i = content.index(key)
portion = content[i:].split(None, 3)
metric = portion[2]
data = float(portion[1])
if metric in ['kB', 'KB']:
data /= 1024.0
metric = 'mb'
riemann_key = 'internal.%d.%s_%s' % (pid, key, metric)
RIEMANN_QUEUE.put((riemann_key, data))
gevent.sleep(10)
except Exception as e:
logging.error(e)
finally:
if status:
status.close()
if get_config('RIEMANN_USE'):
gevent.spawn(infinite_control)
"""
pid = os.getpid()
_proc_fname = '/proc/%d/status' % pid
status = None
mb = 1024 * 1024 * 1024
try:
status = open(_proc_fname, mode='r')
while True:
status.seek(0)
content = status.read()
for key in ['VmSize', 'VmRSS', 'VmStk']:
i = content.index(key)
portion = content[i:].split(None, 3)
metric = portion[2]
data = float(portion[1])
if metric in ['kB', 'KB']:
data /= 1024.0
metric = 'mb'
riemann_key = 'internal.%d.%s_%s' % (pid, key, metric)
RIEMANN_QUEUE.put((riemann_key, data))
gevent.sleep(10)
except Exception as e:
logging.error(e)
finally:
if status:
status.close()
if get_config('RIEMANN_USE'):
gevent.spawn(infinite_control)
def get_partial_digest(self, login):
pw = self.storage.get(login, {})
return self._generate_partial_digest(login, pw,
get_config('DIGEST_REALM'))
def __init__(self, realm=None, timeout=None):
self.realm = realm or get_config('DIGEST_REALM')
self._account_storage = SimpleDigestAuth()
self.timeout = timeout or get_config('DIGEST_TIMEOUT_IN_SECONDS')
self.secret_key = get_config('SECRET_KEY')
# -*- coding: utf-8 -*-
"""
Machinery for launching the wsgi server
"""
from noodles.utils.helpers import get_config
if not get_config('DONT_USE_GEVENT'):
from gevent import monkey
monkey.patch_all()
from logging import Formatter
import os
import re
import sys
import threading
import time
import json
from mako.filters import xml_escape
from config import (URL_RESOLVER, CONTROLLERS, MIDDLEWARES, DEBUG, AUTO_STOP,
HOST, PORT, SERVER_LOGTYPE, EXCEPTION_FLAVOR)
from noodles.dispatcher import Dispatcher
from noodles.http import Request, Error500
from noodles.middleware.middleware import AppMiddlewares
from noodles.websockserver import server
from noodles.utils.datahandler import datahandler
from noodles.utils.logger import log
from noodles.utils.mailer import report_exception, format_exception
resolver = __import__(URL_RESOLVER, globals(), locals())
def pubsub_key(*args):
""" use to create redis pub/sub channel name """
return mkey(*args + (get_config('RDB', ''), ))
# -*- coding: utf-8 -*-
"""
Machinery for launching the wsgi server
"""
from noodles.utils.helpers import get_config
if not get_config('DONT_USE_GEVENT'):
from gevent import monkey
monkey.patch_all()
from logging import Formatter
import os
import re
import sys
import threading
import time
import json
from mako.filters import xml_escape
from config import (URL_RESOLVER, CONTROLLERS, MIDDLEWARES, DEBUG, AUTO_RELOAD,
HOST, PORT, SERVER_LOGTYPE, EXCEPTION_FLAVOR)
from noodles.dispatcher import Dispatcher
from noodles.http import Request, Error500
from noodles.middleware.middleware import AppMiddlewares
from noodles.websockserver import server
from noodles.utils.datahandler import datahandler
from noodles.utils.logger import log
from noodles.utils.mailer import report_exception, format_exception
resolver = __import__(URL_RESOLVER, globals(), locals())
# -*- coding: utf-8 -*-
import sys
import os
import logging
from logging.handlers import WatchedFileHandler
from pythonjsonlogger import jsonlogger
from noodles.utils.helpers import get_config
LOG_DIR = get_config('LOG_DIR',ignore_import_errors=True)
LOG_LEVEL = get_config('LOG_LEVEL',ignore_import_errors=True)
formatter = logging.Formatter('%(asctime)s.%(msecs)d::'
'%(levelname)s::'
'%(filename)s L[%(lineno)d] '
'%(module)s::'
'%(message)s', '%Y-%m-%d %H:%M:%S')
json_formatter = jsonlogger.JsonFormatter('%(asctime)s '
'%(created)f '
'%(filename)s '
'%(funcName)s '
'%(levelname)s '
'%(lineno)d '
'%(module)s '
'%(message)s '
'%(name)s '
'%(pathname)s ')
is_error = lambda level: level >= logging.ERROR
log = logging.getLogger('GameServer')
if LOG_LEVEL:
log.setLevel(LOG_LEVEL)
# -*- coding: utf-8 -*-
"""
filedesc: stores traceback on db connect
@author: Yura A
"""
from gevent import getcurrent
from functools import partial
from traceback import extract_stack, format_list
from noodles.utils.helpers import get_config
CONNECTIONS_INFO_HOLDER = {}
CONNECTIONS_COUNTER = {}
POOL_SIZES = {}
RAW_TRACEBACK = get_config('DEBUG_GREENLET_RAW_TRACEBACK', False)
def exhaust(shard_id):
result = POOL_SIZES[shard_id] <= len(CONNECTIONS_INFO_HOLDER[shard_id]) + 2
return result
def print_holder(req_greenlet_id, shard_id):
slot = CONNECTIONS_INFO_HOLDER[shard_id]
i = 1
print('requested by greenlet_id', req_greenlet_id)
for k, v in slot.items():
(greenlet_id, tb, parallel) = v
print(
i,
',',
# -*- coding: utf-8 -*-
"""
Redis connection wrapper which gives a soft error in case that
noodles is run on a machine without redis
"""
import redis
from redis.client import Lock
from redis.exceptions import LockError
from noodles.utils.helpers import get_config
from functools import wraps
pool = redis.ConnectionPool(**get_config('REDIS_BASE_CONN_DATA'))
RedisConn = redis.Redis(connection_pool=pool)
lock_timeout = get_config('REDIS_LOCK_TIMEOUT', 600)
def single_instance_task(func):
@wraps(func)
def wrapper(*args, **kwargs):
key = "celery-single-instance-%s" % func.__name__
suffix = kwargs.get('sit_suffix')
if suffix:
key = "%s-%s" % (key, suffix)
lock = RedisConn.lock(key, timeout=lock_timeout)
acquire_lock = lambda: lock.acquire(blocking=False)
force = kwargs.get('force', False)
if acquire_lock() or force:
try:
def is_tracked(self):
result = get_config('TRACK_TESTS_EXECUTION', False)
return result
# -*- coding: utf-8 -*-
"""
filedesc: stores traceback on db connect
@author: Yura A
"""
from gevent import getcurrent
from functools import partial
from traceback import extract_stack, format_list
from noodles.utils.helpers import get_config
CONNECTIONS_INFO_HOLDER = {}
CONNECTIONS_COUNTER = {}
POOL_SIZES = {}
RAW_TRACEBACK = get_config('DEBUG_GREENLET_RAW_TRACEBACK', False)
def exhaust(shard_id):
result = POOL_SIZES[shard_id] <= len(CONNECTIONS_INFO_HOLDER[shard_id]) + 2
return result
def print_holder(req_greenlet_id, shard_id):
slot = CONNECTIONS_INFO_HOLDER[shard_id]
i = 1
print('requested by greenlet_id', req_greenlet_id)
for k, v in slot.items():
(greenlet_id, tb, parallel) = v
print(i, ',', 'shard#', shard_id, ' greenlet#', greenlet_id,)
if parallel:
# -*- coding: utf-8 -*-
"""
Redis connection wrapper which gives a soft error in case that
noodles is run on a machine without redis
"""
import redis
from redis.client import Lock
from redis.exceptions import LockError
from noodles.utils.helpers import get_config
from functools import wraps
pool = redis.ConnectionPool(**get_config('REDIS_BASE_CONN_DATA'))
RedisConn = redis.Redis(connection_pool=pool)
lock_timeout = get_config('REDIS_LOCK_TIMEOUT', 600)
def single_instance_task(func):
@wraps(func)
def wrapper(*args, **kwargs):
key = "celery-single-instance-%s" % func.__name__
suffix = kwargs.get('sit_suffix')
if suffix:
key = "%s-%s" % (key, suffix)
lock = RedisConn.lock(key, timeout=lock_timeout)
acquire_lock = lambda: lock.acquire(blocking=False)
force = kwargs.get('force', False)
if acquire_lock() or force:
try:
result = func(*args, **kwargs)
def pubsub_key(*args):
""" use to create redis pub/sub channel name """
return mkey(*args + (get_config('RDB', ''),))
