def decorated_task(task): tasks.append(adict( task=task, seconds=number_of_seconds, last=0, name=task.__name__, )) return task
def __delitem__(self, hash_key): ''' Deletes item from db: del db.user[email] TODO: range_key support: del db.message[topic][date] ''' self.layer2.delete_item(item=adict(table=self, hash_key=hash_key, range_key=None))
def __getitem__(self, hash_key): ''' Gets item from db: user = db.user[email] TODO: range_key support: message = db.message[topic][date] TODO: Option to control consistent_read. ''' return adict(self.get_item(hash_key=hash_key, consistent_read=True))
def code(): with _local.db_conn.cursor(cursor_factory=RealDictCursor) as cursor: if _log: _log((sql, values)) cursor.execute(sql, values) # Use "INSERT ... RETURNING" instead of "cursor.lastrowid": # http://pythonhosted.org/psycopg2/cursor.html#cursor.lastrowid affected = cursor.rowcount try: rows = [adict(row) for row in cursor.fetchall()] if affected else [] except ProgrammingError as e: if 'no results to fetch' in repr(e): rows = [] else: raise return rows, affected
def __init__(self, task_id, mode='train'): self.mode = mode raw_train, raw_test = get_raw_babi(task_id) self.QA = adict() self.QA.VOCAB = {'<PAD>': 0} self.QA.IVOCAB = {0: '<PAD>'} self.count_id = 1 self.tree_dict = {} self.train, self.count_id, tree_dict1 = self.get_indexed_qa( raw_train, self.count_id) self.tree_dict.update(tree_dict1) self.valid = [ self.train[i][int(-len(self.train[i]) / 10):] for i in range(3) ] self.train = [ self.train[i][:int(9 * len(self.train[i]) / 10)] for i in range(3) ] self.test, self.count_id, tree_dict2 = self.get_indexed_qa( raw_test, self.count_id) self.tree_dict.update(tree_dict2)
def db(sql, *values, **params): """ Queries DB and returns result. @param str sql - SQL with %s placehodlers. Do NOT add quotes or IN-brackets around %s. E.g. db('...WHERE `name` IN %s', [value1, value2]) @param tuple values - Values for %s placehodlers. @param dict params: str charset - E.g. "utf8mb4" for tables with Emoji. @return adict: list(adict) rows - All rows in result. adict|NoneType row - First row, if any. int affected - Number of rows affected. """ charset = params.get('charset', db_config.charset) def code(): if _local.db_conn.charset != charset: _local.db_conn.set_charset(charset) with _local.db_conn.cursor() as cursor: with Timeout(db_config.query_timeout): cursor.execute(sql, values) return cursor.fetchall(), cursor.rowcount rows, affected = db_transaction(code) # It's just "rows, affected = code()" if already inside a transaction. return adict( rows=rows if isinstance(rows, list) else list( rows ), # E.g. "DictCursorMixin" creates "list" from "tuple" only if it is not empty. row=rows[0] if rows else None, affected=affected, )
def db(sql, *values, **params): """ Queries DB and returns result. @param str sql - SQL with %s placehodlers. Do NOT add quotes or IN-brackets around %s. E.g. db('...WHERE `name` IN %s', [value1, value2]) @param tuple values - Values for %s placehodlers. @param dict params: str charset - E.g. "utf8mb4" for tables with Emoji. @return adict: list(adict) rows - All rows in result. adict|NoneType row - First row, if any. int affected - Number of rows affected. """ charset = params.get('charset', db_config.charset) def code(): if _local.db_conn.charset != charset: _local.db_conn.set_charset(charset) with _local.db_conn.cursor() as cursor: with Timeout(db_config.query_timeout): cursor.execute(sql, values) return cursor.fetchall(), cursor.rowcount rows, affected = db_transaction(code) # It's just "rows, affected = code()" if already inside a transaction. return adict( rows=rows if isinstance(rows, list) else list(rows), # E.g. "DictCursorMixin" creates "list" from "tuple" only if it is not empty. row=rows[0] if rows else None, affected=affected, )
def __init__(self, field_name, state, explain=''): error = adict(field=field_name, state=state) if explain: error.explain = explain super(BadRequest, self).__init__(400, error)
def test(): pool_size = 2 db_config(name='test', user='******', password='******', pool_size=pool_size, pool_block=pool_size) # Use your database credentials! ### async, reconnect ''' [0.0 seconds] Testing async pool (connections: 2), reconnect and retry: [0.0 seconds] Greenlet 139815404809104 tries to "SELECT pg_sleep(5)". [0.0 seconds] Greenlet 139815404810064 tries to "SELECT pg_sleep(5)". [0.0 seconds] Greenlet 139815404808464 tries to "SELECT pg_sleep(5)". [0.0 seconds] Greenlet 139815401537616 tries to "SELECT pg_sleep(5)". [3.16 seconds] Restarted PostgreSQL in 3.16118597984 seconds. # 4.18 seconds - Reconnected to PostgreSQL, trying to sleep again. [9.18 seconds] Greenlet 139815404809104 stopped sleeping. [9.18 seconds] Greenlet 139815404810064 stopped sleeping. # Pool of 2 connections is ready for pending queries. [14.18 seconds] Greenlet 139815404808464 stopped sleeping. [14.18 seconds] Greenlet 139815401537616 stopped sleeping. [14.18 seconds] All greenlets are done. ''' import gevent, time from subprocess import Popen from thread import get_ident sleep_seconds = 5 start = time.time() def log(text): print('[{seconds} seconds] {text}'.format(seconds=round(time.time() - start, 2), text=text)) log('Testing async pool (connections: {pool_size}), reconnect and retry:'.format(pool_size=pool_size)) def restart_postgresql(): start = time.time() Popen('sudo service postgresql restart', shell=True).communicate() log('Restarted PostgreSQL in {seconds} seconds.'.format(seconds=(time.time() - start))) def sleep_at_postgresql(): greenlet_id = get_ident() log('Greenlet {greenlet_id} tries to "SELECT pg_sleep({seconds})".'.format(greenlet_id=greenlet_id, seconds=sleep_seconds)) db('SELECT pg_sleep(%s)', sleep_seconds) log('Greenlet {greenlet_id} stopped sleeping.'.format(greenlet_id=greenlet_id)) gevent.joinall([ gevent.spawn(sleep_at_postgresql) for _ in xrange(pool_size * 2) ] + [ gevent.spawn(restart_postgresql), ]) log('All greenlets are done.') ### other print('\nTesting other features...') ### db db('DROP TABLE IF EXISTS "test_item"') db(''' CREATE TABLE "test_item" ( "id" serial NOT NULL PRIMARY KEY, "parent_id" integer ) ''') ### db_insert item1_id = db_insert('test_item', _return='id', ) item2_id = db_insert('test_item', parent_id=item1_id, _return='id', ) ### db_insert result, db_transaction, db_rollback, db rows iteration raised = False try: def code(): assert db('SELECT "id" FROM "test_item" WHERE id = %s', item2_id).row.id == item2_id db('DELETE FROM "test_item" WHERE "id" = %s', item2_id) assert db('SELECT "id" FROM "test_item" WHERE id = %s', item2_id).row is None raise db_rollback db_transaction(code) except db_rollback: raised = True assert raised items = db('SELECT "id" FROM "test_item" WHERE "id" IN %s', (item1_id, item2_id)) assert set(item.id for item in items) == set([item1_id, item2_id]) ### db_transaction result, db_update def code(): return db('SELECT "parent_id" FROM "test_item" WHERE "id" = %s', item2_id).row.parent_id assert db_transaction(code) == item1_id assert db_update('test_item', parent_id=None, where=dict(id=item2_id), ) == 1 assert db('SELECT "parent_id" FROM "test_item" WHERE "id" = %s', item2_id).row.parent_id is None ### db_update "IN %s" db_update('test_item', parent_id=0, where=dict(id=(item1_id, item2_id)), ) items = db('SELECT "parent_id" FROM "test_item" WHERE "id" IN %s', (item1_id, item2_id)) assert [item.parent_id for item in items] == [0, 0] ### "cannot run inside a transaction block" db('DROP TYPE IF EXISTS "my_type"') db("""CREATE TYPE "my_type" AS ENUM ('aaa', 'bbb')""") raised = False try: db("""ALTER TYPE "my_type" ADD VALUE 'my_value'""") except Exception as e: raised = True assert 'cannot run inside a transaction block' in repr(e) assert raised db("""ALTER TYPE "my_type" ADD VALUE 'my_value'""", autocommit=True) ### escape_like assert escape_like('\? item_percent = 42%') == r'%\\? item\_percent = 42\%%' ### log state = adict(logged=False) def log(query): sql, values = query assert sql == 'UPDATE "test_item" SET "parent_id" = %s WHERE "id" = %s' assert values == (42, 1) state.logged = True db_config(name=_db_config['database'], user=_db_config['user'], password=_db_config['password'], pool_size=_db_pool.qsize(), log=log) db_update('test_item', parent_id=42, where=dict(id=item1_id)) assert state.logged print('OK')
### adict cursor class AdictCursorMixin(pymysql.cursors.DictCursorMixin): dict_type = adict class AdictCursor(AdictCursorMixin, pymysql.cursors.Cursor): """A cursor which returns results as a dict with attr access to keys.""" ### db_config db_config = adict( host='127.0.0.1', port=3306, user='******', password='******', database='test', charset='utf8', cursor_class=AdictCursor, query_timeout=55, # Less than default web timeout of 60 seconds, but greater than default "innodb_lock_wait_timeout" of 50 seconds. pool_size=10, _pool=None, ) ### db_init def db_init(): db_config._pool = Queue() for _ in xrange(db_config.pool_size): db_config._pool.put(_create_db_conn()) ### _create_db_conn
from traceback import format_exc import sys ### crit_defaults crit_defaults = adict( subject='CRIT', # '{service_name} {host}:{port} CRIT' plugins=[], # [critbot.plugins.syslog.plugin(), ...] crit_in_crit=sys.stderr.write, # logging.getLogger('critbot').critical - if you enabled "syslog" plugin, else your_logger.critical stop_spam_file=adict( # Stop spam from multiple processes at the same host. enabled=False, path=( '/run/lock/critbot' if sys.platform.startswith('linux') # RAM, no disk IO. else '/tmp/critbot' # Mac OS X, etc. ), ), mc=adict( # Memcached to stop spam from multiple hosts. enabled=False, servers=['127.0.0.1'], # http://sendapatch.se/projects/pylibmc/reference.html#pylibmc.Client client_config=adict(binary=True, behaviors=dict(tcp_nodelay=True, distribution='consistent')), pool_size=10, pool=None, # Is initialized on first crit from updated crit_defaults and to avoid useless slowdown in small scripts. ), ) ### utf8 def utf8(value): return (
def app(environ, start_response): status = response = raw_response = '' try: action = routes.get(environ['PATH_INFO']) if not action: raise ApiError(404) raw_response = hasattr(action, 'raw_response') #### raw environ if hasattr(action, 'raw_environ'): if raw_response: response = action(environ, start_response) else: response = action(environ) #### normal request else: if environ['REQUEST_METHOD'] != 'POST': # See README.md. raise ApiError(501) request = environ['wsgi.input'].read() try: request = json.loads(request) if request else {} if not isinstance(request, dict): raise ValueError except ValueError: raise ApiError(400, 'Request content should be JSON Object') # See README.md. request = adict(request) if raw_response: response = action(request, start_response) else: response = action(request) #### normal response if not raw_response: if response is None: response = {} if not isinstance(response, dict): raise Exception('Response content should be JSON Object') # See README.md. status = status_by_code[200] #### Expected error, no logging. except ApiError as e: status = status_by_code[e.status_code] response = dict(error=(e.error or status)) #### Unexpected error, traceback is logged. except: print_exc() # To sys.stderr. status = status_by_code[500] response = dict(error=status) # Server error details are saved to log and not disclosed to a client. #### Response. finally: #### raw response if raw_response: return response #### normal response start_response(status, headers=[ ('Content-Type', 'application/json'), ]) return [ json.dumps(response), ]
#### import from adict import adict, ajson import json, requests #### config config = adict( host='127.0.0.1', port=8888, version='v0', ) # Can be updated from a caller script. #### test def test(target, action, request_json, status_code, response_json, method='POST'): url = 'http://{host}:{port}/api/{version}/{target}/{action}'.format(target=target, action=action, **config) request_json = json.dumps(request_json) if request_json else None print('{method} {url} {request_json} --> {status_code} {response_json}'.format(**locals())) response = requests.request(method, url, data=request_json) assert response.status_code == status_code, (response.status_code, status_code, response.text) response.ajson = ajson(response.json()) assert response.ajson == response_json, (response.text, response_json) # NOTE: response.text is used instead of response.ajson|json() to easily compare without extra u'...'-s. return response
dict_type = adict class AdictCursor(AdictCursorMixin, pymysql.cursors.Cursor): """A cursor which returns results as a dict with attr access to keys.""" ### db_config db_config = adict( host='127.0.0.1', port=3306, user='******', password='******', database='test', charset='utf8', cursor_class=AdictCursor, query_timeout= 55, # Less than default web timeout of 60 seconds, but greater than default "innodb_lock_wait_timeout" of 50 seconds. pool_size=10, _pool=None, ) ### db_init def db_init(): db_config._pool = Queue() for _ in xrange(db_config.pool_size): db_config._pool.put(_create_db_conn())