def get_record(arg):
""" Parse data for one urlencoded record.
Useful for turning incoming serialized data into structure usable for manipulation.
"""
if not arg:
return dbdict()
# allow array of single record
if arg[0] in ('{', '['):
lst = skytools.parse_pgarray(arg)
if len(lst) != 1:
raise ValueError('get_record() expects exactly 1 row, got %d' % len(lst))
arg = lst[0]
# parse record
return dbdict(skytools.db_urldecode(arg))
def retval(self, service_name=None, params=None, **kvargs):
""" Return collected resultsets and append to the end messages to the users
Method is called usually as last statment in dbservice to return the results
Also converts results into desired format
"""
params = params or kvargs
self.raise_if_errors()
if len(self.messages):
self.return_next(self.messages, "_status")
if self.sqls is not None and len(self.sqls):
self.return_next(self.sqls, "_sql")
results = []
for r in self._retval:
res_name = r[0]
rows = r[1]
res_count = str(len(rows))
if self._is_test and len(rows) > 0:
results.append([res_name, res_count, res_name])
n = 1
for trow in render_table(rows, rows[0].keys()):
results.append([res_name, n, trow])
n += 1
else:
res_rows = make_record_array(rows)
results.append([res_name, res_count, res_rows])
if service_name:
sql = "select * from %s( {i_context}, {i_params} );" % skytools.quote_fqident(
service_name)
par = dbdict(i_context=self._context, i_params=make_record(params))
res = self.run_query(sql, par)
for r in res:
results.append((r.res_code, r.res_text, r.res_rows))
return results
def retval(self, service_name = None, params = None, **kvargs):
""" Return collected resultsets and append to the end messages to the users
Method is called usually as last statment in dbservice to return the results
Also converts results into desired format
"""
params = params or kvargs
self.raise_if_errors()
if len( self.messages ):
self.return_next( self.messages, "_status" )
if self.sqls is not None and len( self.sqls ):
self.return_next( self.sqls, "_sql" )
results = []
for r in self._retval:
res_name = r[0]
rows = r[1]
res_count = str(len(rows))
if self._is_test and len(rows) > 0:
results.append([res_name, res_count, res_name])
n = 1
for trow in render_table(rows, rows[0].keys()):
results.append([res_name, n, trow])
n += 1
else:
res_rows = make_record_array(rows)
results.append([res_name, res_count, res_rows])
if service_name:
sql = "select * from %s( {i_context}, {i_params} );" % skytools.quote_fqident(service_name)
par = dbdict( i_context = self._context, i_params = make_record(params) )
res = self.run_query( sql, par )
for r in res:
results.append((r.res_code, r.res_text, r.res_rows))
return results
def execute(self):
"""Server-side query execution via plpy.
Query can be run either cached or uncached, depending
on C{plan_cache} setting given to L{__init__}.
Returns result of plpy.execute().
"""
args = self._arg_value_list
types = self._arg_type_list
if self._sqls is not None:
self._sqls.append( { "sql": self.get_sql(PARAM_INLINE) } )
if self._plan_cache is not None:
sql = self.get_sql(PARAM_PLPY)
plan = self._plan_cache.get_plan(sql, types)
res = plpy.execute(plan, args)
else:
sql = self.get_sql(PARAM_INLINE)
res = plpy.execute(sql)
if res:
res = [skytools.dbdict(r) for r in res]
return res
def execute(self):
"""Server-side query execution via plpy.
Query can be run either cached or uncached, depending
on C{plan_cache} setting given to L{__init__}.
Returns result of plpy.execute().
"""
args = self._arg_value_list
types = self._arg_type_list
if self._sqls is not None:
self._sqls.append({"sql": self.get_sql(PARAM_INLINE)})
if self._plan_cache is not None:
sql = self.get_sql(PARAM_PLPY)
plan = self._plan_cache.get_plan(sql, types)
res = plpy.execute(plan, args)
else:
sql = self.get_sql(PARAM_INLINE)
res = plpy.execute(sql)
if res:
res = [skytools.dbdict(r) for r in res]
return res
def tapi_do(self, tablename, row, **fields):
""" Convenience function for just doing the change without creating tapi object first
Fields object may contain aditional overriding values that are aplied before do
"""
tapi = TableAPI(self, tablename, self._changelog(fields))
row = row or dbdict()
fields and row.update(fields)
return tapi.do(row)
def tapi_do(self, tablename, row, **fields):
""" Convenience function for just doing the change without creating tapi object first
Fields object may contain aditional overriding values that are aplied before do
"""
tapi = TableAPI(self, tablename, self._changelog(fields))
row = row or dbdict()
fields and row.update(fields)
return tapi.do( row )
def get_record_lists(tbl, field):
""" Create dictionary of lists from given list using field as grouping criteria
Used for master detail operatons to group detail records according to master id
"""
dict = dbdict()
for rec in tbl:
id = str(rec[field])
dict.setdefault(id, []).append(rec)
return dict
def field_copy(self, dict, *keys):
""" Used to copy subset of fields from one record into another
example: dbs.copy(record, hosting) "start_date", "key_colo", "key_rack")
"""
retval = dbdict()
for key in keys:
if key in dict:
retval[key] = dict[key]
return retval
def get_record_lists(tbl, field):
""" Create dictionary of lists from given list using field as grouping criteria
Used for master detail operatons to group detail records according to master id
"""
dict = dbdict()
for rec in tbl:
id = str( rec[field] )
dict.setdefault( id, [] ).append(rec)
return dict
def field_copy(self, dict, *keys):
""" Used to copy subset of fields from one record into another
example: dbs.copy(record, hosting) "start_date", "key_colo", "key_rack")
"""
retval = dbdict()
for key in keys:
if key in dict:
retval[key] = dict[key]
return retval
def run_query(cur, sql, params = None, **kwargs):
""" Helper function if everything you need is just paramertisized execute
Sets rows_found that is coneninet to use when you don't need result just
want to know how many rows were affected
"""
params = params or kwargs
sql = QueryBuilder(sql, params).get_sql(0)
cur.execute(sql)
rows = cur.fetchall()
# convert result rows to dbdict
if rows:
rows = [skytools.dbdict(r) for r in rows]
return rows
def run_query(cur, sql, params=None, **kwargs):
""" Helper function if everything you need is just paramertisized execute
Sets rows_found that is coneninet to use when you don't need result just
want to know how many rows were affected
"""
params = params or kwargs
sql = QueryBuilder(sql, params).get_sql(0)
cur.execute(sql)
rows = cur.fetchall()
# convert result rows to dbdict
if rows:
rows = [skytools.dbdict(r) for r in rows]
return rows
def get_config(self):
"""Processes args dict"""
conf = skytools.dbdict()
# set table mode
conf.table_mode = self.get_arg('table_mode', TABLE_MODES)
conf.analyze = self.get_arg('analyze', [0, 1])
if conf.table_mode == 'part':
conf.part_mode = self.get_arg('part_mode', PART_MODES)
conf.part_field = self.args.get('part_field')
if conf.part_mode == 'date_field' and not conf.part_field:
raise Exception('part_mode date_field requires part_field!')
conf.period = self.get_arg('period', PERIODS)
conf.part_name = self.args.get('part_name')
conf.part_template = self.args.get('part_template')
conf.pre_part = self.args.get('pre_part')
conf.post_part = self.args.get('post_part')
conf.part_func = self.args.get('part_func', PART_FUNC_NEW)
# set row mode and event types to process
conf.row_mode = self.get_arg('row_mode', ROW_MODES)
event_types = self.args.get('event_types', '*')
if event_types == '*':
event_types = EVENT_TYPES
else:
event_types = [evt.upper() for evt in event_types.split(',')]
for evt in event_types:
if evt not in EVENT_TYPES:
raise Exception('Unsupported operation: %s' % evt)
conf.event_types = event_types
# set load handler
conf.load_mode = self.get_arg('load_mode', LOAD_MODES)
conf.method = self.get_arg('method', METHODS)
# fields to skip
conf.skip_fields = [
f.strip().lower()
for f in self.args.get('skip_fields', '').split(',')
]
# get fields map (obsolete, for compatibility reasons)
fields = self.args.get('fields', '*')
if fields == "*":
conf.field_map = None
else:
conf.field_map = {}
for fval in fields.split(','):
tmp = fval.split(':')
if len(tmp) == 1:
conf.field_map[tmp[0]] = tmp[0]
else:
conf.field_map[tmp[0]] = tmp[1]
# encoding validator
conf.encoding = self.args.get('encoding')
return conf
def run_query(self, sql, params=None, **kvargs):
""" Helper function if everything you need is just paramertisized execute
Sets rows_found that is coneninet to use when you don't need result just
want to know how many rows were affected
"""
params = params or kvargs
rows = skytools.plpy_exec(self.global_dict, sql, params)
# convert result rows to dbdict
if rows:
rows = [dbdict(r) for r in rows]
self.rows_found = len(rows)
else:
self.rows_found = 0
return rows
def get_config(self):
"""Processes args dict"""
conf = skytools.dbdict()
# set table mode
conf.table_mode = self.get_arg('table_mode', TABLE_MODES)
conf.analyze = self.get_arg('analyze', [0, 1])
if conf.table_mode == 'part':
conf.part_mode = self.get_arg('part_mode', PART_MODES)
conf.part_field = self.args.get('part_field')
if conf.part_mode == 'date_field' and not conf.part_field :
raise Exception('part_mode date_field requires part_field!')
conf.period = self.get_arg('period', PERIODS)
conf.part_name = self.args.get('part_name')
conf.part_template = self.args.get('part_template')
conf.pre_part = self.args.get('pre_part')
conf.post_part = self.args.get('post_part')
conf.part_func = self.args.get('part_func', PART_FUNC_NEW)
conf.retention_period = self.args.get('retention_period')
# set row mode and event types to process
conf.row_mode = self.get_arg('row_mode', ROW_MODES)
event_types = self.args.get('event_types', '*')
if event_types == '*':
event_types = EVENT_TYPES
else:
event_types = [evt.upper() for evt in event_types.split(',')]
for evt in event_types:
if evt not in EVENT_TYPES:
raise Exception('Unsupported operation: %s' % evt)
conf.event_types = event_types
# set load handler
conf.load_mode = self.get_arg('load_mode', LOAD_MODES)
conf.method = self.get_arg('method', METHODS)
# fields to skip
conf.skip_fields = [f.strip().lower()
for f in self.args.get('skip_fields','').split(',')]
# get fields map (obsolete, for compatibility reasons)
fields = self.args.get('fields', '*')
if fields == "*":
conf.field_map = None
else:
conf.field_map = {}
for fval in fields.split(','):
tmp = fval.split(':')
if len(tmp) == 1:
conf.field_map[tmp[0]] = tmp[0]
else:
conf.field_map[tmp[0]] = tmp[1]
# encoding validator
conf.encoding = self.args.get('encoding')
return conf
def run_query(self, sql, params = None, **kvargs):
""" Helper function if everything you need is just paramertisized execute
Sets rows_found that is coneninet to use when you don't need result just
want to know how many rows were affected
"""
params = params or kvargs
rows = skytools.plpy_exec(self.global_dict, sql, params)
# convert result rows to dbdict
if rows:
rows = [dbdict(r) for r in rows]
self.rows_found = len(rows)
else:
self.rows_found = 0
return rows
def get_config(self):
"""Processes args dict"""
conf = skytools.dbdict()
# set table mode
conf.table_mode = self.get_arg("table_mode", TABLE_MODES)
if conf.table_mode == "part":
conf.part_mode = self.get_arg("part_mode", PART_MODES)
conf.part_field = self.args.get("part_field")
if conf.part_mode == "date_field" and not conf.part_field:
raise Exception("part_mode date_field requires part_field!")
conf.period = self.get_arg("period", PERIODS)
conf.part_name = self.args.get("part_name")
conf.part_template = self.args.get("part_template")
conf.pre_part = self.args.get("pre_part")
conf.post_part = self.args.get("post_part")
# set row mode and event types to process
conf.row_mode = self.get_arg("row_mode", ROW_MODES)
event_types = self.args.get("event_types", "*")
if event_types == "*":
event_types = EVENT_TYPES
else:
event_types = [evt.upper() for evt in event_types.split(",")]
for evt in event_types:
if evt not in EVENT_TYPES:
raise Exception("Unsupported operation: %s" % evt)
conf.event_types = event_types
# set load handler
conf.load_mode = self.get_arg("load_mode", LOAD_MODES)
conf.method = self.get_arg("method", METHODS)
# fields to skip
conf.skip_fields = [f.strip().lower() for f in self.args.get("skip_fields", "").split(",")]
# get fields map (obsolete, for compatibility reasons)
fields = self.args.get("fields", "*")
if fields == "*":
conf.field_map = None
else:
conf.field_map = {}
for fval in fields.split(","):
tmp = fval.split(":")
if len(tmp) == 1:
conf.field_map[tmp[0]] = tmp[0]
else:
conf.field_map[tmp[0]] = tmp[1]
# encoding validator
conf.encoding = self.args.get("encoding")
return conf
def parse_sql(self, op, sql, pklist=None):
"""Main entry point."""
if pklist is None:
self.pklist = []
else:
self.pklist = pklist
tk = self.tokenizer(sql)
fields = []
values = []
try:
if op == "I":
self.parse_insert(tk, fields, values)
elif op == "U":
self.parse_update(tk, fields, values)
elif op == "D":
self.parse_delete(tk, fields, values)
raise Exception("syntax error")
except StopIteration:
# last sanity check
if len(fields) == 0 or len(fields) != len(values):
raise Exception("syntax error, fields do not match values")
fields = [skytools.unquote_ident(f) for f in fields]
values = [skytools.unquote_literal(f) for f in values]
return skytools.dbdict(zip(fields, values))
def copy(self):
"""Return regular dict."""
return skytools.dbdict(self.iteritems())
def get_config(self):
""" Process args dict (into handler config). """
conf = skytools.dbdict()
return conf
# confusion among older and newer libzmq and their forks
if not hasattr(zmq, 'XREQ'): zmq.XREQ = zmq.DEALER
if not hasattr(zmq, 'XREP'): zmq.XREP = zmq.ROUTER
from cc import __version__
from cc.crypto import CryptoContext
from cc.handler import cc_handler_lookup
from cc.message import CCMessage
from cc.stream import CCStream
from cc.util import hsize_to_bytes, reset_stats, write_atomic
LOG = skytools.dbdict(
fmt = '%(asctime)s %(process)s %(levelname)s %(name)s.%(funcName)s: %(message)s',
datefmt = '',
fmt_v = '%(asctime)s,%(msecs)03d %(levelname)s %(name)s.%(funcName)s: %(message)s',
datefmt_v = '%H:%M:%S',
)
for k in LOG:
LOG[k] = LOG[k].replace('%', '%%')
class CCServer(skytools.BaseScript):
"""Listens on single ZMQ sockets, dispatches messages to handlers.
Config::
## Parameters for CCServer ##
# listening socket for this CC instance
cc-socket = tcp://127.0.0.1:22632
def copy(self):
"""Return regular dict."""
return skytools.dbdict(self.items())
def _create_dbdict(self, fields, values):
fields = [skytools.unquote_ident(f) for f in fields]
values = [skytools.unquote_literal(f) for f in values]
return skytools.dbdict(zip(fields, values))
def get_config(self):
""" Process args dict (into handler config). """
conf = skytools.dbdict()
return conf
import skytools
import zmq
from zmq.eventloop.ioloop import IOLoop, PeriodicCallback
from cc import __version__
from cc.crypto import CryptoContext
from cc.handler import cc_handler_lookup
from cc.message import CCMessage
from cc.stream import CCStream
from cc.util import hsize_to_bytes, reset_stats, write_atomic
LOG = skytools.dbdict(
fmt="%(asctime)s %(process)s %(levelname)s %(name)s.%(funcName)s: %(message)s",
datefmt="",
fmt_v="%(asctime)s,%(msecs)03d %(levelname)s %(name)s.%(funcName)s: %(message)s",
datefmt_v="%H:%M:%S",
)
for k in LOG:
LOG[k] = LOG[k].replace("%", "%%")
class CCServer(skytools.BaseScript):
"""Listens on single ZMQ sockets, dispatches messages to handlers.
Config::
## Parameters for CCServer ##
# listening socket for this CC instance
cc-socket = tcp://127.0.0.1:22632
import skytools
import zmq
from zmq.eventloop.ioloop import IOLoop, PeriodicCallback
from cc import __version__
from cc.crypto import CryptoContext
from cc.handler import cc_handler_lookup
from cc.message import CCMessage
from cc.stream import CCStream
from cc.util import hsize_to_bytes, reset_stats
LOG = skytools.dbdict(
fmt=
'%(asctime)s %(process)s %(levelname)s %(name)s.%(funcName)s: %(message)s',
datefmt='',
fmt_v=
'%(asctime)s,%(msecs)03d %(levelname)s %(name)s.%(funcName)s: %(message)s',
datefmt_v='%H:%M:%S',
)
for k in LOG:
LOG[k] = LOG[k].replace('%', '%%')
class CCServer(skytools.BaseScript):
"""Listens on single ZMQ sockets, dispatches messages to handlers.
Config::
## Parameters for CCServer ##
# listening socket for this CC instance
cc-socket = tcp://127.0.0.1:22632
def _create_dbdict(self, fields, values):
fields = [skytools.unquote_ident(f) for f in fields]
values = [skytools.unquote_literal(f) for f in values]
return skytools.dbdict(zip(fields, values))
