def __init__(self, table_name, args, log):
TableHandler.__init__(self, table_name, args, log)
self.max_part = None # max part number
self.local_part = None # part number of local node
self.key = args.get('key')
if self.key is None:
raise Exception('Specify key field as key agument')
def __init__(self, table_name, args, log):
TableHandler.__init__(self, table_name, args, log)
self.max_part = None # max part number
self.local_part = None # part number of local node
self.key = args.get('key')
if self.key is None:
raise Exception('Specify key field as key agument')
def add(self, trigger_arg_list):
"""Let trigger put hash into extra3"""
arg = "ev_extra3='hash='||hashtext(%s)" % skytools.quote_ident(
self.key)
trigger_arg_list.append(arg)
TableHandler.add(self, trigger_arg_list)
def process_event(self, ev, sql_queue_func, arg):
"""Filter event by hash in extra3, apply only local part."""
if ev.extra3:
meta = skytools.db_urldecode(ev.extra3)
self.log.debug('part.process_event: hash=%d, max_part=%s, local_part=%d',
int(meta['hash']), self.max_part, self.local_part)
if (int(meta['hash']) & self.max_part) != self.local_part:
self.log.debug('part.process_event: not my event')
return
self.log.debug('part.process_event: my event, processing')
TableHandler.process_event(self, ev, sql_queue_func, arg)
def process_event(self, ev, sql_queue_func, arg):
"""Filter event by hash in extra3, apply only local part."""
if ev.extra3:
meta = skytools.db_urldecode(ev.extra3)
self.log.debug('part.process_event: hash=%d, max_part=%s, local_part=%d' %\
(int(meta['hash']), self.max_part, self.local_part))
if (int(meta['hash']) & self.max_part) != self.local_part:
self.log.debug('part.process_event: not my event')
return
self.log.debug('part.process_event: my event, processing')
TableHandler.process_event(self, ev, sql_queue_func, arg)
def __init__(self, table_name, args, dest_table):
TableHandler.__init__(self, table_name, args, dest_table)
self.hash_mask = None # aka max part number (atm)
self.shard_nr = None # part number of local node
# primary key columns
self.hash_key = args.get('hash_key', args.get('key'))
self._validate_hash_key()
# hash function & full expression
hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC)
self.hashexpr = self.DEFAULT_HASHEXPR % (skytools.quote_fqident(
hashfunc), skytools.quote_ident(self.hash_key or ''))
self.hashexpr = args.get('hashexpr', self.hashexpr)
def __init__(self, table_name, args, dest_table):
TableHandler.__init__(self, table_name, args, dest_table)
self.max_part = None # max part number
self.local_part = None # part number of local node
# primary key columns
self.key = args.get('key')
if self.key is None:
raise Exception('Specify key field as key agument')
# hash function & full expression
hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC)
self.hashexpr = self.DEFAULT_HASHEXPR % (
skytools.quote_fqident(hashfunc), skytools.quote_ident(self.key))
self.hashexpr = args.get('hashexpr', self.hashexpr)
def __init__(self, table_name, args, dest_table):
TableHandler.__init__(self, table_name, args, dest_table)
self.hash_mask = None # aka max part number (atm)
self.shard_nr = None # part number of local node
# primary key columns
self.hash_key = args.get('hash_key', args.get('key'))
self._validate_hash_key()
# hash function & full expression
hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC)
self.hashexpr = self.DEFAULT_HASHEXPR % (
skytools.quote_fqident(hashfunc),
skytools.quote_ident(self.hash_key or ''))
self.hashexpr = args.get('hashexpr', self.hashexpr)
def __init__(self, table_name, args, dest_table):
TableHandler.__init__(self, table_name, args, dest_table)
self.max_part = None # max part number
self.local_part = None # part number of local node
# primary key columns
self.key = args.get('key')
if self.key is None:
raise Exception('Specify key field as key argument')
# hash function & full expression
hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC)
self.hashexpr = self.DEFAULT_HASHEXPR % (
skytools.quote_fqident(hashfunc),
skytools.quote_ident(self.key))
self.hashexpr = args.get('hashexpr', self.hashexpr)
def get_copy_condition(self, src_curs, dst_curs):
"""Prepare the where condition for copy and replay filtering"""
if self.hash_key is None:
return TableHandler.get_copy_condition(self, src_curs, dst_curs)
self.load_shard_info(dst_curs)
w = "(%s & %d) = %d" % (self.hashexpr, self.hash_mask, self.shard_nr)
self.log.debug('shard: copy_condition=%r', w)
return w
def get_copy_condition(self, src_curs, dst_curs):
"""Prepare the where condition for copy and replay filtering"""
if self.hash_key is None:
return TableHandler.get_copy_condition(self, src_curs, dst_curs)
self.load_shard_info(dst_curs)
w = "(%s & %d) = %d" % (self.hashexpr, self.hash_mask, self.shard_nr)
self.log.debug('shard: copy_condition=%r', w)
return w
def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list):
"""Copy only slots needed locally."""
self.load_part_info(dst_curs)
fn = 'hashtext(%s)' % skytools.quote_ident(self.key)
w = "%s & %d = %d" % (fn, self.max_part, self.local_part)
self.log.debug('part: copy_condition=%s' % w)
cond_list.append(w)
return TableHandler.real_copy(self, tablename, src_curs, dst_curs,
column_list, cond_list)
def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list):
"""Copy only slots needed locally."""
self.load_part_info(dst_curs)
fn = 'hashtext(%s)' % skytools.quote_ident(self.key)
w = "%s & %d = %d" % (fn, self.max_part, self.local_part)
self.log.debug('part: copy_condition=%s' % w)
cond_list.append(w)
return TableHandler.real_copy(self, tablename, src_curs, dst_curs,
column_list, cond_list)
def prepare_batch(self, batch_info, dst_curs):
"""Called on first event for this table in current batch."""
if not self.max_part:
self.load_part_info(dst_curs)
TableHandler.prepare_batch(self, batch_info, dst_curs)
def _process_event(self, ev, sql_queue_func, arg):
self.log.debug('shard.process_event: my event, processing')
TableHandler.process_event(self, ev, sql_queue_func, arg)
def prepare_batch(self, batch_info, dst_curs):
"""Called on first event for this table in current batch."""
if self.hash_key is not None:
if not self.hash_mask:
self.load_shard_info(dst_curs)
TableHandler.prepare_batch(self, batch_info, dst_curs)
def add(self, trigger_arg_list):
"""Let trigger put hash into extra3"""
arg = "ev_extra3='hash='||%s" % self.hashexpr
trigger_arg_list.append(arg)
TableHandler.add(self, trigger_arg_list)
def reset(self):
"""Forget config info."""
self.hash_mask = None
self.shard_nr = None
TableHandler.reset(self)
def prepare_batch(self, batch_info, dst_curs):
"""Called on first event for this table in current batch."""
if self.hash_key is not None:
if not self.hash_mask:
self.load_shard_info(dst_curs)
TableHandler.prepare_batch(self, batch_info, dst_curs)
def _process_event(self, ev, sql_queue_func, arg):
self.log.debug('shard.process_event: my event, processing')
TableHandler.process_event(self, ev, sql_queue_func, arg)
def prepare_batch(self, batch_info, dst_curs):
"""Called on first event for this table in current batch."""
if not self.max_part:
self.load_part_info(dst_curs)
TableHandler.prepare_batch(self, batch_info, dst_curs)
def add(self, trigger_arg_list):
"""Let trigger put hash into extra3"""
arg = "ev_extra3='hash='||%s" % self.hashexpr
trigger_arg_list.append(arg)
TableHandler.add(self, trigger_arg_list)
def reset(self):
"""Forget config info."""
self.max_part = None
self.local_part = None
TableHandler.reset(self)
def add(self, trigger_arg_list):
"""Let trigger put hash into extra3"""
arg = "ev_extra3='hash='||hashtext(%s)" % skytools.quote_ident(self.key)
trigger_arg_list.append(arg)
TableHandler.add(self, trigger_arg_list)
def reset(self):
"""Forget config info."""
self.hash_mask = None
self.shard_nr = None
TableHandler.reset(self)
def reset(self):
"""Forget config info."""
self.max_part = None
self.local_part = None
TableHandler.reset(self)
