def __init__(self, table_name, args, log): TableHandler.__init__(self, table_name, args, log) self.max_part = None # max part number self.local_part = None # part number of local node self.key = args.get('key') if self.key is None: raise Exception('Specify key field as key agument')
def add(self, trigger_arg_list): """Let trigger put hash into extra3""" arg = "ev_extra3='hash='||hashtext(%s)" % skytools.quote_ident( self.key) trigger_arg_list.append(arg) TableHandler.add(self, trigger_arg_list)
def process_event(self, ev, sql_queue_func, arg): """Filter event by hash in extra3, apply only local part.""" if ev.extra3: meta = skytools.db_urldecode(ev.extra3) self.log.debug('part.process_event: hash=%d, max_part=%s, local_part=%d', int(meta['hash']), self.max_part, self.local_part) if (int(meta['hash']) & self.max_part) != self.local_part: self.log.debug('part.process_event: not my event') return self.log.debug('part.process_event: my event, processing') TableHandler.process_event(self, ev, sql_queue_func, arg)
def process_event(self, ev, sql_queue_func, arg): """Filter event by hash in extra3, apply only local part.""" if ev.extra3: meta = skytools.db_urldecode(ev.extra3) self.log.debug('part.process_event: hash=%d, max_part=%s, local_part=%d' %\ (int(meta['hash']), self.max_part, self.local_part)) if (int(meta['hash']) & self.max_part) != self.local_part: self.log.debug('part.process_event: not my event') return self.log.debug('part.process_event: my event, processing') TableHandler.process_event(self, ev, sql_queue_func, arg)
def __init__(self, table_name, args, dest_table): TableHandler.__init__(self, table_name, args, dest_table) self.hash_mask = None # aka max part number (atm) self.shard_nr = None # part number of local node # primary key columns self.hash_key = args.get('hash_key', args.get('key')) self._validate_hash_key() # hash function & full expression hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC) self.hashexpr = self.DEFAULT_HASHEXPR % (skytools.quote_fqident( hashfunc), skytools.quote_ident(self.hash_key or '')) self.hashexpr = args.get('hashexpr', self.hashexpr)
def __init__(self, table_name, args, dest_table): TableHandler.__init__(self, table_name, args, dest_table) self.max_part = None # max part number self.local_part = None # part number of local node # primary key columns self.key = args.get('key') if self.key is None: raise Exception('Specify key field as key agument') # hash function & full expression hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC) self.hashexpr = self.DEFAULT_HASHEXPR % ( skytools.quote_fqident(hashfunc), skytools.quote_ident(self.key)) self.hashexpr = args.get('hashexpr', self.hashexpr)
def __init__(self, table_name, args, dest_table): TableHandler.__init__(self, table_name, args, dest_table) self.hash_mask = None # aka max part number (atm) self.shard_nr = None # part number of local node # primary key columns self.hash_key = args.get('hash_key', args.get('key')) self._validate_hash_key() # hash function & full expression hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC) self.hashexpr = self.DEFAULT_HASHEXPR % ( skytools.quote_fqident(hashfunc), skytools.quote_ident(self.hash_key or '')) self.hashexpr = args.get('hashexpr', self.hashexpr)
def __init__(self, table_name, args, dest_table): TableHandler.__init__(self, table_name, args, dest_table) self.max_part = None # max part number self.local_part = None # part number of local node # primary key columns self.key = args.get('key') if self.key is None: raise Exception('Specify key field as key argument') # hash function & full expression hashfunc = args.get('hashfunc', self.DEFAULT_HASHFUNC) self.hashexpr = self.DEFAULT_HASHEXPR % ( skytools.quote_fqident(hashfunc), skytools.quote_ident(self.key)) self.hashexpr = args.get('hashexpr', self.hashexpr)
def get_copy_condition(self, src_curs, dst_curs): """Prepare the where condition for copy and replay filtering""" if self.hash_key is None: return TableHandler.get_copy_condition(self, src_curs, dst_curs) self.load_shard_info(dst_curs) w = "(%s & %d) = %d" % (self.hashexpr, self.hash_mask, self.shard_nr) self.log.debug('shard: copy_condition=%r', w) return w
def real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list): """Copy only slots needed locally.""" self.load_part_info(dst_curs) fn = 'hashtext(%s)' % skytools.quote_ident(self.key) w = "%s & %d = %d" % (fn, self.max_part, self.local_part) self.log.debug('part: copy_condition=%s' % w) cond_list.append(w) return TableHandler.real_copy(self, tablename, src_curs, dst_curs, column_list, cond_list)
def prepare_batch(self, batch_info, dst_curs): """Called on first event for this table in current batch.""" if not self.max_part: self.load_part_info(dst_curs) TableHandler.prepare_batch(self, batch_info, dst_curs)
def _process_event(self, ev, sql_queue_func, arg): self.log.debug('shard.process_event: my event, processing') TableHandler.process_event(self, ev, sql_queue_func, arg)
def prepare_batch(self, batch_info, dst_curs): """Called on first event for this table in current batch.""" if self.hash_key is not None: if not self.hash_mask: self.load_shard_info(dst_curs) TableHandler.prepare_batch(self, batch_info, dst_curs)
def add(self, trigger_arg_list): """Let trigger put hash into extra3""" arg = "ev_extra3='hash='||%s" % self.hashexpr trigger_arg_list.append(arg) TableHandler.add(self, trigger_arg_list)
def reset(self): """Forget config info.""" self.hash_mask = None self.shard_nr = None TableHandler.reset(self)
def reset(self): """Forget config info.""" self.max_part = None self.local_part = None TableHandler.reset(self)
def add(self, trigger_arg_list): """Let trigger put hash into extra3""" arg = "ev_extra3='hash='||hashtext(%s)" % skytools.quote_ident(self.key) trigger_arg_list.append(arg) TableHandler.add(self, trigger_arg_list)