def create_table_to_hbase(self, table, families): try: with self._hb_pool.connection() as connection: connection.create_table(table, families) except Exception, e: TDDCLogging.error(e) return False
def put_to_hbase(self, table, row_key, items): ''' 单个存储 params: items: EXP: {'familyxxx': {'column': data}, 'familyooo': {'column': data}} ''' try: with self._hb_pool.connection() as connection: self._auto_create_table(connection, table, items) table = connection.table(table) bool_trans_to_numbre_dict = {True: 1, False: 0} for family, data in items.items(): cf_fmt = family + ':' values = {} for column, value in data.items(): if isinstance(value, dict) or isinstance(value, list): if type(value) == type(True): value = bool_trans_to_numbre_dict.get(value) value = json.dumps(value) values[cf_fmt + column] = value table.put(row_key, values) return True except Exception, e: TDDCLogging.error(e) return False
def start(self, nodes): self._nodes = nodes TDDCLogging.info('-->Exception Manager Is Starting.') self._exception_producer = KafkaHelper.make_producer(self._nodes) gevent.spawn(self._send) gevent.sleep() TDDCLogging.info('-->Exception Manager Was Ready.')
def _dispatch(self): while True: event = self._event_queue.get() callback = self._event_call.get(event.event_type, None) if callback: callback(event) else: TDDCLogging.warning('Event Exception: %d Not Register.' % event.event_type)
def update_status(self, table, key, new_status): try: old_status = self.hget(table, key) status = old_status.split(',') if old_status else [] status.append('[host:%s | status: %s]' % (socket.gethostname(), new_status)) self.hset(table, key, ','.join(status)) except Exception, e: TDDCLogging.error(e) return False
def _auto_create_table(self, connection, table, items=None): keys = items.keys() if items else ['source', 'valuable', 'task'] for cnt in range(2): if table not in self._tables: if cnt == 1: connection.create_table(table, {k:{} for k in keys}) TDDCLogging.warning('Create New Table(%s) to HBase.' % table) self._tables = connection.tables() else: break
def psubscribe(self, pattern): ''' 匹配订阅 ''' ps = self.pubsub() ps.psubscribe(pattern) TDDCLogging.info('--->Pubsub Was Ready.') for item in ps.listen(): yield item ps.unsubscribe('spub') TDDCLogging.info('-->Pubsub Is Exit.')
def _consume_msg_exp(self, exp_type, info, exception=None): if 'JSON_ERR' in exp_type: TDDCLogging.error('*'*5+exp_type+'*'*5+ '\nException: '+info+'\n'+ exception.message+'\n'+ '*'*(10+len(exp_type))+'\n') elif 'TASK_ERR' in exp_type or 'EVENT_ERR' in exp_type: TDDCLogging.error('*'*5+exp_type+'*'*5+ '\nException: '+ 'item={item}\n'.format(item=info)+ 'item_type={item_type}\n'.format(item_type=type(info))+ '*'*(10+len(exp_type))+'\n')
def get_from_hbase(self, table, row_key, family=None, qualifier=None): try: with self._hb_pool.connection() as connection: table = connection.table(table) if family and qualifier: cf = family + ':' + qualifier elif family and not qualifier: cf = family else: return False, None return True, table.row(row_key, columns=[cf]) except Exception, e: TDDCLogging.error(e) return False, None
def __init__(self, nodes, push=True, pull=False): ''' Constructor ''' self._nodes = nodes TDDCLogging.info('-->Storager Manager Is Starting.') self._db = DBManager(self._nodes) if push: gevent.spawn(self._push) gevent.sleep() if pull: gevent.spawn(self._pull) gevent.sleep() TDDCLogging.info('-->Storager Manager Was Ready.')
def start(self, site): self._nodes = site.KAFKA_NODES self._topic = site.EVENT_TOPIC self._group = site.EVENT_TOPIC_GROUP self._events_cls = site.EVENT_TABLES self._cache_manager = EventCacheManager(site) TDDCLogging.info('-->Event Manager Is Starting.') self._event_consumer = KafkaHelper.make_consumer(self._nodes, self._topic, self._group) self._event_queue = gevent.queue.Queue() self._event_call = {} gevent.spawn(self._recv) gevent.sleep() gevent.spawn(self._dispatch) gevent.sleep() TDDCLogging.info('-->Event Manager Was Ready.')
def puts_to_hbase(self, table_rows): ''' 批量存储 params: table_rows: EXP: {'platformxxx': {'row_key1': {'familyxxx': {'column': data}, {'familyooo': {'column': data}}, {'row_key2': {'familyxxx': {'column': data}, {'familyooo': {'column': data}}}} ''' try: with self._hb_pool.connection() as connection: for table, rows in table_rows.items(): self._auto_create_table(connection, table) table = connection.table(table) b = table.batch() self._puts(b, rows) b.send() return True except Exception, e: TDDCLogging.error(e) return False
def _push(self): cnt = 0 platform_rows = {} while True: try: task, storage_info = PublicQueues.STORAGE.get() items = {self.FAMILY: storage_info, 'task': {'task': task.to_json()}} if not platform_rows.get(task.platform): platform_rows[task.platform] = {} platform_rows[task.platform][task.row_key] = items cnt += 1 if PublicQueues.STORAGE.qsize() and not cnt % 5: gevent.sleep(0.01) continue if self._db.puts_to_hbase(platform_rows): self._pushed(platform_rows, True) else: self._pushed(platform_rows, False) gevent.sleep(1) platform_rows = {} except Exception, e: TDDCLogging.error(e)
def __init__(self, host_port=None): ''' Constructor params: host_port: EXP: 'localhost:8888' DES: HBase的IP、PORT ''' TDDCLogging.info('---->DB Manager Is Starting.') self._tables = [] host, port = host_port.split(':') self._hb_pool = happybase.ConnectionPool(size=8, host=host, port=int(port), transport='framed', protocol='compact') TDDCLogging.info('----->HBase(%s:%s) Was Ready.' % (host, port)) TDDCLogging.info('---->DB Manager Was Ready.')
def create_record(self, table, key, value): try: self.hset(table, key, value) except Exception, e: TDDCLogging.error(e) return False
gevent.sleep(5) def _event_parse(self, record): try: item = json.loads(record.value) except Exception, e: self._consume_msg_exp('EVENT_JSON_ERR', record.value, e) else: if item and isinstance(item, dict): event_type = item.get('event_type') if not event_type: self._consume_msg_exp('EVENT_ERR', item) return cls = self._events_cls.get(event_type) if not cls: TDDCLogging.error('Undefine Event Type: %d <%s>' % (event_type, json.dumps(item))) return event = cls(**item) self._event_queue.put(event) else: self._consume_msg_exp('EVENT_ERR', item) def _dispatch(self): while True: event = self._event_queue.get() callback = self._event_call.get(event.event_type, None) if callback: callback(event) else: TDDCLogging.warning('Event Exception: %d Not Register.' % event.event_type)
def get_record(self, table, key): try: return self.hget(table, key) except Exception, e: TDDCLogging.error(e) return None