def create_table_to_hbase(self, table, families):
try:
with self._hb_pool.connection() as connection:
connection.create_table(table, families)
except Exception, e:
TDDCLogging.error(e)
return False
def put_to_hbase(self, table, row_key, items):
'''
单个存储
params:
items:
EXP: {'familyxxx': {'column': data},
'familyooo': {'column': data}}
'''
try:
with self._hb_pool.connection() as connection:
self._auto_create_table(connection, table, items)
table = connection.table(table)
bool_trans_to_numbre_dict = {True: 1, False: 0}
for family, data in items.items():
cf_fmt = family + ':'
values = {}
for column, value in data.items():
if isinstance(value, dict) or isinstance(value, list):
if type(value) == type(True):
value = bool_trans_to_numbre_dict.get(value)
value = json.dumps(value)
values[cf_fmt + column] = value
table.put(row_key, values)
return True
except Exception, e:
TDDCLogging.error(e)
return False
def start(self, nodes):
self._nodes = nodes
TDDCLogging.info('-->Exception Manager Is Starting.')
self._exception_producer = KafkaHelper.make_producer(self._nodes)
gevent.spawn(self._send)
gevent.sleep()
TDDCLogging.info('-->Exception Manager Was Ready.')
def _dispatch(self):
while True:
event = self._event_queue.get()
callback = self._event_call.get(event.event_type, None)
if callback:
callback(event)
else:
TDDCLogging.warning('Event Exception: %d Not Register.' % event.event_type)
def update_status(self, table, key, new_status):
try:
old_status = self.hget(table, key)
status = old_status.split(',') if old_status else []
status.append('[host:%s | status: %s]' % (socket.gethostname(), new_status))
self.hset(table, key, ','.join(status))
except Exception, e:
TDDCLogging.error(e)
return False
def _auto_create_table(self, connection, table, items=None):
keys = items.keys() if items else ['source', 'valuable', 'task']
for cnt in range(2):
if table not in self._tables:
if cnt == 1:
connection.create_table(table, {k:{} for k in keys})
TDDCLogging.warning('Create New Table(%s) to HBase.' % table)
self._tables = connection.tables()
else:
break
def psubscribe(self, pattern):
'''
匹配订阅
'''
ps = self.pubsub()
ps.psubscribe(pattern)
TDDCLogging.info('--->Pubsub Was Ready.')
for item in ps.listen():
yield item
ps.unsubscribe('spub')
TDDCLogging.info('-->Pubsub Is Exit.')
def _consume_msg_exp(self, exp_type, info, exception=None):
if 'JSON_ERR' in exp_type:
TDDCLogging.error('*'*5+exp_type+'*'*5+
'\nException: '+info+'\n'+
exception.message+'\n'+
'*'*(10+len(exp_type))+'\n')
elif 'TASK_ERR' in exp_type or 'EVENT_ERR' in exp_type:
TDDCLogging.error('*'*5+exp_type+'*'*5+
'\nException: '+
'item={item}\n'.format(item=info)+
'item_type={item_type}\n'.format(item_type=type(info))+
'*'*(10+len(exp_type))+'\n')
def get_from_hbase(self, table, row_key, family=None, qualifier=None):
try:
with self._hb_pool.connection() as connection:
table = connection.table(table)
if family and qualifier:
cf = family + ':' + qualifier
elif family and not qualifier:
cf = family
else:
return False, None
return True, table.row(row_key, columns=[cf])
except Exception, e:
TDDCLogging.error(e)
return False, None
def __init__(self, nodes, push=True, pull=False):
'''
Constructor
'''
self._nodes = nodes
TDDCLogging.info('-->Storager Manager Is Starting.')
self._db = DBManager(self._nodes)
if push:
gevent.spawn(self._push)
gevent.sleep()
if pull:
gevent.spawn(self._pull)
gevent.sleep()
TDDCLogging.info('-->Storager Manager Was Ready.')
def start(self, site):
self._nodes = site.KAFKA_NODES
self._topic = site.EVENT_TOPIC
self._group = site.EVENT_TOPIC_GROUP
self._events_cls = site.EVENT_TABLES
self._cache_manager = EventCacheManager(site)
TDDCLogging.info('-->Event Manager Is Starting.')
self._event_consumer = KafkaHelper.make_consumer(self._nodes,
self._topic,
self._group)
self._event_queue = gevent.queue.Queue()
self._event_call = {}
gevent.spawn(self._recv)
gevent.sleep()
gevent.spawn(self._dispatch)
gevent.sleep()
TDDCLogging.info('-->Event Manager Was Ready.')
def puts_to_hbase(self, table_rows):
'''
批量存储
params:
table_rows:
EXP: {'platformxxx': {'row_key1': {'familyxxx': {'column': data},
{'familyooo': {'column': data}},
{'row_key2': {'familyxxx': {'column': data},
{'familyooo': {'column': data}}}}
'''
try:
with self._hb_pool.connection() as connection:
for table, rows in table_rows.items():
self._auto_create_table(connection, table)
table = connection.table(table)
b = table.batch()
self._puts(b, rows)
b.send()
return True
except Exception, e:
TDDCLogging.error(e)
return False
def _push(self):
cnt = 0
platform_rows = {}
while True:
try:
task, storage_info = PublicQueues.STORAGE.get()
items = {self.FAMILY: storage_info,
'task': {'task': task.to_json()}}
if not platform_rows.get(task.platform):
platform_rows[task.platform] = {}
platform_rows[task.platform][task.row_key] = items
cnt += 1
if PublicQueues.STORAGE.qsize() and not cnt % 5:
gevent.sleep(0.01)
continue
if self._db.puts_to_hbase(platform_rows):
self._pushed(platform_rows, True)
else:
self._pushed(platform_rows, False)
gevent.sleep(1)
platform_rows = {}
except Exception, e:
TDDCLogging.error(e)
def __init__(self, host_port=None):
'''
Constructor
params:
host_port:
EXP: 'localhost:8888'
DES: HBase的IP、PORT
'''
TDDCLogging.info('---->DB Manager Is Starting.')
self._tables = []
host, port = host_port.split(':')
self._hb_pool = happybase.ConnectionPool(size=8,
host=host,
port=int(port),
transport='framed',
protocol='compact')
TDDCLogging.info('----->HBase(%s:%s) Was Ready.' % (host, port))
TDDCLogging.info('---->DB Manager Was Ready.')
def create_record(self, table, key, value):
try:
self.hset(table, key, value)
except Exception, e:
TDDCLogging.error(e)
return False
gevent.sleep(5)
def _event_parse(self, record):
try:
item = json.loads(record.value)
except Exception, e:
self._consume_msg_exp('EVENT_JSON_ERR', record.value, e)
else:
if item and isinstance(item, dict):
event_type = item.get('event_type')
if not event_type:
self._consume_msg_exp('EVENT_ERR', item)
return
cls = self._events_cls.get(event_type)
if not cls:
TDDCLogging.error('Undefine Event Type: %d <%s>' % (event_type,
json.dumps(item)))
return
event = cls(**item)
self._event_queue.put(event)
else:
self._consume_msg_exp('EVENT_ERR', item)
def _dispatch(self):
while True:
event = self._event_queue.get()
callback = self._event_call.get(event.event_type, None)
if callback:
callback(event)
else:
TDDCLogging.warning('Event Exception: %d Not Register.' % event.event_type)
def get_record(self, table, key):
try:
return self.hget(table, key)
except Exception, e:
TDDCLogging.error(e)
return None
