def create_table_to_hbase(self, table, families):
try:
with self._hb_pool.connection() as connection:
connection.create_table(table, families)
except Exception, e:
TDDCLogging.error(e)
return False
def put_to_hbase(self, table, row_key, items):
'''
单个存储
params:
items:
EXP: {'familyxxx': {'column': data},
'familyooo': {'column': data}}
'''
try:
with self._hb_pool.connection() as connection:
self._auto_create_table(connection, table, items)
table = connection.table(table)
bool_trans_to_numbre_dict = {True: 1, False: 0}
for family, data in items.items():
cf_fmt = family + ':'
values = {}
for column, value in data.items():
if isinstance(value, dict) or isinstance(value, list):
if type(value) == type(True):
value = bool_trans_to_numbre_dict.get(value)
value = json.dumps(value)
values[cf_fmt + column] = value
table.put(row_key, values)
return True
except Exception, e:
TDDCLogging.error(e)
return False
def update_status(self, table, key, new_status):
try:
old_status = self.hget(table, key)
status = old_status.split(',') if old_status else []
status.append('[host:%s | status: %s]' % (socket.gethostname(), new_status))
self.hset(table, key, ','.join(status))
except Exception, e:
TDDCLogging.error(e)
return False
def _consume_msg_exp(self, exp_type, info, exception=None):
if 'JSON_ERR' in exp_type:
TDDCLogging.error('*'*5+exp_type+'*'*5+
'\nException: '+info+'\n'+
exception.message+'\n'+
'*'*(10+len(exp_type))+'\n')
elif 'TASK_ERR' in exp_type or 'EVENT_ERR' in exp_type:
TDDCLogging.error('*'*5+exp_type+'*'*5+
'\nException: '+
'item={item}\n'.format(item=info)+
'item_type={item_type}\n'.format(item_type=type(info))+
'*'*(10+len(exp_type))+'\n')
def get_from_hbase(self, table, row_key, family=None, qualifier=None):
try:
with self._hb_pool.connection() as connection:
table = connection.table(table)
if family and qualifier:
cf = family + ':' + qualifier
elif family and not qualifier:
cf = family
else:
return False, None
return True, table.row(row_key, columns=[cf])
except Exception, e:
TDDCLogging.error(e)
return False, None
def puts_to_hbase(self, table_rows):
'''
批量存储
params:
table_rows:
EXP: {'platformxxx': {'row_key1': {'familyxxx': {'column': data},
{'familyooo': {'column': data}},
{'row_key2': {'familyxxx': {'column': data},
{'familyooo': {'column': data}}}}
'''
try:
with self._hb_pool.connection() as connection:
for table, rows in table_rows.items():
self._auto_create_table(connection, table)
table = connection.table(table)
b = table.batch()
self._puts(b, rows)
b.send()
return True
except Exception, e:
TDDCLogging.error(e)
return False
def _push(self):
cnt = 0
platform_rows = {}
while True:
try:
task, storage_info = PublicQueues.STORAGE.get()
items = {self.FAMILY: storage_info,
'task': {'task': task.to_json()}}
if not platform_rows.get(task.platform):
platform_rows[task.platform] = {}
platform_rows[task.platform][task.row_key] = items
cnt += 1
if PublicQueues.STORAGE.qsize() and not cnt % 5:
gevent.sleep(0.01)
continue
if self._db.puts_to_hbase(platform_rows):
self._pushed(platform_rows, True)
else:
self._pushed(platform_rows, False)
gevent.sleep(1)
platform_rows = {}
except Exception, e:
TDDCLogging.error(e)
def get_record(self, table, key):
try:
return self.hget(table, key)
except Exception, e:
TDDCLogging.error(e)
return None
def create_record(self, table, key, value):
try:
self.hset(table, key, value)
except Exception, e:
TDDCLogging.error(e)
return False
gevent.sleep(5)
def _event_parse(self, record):
try:
item = json.loads(record.value)
except Exception, e:
self._consume_msg_exp('EVENT_JSON_ERR', record.value, e)
else:
if item and isinstance(item, dict):
event_type = item.get('event_type')
if not event_type:
self._consume_msg_exp('EVENT_ERR', item)
return
cls = self._events_cls.get(event_type)
if not cls:
TDDCLogging.error('Undefine Event Type: %d <%s>' % (event_type,
json.dumps(item)))
return
event = cls(**item)
self._event_queue.put(event)
else:
self._consume_msg_exp('EVENT_ERR', item)
def _dispatch(self):
while True:
event = self._event_queue.get()
callback = self._event_call.get(event.event_type, None)
if callback:
callback(event)
else:
TDDCLogging.warning('Event Exception: %d Not Register.' % event.event_type)