def main(filename): dirname = os.path.dirname(filename) parser = YBinlogP(filename, always_update=True) while True: for i, event in enumerate(parser): print event if event.event_type == "ROTATE_EVENT": next_file = os.path.join(dirname, event.data.file_name) parser.close() parser = YBinlogP(next_file, always_update=True) else: print "Got to end at %r" % (parser.tell(),) break parser.close()
def main(filename): dirname = os.path.dirname(filename) parser = YBinlogP(filename, always_update=True) while True: for i, event in enumerate(parser): print event if event.event_type == "ROTATE_EVENT": next_file = os.path.join(dirname, event.data.file_name) parser.close() parser = YBinlogP(next_file, always_update=True) else: print "Got to end at %r" % (parser.tell(), ) break parser.close()
from ybinlogp import YBinlogP import sys bp = YBinlogP(sys.argv[1], always_update=True) while True: for i, event in enumerate(bp): if event.event_type == "ROTATE_EVENT": next_file = event.data.file_name bp.clean_up() bp = YBinlogP(next_file, always_update=True) break else: print event.event_type else: print "Got to end at %r" % (bp.tell(), ) break bp.clean_up()
name = "%s:%s" %(day, itemid) redis_value = "%s:%s" %(clock, value) print name, redis_value #if not redis_conn.exists(name): # redis_conn.lpush(name, redis_value) # redis_conn.expire(name, EXPIRE) #else: # redis_conn.lpush(name, redis_value) timer_now = time.time() if timer_now - timer >= 30: flush_cache_to_redis() buffer_dict = defaultdict(list) timer = timer_now else: buffer_dict[name].append(redis_value) flush_cache_to_redis() print "Got to end at %r" % (bp.tell(),) time.sleep(1) if next_file_exists(get_next_filename(g_current_file)): print "check next file g_current_file", g_current_file next_file = get_next_filename(g_current_file) g_current_file = next_file bp.clean_up() bp = YBinlogP(next_file, always_update=True) bp.clean_up()