def insert_data(table_name, row_format): ''' insert test data ''' model_parser = ModelParser() table = connection.table(table_name) cursor = MONGO_INSTANCE[table_name].find().limit(10) for cur_item in cursor: table.put( row_format % cur_item, model_parser.deserialized( table_name, cur_item, ) )
def test_hbase_user(): ''' 测试hbase init user ''' hbase_instance = InitTestData('192.168.122.103') hi = HbaseInit('192.168.122.103') hi.init_user() tmp_record = hbase_instance.insert_test_user() mp = ModelParser() result = mp.parse('followers', tmp_record) print result import ipdb;ipdb.set_trace() print 'done!'
def insert_test_user(self): ''' init a user test module. ''' model_parser = ModelParser() table = self.connection.table('users') users = MONGO_INSTANCE.users.find() for user in users: print user.get('_id'), model_parser.de_parse('users', 'user', user) table.put( str(user.get('_id')), model_parser.de_parse('attrs', 'user', user), )
def insert_test_followers(self): ''' init a test followers ''' model_parser = ModelParser() table = self.connection.table('followers') followers = MONGO_INSTANCE.followers.find() from datetime import datetime a = datetime.now() for cur_follower in followers: print cur_follower.get('_id'), model_parser.de_parse('followers', 'followers', cur_follower) table.put( str(cur_follower.get('_id')), model_parser.de_parse( 'followers', 'followers', cur_follower, ) ) print followers.count() print datetime.now() - a
def insert_test_follow_relations(self): ''' insert a new follow_relations test data. ''' model_parser = ModelParser() table = self.connection.table('follow_relations') follow_relations = MONGO_INSTANCE.follow_relations.find() from datetime import datetime a = datetime.now() for cur_relation in follow_relations: print cur_relation.get('user_id'), model_parser.de_parse('follow_relation', 'follow_relation', cur_relation) table.put( '_'.join([ str(cur_relation.get('user_id')), str(cur_relation.get('follower_id')), ]), model_parser.de_parse( 'follow_relation', 'follow_relation', cur_relation, ) ) print follow_relations.count() print datetime.now() - a
def insert_data(table_name, row_format, batch_size=100): ''' insert test data ''' if table_name == 'followbrand_flwrs': table_name = 'followers' table = connection.table(table_name) # batch mode batch = table.batch(batch_size=batch_size) model_parser = ModelParser() if table_name in [ 'followers', 'follow_relations', 'followbrand_flwrs', 'followbrand_flwr_relations', ]: db = MONGO_EXT_INSTANCE else: db = MONGO_INSTANCE cursor = db[table_name].find(timeout=False).limit(10) length = float(cursor.count()) i = 0 for cur_item in cursor: if table_name == 'follow_relations': try: tmp_flwr = db.followers.find_one({'_id': cur_item.get('follower_id')}) cur_item.update(tmp_flwr) except: continue elif table_name == 'followbrand_flwr_relations': try: tmp_flwr = db.followbrand_flwrs.find_one({'_id': cur_item.get('follower_id')}) cur_item.update(tmp_flwr) except: continue ''' table.put( row_format % cur_item, model_parser.deserialized(table_name, cur_item), ) ''' while 1: if add_batch(batch, row_format % cur_item, model_parser.deserialized(table_name, cur_item)): print row_format % cur_item break else: time.sleep(5) i+=1 if i % 100 == 0: while 1: if send_batch(batch): break else: time.sleep(5) print table_name, "=========>", "%0.4f" % (i/length, ) else: while 1: print 1 if send_batch(batch): break else: time.sleep(5)