def insert_data(table_name, row_format):
''' insert test data '''
model_parser = ModelParser()
table = connection.table(table_name)
cursor = MONGO_INSTANCE[table_name].find().limit(10)
for cur_item in cursor:
table.put(
row_format % cur_item,
model_parser.deserialized(
table_name,
cur_item,
)
)
def test_hbase_user():
'''
测试hbase init user
'''
hbase_instance = InitTestData('192.168.122.103')
hi = HbaseInit('192.168.122.103')
hi.init_user()
tmp_record = hbase_instance.insert_test_user()
mp = ModelParser()
result = mp.parse('followers', tmp_record)
print result
import ipdb;ipdb.set_trace()
print 'done!'
def insert_test_user(self):
'''
init a user test module.
'''
model_parser = ModelParser()
table = self.connection.table('users')
users = MONGO_INSTANCE.users.find()
for user in users:
print user.get('_id'), model_parser.de_parse('users', 'user', user)
table.put(
str(user.get('_id')),
model_parser.de_parse('attrs', 'user', user),
)
def insert_test_followers(self):
'''
init a test followers
'''
model_parser = ModelParser()
table = self.connection.table('followers')
followers = MONGO_INSTANCE.followers.find()
from datetime import datetime
a = datetime.now()
for cur_follower in followers:
print cur_follower.get('_id'), model_parser.de_parse('followers', 'followers', cur_follower)
table.put(
str(cur_follower.get('_id')),
model_parser.de_parse(
'followers',
'followers',
cur_follower,
)
)
print followers.count()
print datetime.now() - a
def insert_test_follow_relations(self):
'''
insert a new follow_relations test data.
'''
model_parser = ModelParser()
table = self.connection.table('follow_relations')
follow_relations = MONGO_INSTANCE.follow_relations.find()
from datetime import datetime
a = datetime.now()
for cur_relation in follow_relations:
print cur_relation.get('user_id'), model_parser.de_parse('follow_relation', 'follow_relation', cur_relation)
table.put(
'_'.join([
str(cur_relation.get('user_id')),
str(cur_relation.get('follower_id')),
]),
model_parser.de_parse(
'follow_relation',
'follow_relation',
cur_relation,
)
)
print follow_relations.count()
print datetime.now() - a
def insert_data(table_name, row_format, batch_size=100):
''' insert test data '''
if table_name == 'followbrand_flwrs':
table_name = 'followers'
table = connection.table(table_name)
# batch mode
batch = table.batch(batch_size=batch_size)
model_parser = ModelParser()
if table_name in [
'followers',
'follow_relations',
'followbrand_flwrs',
'followbrand_flwr_relations',
]:
db = MONGO_EXT_INSTANCE
else:
db = MONGO_INSTANCE
cursor = db[table_name].find(timeout=False).limit(10)
length = float(cursor.count())
i = 0
for cur_item in cursor:
if table_name == 'follow_relations':
try:
tmp_flwr = db.followers.find_one({'_id': cur_item.get('follower_id')})
cur_item.update(tmp_flwr)
except:
continue
elif table_name == 'followbrand_flwr_relations':
try:
tmp_flwr = db.followbrand_flwrs.find_one({'_id': cur_item.get('follower_id')})
cur_item.update(tmp_flwr)
except:
continue
'''
table.put(
row_format % cur_item,
model_parser.deserialized(table_name, cur_item),
)
'''
while 1:
if add_batch(batch, row_format % cur_item, model_parser.deserialized(table_name, cur_item)):
print row_format % cur_item
break
else:
time.sleep(5)
i+=1
if i % 100 == 0:
while 1:
if send_batch(batch):
break
else:
time.sleep(5)
print table_name, "=========>", "%0.4f" % (i/length, )
else:
while 1:
print 1
if send_batch(batch):
break
else:
time.sleep(5)
