def insert_data(table_name, row_format):
''' insert test data '''
model_parser = ModelParser()
table = connection.table(table_name)
cursor = MONGO_INSTANCE[table_name].find().limit(10)
for cur_item in cursor:
table.put(
row_format % cur_item,
model_parser.deserialized(
table_name,
cur_item,
)
)
def insert_data(table_name, row_format, batch_size=100):
''' insert test data '''
if table_name == 'followbrand_flwrs':
table_name = 'followers'
table = connection.table(table_name)
# batch mode
batch = table.batch(batch_size=batch_size)
model_parser = ModelParser()
if table_name in [
'followers',
'follow_relations',
'followbrand_flwrs',
'followbrand_flwr_relations',
]:
db = MONGO_EXT_INSTANCE
else:
db = MONGO_INSTANCE
cursor = db[table_name].find(timeout=False).limit(10)
length = float(cursor.count())
i = 0
for cur_item in cursor:
if table_name == 'follow_relations':
try:
tmp_flwr = db.followers.find_one({'_id': cur_item.get('follower_id')})
cur_item.update(tmp_flwr)
except:
continue
elif table_name == 'followbrand_flwr_relations':
try:
tmp_flwr = db.followbrand_flwrs.find_one({'_id': cur_item.get('follower_id')})
cur_item.update(tmp_flwr)
except:
continue
'''
table.put(
row_format % cur_item,
model_parser.deserialized(table_name, cur_item),
)
'''
while 1:
if add_batch(batch, row_format % cur_item, model_parser.deserialized(table_name, cur_item)):
print row_format % cur_item
break
else:
time.sleep(5)
i+=1
if i % 100 == 0:
while 1:
if send_batch(batch):
break
else:
time.sleep(5)
print table_name, "=========>", "%0.4f" % (i/length, )
else:
while 1:
print 1
if send_batch(batch):
break
else:
time.sleep(5)
