def insert_data(table_name, row_format): ''' insert test data ''' model_parser = ModelParser() table = connection.table(table_name) cursor = MONGO_INSTANCE[table_name].find().limit(10) for cur_item in cursor: table.put( row_format % cur_item, model_parser.deserialized( table_name, cur_item, ) )
def insert_data(table_name, row_format, batch_size=100): ''' insert test data ''' if table_name == 'followbrand_flwrs': table_name = 'followers' table = connection.table(table_name) # batch mode batch = table.batch(batch_size=batch_size) model_parser = ModelParser() if table_name in [ 'followers', 'follow_relations', 'followbrand_flwrs', 'followbrand_flwr_relations', ]: db = MONGO_EXT_INSTANCE else: db = MONGO_INSTANCE cursor = db[table_name].find(timeout=False).limit(10) length = float(cursor.count()) i = 0 for cur_item in cursor: if table_name == 'follow_relations': try: tmp_flwr = db.followers.find_one({'_id': cur_item.get('follower_id')}) cur_item.update(tmp_flwr) except: continue elif table_name == 'followbrand_flwr_relations': try: tmp_flwr = db.followbrand_flwrs.find_one({'_id': cur_item.get('follower_id')}) cur_item.update(tmp_flwr) except: continue ''' table.put( row_format % cur_item, model_parser.deserialized(table_name, cur_item), ) ''' while 1: if add_batch(batch, row_format % cur_item, model_parser.deserialized(table_name, cur_item)): print row_format % cur_item break else: time.sleep(5) i+=1 if i % 100 == 0: while 1: if send_batch(batch): break else: time.sleep(5) print table_name, "=========>", "%0.4f" % (i/length, ) else: while 1: print 1 if send_batch(batch): break else: time.sleep(5)