def train(numIters):
global users
checkUserPartitionMapping()
kafka = KafkaClient(kafkaHost, timeout=None)
producer = UserProducer(kafka, kafkaTopic, users, partitions, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
for i in range(numIters):
for user, partitionId in users.iteritems():
if user == '' or user == 'monk':
continue
encodedMessage = simplejson.dumps({'turtleName':turtleName,
'user':user,
'operation':'train'})
print i, producer.send(user, encodedMessage)
producer.stop(1)
kafka.close()
def set_mantis_parameter(para, value):
global users
checkUserPartitionMapping()
kafka = KafkaClient(kafkaHost, timeout=None)
producer = UserProducer(kafka, kafkaTopic, users, partitions, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
for user, partitionId in users.iteritems():
# if not partitionId == 4:
# continue
encodedMessage = simplejson.dumps({'turtleName':turtleName,
'user':user,
'operation':'set_mantis_parameter',
'para':para,
'value':value})
print producer.send(user, encodedMessage)
producer.stop(1)
kafka.close()
def reset_all_data():
global users
checkUserPartitionMapping()
kafka = KafkaClient(kafkaHost, timeout=None)
producer = UserProducer(kafka, kafkaTopic, users, partitions, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
for user, partitionId in users.iteritems():
encodedMessage = simplejson.dumps({'turtleName':turtleName,
'user':user,
'operation':'reset_all_data'})
print producer.send(user, encodedMessage)
users['monk'] = 8
encodedMessage = simplejson.dumps({'turtleName':turtleName,
'user':'******',
'operation':'reset_all_data'})
print producer.send('monk', encodedMessage)
producer.stop(1)
kafka.close()
def train(numIters):
global users
try:
mcl = pm.MongoClient('10.137.168.196:27017')
userColl = mcl.DataSet['PMLUsers']
users = {user['userId']:user['partitionId'] for user in userColl.find()}
mcl.close()
kafka = KafkaClient('mozo.cloudapp.net:9092', timeout=None)
producer = UserProducer(kafka, kafkaTopic, users, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
for userId, partitionId in users.iteritems():
if userId in UoI.keys():
for i in range(numIters):
#print "iteration " + str(i)
encodedMessage = simplejson.dumps({'turtleId':turtleId,
'userId':userId,
'operation':'train_one'})
print i, producer.send(userId, encodedMessage)
finally:
producer.stop()
kafka.close()
def add_data():
global users
try:
mcl = pm.MongoClient('10.137.168.196:27017')
kafka = KafkaClient('mozo.cloudapp.net:9092', timeout=None)
producer = UserProducer(kafka, kafkaTopic, users, parts, async=False,
req_acks=UserProducer.ACK_AFTER_LOCAL_WRITE,
ack_timeout=200)
coll = mcl.DataSet['PMLExpression']
ii = 0 # max is 151413 (number of doc in PMLExpression)
for ent in coll.find({'userId': {'$in': UoI.keys()}}, {'_id':True, 'userId':True}, timeout=False):
ii += 1
entity = str(ent['_id'])
userId = ent['userId']
if (stop_add_data(userId)):
continue
UoI[userId] += 1
encodedMessage = simplejson.dumps({'turtleId':turtleId,
'userId':userId,
'entity':entity,
'operation':'add_data'})
print producer.send(userId, encodedMessage)
for userId, partitionId in users.iteritems():
encodedMessage = simplejson.dumps({'turtleId':turtleId,
'userId':userId,
'operation':'save_one'})
print producer.send(userId, encodedMessage)
userColl = mcl.DataSet['PMLUsers']
if users:
userColl.insert([{'userId':userId, 'partitionId':partitionId} for userId, partitionId in users.iteritems()])
finally:
producer.stop()
mcl.close()
kafka.close()
