def remoteWorker(queueName,worker,WorkFile):
connection_ob = boto.connect_sqs(AWS_KEY,AWS_ACCESS_KEY)
connection = boto.connect_dynamodb(AWS_KEY,AWS_ACCESS_KEY)
myschema=connection.create_schema(hash_key_name='task_id',hash_key_proto_value='S')
queue = connection_ob.create_queue(queueName)
try:
print "Creating Table.."
table=connection.create_table(name='task_table', schema=myschema, read_units=100, write_units=100)
print "Table Created Successfully...."
except:
print "Table already exist"
msg = RawMessage()
f = open(WorkFile)
for line in iter(f):
rand = random.randrange(0,9999)
msg.set_body(line)
msg.message_attributes = {"Values": {
"data_type":"String",
"string_value":str(rand)
}
}
queue.write(msg)
f.close()
print "Data Inserted into Queue"
def insert_to_queue(queue, message_body, message_attributes):
message = RawMessage()
message.set_body(message_body)
message.message_attributes = message_attributes
queue.write(message)
return None
def queue(self, msg, headers):
msg_src = "Needs to forward header info"
m = RawMessage()
m.message_attributes = {
"instance-id":{"data_type": "String", "string_value": iid},
"message-source":{"data_type": "String", "string_value": msg_src}}
m.set_body(msg)
q.write(m)
logging.info("Waiting...")
while True:
time.sleep(5)
count=q.count()
logging.info(count)
firstiid=""
cmsg=""
if count > 10:
num=10
else:
num=count
if count != 0:
rs = q.get_messages(num_messages=num, attributes='All', message_attributes=['instance-id'])
oldest_date = 99999999999999999
for n in range(num):
timestamp=int(rs[n].attributes['SentTimestamp'])
logging.info(timestamp)
miid=rs[n].message_attributes['instance-id']['string_value']
logging.info(miid)
## Checks to see who is first
if timestamp < oldest_date:
firstiid=miid
cmsg=rs[n].get_body()
oldest_date = timestamp
logging.info(oldest_date)
global msgid
msgid=rs[n]
## If first, start updating
if firstiid == iid and cmsg == msg:
logging.info(firstiid)
logging.info(msgid)
logging.info("I'm going to start updating now because it's my turn")
logging.info("And here's what I'm going to do: %s", cmsg)
return "ready"
else:
msgcount=str(count)
logging.info("I'm in the queue! My message was %s and so are %s other people", msg, msgcount)
dynamodb = createDynamoDB()
rand = random.sample(range(1, 18000), len(tasks.readlines())) # Generating Unique numbers..
#print rand
tasks.seek(0)
start = time()
for i, data in enumerate(tasks.readlines()):
m = RawMessage()
print data
m.set_body(data) # Set the message in queue as a body.
# q.write(m)
m.message_attributes = {
"Task": {
"data_type": "String",
"string_value": str(rand[i])
}
}
taskQueue.write(m) # to write the message in queue
print 'All tasks have been sent to SQS successfully.\n'
tasks.seek(0)
resQueue = conn.get_queue("resQueue") # Result queue to get data from Worker.
#tasks.seek(0)
i = 0
rs = resQueue.get_messages()
#print len(rs)
#print len(tasks.readlines())
#print taskId
taskContent = m.get_body() # To get the message body
if myTable.has_item(hash_key=taskId): # First Check the task id and if available then discard from queue.
q.delete_message(m)
else:
print 'Storing task {} into DynamoDB...' .format(taskId)
item_data={'taskContent':taskContent}
item = myTable.new_item(hash_key=taskId, attrs=item_data) # To add new item in table.
item.put()
# Execute task
print 'Executing task {} ...' .format(taskId)
arg1 = float(str(arg[1])) / 1000
arg = arg[0] + " " + str(arg1)
print arg
os.system(arg)
dm = RawMessage()
dm.set_body(m.message_attributes['Task']['string_value']) # to set the value in message body.
dm.message_attributes = {
"Result": {
"data_type": "Number",
"string_value": 0
}
}
rq.write(dm)
print 'Sending result ({}) to resultQueue...\n\n' .format(arg)
q.delete_message(m)
dynamodb = createDynamoDB()
rand = random.sample(range(1, 18000), len(tasks.readlines())) # Generating Unique numbers..
#print rand
tasks.seek(0)
start = time()
for i, data in enumerate(tasks.readlines()):
m = RawMessage()
print data
m.set_body(data) # Set the message in queue as a body.
# q.write(m)
m.message_attributes = {
"Task": {
"data_type": "String",
"string_value": str(rand[i])
}
}
taskQueue.write(m) # to write the message in queue
print 'All tasks have been sent to SQS successfully.\n'
tasks.seek(0)
resQueue = conn.get_queue("resQueue") # Result queue to get data from Worker.
#tasks.seek(0)
i = 0
rs = resQueue.get_messages()
#print len(rs)
#print len(tasks.readlines())