예제 #1
0
def remoteWorker(queueName,worker,WorkFile):

    connection_ob = boto.connect_sqs(AWS_KEY,AWS_ACCESS_KEY)
    connection = boto.connect_dynamodb(AWS_KEY,AWS_ACCESS_KEY)
    myschema=connection.create_schema(hash_key_name='task_id',hash_key_proto_value='S')
    queue = connection_ob.create_queue(queueName)
    try:
        print "Creating Table.."
        table=connection.create_table(name='task_table', schema=myschema, read_units=100, write_units=100)
        print "Table Created Successfully...."
    except:
        print "Table already exist"

    msg = RawMessage()
    f = open(WorkFile)

    for line in iter(f):
        rand = random.randrange(0,9999)
        msg.set_body(line)
        msg.message_attributes = {"Values": {
                                            "data_type":"String",
                                             "string_value":str(rand)
                                             }
                                        }
        queue.write(msg)
    f.close()
    print "Data Inserted into Queue"
예제 #2
0
def insert_to_queue(queue, message_body, message_attributes):

    message = RawMessage()
    message.set_body(message_body)
    message.message_attributes = message_attributes

    queue.write(message)

    return None
예제 #3
0
파일: updater.py 프로젝트: msirull/skynet
 def queue(self, msg, headers):
     msg_src = "Needs to forward header info"
     m = RawMessage()
     m.message_attributes = {
                         "instance-id":{"data_type": "String", "string_value": iid},
                         "message-source":{"data_type": "String", "string_value": msg_src}}
     m.set_body(msg)
     q.write(m)
     logging.info("Waiting...")
     while True:
         time.sleep(5)
         count=q.count()
         logging.info(count)
         firstiid=""
         cmsg=""
         if count > 10:
             num=10
         else:
             num=count
         if count != 0:
             rs = q.get_messages(num_messages=num, attributes='All', message_attributes=['instance-id'])
             oldest_date = 99999999999999999
             for n in range(num):
                 timestamp=int(rs[n].attributes['SentTimestamp'])
                 logging.info(timestamp)
                 miid=rs[n].message_attributes['instance-id']['string_value']
                 logging.info(miid)
                 ## Checks to see who is first
                 if timestamp < oldest_date:
                     firstiid=miid
                     cmsg=rs[n].get_body()
                     oldest_date = timestamp
                     logging.info(oldest_date)
                     global msgid
                     msgid=rs[n]
         ## If first, start updating
             if firstiid == iid and cmsg == msg:
                 logging.info(firstiid)
                 logging.info(msgid)
                 logging.info("I'm going to start updating now because it's my turn")
                 logging.info("And here's what I'm going to do: %s", cmsg)
                 return "ready"
             else:
                 msgcount=str(count)
                 logging.info("I'm in the queue! My message was %s and so are %s other people", msg, msgcount)
예제 #4
0
        dynamodb = createDynamoDB()

        rand = random.sample(range(1, 18000), len(tasks.readlines()))													# Generating Unique numbers.. 
        #print rand
        tasks.seek(0)
        start = time()
        for i, data in enumerate(tasks.readlines()):
            m = RawMessage()
            print data
            m.set_body(data)																							# Set the message in queue as a body. 
            # q.write(m)

            m.message_attributes = {
                "Task": {
                    "data_type": "String",
                    "string_value": str(rand[i])
                }

            }
            taskQueue.write(m)																							# to write the message in queue

        print 'All tasks have been sent to SQS successfully.\n'

        tasks.seek(0)
        resQueue = conn.get_queue("resQueue")																			# Result queue to get data from Worker. 		

        #tasks.seek(0)
        i = 0
        rs = resQueue.get_messages()
        #print len(rs)
        #print len(tasks.readlines())
예제 #5
0
            #print taskId
            taskContent = m.get_body()																				# To get the message body

            if myTable.has_item(hash_key=taskId):																	# First Check the task id and if available then discard from queue. 
                q.delete_message(m)

            else:
                print 'Storing task {} into DynamoDB...' .format(taskId)
                item_data={'taskContent':taskContent}
                item = myTable.new_item(hash_key=taskId, attrs=item_data)											# To add new item in table. 
                item.put()
                # Execute task
                print 'Executing task {} ...' .format(taskId)
                arg1 = float(str(arg[1])) / 1000
                arg = arg[0] + " " + str(arg1)
                print arg
                os.system(arg)

                dm = RawMessage()
                dm.set_body(m.message_attributes['Task']['string_value'])											# to set the value in message body. 
                dm.message_attributes = {
                    "Result": {
                        "data_type": "Number",
                        "string_value": 0
                    }
                }
                rq.write(dm)
                print 'Sending result ({}) to resultQueue...\n\n' .format(arg)
                q.delete_message(m)

        dynamodb = createDynamoDB()

        rand = random.sample(range(1, 18000), len(tasks.readlines()))													# Generating Unique numbers.. 
        #print rand
        tasks.seek(0)
        start = time()
        for i, data in enumerate(tasks.readlines()):
            m = RawMessage()
            print data
            m.set_body(data)																							# Set the message in queue as a body. 
            # q.write(m)

            m.message_attributes = {
                "Task": {
                    "data_type": "String",
                    "string_value": str(rand[i])
                }

            }
            taskQueue.write(m)																							# to write the message in queue

        print 'All tasks have been sent to SQS successfully.\n'

        tasks.seek(0)
        resQueue = conn.get_queue("resQueue")																			# Result queue to get data from Worker. 		

        #tasks.seek(0)
        i = 0
        rs = resQueue.get_messages()
        #print len(rs)
        #print len(tasks.readlines())