def test_worker_fills_internal_queue_from_celery_task(): """ Test read workers fill internal queue with celery tasks """ conn = boto.connect_sqs() queue = conn.create_queue("tester") message = Message() body = '{"body": "KGRwMApTJ3Rhc2snCnAxClMndGVzdHMudGFza3MuaW5kZXhfaW5jcmVtZW50ZXInCnAyCnNTJ2Fy\\nZ3MnCnAzCihscDQKc1Mna3dhcmdzJwpwNQooZHA2ClMnbWVzc2FnZScKcDcKUydUZXN0IG1lc3Nh\\nZ2UyJwpwOApzcy4=\\n", "some stuff": "asdfasf"}' message.set_body(body) queue.write(message) internal_queue = Queue() worker = ReadWorker(queue, internal_queue) worker.read_message() packed_message = internal_queue.get(timeout=1) found_message_body = decode_message(packed_message['message']) found_message_body.should.equal({ 'task': 'tests.tasks.index_incrementer', 'args': [], 'kwargs': { 'message': 'Test message2', }, })
def test_worker_fills_internal_queue(): """ Test read workers fill internal queue """ conn = boto.connect_sqs() queue = conn.create_queue("tester") message = Message() body = json.dumps({ 'task': 'tests.tasks.index_incrementer', 'args': [], 'kwargs': { 'message': 'Test message', }, }) message.set_body(body) queue.write(message) internal_queue = Queue() worker = ReadWorker(queue, internal_queue) worker.read_message() packed_message = internal_queue.get(timeout=1) found_message_body = decode_message(packed_message['message']) found_message_body.should.equal({ 'task': 'tests.tasks.index_incrementer', 'args': [], 'kwargs': { 'message': 'Test message', }, })
def test_worker_fills_internal_queue_only_until_maximum_queue_size(): """ Test read workers fill internal queue only to maximum size """ conn = boto.connect_sqs() queue = conn.create_queue("tester") queue.set_timeout(1) # Set visibility timeout low to improve test speed message = Message() body = json.dumps({ 'task': 'tests.tasks.index_incrementer', 'args': [], 'kwargs': { 'message': 'Test message', }, }) message.set_body(body) for i in range(3): queue.write(message) internal_queue = Queue(maxsize=2) worker = ReadWorker(queue, internal_queue) worker.read_message() # The internal queue should only have two messages on it internal_queue.get(timeout=1) internal_queue.get(timeout=1) try: internal_queue.get(timeout=1) except Empty: pass else: raise AssertionError("The internal queue should be empty")
def test_worker_fills_internal_queue_and_respects_visibility_timeouts(): """ Test read workers respect visibility timeouts """ # Setup logging logger = logging.getLogger("pyqs") logger.handlers.append(MockLoggingHandler()) # Setup SQS Queue conn = boto.connect_sqs() queue = conn.create_queue("tester") queue.set_timeout(1) # Add MEssages message = Message() body = '{"body": "KGRwMApTJ3Rhc2snCnAxClMndGVzdHMudGFza3MuaW5kZXhfaW5jcmVtZW50ZXInCnAyCnNTJ2Fy\\nZ3MnCnAzCihscDQKc1Mna3dhcmdzJwpwNQooZHA2ClMnbWVzc2FnZScKcDcKUydUZXN0IG1lc3Nh\\nZ2UyJwpwOApzcy4=\\n", "some stuff": "asdfasf"}' message.set_body(body) queue.write(message) queue.write(message) queue.write(message) # Run Reader internal_queue = Queue(maxsize=1) worker = ReadWorker(queue, internal_queue) worker.read_message() # Check log messages logger.handlers[0].messages['warning'][0].should.contain("Timed out trying to add the following message to the internal queue") logger.handlers[0].messages['warning'][1].should.contain("Clearing Local messages since we exceeded their visibility_timeout")
def post_count_tweets_result(self, job_id, coordinate_box, count): ''' Submits the results of the tweet count. @param job_id Tracking id of the job @paramType uuid/string @param coordinate_box Box in which the tweets were counted @paramType dictionary @param count # of tweets in the coordinate_box @paramType int @returns n/a ''' assert job_id is not None assert coordinate_box is not None assert count is not None message = Message() # Set up the message message.set_body(json.dumps({ 'job_id' : job_id, 'task' : 'count_tweets', 'result' : count, 'coordinate_box' : coordinate_box })) result = self.queue.write(message) # Write out the request assert result is not None, 'Failed to push results to queue!'
def submit_task(self, task, **kwargs): queue = self._choose_queue(task, **kwargs) logger.debug("Sending task '%s' to queue '%s'.", task.id, queue.name) m = Message() m.set_body(json.dumps(task.to_primitive())) return queue.write(m)
def resubmit_task(self, task, delay, **kwargs): task.increment_attempt() logger.debug("Resubmitting task %s with %d second delay.", task.id, delay) m = Message() m.set_body(json.dumps(task.serialize())) return self.queue.write(m, delay_seconds=delay)
def queue_msgwrite(queue): """ Writes a message to the queue curl -X POST -H 'Content-Type: application/json' http://localhost:5000/queue/<mytestqueue>/msqs -d '{"content":"message"}' """ conn = get_conn() body = request.get_json(force=True) q = conn.get_queue("C13765235-%s" % queue) resp = {} if q != None: m = Message() m.set_body(body["content"]) q.write(m) resp["id"] = q.id resp["message"] = m.get_body() else: resp["status"] = "Not Found" return Response( response=json.dumps(resp), mimetype="application/json" )
def run(self): conn = boto.sqs.connect_to_region(self.mysqsZone) if not conn: print "NotificationManager.py: error while connecting at"+self.mysqsZone+"zone" fakelist = range(int(self.rEnd)-int(self.rStart)) while True: now = time.time() currentID = int(self.rStart) for item in fakelist: aRequestId = int(time.time()) JsonRequest = jm.createOverviewRequest(aRequestId,"_APPosto_SDCC_notification_poller",currentID) destinationQueueName = "_APPosto_requests_queue" dest_queue = conn.get_queue(destinationQueueName) while dest_queue == None: dest_queue = conn.create_queue(str(destinationQueueName)) if dest_queue==None: print "queue creation failed" m = Message() m.set_body(str(JsonRequest)) dest_queue.write(m) currentID = currentID+1 duration = time.time()-now slack = int(self.frequency)-int(duration) if slack>0: print "NotificationManager: prossimo polling tra almeno "+str(slack)+" secondi" time.sleep(float(slack))
def sqs_enque(data): ''' Send (string) data to an Amazon SQS queue. Good uses: * Send a list of filenames of tracking files stored on S3. ~50k requests for around 3 pennies. Bad uses: * Send all events to Amazon SQS as part of a pipeline. Each billion events runs us $500. TODO: Batch operations. Would improve performance and cut cost 10x. (Untested) ''' sqs_conn = boto.sqs.connect_to_region( "us-east-1", aws_access_key_id=settings['edx-aws-access-key-id'], aws_secret_access_key=settings['edx-aws-secret-key'] ) q = sqs_conn.get_queue(settings["tracking-logs-queue"]) for item in data: m = Message() m.set_body(item) q.write(m)
def test_count(self): q = SQSQueueMock('test') m = Message() m.set_body('this is a test') status = q.write(m) self.assertTrue(status) self.assertTrue(q.count() == 1) status = q.write(m) self.assertTrue(status) self.assertTrue(q.count() == 2) status = q.write(m) self.assertTrue(status) self.assertTrue(q.count() == 3) messages = q.get_messages(num_messages=1) self.assertTrue(q.delete_message(messages[0])) self.assertTrue(q.count() == 2) messages = q.get_messages(num_messages=1) self.assertTrue(q.delete_message(messages[0])) self.assertTrue(q.count() == 1) messages = q.get_messages(num_messages=1) self.assertTrue(q.delete_message(messages[0])) self.assertTrue(q.count() == 0)
def processLogContent(filename, logContent): for line in logContent.split('\n'): # line should start with a timestamp: 2014 if not line.startswith("20"): continue # line should have an exception in it if "Exception" not in line: continue parsedLine = line.split(" ") exceptionDate = parsedLine[0] exceptionTime = parsedLine[1].split(",")[0] # no milliseconds exceptionType = parsedLine[2] exceptionTask = parsedLine[3].strip("[]") exceptionClass = parsedLine[4].strip("[]") exceptionMessage = " ".join(parsedLine[5::]) message = {"file": filename, "date": exceptionDate, "time": exceptionTime, "type": exceptionType, "task": exceptionTask, "class": exceptionClass, "message": exceptionMessage} sqsMessage = Message() sqsMessage.set_body(json.dumps(message)) sendMessageToQueue(sqsMessage)
def readtaskSQS(filename, queue_name, process_queue): aws_conn=boto.sqs.connect_to_region("us-east-1", aws_access_key_id='{aws_access_key_id}', aws_secret_access_key = '{aws_secret_access_key}') dynamo_conn = boto.dynamodb.connect_to_region("us-east-1", aws_access_key_id='{aws_access_key_id}', aws_secret_access_key = '{aws_secret_access_key}') SQS_queue=aws_conn.get_queue(queue_name) SQS_process_queue=aws_conn.get_queue(process_queue) task_id = 0; try: task_table_schema = conn_dynamo.create_schema(hash_key_name='task_id',hash_key_proto_value=str) table = conn_dynamo.create_table(name = 'Dynamo_Table', schema = task_table_schema,read_units = 10,write_units = 10) print 'Table Dynamo_Table has been created' except Exception as e: print 'Dynamo_Table already exists.' with open(filename) as f: task_list=f.readlines() for i in task_list: msg = Message() json_msg = {} json_msg["task_id"] = task_id json_msg["task"] = i msg.set_body(json.dumps(json_msg)) SQS_queue.write(msg); task_id=task_id+1 return SQS_queue, SQS_process_queue, task_id;
def enqueue(queue, files): ''' loops through an array of CSV file paths in order to convert each row to a JSON object, and then add it to the SQS Queue ''' counter = 0 # loop through files for f in files: # create a reader for the CSV object that will provide each row as # a dictionary reader = unicodecsv.DictReader(open(f)) # loop through rows of the CSV for data in reader: # create a json version of the csv data package = json.dumps(data) # create and submit a new message to the queue m = Message() m.set_body(package) queue.write(m) # increment counter and draw a '.' to screen to show progress counter += 1 ping() print "\n\nExiting - End of data files. %d rows queued.\n" % counter
def ProcessDelta(): if request.headers.get('X-AppEngine-QueueName') is None: # Ignore if not from AppEngine abort(403) client = dropbox.client.DropboxClient(token) try: #see if we can grab the cursor file from dropbox f = client.get_file('/.cursor') cursor = f.read() except: #if we can't, set cursor to none and it will #get all of the changes, then we write to it cursor = None delta = client.delta(cursor) for filepath, data in delta['entries']: #check that there is an entry, and that it isn't a cursor if (data is not None) and (filepath != '/.cursor'): #check for directories and compressed files if not data['is_dir'] and not filepath.endswith('.gz'): #we have something we want to process q = aws_sqs.create_queue(queue_name, 30) message = Message() message.set_body(filepath) q.write(message) #write the changes to the cursor file in dropbox cursor = delta['cursor'] client.put_file('/.cursor', cursor, overwrite=True)
def post(self): if request.content_type != 'application/json': abort(400, message="Invalid Content-Type") data = request.get_json() if not isinstance(data, dict): data = {} url = data.get('video_url', '') owner = data.get('owner', '') s = Session(url) s.owner = owner db.session.add(s) db.session.commit() if app.config['SEND_MESSAGES']: msg = Message() msg.set_body(json.dumps(marshal(s, SessionAPI.fields))) app.session_queue.write(msg) increment_asg('video-processing-group') return {"message": "Success", "id": s.id}
def create_queue_order ( pair, side, units ): # Create connection to SQS queue. conn = boto.sqs.connect_to_region('us-east-1') queue = conn.get_queue('forex_moving_average_orders') # Set queue message. message = Message() message.set_body( pair + ' ' + side + ' ' + str(units) ) # Set queue message attributes. message.message_attributes = { "pair": { "data_type": "String", "string_value": pair }, "side": { "data_type": "String", "string_value": side }, "units": { "data_type": "Number", "string_value": str(units) } } # Write message to queue. queue.write(message)
def parse_add_activities(): json_api_spec = {} rq_id = request.query.id rq_name = request.query.name rq_activities = request.query.activities global request_count print '\n' request_count += 1 dict_sqs = { 'id': rq_id, 'name': rq_name, 'activities': rq_activities, 'request_type': 'add_activities', 'rq_count': request_count, } m = Message() js = json.dumps(dict_sqs) m.set_body(js) sqs_in.write(m) json_ret = \ json.dumps('{"data": {"type": "Notification","msg": "Accepted"}}' ) json_full_spec = {} json_full_spec['type'] = 'Notification' json_full_spec['msg'] = 'Accepted' json_api_spec['data'] = json_full_spec return json_api_spec
def POST(self): global q data = web.data() print data m = Message() m.set_body(data) q.write(m)
def run(message): try_count = 0 while True: if try_count != MAX_TRY: launch_result = launch_vm(message) #print launch_result result = json.loads(launch_result) if result['rescode'] == '1': sqs_conn = boto.connect_sqs() request_queue = sqs_conn.create_queue(REQUEST_QUEUE) request_queue.delete_message(message) print "Delete message from dev queue :(" return 1 break elif result['rescode'] == '2': print 'job fail' try_count = try_count +1 else: print try_count print "last job fail" sqs_conn = boto.connect_sqs() request_queue = sqs_conn.create_queue(REQUEST_QUEUE) request_queue.delete_message(m) new_m = Message() new_m.set_body('new') status = request_queue.write(new_m) print "Re-insert queue message (fail) :(" return 2 break
def queue_article_publication(article_id, version, run): if version is None or run is None: return { 'publication-status': 'error', 'id': article_id, 'version': str(version), 'run': run } queue_provider = QueueProvider() out_queue = queue_provider.get_queue(settings.workflow_starter_queue) article = get_article(article_id) article_versions = article.get('versions') result = {} if isinstance(article_versions, dict): version_data = article_versions.get(int(version)) if isinstance(version_data, dict): version_properties = version_data.get('properties') if isinstance(version_properties, dict): # TODO : get publication status and check still valid # also see http://jira.elifesciences.org:8080/browse/ELPP-613 status = 'queued' status_message = 'article queued' publication_data = version_properties.get('_publication-data') # This string is a base64 encoded message which allows initiation of the PostPerfectPublciation # workflow. # This class now needs constructs a workflow starter message which initiates the ApprovePerfectArticle # workflow. That workflow needs data to give drupal to publish the version <version> of article # <article_id> and also requires the data in this encoded string to initiate PostPerfectPublication upon # successful publication so pass it article_id, version and the base 64 encoded string via the starter # mechanism follow_on_data = { 'article_id': article_id, 'version': version, 'run': run, 'publication_data': publication_data.get('value') } message = { 'workflow_name': 'ApproveArticlePublication', 'workflow_data': follow_on_data } m = Message() m.set_body(json.dumps(message)) out_queue.write(m) result = { 'publication-status': status, 'id': article_id, 'version': str(version), 'run': run } return result
def add_ids_to_queue(args, and_expression): engine = create_engine(DB_LOGIN) connection = engine.connect() # noinspection PyBroadException try: queue = None if not args.test: sqs_helper = SqsHelper('us-east-1') queue = sqs_helper.get_queue(args.queue_name) select_galaxy = select([GALAXY.c.galaxy_id]).where(and_expression).order_by(func.RAND()) counter = 1 for galaxy in connection.execute(select_galaxy): if counter % 100 == 0: LOG.info('Added {0}'.format(counter)) counter += 1 message_text = '{0}'.format(galaxy[GALAXY.c.galaxy_id]) if args.test: LOG.info(message_text) else: message = Message() message.set_body(message_text) queue.write(message) except Exception: LOG.exception('Error adding ids')
def on_data(self,data): print data ms=cPickle.dumps(data) m=Message() m.set_body(msg) status=q[0].write(m) return True
def write(self, message): try: m = Message() m.set_body(message) return self._queue.write(m) except SQSError, e: raise QueueException, "%s" % e.code
def poll_queue_test(self, mock_process_message): example_message_body = "---\ntest_job_name:\n request_id: testrequestid\n" message = Message() message.set_body(example_message_body) self.sqslistener.queue.write(message) self.sqslistener.poll_queue() assert mock_process_message.called
def send_to_sqs(self, msg): global log conn = boto.sqs.connect_to_region(settings.AWS_CONF['region'], aws_access_key_id=settings.AWS_CONF['awskey'], aws_secret_access_key=settings.AWS_CONF['awssecret'], is_secure=False) try: parsed_json = simplejson.loads(msg) except Exception as e: logger.critical("Invalid message:%s Reason:%s" % (msg, e)) # wrap with timestamp if 'timestamp' not in parsed_json: msg = '{"event":%s , "timestamp":"%d"}' % (msg, int(time())) logger.info('Message %s' % msg) # Send to SQS if conn is None: logger.critical("Could not connect to AWS: region=%s" % settings.AWS_CONF['region']) else: q = conn.get_queue(settings.AWS_CONF['sqsqueue']) if q is None: logger.critical("Error connecting to SQS queue: %s" % settings.AWS_CONF['sqsqueue']) return m = Message() m.set_body(msg) q.write(m) if m.id is None: logger.critical("Could not send this msg to queue: %s" % m.get_body()) else: logger.info('Wrote %s' % m.id)
def wake_up_message_send(): sqs_conn = boto.connect_sqs() request_queue = sqs_conn.create_queue(master_queue_name) rvm_host = get_rvm_hostname() if True: for i in range(0, 5): try: log.debug("Attempted to get hostname") fp = urllib.urlopen('http://%s/latest/meta-data/local-hostname' % rvm_host) local_hostname = fp.read() fp.close() if local_hostname: break except IOError: pass for i in range(0, 5): try: log.debug("Attempted to get hostip") fp = urllib.urlopen('http://%s/latest/meta-data/local-ipv4' % rvm_host) local_ipaddress = fp.read() fp.close() if local_ipaddress: break except IOError: pass new_message = Message() msg = "MASTER|%s|%s" % (local_hostname, local_ipaddress) new_message.set_body(msg) status = request_queue.write(new_message) log.debug("sending message '%s'" % msg)
def on_status(self, status): if status: json_data = json.loads(status) id_str=json_data['id_str'] location=re.escape(json_data['user']['location']) coordinates=json_data['coordinates'] text=json_data['text'] if coordinates: coordinates_list=coordinates.items() longitude=coordinates_list[1][1][0] latitude=coordinates_list[1][1][1] if not location: location="" location_ascii = ''.join([i if ord(i) < 128 else ' ' for i in location]) text_ascii = ''.join([j if ord(j) < 128 else ' ' for j in text]) category = "" for substring in track: if substring in status.lower(): category = substring break if id_str and coordinates and category: sql='insert ignore into twittmapapp_tweetdata (id_str,location,category,longitude,latitude) values ("'+id_str+'","'+location_ascii+'","'+category+'",'+str(longitude)+','+str(latitude)+');' t=self.cur.execute(sql) m = Message() m.set_body("This message contains text,id,latitude and longitude") m.message_attributes = {"text":{"data_type": "String","string_value":text_ascii}, "id":{"data_type": "String","string_value":id_str},"category":{"data_type": "String","string_value":category},"longitude":{"data_type": "String","string_value":str(longitude)},"latitude":{"data_type": "String","string_value":str(latitude)}} self.q.write(m) # print "hi" # response = self.alchemyapi.sentiment("text", text_ascii) # if response['status']!="ERROR": # print "Sentiment: ", response["docSentiment"]["type"] # self.sns.publish(self.topic, response["docSentiment"]["type"], self.subject); return
def publish_node_stats(self): try: # create a connection to SQS conn = SQSConnection(); # ask for the QUEUE q = conn.get_queue(NODE_STATS_QUEUE); # create a new message m = Message(); # populate the message with stats m.set_body(self.node_stats()); # publish the message to SQS q.write(m); # schedule another publish self.schedule_node_stats(); except Exception as e: # blab about the err on std err os.sys.stderr.write(str(e)+'\n'); # log error message to the error queue self.publish_error('publish_node_stats: '+str(e));
def callback(self, filename, lines, **kwargs): timestamp = self.get_timestamp(**kwargs) if kwargs.get('timestamp', False): del kwargs['timestamp'] message_batch = [] message_batch_size = 0 message_batch_size_max = 250000 # Max 256KiB but leave some headroom for line in lines: m = Message() m.set_body(self.format(filename, line, timestamp, **kwargs)) message_size = len(m) if (message_size > message_batch_size_max): self._logger.debug('Dropping the message as it is too large to send ({0} bytes)'.format(message_size)) continue # SQS can only handle up to 10 messages in batch send and it can not exceed 256KiB (see above) # Check the new total size before adding a new message and don't try to send an empty batch if (len(message_batch) > 0) and (((message_batch_size + message_size) >= message_batch_size_max) or (len(message_batch) == 10)): self._logger.debug('Flushing {0} messages to SQS queue {1} bytes'.format(len(message_batch), message_batch_size)) self._send_message_batch(message_batch) message_batch = [] message_batch_size = 0 message_batch_size = message_batch_size + message_size message_batch.append((uuid.uuid4(), self.format(filename, line, timestamp, **kwargs), 0)) if len(message_batch) > 0: self._logger.debug('Flushing the last {0} messages to SQS queue {1} bytes'.format(len(message_batch), message_batch_size)) self._send_message_batch(message_batch) return True
def load_from_file(self, fp, sep='\n'): """Utility function to load messages from a file-like object to a queue""" n = 0 body = '' l = fp.readline() while l: if l == sep: m = Message(self, body) self.write(m) n += 1 print 'writing message %d' % n body = '' else: body = body + l l = fp.readline() return n
def test_send_receive_message(self): q = self.conn.create_queue('message_test') msg_created = Message() msg_created.set_body('hello world') msg_sent = q.write(msg_created) self.assertEquals(msg_created, msg_sent) read_msg = q.read() self.assertIsNotNone(read_msg) print 'written', msg_created.get_body() print 'read', read_msg.get_body() self.assertEquals(msg_created.get_body(), read_msg.get_body())
def load(self, file_name, sep='\n'): """Utility function to load messages from a file to a queue""" fp = open(file_name, 'rb') n = 0 body = '' l = fp.readline() while l: if l == sep: m = Message(self, body) self.write(m) n += 1 print 'writing message %d' % n body = '' else: body = body + l l = fp.readline() fp.close() return n
def __init__(self, callable, *args, **kwargs): """ Create a new Job, :param obj callable: [optional] A callable to run. """ self.start_time = None self.stop_time = None self.run_time = None self.exception = None self.result = None self.callable = callable self.args = args self.kwargs = kwargs self.message = Message(body=dumps({ 'callable': self.callable, 'args': self.args, 'kwargs': self.kwargs, }))
def test_get_messages_attributes(self): conn = SQSConnection() current_timestamp = int(time.time()) queue_name = 'test%d' % int(time.time()) test = conn.create_queue(queue_name) self.addCleanup(conn.delete_queue, test) time.sleep(65) # Put a message in the queue. m1 = Message() m1.set_body('This is a test message.') test.write(m1) self.assertEqual(test.count(), 1) # Check all attributes. msgs = test.get_messages( num_messages=1, attributes='All' ) for msg in msgs: self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1') first_rec = msg.attributes['ApproximateFirstReceiveTimestamp'] first_rec = int(first_rec) / 1000 self.assertTrue(first_rec >= current_timestamp) # Put another message in the queue. m2 = Message() m2.set_body('This is another test message.') test.write(m2) self.assertEqual(test.count(), 1) # Check a specific attribute. msgs = test.get_messages( num_messages=1, attributes='ApproximateReceiveCount' ) for msg in msgs: self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1') with self.assertRaises(KeyError): msg.attributes['ApproximateFirstReceiveTimestamp']
def process_links(article_title): print "processing links for " + article_title links_tree = lxml.etree.parse(LINKS_ROOT + article_title) link_nodes = links_tree.xpath("//pl[@ns='0' and @exists='']") links = [l.text for l in link_nodes] #add links to the queue if they haven't been seen before if(LOCAL_TEST): for title in links: if (not status.has_key(title)): status[title] = (0, 0.0) message = Message() message.set_body(title.encode('utf-8')) q.write(message) print "adding article to queue: " + title else: request_list = [] for t in links: request_list = request_list + [dict(title=t)] #returns list of dicts - each dict is one item from the database present_links = database.batch_get(keys=request_list) with database.batch_write() as batch: for r in request_list: #find titles in the returned list that match requested titles # if none exist, add the item to the database and queue matches = [item for item in present_links if item['title'] == r['title']] if len(matches) == 0: add_time = str(datetime.datetime.now()) batch.put_item(data={ 'title': r['title'], 'status': 'queued', 'difficulty': Decimal(0.0), 'time': add_time, }) message = Message() message.set_body(r['title'].encode('utf-8')) q.write(message) print "adding article to queue: " + r['title']
def createQ(): conn = boto.sqs.connect_to_region( "eu-west-1", aws_access_key_id='AKIAIR7EH3TNSTDUCWKA', aws_secret_access_key='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') q = conn.create_queue("queue_jabba") #write 1 message to queue m = Message() m.set_body('first message') q.write(m) #write 100 messages to queue for x in range(1, 100): mx = Message() mx.set_body('Message' + x) q.write(mx) #read message from queue rs = q.get_messages() mread = rs[0] mread.get_body() #delete message from queue q.delete_message(m) return mread
def timeout(self): from boto.sqs.message import Message if not self._queue and self._connected: try: from boto.sqs.connection import SQSConnection sqs_connection = SQSConnection(self._access_key, self._secret_access_key, **self._connection_kwargs) queues = sqs_connection.get_all_queues(prefix=self._queue_name) if len(queues) == 0: raise Exception('Queue %s does not exist' % self._queue_name) self._queue = queues[0] except Exception as e: LOGGER.error('Could not connect to logging queue %s' % self._queue_name, exc_info=e) self._connected = False while len(self._records_to_emit) and self._connected: record = self._records_to_emit.pop() self._queue.write(Message(body=record))
def on_data(self, data): try: status_wrapper = TextWrapper(width=60, initial_indent=' ', subsequent_indent=' ') twitter_data = json.loads(data) m = Message() if ('coordinates' in twitter_data.keys()): if (twitter_data['coordinates'] is not None): tweet = { 'id': twitter_data['id'], 'time': twitter_data['timestamp_ms'], 'text': twitter_data['text'].lower().encode( 'ascii', 'ignore').decode('ascii'), 'coordinates': twitter_data['coordinates'], 'place': twitter_data['place'], 'handle': twitter_data['user']['screen_name'], 'sentiment': "" } global count count += 1 print(count) print(tweet) sqs.send_message( QueueUrl='', MessageBody=json.dumps(tweet)) #Adding data to Queue1 return True except BaseException as e: print("Error on_data: %s" % str(e)) return True
def taskProcess(msg, jObj): print jObj["task"] job = jObj["task"].split(' ') try: exe = 'time.' + job[0] + '(' + str(float(job[1]) / 1000) + ')' print 'Executing ... ' + exe exec(exe) print 'successful' m1 = Message() j = responseMessage(jObj["clientID"], jObj["jobID"], "1") m1.set_body(j) processed_queue.write(m1) except Exception as e: m = Message() m.set_body(jObj) task_queue.write(m) key = jObj["clientID"] + ";" + jObj["jobID"] item = table.get_item(hash_key=key) item['Body'] = 'False' item.put() print 'Interrupted' + str(e) global threadCount threadCount = threadCount - 1
parser.add_argument("qname") args = parser.parse_args() conn = boto.sqs.connect_to_region("us-west-2a") q = conn.get_queue(args.qname) try: m = Message() m = q.read(60) str1 = m.get_body() print "Message read = ", str1 except: print "Could not read message"
def do_activity(self, data=None): """ Do the work """ if self.logger: self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4)) run = data['run'] session = Session(self.settings) version = session.get_value(run, 'version') article_id = session.get_value(run, 'article_id') self.emit_monitor_event( self.settings, article_id, version, run, self.pretty_name, "start", "Starting preparation of article for EIF " + article_id) try: eif_location = session.get_value(run, 'eif_location') eif_bucket = self.settings.publishing_buckets_prefix + self.settings.eif_bucket article_path = session.get_value(run, 'article_path') self.set_monitor_property(self.settings, article_id, 'path', article_path, 'text', version=version) expanded_folder = session.get_value(run, 'expanded_folder') status = session.get_value(run, 'status') update_date = session.get_value(run, 'update_date') carry_over_data = { 'eif_location': eif_location, 'eif_bucket': eif_bucket, 'passthrough': { 'article_id': article_id, 'version': version, 'run': run, 'article_path': article_path, 'expanded_folder': expanded_folder, 'status': status, 'update_date': update_date, } } message = carry_over_data sqs_conn = boto.sqs.connect_to_region( self.settings.sqs_region, aws_access_key_id=self.settings.aws_access_key_id, aws_secret_access_key=self.settings.aws_secret_access_key) out_queue = sqs_conn.get_queue(self.settings.website_ingest_queue) m = Message() m.set_body(json.dumps(message)) out_queue.write(m) ######### except Exception as e: self.logger.exception("Exception when Preparing for PostEIF") self.emit_monitor_event( self.settings, article_id, version, run, self.pretty_name, "error", "Error submitting EIF For article" + article_id + " message:" + str(e.message)) return False self.emit_monitor_event( self.settings, article_id, version, run, self.pretty_name, "end", "Finished preparation of article for EIF " + article_id) return True
def sqsput(filename): print filename m = Message() m.set_body(filename) q.write(m)
def write_message_to_queue(queue): m = Message() m.set_body('request_ip') queue.write(m) print m
# this script expects 2 environment variables # 1. SQS_KEY_ID (preferably an IAM user with limited rights) # 2. SQS_SECRET_KEY (accompanying secret key) # 3. SQS_TASK_QUEUE (the queue to use) import os import sys from boto.sqs.connection import SQSConnection from boto.sqs.message import Message import utils # your amazon keys key = os.environ['SQS_KEY_ID'] access = os.environ['SQS_ACCESS_KEY'] queue = os.environ['SQS_TASK_QUEUE'] if __name__ == '__main__': region_info = utils.get_region_info() sqs = SQSConnection(key, access, region=region_info) tasks = sqs.create_queue(queue) m = Message() m.set_body(sys.argv[1]) tasks.write(m)
import sys import signal import time from boto.sqs.message import Message from subprocess import call parser = argparse.ArgumentParser() parser.add_argument("echo") args = parser.parse_args() conn = boto.sqs.connect_to_region( "us-east-1", aws_access_key_id='AKIAINWVSI3MIXIB5N3Q', aws_secret_access_key='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') my_queue = conn.get_queue(args.echo) m = Message() print "Reading Queue " + args.echo + " Messages read are deleted" while True: try: try: rs = my_queue.get_messages(num_messages=10, visibility_timeout=12000, wait_time_seconds=5) if (len(rs) == 0): print args.echo + " len = 0 now empty" sys.exit(0) for msgcounter in range(len(rs)): m = rs[msgcounter - 1]
def _put(self, queue, message, **kwargs): """Put message onto queue.""" q = self._new_queue(queue) m = Message() m.set_body(dumps(message)) q.write(m)
while True: ''' EXTEND: Replace the following line with code to read a message off the input queue, convert from JSON to a Python dict, and assign to `req`. ''' #rs is queue from front end, going to read from it rs = q1.get_messages() if (len(rs) > 0): m = rs[0] msg = m.get_body() #Convert json object from sqs and convert to dict so we can code.. ds = json.loads(msg) actual_s = random.randint(0, ds['seconds']) time.sleep(actual_s) ds['actual_s'] = actual_s q1.delete_message(m) ''' EXTEND: Replace the following line with code to put the response on the output queue, in JSON representation. ''' if (len(rs) > 0): m = Message() js = json.dumps(ds) m.set_body(js) q2.write( m) #output queue from worker so we can get backend.py to read it. print "Output Queue Written"
JC_key = bucket.get_key(folder + '/' + JC_fname) if JC_key is None: JC_key = bucket.new_key(folder + '/' + JC_fname) try: JC_key.set_contents_from_filename(JC_fname) except S3ResponseError as e: sys.exit(-1) bucket.set_acl('public-read', JC_key.name) reskey = bucket.get_key(folder + '/' + res_fname) if reskey is not None: reskey.delete() mtxt = json.dumps({'bucket': bucket.name, 'folder': folder, 'argv': argv}) m = Message() m.set_body(mtxt) status = q.write(m) reskey = bucket.get_key(folder + '/' + res_fname) while reskey is None: time.sleep(polling_wait_time) reskey = bucket.get_key(folder + '/' + res_fname) reskey.get_contents_to_filename(res_fname) res_f = open(res_fname, 'r') exitcode = int(res_f.readline()) res_f.close() logkey = bucket.get_key(folder + '/' + log_fname) if logkey is not None:
def writeToSQS(messageBody): # Note that messages are base64 encoded. m1 = Message() m1.set_body(messageBody) q.write(m1)
def add(self, message): message = to_data(message) m = Message() m.set_body(value2json(message)) self.queue.write(m)
def on_data(self, data): msg = pickle.dumps(data) m = Message() m.set_body(msg) status = q[0].write(m) return True
def send_message(self, queue_name=None, msg=None): q = self.conn.lookup(queue_name) m = Message() m.set_body(msg) q.write(m)
def add(self, name, **spider_args): d = spider_args.copy() d['name'] = name msg = Message(body=json.dumps(d)) return threads.deferToThread(self._queue_method, 'write', msg)
def write_message_to_queue(queue): m = Message() m.set_body(ipgetter.myip()) queue.write(m) print m
def send(message): global q m = Message() m.set_body(message) q.write(m)
import boto.sqs from boto.sqs.message import Message import os import config conn = boto.sqs.connect_to_region("us-west-2", aws_access_key_id=config.sqs_access_key, aws_secret_access_key=config.sqs_access_secret) queue = conn.get_queue("Image") queuepkg = conn.get_queue("Package") while (True): messages = queue.get_messages() for mes in messages: dname = mes.get_body() print dname res = os.popen('./script.sh ' + dname) for i in range(4): res.readline() pkglist = res.read() # print pkglist # txt = open("file.txt", "r") # pkglist = txt.read() mes = Message() mes.set_body(pkglist) queuepkg.write(mes) if (len(messages) != 0): queue.delete_message_batch(messages)
def test_single_path_document_zip(registry_mock, connection_mock, document_mock, job_mock, parent_url_mock, get_document_mock): conn_s3 = boto.connect_s3() bucket = conn_s3.create_bucket('storage-bucket') conn_s3.create_bucket('files-bucket') registry = MagicMock() registry.get = MagicMock(side_effect=side_effect) registry_mock.return_value = registry message = Message() message.set_body( json.dumps({ 'Message': '9bd96ca7-3d0a-4e74-b523-b3bd38e9862e', 'Subject': 'Test Subject' })) job = MagicMock( **{ 'uuid': '9bd96ca7-3d0a-4e74-b523-b3bd38e9862e', 'name': 'Migration Download', 'status': 'pending', 'message': { 'documents': [{ 'parent_id': '56d3c182-f72f-4216-9e94-1756bf67564d' }] } }) job.set = MagicMock() document_mock.query.return_value = [ MagicMock( **{ 'uuid': '56d3c182-f72f-4216-9e94-1756bf67564d', 'created': datetime(2015, 5, 17), 'url': '/' }), MagicMock( **{ 'uuid': '79254d0b-0902-4697-89d1-4be8ff3acd69', 'created': datetime(2015, 5, 17), 'url': 'test', 'type': 'File' }) ] key1 = Key(bucket, '17/5/2015/56d3c182-f72f-4216-9e94-1756bf67564d') key1.set_contents_from_string( json.dumps({ 'document': { 'id': 1, 'uuid': '56d3c182-f72f-4216-9e94-1756bf67564d', 'created': str(datetime(2015, 5, 17)), 'url': '/', 'parent': 0, 'path': '1/', } })) key2 = Key(bucket, '17/5/2015/79254d0b-0902-4697-89d1-4be8ff3acd69') key2.set_contents_from_string( json.dumps({ 'document': { 'id': 2, 'uuid': '79254d0b-0902-4697-89d1-4be8ff3acd69', 'created': str(datetime(2015, 5, 17)), 'url': 'test', 'parent': 1, 'path': '1/2', 'type': 'File' }, 'file': { "bucket": "storage-bucket", 'key': '17/5/2015/a984dea7-8140-44cb-80a0-7e832ff1ff19' } })) key3 = Key(bucket, '17/5/2015/a984dea7-8140-44cb-80a0-7e832ff1ff19') key3.set_contents_from_string('Hello World') job_mock.selectBy.return_value.getOne.return_value = job service = MigrationDownloadJob() get_document_mock.return_value = IN('uuid', [ '56d3c182-f72f-4216-9e94-1756bf67564d', '79254d0b-0902-4697-89d1-4be8ff3acd69' ]) def parent_side_effects(parent_id): return MagicMock( url='/', uuid='56d3c182-f72f-4216-9e94-1756bf67564d') if parent_id else None parent_url_mock.side_effect = parent_side_effects service.do_work(message) key = Key(bucket, '9bd96ca7-3d0a-4e74-b523-b3bd38e9862e') contents = StringIO(key.get_contents_as_string()) handle = zipfile.ZipFile(contents, 'r', compression=zipfile.ZIP_DEFLATED) assert key.exists() assert handle.namelist() == [ '56d3c182-f72f-4216-9e94-1756bf67564d', '79254d0b-0902-4697-89d1-4be8ff3acd69', '17/5/2015/a984dea7-8140-44cb-80a0-7e832ff1ff19', 'manifest' ] assert job.set.call_args_list == [ call(status='running'), call(message={ 'documents': [{ 'parent_id': '56d3c182-f72f-4216-9e94-1756bf67564d' }], 'download': { 'key': '9bd96ca7-3d0a-4e74-b523-b3bd38e9862e', 'bucket': 'storage-bucket' } }, status='complete') ]
def emit(self, record): m = Message() m.set_body(record.msg) self.q.write(m)
def publish_to_sqs(data): m = Message() m.set_body(data) status = q[0].write(m) return status