class TaskSQS:
TASK_QUEUE = "tasks"
RESULT_QUEUE = "results"
VISIBILITY_TIMEOUT = 120
def __init__(self):
self.conn = None
self.taskq = None
self.resultq = None
def connect(self):
# open connection
self.conn = SQSConnection()
# initialize queues
self.taskq = self.conn.create_queue(self.TASK_QUEUE, self.VISIBILITY_TIMEOUT)
self.taskq.set_message_class(JSONMessage)
self.resultq = self.conn.create_queue(self.RESULT_QUEUE, self.VISIBILITY_TIMEOUT)
self.resultq.set_message_class(JSONMessage)
def clear(self):
self.taskq.clear()
self.resultq.clear()
# self.conn.delete_queue(self.taskq)
# self.conn.delete_queue(self.resultq)
self.taskq = None
self.resultq = None
def new_task(self, task):
return JSONMessage(self.taskq, task)
def new_result(self, result):
return JSONMessage(self.resultq, result)
def put_task(self, task):
self.taskq.write(task)
def next_task(self):
next = None
while next is None:
next = self.taskq.read()
return next
def complete(self, task, result=None):
self.taskq.delete_message(task)
if result is not None:
self.put_result(result)
def put_result(self, result):
self.resultq.write(result)
def get_result(self):
result = self.resultq.read()
if result is not None:
self.resultq.delete_message(result)
return result
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'hcq:o:t:',
['help', 'clear', 'queue',
'output', 'timeout'])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError, e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
def __init__(self, *args, **kwargs):
Queue.__init__(self, *args, **kwargs)
c = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
self.queue = c.lookup(self.name)
if not self.queue:
self.queue = c.create_queue(self.name, 1)
self.connection = c
class SqsApi(object):
def __init__(self):
self.conn = SQSConnection(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
def create_queue(self, name="MuscleOpsQ"):
return self.conn.create_queue(name)
def list_queues(self):
return self.conn.get_all_queues()
def write_message(self,
q,
msg="If you're sleepy and you know it; clap your hands!"
):
m = RawMessage()
m.set_body(msg)
q.write(m)
def read_message(self,
q,
msg="If you're sleepy and you know it; clap your hands!"):
rs = q.get_messages()
m = rs[0].get_body() if len(rs) else msg
return m
def test_sqs_timeout(self):
c = SQSConnection()
queue_name = 'test_sqs_timeout_%s' % int(time.time())
queue = c.create_queue(queue_name)
self.addCleanup(c.delete_queue, queue, True)
start = time.time()
poll_seconds = 2
response = queue.read(visibility_timeout=None,
wait_time_seconds=poll_seconds)
total_time = time.time() - start
self.assertTrue(
total_time > poll_seconds,
"SQS queue did not block for at least %s seconds: %s" %
(poll_seconds, total_time))
self.assertIsNone(response)
# Now that there's an element in the queue, we should not block for 2
# seconds.
c.send_message(queue, 'test message')
start = time.time()
poll_seconds = 2
message = c.receive_message(queue,
number_messages=1,
visibility_timeout=None,
attributes=None,
wait_time_seconds=poll_seconds)[0]
total_time = time.time() - start
self.assertTrue(
total_time < poll_seconds,
"SQS queue blocked longer than %s seconds: %s" %
(poll_seconds, total_time))
self.assertEqual(message.get_body(), 'test message')
attrs = c.get_queue_attributes(queue, 'ReceiveMessageWaitTimeSeconds')
self.assertEqual(attrs['ReceiveMessageWaitTimeSeconds'], '0')
def test_worker_consumes_queue(self):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
queue = sqs.create_queue('test_events')
queue.write(make_message({'test': '1'}))
self.assertEqual(queue.count(), 1)
worker = Worker()
worker.ctrl.wait_time_seconds = 0.1
worker.idle_time_seconds = 0.1
worker.add_consumer('test_events', dummy_consumer)
with spawn_worker(worker):
time.sleep(.2)
self.assertEqual(queue.count(), 0)
queue.write(make_message({'test': '2'}))
self.assertEqual(queue.count(), 1)
time.sleep(.2)
self.assertEqual(queue.count(), 0)
def test_sqs_timeout(self):
c = SQSConnection()
queue_name = "test_sqs_timeout_%s" % int(time.time())
queue = c.create_queue(queue_name)
self.addCleanup(c.delete_queue, queue, True)
start = time.time()
poll_seconds = 2
response = queue.read(visibility_timeout=None, wait_time_seconds=poll_seconds)
total_time = time.time() - start
self.assertTrue(
total_time > poll_seconds,
"SQS queue did not block for at least %s seconds: %s" % (poll_seconds, total_time),
)
self.assertIsNone(response)
# Now that there's an element in the queue, we should not block for 2
# seconds.
c.send_message(queue, "test message")
start = time.time()
poll_seconds = 2
message = c.receive_message(
queue, number_messages=1, visibility_timeout=None, attributes=None, wait_time_seconds=poll_seconds
)[0]
total_time = time.time() - start
self.assertTrue(
total_time < poll_seconds, "SQS queue blocked longer than %s seconds: %s" % (poll_seconds, total_time)
)
self.assertEqual(message.get_body(), "test message")
attrs = c.get_queue_attributes(queue, "ReceiveMessageWaitTimeSeconds")
self.assertEqual(attrs["ReceiveMessageWaitTimeSeconds"], "0")
def request_images(self, params):
if len(params)==0: return
scene = params[0]['scene']
scene = scene.replace('.3dm','')
lowpriority = ""
#print "low_priority"
#print params[0]['low_priority']
if params[0]['low_priority']==True:
lowpriority = "_lowpriority"
q_name = "%s%s_%s_%s" % (self.site_name, lowpriority, scene, 'request')
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
q = conn.create_queue(q_name)
q.set_message_class(Message)
params = self.trunc_params(params)
messages = []
for i in range(len(params)):
#params[i]['params']['textParam'] = 'sunsun'
body = simplejson.dumps(params[i])
sys.stderr.write(body + "\n")
messages.append((i,base64.b64encode(body),0))
for j in range(int(math.ceil(float(len(params))/10.0))):
conn.send_message_batch(q, messages[j*10:(j+1)*10])
sys.stderr.write("\n\n\nSent messages\n\n\n")
return
def __init__(self, conf, name):
super(MessageQueue, self).__init__(conf, name)
conn = SQSConnection(conf['access_key'], conf['secret_access_key'])
self._queue = conn.create_queue(self.name)
self._queue.set_attribute('MessageRetentionPeriod', base.FOURTEEN_DAYS)
self._vtime = base.get_vtime(conf)
class UploadCompleteHandler(tornado.web.RequestHandler):
def prepare(self):
from boto.sqs.connection import SQSConnection
self.conn = SQSConnection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
self._q = self.conn.create_queue('FZconvertQueue', 120)
from boto.s3.connection import S3Connection
self.s3conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
def get(self):
completeDebugDict = {}
# completeDebugDict['bucket'] = self.get_arguments('bucket')[0]
completeDebugDict['key'] = self.get_arguments('key')[0]
completeDebugDict['parseID'] = self.get_arguments('id')[0]
self._bucket = self.s3conn.create_bucket('figurezero')
k = self._bucket.get_key(completeDebugDict['key'])
rootID = completeDebugDict['key'].split('/')[1]
completeDebugDict['tag'] = k.get_metadata('tag')
completeDebugDict['rootID'] = rootID
# completeDebugDict['parseID'] = k.get_metadata('parse')
completeDebugDict['emailAddress'] = k.get_metadata('email')
completeDebugDict['size'] = k.size
from boto.sqs.message import Message
m = Message()
m.set_body(json.dumps(completeDebugDict))
status = self._q.write(m)
self.redirect('/approve/%s' % (completeDebugDict['parseID']))
def queue(self):
if self._queue:
return self._queue
conn = SQSConnection(self._aws_access_key_id, self._aws_secret_access_key, self._is_secure, self._port, region=self._region)
self._queue = conn.create_queue(self._queue_name)
self._queue.set_message_class(self._message_class)
return self._queue
def test_sqs_longpoll(self):
c = SQSConnection()
queue_name = "test_sqs_longpoll_%s" % int(time.time())
queue = c.create_queue(queue_name)
self.addCleanup(c.delete_queue, queue, True)
messages = []
# The basic idea is to spawn a timer thread that will put something
# on the queue in 5 seconds and verify that our long polling client
# sees the message after waiting for approximately that long.
def send_message():
messages.append(queue.write(queue.new_message("this is a test message")))
t = Timer(5.0, send_message)
t.start()
self.addCleanup(t.join)
start = time.time()
response = queue.read(wait_time_seconds=10)
end = time.time()
t.join()
self.assertEqual(response.id, messages[0].id)
self.assertEqual(response.get_body(), messages[0].get_body())
# The timer thread should send the message in 5 seconds, so
# we're giving +- .5 seconds for the total time the queue
# was blocked on the read call.
self.assertTrue(4.5 <= (end - start) <= 5.5)
def test_get_messages_attributes(self):
conn = SQSConnection()
current_timestamp = int(time.time())
queue_name = 'test%d' % int(time.time())
test = conn.create_queue(queue_name)
self.addCleanup(conn.delete_queue, test)
time.sleep(65)
# Put a message in the queue.
m1 = Message()
m1.set_body('This is a test message.')
test.write(m1)
self.assertEqual(test.count(), 1)
# Check all attributes.
msgs = test.get_messages(num_messages=1, attributes='All')
for msg in msgs:
self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1')
first_rec = msg.attributes['ApproximateFirstReceiveTimestamp']
first_rec = int(first_rec) / 1000
self.assertTrue(first_rec >= current_timestamp)
# Put another message in the queue.
m2 = Message()
m2.set_body('This is another test message.')
test.write(m2)
self.assertEqual(test.count(), 1)
# Check a specific attribute.
msgs = test.get_messages(num_messages=1,
attributes='ApproximateReceiveCount')
for msg in msgs:
self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1')
with self.assertRaises(KeyError):
msg.attributes['ApproximateFirstReceiveTimestamp']
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:], 'hcq:o:t:',
['help', 'clear', 'queue', 'output', 'timeout'])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError, e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
def __init__(self, *args, **kwargs):
Queue.__init__(self, *args, **kwargs)
c = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
self.queue = c.lookup(self.name)
if not self.queue:
self.queue = c.create_queue(self.name, 1)
self.connection = c
def test_sqs_longpoll(self):
c = SQSConnection()
queue_name = 'test_sqs_longpoll_%s' % int(time.time())
queue = c.create_queue(queue_name)
self.addCleanup(c.delete_queue, queue, True)
messages = []
# The basic idea is to spawn a timer thread that will put something
# on the queue in 5 seconds and verify that our long polling client
# sees the message after waiting for approximately that long.
def send_message():
messages.append(
queue.write(queue.new_message('this is a test message')))
t = Timer(5.0, send_message)
t.start()
self.addCleanup(t.join)
start = time.time()
response = queue.read(wait_time_seconds=10)
end = time.time()
t.join()
self.assertEqual(response.id, messages[0].id)
self.assertEqual(response.get_body(), messages[0].get_body())
# The timer thread should send the message in 5 seconds, so
# we're giving +- .5 seconds for the total time the queue
# was blocked on the read call.
self.assertTrue(4.5 <= (end - start) <= 5.5)
def test_credentialled_SQS_access(self):
"check that the credentials can access the SQS service"
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0], aws_secret_access_key=creds[1], region=region)
q = conn.create_queue("PLEASE_KEEP_FOR_TESTING", 30)
assert q
def get_lowpriority_wait_count(self, scenes):
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
count = 0
for scene in scenes:
q_name = "%s%s_%s_%s" % (self.site_name, '_lowpriority', scene, 'request')
q = conn.create_queue(q_name)
count += q.count()
return count
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:], 'hcq:o:t:r:',
['help', 'clear', 'queue=', 'output=', 'timeout=', 'region='])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
region = ''
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
if o in ('-r', '--region'):
region = a
if region:
c = boto.sqs.connect_to_region(region)
if c is None:
print 'Invalid region (%s)' % region
sys.exit(1)
else:
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError as e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
else:
try:
rs = c.get_all_queues()
except SQSError as e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
for q in rs:
if clear:
n = q.clear()
print 'clearing %d messages from %s' % (n, q.id)
elif output_file:
q.dump(output_file)
else:
print q.id, q.count(vtimeout=timeout)
def setup_sqs_queue(self):
conn = SQSConnection()
q = conn.create_queue('some-queue')
m = Message()
m.set_body('This is my first message.')
q.write(m)
self.assertEqual(q.count(), 1)
def adjust_ghx(self, file_name, scene):
scene = scene.replace('.3dm','')
q_name = "%s_%s_%s" % (self.site_name, scene, 'request')
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
q = conn.create_queue(q_name)
q.set_message_class(Message)
body = simplejson.dumps({'operation': 'adjust_ghx', 'gh_file': file_name})
sys.stderr.write(body + "\n")
conn.send_message_batch(q, [(0,base64.b64encode(body),0)])
def __init__(self, conf, name):
super(MessageQueue, self).__init__(conf, name)
conn = SQSConnection(
conf['access_key'], conf['secret_access_key'])
self._queue = conn.create_queue(self.name)
self._queue.set_attribute(
'MessageRetentionPeriod', base.FOURTEEN_DAYS)
self._vtime = base.get_vtime(conf)
def push(body, queue='pi-status'):
'''
Create a JSON-encoded boto-style Message object and write it to the queue.
'''
sqs = SQSConnection()
sqs_queue = sqs.create_queue(queue)
message = JSONMessage(body=body)
sqs_queue.write(message)
def setup_sqs_queue(self):
conn = SQSConnection()
q = conn.create_queue('some-queue')
m = Message()
m.set_body('This is my first message.')
q.write(m)
self.assertEqual(q.count(), 1)
class TweetQueue:
def __init__(self,config):
self.conn = SQSConnection(config["aws-s3"]["access-key-id"], config["aws-s3"]["secret-access-key"])
self.q = self.conn.create_queue('condor-tweets')
def enqueueTweets(self,tweets):
if len(tweets) > 0:
m = Message()
m.set_body(json.dumps(map(slimTweet,tweets)))
self.q.write(m)
class UserQueue:
def __init__(self,config):
self.conn = SQSConnection(config["aws-s3"]["access-key-id"], config["aws-s3"]["secret-access-key"])
self.q = self.conn.create_queue('condor-users')
def enqueueUsers(self,users):
if len(users) > 0:
m = Message()
m.set_body(json.dumps(users))
self.q.write(m)
def queue(q_name, msg):
conn_logger.debug('queue')
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
m = Message()
m.set_body(msg)
rs = q.write(m)
conn_logger.debug('put "%s" to %s queue: ' % (msg, q_name))
def queue(q_name, msg):
conn_logger.debug('queue')
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
m = Message()
m.set_body(msg)
rs = q.write(m)
conn_logger.debug('put "%s" to %s queue: ' % (msg, q_name))
class islandoraEncodingDaemon(Daemon):
def run(self, config_file):
self.init_config(config_file)
self.init_logger()
self.init_sqs()
self.max_workers = self.config.getint('Threading', 'number_workers')
self.sleep_time = self.config.getint('Threading', 'sleep_time')
while True:
self.logger.info('Daemon looking for jobs for workers.')
for worker_id in range(self.max_workers):
self.logger.info('Daemon reports queue length is currently %s.', self.queue.count())
next_queue_message = self.queue.read()
if next_queue_message is not None:
self.logger.info('Daemon found a job - assigning to worker %s.', worker_id)
worker = islandoraEncodingWorker(
worker_id,
next_queue_message,
self.queue,
self.logger,
self.config
)
worker.start()
for thread in threading.enumerate():
if thread is not threading.currentThread():
thread.join()
self.logger.info('All workers retired, daemon sleeping for %s seconds.', self.sleep_time)
time.sleep(self.sleep_time)
def init_config(self, config_filepath):
self.config = ConfigParser.SafeConfigParser()
self.config.read(config_filepath)
def init_logger(self):
self.logger = logging.getLogger('newspaper_encoding')
self.hdlr = logging.FileHandler(self.config.get('Logging', 'log_file'))
logLevelValue = getattr(
logging,
self.config.get('Logging', 'log_level')
)
self.logger.setLevel(logLevelValue)
self.hdlr.setLevel(logLevelValue)
self.formatter = logging.Formatter(self.config.get('Logging', 'log_format'))
self.hdlr.setFormatter(self.formatter)
self.logger.addHandler(self.hdlr)
def init_sqs(self):
try:
self.sqsConn = SQSConnection(
self.config.get('SQS', 'aws_access_key'),
self.config.get('SQS', 'aws_secret_key')
)
self.queue = self.sqsConn.create_queue(self.config.get('SQS', 'sqs_queue_name'))
except:
self.logger.error('Cannot initialize SQS queue (check credentials?)')
def sendMessageToSQSQueue(self, message_body, queue_name):
try:
sqsConn = SQSConnection(self.access_key, self.secret_key)
queue = sqsConn.create_queue(queue_name)
new_item_message = Message()
new_item_message.set_body(message_body)
if queue.write(new_item_message) :
self.logger.info('Message added to processing queue.')
else :
self.logger.error('SQS service did not accept object into the queue.')
except Exception, e:
self.logger.error('Message could not be added to processing queue. ' + str(e))
def pop_loop(queue='pi-status', wait=5):
# like pop(), but iterate
sqs = SQSConnection()
sqs_queue = sqs.create_queue(queue)
sqs_queue.set_message_class(JSONMessage)
while True:
message = sqs_queue.read(wait_time_seconds=wait)
if message is not None:
body = message.get_body()
message.delete()
yield body
def create_queue(name):
if settings.OPENSOCIAL_DEBUG: # ローカル環境の場合なにもしない
return None
if not name:
raise JobQueueError('Not specified queue name')
# Connect to SQS queue
sri = SQSRegionInfo(name='ap-northeast-1', endpoint='ap-northeast-1.queue.amazonaws.com')
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=sri)
q = conn.create_queue(name)
return q
def process_config():
config = ConfigParser.ConfigParser()
config.read(config_file)
try:
tmp_dir = config.get("global", "tmp_dir")
log_file = config.get("global", "log_file")
log_level = config.get("global", "log_level")
except:
pass
s3logger.setLevel(LEVELS.get(log_level, logging.INFO))
file_handler = logging.handlers.WatchedFileHandler(log_file)
s3logger.addHandler(file_handler)
log_label("starting ..." )
log_msg( "Temp dir: %s" % tmp_dir)
try:
# AWS auth details
aws_access_key = config.get("AWSauth", "aws_access_key")
aws_secret_key = config.get("AWSauth", "aws_secret_key")
# S3 configuration details
s3_bucket = config.get("S3config", "s3_bucket")
# SQS job queue configuration details
queue_name = config.get("SQSconfig", "queue_name")
conn = SQSConnection(aws_access_key, aws_secret_key)
queue = conn.create_queue(queue_name)
queue.set_message_class(MHMessage)
sleep_time = float(config.get("SQSconfig", "sleep_time"))
log_msg("sleep time: %s" % sleep_time)
for section in config.sections():
if re.search("job", section):
suffix = config.get(section, "match")
log_msg("processing section [%s]" % section)
job_match[suffix] = []
for option in config.options(section):
if re.search("exec", option):
job_match[suffix].append( config.get(section, option) )
log_debug("CMDs for suffix [png]")
for cmd in job_match[suffix]:
log_debug(" %s" % cmd)
except Exception, e:
log_err("Error reading config file [%s]: %s" % (config_file, e))
sys.exit(1)
def __init__(self, queue="sqs_handler_debug", aws_key=None, secret_key=None):
''' Sends log messages to SNS. Parameters:
* queue is the SQS queue. This will be created if it does not exist.
* Optional: aws_key and secret_key. If these don't exist, it will look
at the appropriate environment variables.
'''
logging.Handler.__init__(self)
if aws_key and secret_key:
conn = SQSConnection(aws_key, secret_key)
else:
conn = SQSConnection()
self.q = conn.create_queue(queue)
def send_ds_encode_sqs(pid, dsid):
config = ConfigParser.SafeConfigParser()
config.read("awsMiscTools.conf")
message_body = json.dumps({"pid": pid, "dsid": dsid, "language": "eng"})
try:
sqsConn = SQSConnection(config.get("SQS", "access_key"), config.get("SQS", "secret_key"))
queue = sqsConn.create_queue(config.get("SQS", "queue_name"))
except:
sys.exit("Cannot initialize SQS queue (check credentials?)")
message = Message()
message.set_body(message_body)
print message_body
queue.write(message)
class SQS(object):
def __init__(self, config="config.ini"):
if isinstance(config, basestring):
config = credentials.ConfigFileCredentials(config)
elif not isinstance(config, credentials.Credentials):
raise TypeError("Unsupported config parameter type")
aws_access_key_id, aws_secret_access_key, aws_queue = config.get_data()
try:
self.conn = SQSConnection(aws_access_key_id, aws_secret_access_key)
self.set_queue(aws_queue)
except:
print 'Error connection'
def get_all_queues(self):
return self.conn.get_all_queues()
def get_queue_attributes(self):
return self.conn.get_queue_attributes(self.queue, attribute='All')
def create_queue(self, queue, timeout):
return self.conn.create_queue(queue, timeout)
def set_queue(self, queue):
self.queue = self.conn.get_queue(queue)
return True
def get_messages(self, limit=10):
return self.queue.get_messages(limit)
def count(self):
#print "Count: %s" % self.queue.count()
return self.queue.count()
def write(self, data):
m = Message()
m.set_body(json.dumps(data))
return self.queue.write(m)
def delete(self, id):
#print "Eliminando %s" % id
self.queue.delete_message(id)
def clear(self):
return self.queue.clear()
def delete_queue(self):
return self.conn.delete_queue(self.queue)
def test_queue_deletion_affects_full_queues(self):
conn = SQSConnection()
initial_count = len(conn.get_all_queues())
empty = conn.create_queue("empty%d" % int(time.time()))
full = conn.create_queue("full%d" % int(time.time()))
time.sleep(60)
# Make sure they're both around.
self.assertEqual(len(conn.get_all_queues()), initial_count + 2)
# Put a message in the full queue.
m1 = Message()
m1.set_body("This is a test message.")
full.write(m1)
self.assertEqual(full.count(), 1)
self.assertTrue(conn.delete_queue(empty))
# Here's the regression for the docs. SQS will delete a queue with
# messages in it, no ``force_deletion`` needed.
self.assertTrue(conn.delete_queue(full))
# Wait long enough for SQS to finally remove the queues.
time.sleep(90)
self.assertEqual(len(conn.get_all_queues()), initial_count)
def test_queue_deletion_affects_full_queues(self):
conn = SQSConnection()
initial_count = len(conn.get_all_queues())
empty = conn.create_queue('empty%d' % int(time.time()))
full = conn.create_queue('full%d' % int(time.time()))
time.sleep(60)
# Make sure they're both around.
self.assertEqual(len(conn.get_all_queues()), initial_count + 2)
# Put a message in the full queue.
m1 = Message()
m1.set_body('This is a test message.')
full.write(m1)
self.assertEqual(full.count(), 1)
self.assertTrue(conn.delete_queue(empty))
# Here's the regression for the docs. SQS will delete a queue with
# messages in it, no ``force_deletion`` needed.
self.assertTrue(conn.delete_queue(full))
# Wait long enough for SQS to finally remove the queues.
time.sleep(90)
self.assertEqual(len(conn.get_all_queues()), initial_count)
def setUp(self):
# Ensure the queue is clear before we start, or we'll lose more hair
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0],
aws_secret_access_key=creds[1],
region=region)
q = conn.create_queue(self.qname, 30)
cruft = q.get_messages(10)
while cruft:
for c in cruft:
print 'deleting old message %s', c.__dict__
q.delete_message(c)
cruft = q.get_messages(10)
def dequeue(q_name, func, conn=None):
conn_logger.debug('dequeuing %s' % q_name)
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
#q.clear()
rs = q.get_messages(5)
if len(rs) == 0:
conn_logger.info('%s queue is empty' % q_name)
else:
for m in rs:
msg = m.get_body()
if func(msg, conn):
q.delete_message(m)
conn_logger.info('dequeue done: %s from %s' % (msg, q_name))
def setUp(self):
# Ensure the queue is clear before we start, or we'll lose more hair
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0],
aws_secret_access_key=creds[1],
region=region)
q = conn.create_queue(self.qname, 30)
cruft = q.get_messages(10)
while cruft:
for c in cruft:
print 'deleting old message %s', c.__dict__
q.delete_message(c)
cruft = q.get_messages(10)
def dequeue(q_name, func, conn=None):
conn_logger.debug('dequeuing %s' % q_name)
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
#q.clear()
rs = q.get_messages(5)
if len(rs) == 0:
conn_logger.info('%s queue is empty' % q_name)
else:
for m in rs:
msg = m.get_body()
if func(msg, conn):
q.delete_message(m)
conn_logger.info('dequeue done: %s from %s' % (msg, q_name))
class awssqs:
def __init__(self, name, visibility_timeout=60):
self.visibility_timeout = visibility_timeout
self.conn = SQSConnection(region=boto.sqs.regions()[1]) # eu-west1
self.q = self.conn.create_queue(name)
if self.q is None:
raise Exception("Could not get that queue " + name)
self.name = name
def write(self, message):
if self.q is None:
raise Exception("Queue is none " + self.name)
m = Message()
m.set_body(message)
success = self.q.write(m)
failed = 0
while not success:
time.sleep(5)
success = self.q.write(m) # Keep trying until success
failed += 1
if failed > 10:
raise Exception("Failed over 10 times to write to queue %s!" %
self.name)
# Return a Message, use m.get_body() to get the text
def read(self):
if self.q is None:
raise Exception("Queue is none " + self.name)
rs = self.q.get_messages(visibility_timeout=self.visibility_timeout)
if len(rs) > 0:
m = rs[0]
return m
return None
def length(self):
if self.q is None:
raise Exception("Queue is none " + self.name)
rs = self.q.get_messages(visibility_timeout=self.visibility_timeout)
return len(rs)
def delete(self, m):
self.q.delete_message(m)
def deleteQueue(self):
self.conn.delete_queue(self.q, force_deletion=True)
self.q = None
self.conn.close()
class awssqs:
def __init__(self, name, visibility_timeout=60):
self.visibility_timeout = visibility_timeout
self.conn = SQSConnection(region=boto.sqs.regions()[1]) # eu-west1
self.q = self.conn.create_queue(name)
if self.q is None:
raise Exception("Could not get that queue " + name)
self.name = name
def write(self, message):
if self.q is None:
raise Exception("Queue is none " + self.name)
m = Message()
m.set_body(message)
success = self.q.write(m)
failed = 0
while not success:
time.sleep(5)
success = self.q.write(m) # Keep trying until success
failed +=1
if failed > 10:
raise Exception("Failed over 10 times to write to queue %s!" % self.name)
# Return a Message, use m.get_body() to get the text
def read(self):
if self.q is None:
raise Exception("Queue is none " + self.name)
rs = self.q.get_messages(visibility_timeout = self.visibility_timeout)
if len(rs) > 0:
m = rs[0]
return m
return None
def length(self):
if self.q is None:
raise Exception("Queue is none " + self.name)
rs = self.q.get_messages(visibility_timeout = self.visibility_timeout)
return len(rs)
def delete(self, m):
self.q.delete_message(m)
def deleteQueue(self):
self.conn.delete_queue(self.q, force_deletion = True)
self.q = None
self.conn.close()
def pop(queue='pi-status'):
'''
Pop the next Message off the queue, immediately delete it
(mark it as consumed), and return the body string.
Presumably, these will be a JSONMessage instances, and get_body() will return a dict/list/etc.
'''
sqs = SQSConnection()
sqs_queue = sqs.create_queue(queue)
sqs_queue.set_message_class(JSONMessage)
message = sqs_queue.read()
if message is not None:
body = message.get_body()
message.delete()
return body
class SQSQueue(object):
implements(interface.IQueue)
def __init__(self, name, acc_sec_pair=None, visibility_timeout=30):
self.name = name
if acc_sec_pair is None:
acc_sec_pair = aws_credentials.get_credentials()
self.region_name = 'eu-west-1'
self.region = [r for r in regions() if r.name == self.region_name][0]
self.conn = SQSConnection(aws_access_key_id=acc_sec_pair[0],
aws_secret_access_key=acc_sec_pair[1],
region=self.region)
self.q = self.conn.create_queue(name, visibility_timeout)
self.q.set_message_class(event.SQSEvent)
def put_event(self, event_in):
assert isinstance(event_in, event.SQSEvent)
self.q.write(event_in)
def get_event(self, block=True, timeout=None):
ev = self.q.read()
return ev
def event_factory(self, event_id, data={}):
return event.SQSEvent(event_id=event_id, data=data)
def _expand_decimal_values(self, data):
"""expand Decimals which otherwise cause msgpack exception"""
for k, v in data.items():
if type(v) is decimal.Decimal:
data[k] = unicode(v)
def create_and_send(self, event_id, data={}):
self._expand_decimal_values(data)
ev = self.event_factory(event_id=event_id, data=data)
self.put_event(ev)
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad*queue*name')
self.fail('queue name should have been bad')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues+1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
time.sleep(60)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(60)
assert queue.count_slow() == 1
time.sleep(90)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(30)
assert queue.count_slow() == 0
# try a batch write
num_msgs = 10
msgs = [(i, 'This is message %d' % i, 0) for i in range(num_msgs)]
queue.write_batch(msgs)
# try to delete all of the messages using batch delete
deleted = 0
while deleted < num_msgs:
time.sleep(5)
msgs = queue.get_messages(num_msgs)
if msgs:
br = queue.delete_message_batch(msgs)
deleted += len(br.results)
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
queue.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue.new_message()
message['foo'] = 'bar'
queue.write(message)
message_body = {'fie' : 'baz', 'foo' : 'bar'}
message = queue.new_message(body=message_body)
queue.write(message)
time.sleep(30)
m = queue.read()
assert m['foo'] == 'bar'
# now delete that queue and messages
c.delete_queue(queue, True)
print '--- tests completed ---'
class AmazonSQS(StorageService):
def __init__(self):
self.last_msg = -1
self.buffered = []
self.msg_num = 0
self.access_key_id = ""
self.secret_access_key = ""
self.conn = SQSConnection(self.access_key_id, self.secret_access_key,
True, None, None, None, None, None)
def new_connection(self, conn_id, conn_name):
conns = self.conn.create_queue("connections")
self.put(conns, conn_name)
return (self.get_requests_loc(conn_id),
self.get_responses_loc(conn_id))
def get_place(self, place_name):
return self.create_queue(place_name)
def get_connection(self):
conns = self.conn.create_queue("connections")
return self.get(conns)
def get_connections_loc(self):
return self.conn.create_queue("connections")
def get_responses_loc(self, conn_id):
return self.conn.create_queue("%s_response" % conn_id)
def get_requests_loc(self, conn_id):
return self.conn.create_queue("%s_request" % conn_id)
def put(self, q, data, use_seq_num=False):
m = Message()
if (use_seq_num):
data = str(self.msg_num) + " " + data
self.msg_num += 1
m.set_body(data)
# print "Putting data: %s"%data
status = q.write(m)
if (status == False):
print "Put failed"
return False
return True
def get(self, q, use_seq_num=False):
if (len(self.buffered) > 0
and self.buffered[0][0] == self.last_msg + 1):
data = self.buffered[0][1]
self.buffered = self.buffered[1:]
self.last_msg += 1
return data
m = q.read()
if (m == None):
return False
else:
q.delete_message(m)
if (not use_seq_num):
return m.get_body()
else:
msg_num, sep, msg = m.get_body().partition(" ")
msg_num = int(msg_num)
if (msg_num == self.last_msg + 1):
# print "Using msg %d"%msg_num
self.last_msg = msg_num
return msg
else:
# print "Buffering msg %d, last_msg: %d"%(msg_num, self.last_msg)
self.buffered.append((msg_num, msg))
self.buffered.sort()
def create_queue(self, name):
return self.conn.create_queue(name)
def delete(self, place, data):
msgs = place.get_messages()
for m in msgs:
if (m.get_body() == data):
place.delete_message(m)
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad_queue_name')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues + 1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
time.sleep(60)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(30)
assert queue.count_slow() == 1
time.sleep(30)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue.read()
assert message
if c.APIVersion == '2007-05-01':
# now terminate the visibility timeout for this message
message.change_visibility(0)
# now see if we can read it in the queue
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(30)
assert queue.count_slow() == 0
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
queue.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue.new_message()
message['foo'] = 'bar'
queue.write(message)
message_body = {'fie': 'baz', 'foo': 'bar'}
message = queue.new_message(body=message_body)
queue.write(message)
time.sleep(30)
m = queue.read()
assert m['foo'] == 'bar'
# now delete that queue and messages
c.delete_queue(queue, True)
print '--- tests completed ---'
config = ConfigParser.RawConfigParser()
config.read('/home/ubuntu/fz/conf/app.cfg')
AWS_ACCESS_KEY = config.get('aws', 'accesskey')
AWS_SECRET_KEY = config.get('aws', 'secretkey')
ParsePy.APPLICATION_ID = config.get('parse', 'P_APP_ID')
ParsePy.MASTER_KEY = config.get('parse', 'P_MASTER_KEY')
_convert = config.get('fz', 'convertQueueName')
_upload = config.get('fz', 'uploadQueueName')
activedomain = 'http://%s' % (config.get('fz', 'activedomain'))
conn = SQSConnection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
convert_q = conn.create_queue(_convert, 120)
upload_q = conn.create_queue(_upload, 120)
def processMessageFromSQS(temp_message):
message_dict = json.loads(temp_message.get_body())
convert_q.delete_message(temp_message)
pprint.pprint(message_dict)
figureObject = ParsePy.ParseQuery("UploadObject").get(
message_dict['parseID'])
if figureObject == None:
print 'object not found'
class SQSAuthParams(AWSMockServiceTestCase):
connection_class = SQSConnection
def setUp(self):
super(SQSAuthParams, self).setUp()
def default_body(self):
return """<?xml version="1.0"?>
<CreateQueueResponse>
<CreateQueueResult>
<QueueUrl>
https://queue.amazonaws.com/599169622985/myqueue1
</QueueUrl>
</CreateQueueResult>
<ResponseMetadata>
<RequestId>54d4c94d-2307-54a8-bb27-806a682a5abd</RequestId>
</ResponseMetadata>
</CreateQueueResponse>"""
def test_auth_service_name_override(self):
self.set_http_response(status_code=200)
# We can use the auth_service_name to change what service
# name to use for the credential scope for sigv4.
self.service_connection.auth_service_name = 'service_override'
self.service_connection.create_queue('my_queue')
# Note the service_override value instead.
self.assertIn('us-east-1/service_override/aws4_request',
self.actual_request.headers['Authorization'])
def test_class_attribute_can_set_service_name(self):
self.set_http_response(status_code=200)
# The SQS class has an 'AuthServiceName' param of 'sqs':
self.assertEqual(self.service_connection.AuthServiceName, 'sqs')
self.service_connection.create_queue('my_queue')
# And because of this, the value of 'sqs' will be used instead of
# 'queue' for the credential scope:
self.assertIn('us-east-1/sqs/aws4_request',
self.actual_request.headers['Authorization'])
def test_auth_region_name_is_automatically_updated(self):
region = SQSRegionInfo(name='us-west-2',
endpoint='us-west-2.queue.amazonaws.com')
self.service_connection = SQSConnection(
https_connection_factory=self.https_connection_factory,
aws_access_key_id='aws_access_key_id',
aws_secret_access_key='aws_secret_access_key',
region=region)
self.initialize_service_connection()
self.set_http_response(status_code=200)
self.service_connection.create_queue('my_queue')
# Note the region name below is 'us-west-2'.
self.assertIn('us-west-2/sqs/aws4_request',
self.actual_request.headers['Authorization'])
def test_set_get_auth_service_and_region_names(self):
self.service_connection.auth_service_name = 'service_name'
self.service_connection.auth_region_name = 'region_name'
self.assertEqual(self.service_connection.auth_service_name,
'service_name')
self.assertEqual(self.service_connection.auth_region_name,
'region_name')
def test_get_queue_with_owner_account_id_returns_queue(self):
self.set_http_response(status_code=200)
self.service_connection.create_queue('my_queue')
self.service_connection.get_queue('my_queue', '599169622985')
assert 'QueueOwnerAWSAccountId' in self.actual_request.params.keys()
self.assertEquals(self.actual_request.params['QueueOwnerAWSAccountId'],
'599169622985')
class SNSSubcribeSQSTest(unittest.TestCase):
sqs = True
sns = True
def setUp(self):
self.sqsc = SQSConnection()
self.snsc = SNSConnection()
def get_policy_statements(self, queue):
attrs = queue.get_attributes('Policy')
policy = json.loads(attrs.get('Policy', "{}"))
return policy.get('Statement', {})
def test_correct_sid(self):
now = time.time()
topic_name = queue_name = "test_correct_sid%d" % (now)
timeout = 60
queue = self.sqsc.create_queue(queue_name, timeout)
self.addCleanup(self.sqsc.delete_queue, queue, True)
queue_arn = queue.arn
topic = self.snsc.create_topic(topic_name)
topic_arn = topic['CreateTopicResponse']['CreateTopicResult']\
['TopicArn']
self.addCleanup(self.snsc.delete_topic, topic_arn)
expected_sid = hashlib.md5(
(topic_arn + queue_arn).encode('utf-8')).hexdigest()
resp = self.snsc.subscribe_sqs_queue(topic_arn, queue)
found_expected_sid = False
statements = self.get_policy_statements(queue)
for statement in statements:
if statement['Sid'] == expected_sid:
found_expected_sid = True
break
self.assertTrue(found_expected_sid)
def test_idempotent_subscribe(self):
now = time.time()
topic_name = queue_name = "test_idempotent_subscribe%d" % (now)
timeout = 60
queue = self.sqsc.create_queue(queue_name, timeout)
self.addCleanup(self.sqsc.delete_queue, queue, True)
initial_statements = self.get_policy_statements(queue)
queue_arn = queue.arn
topic = self.snsc.create_topic(topic_name)
topic_arn = topic['CreateTopicResponse']['CreateTopicResult']\
['TopicArn']
self.addCleanup(self.snsc.delete_topic, topic_arn)
resp = self.snsc.subscribe_sqs_queue(topic_arn, queue)
time.sleep(3)
first_subscribe_statements = self.get_policy_statements(queue)
self.assertEqual(len(first_subscribe_statements),
len(initial_statements) + 1)
resp2 = self.snsc.subscribe_sqs_queue(topic_arn, queue)
time.sleep(3)
second_subscribe_statements = self.get_policy_statements(queue)
self.assertEqual(len(second_subscribe_statements),
len(first_subscribe_statements))
import ParsePy
from boto.s3.key import Key
from boto.sqs.message import Message
from boto.sqs.connection import SQSConnection
from boto.s3.connection import S3Connection
AWS_ACCESS_KEY = ''
AWS_SECRET_KEY = ''
ParsePy.APPLICATION_ID = ""
ParsePy.MASTER_KEY = ""
conn = SQSConnection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
convert_q = conn.create_queue('FZconvertQueue', 120)
upload_q = conn.create_queue('FZuploadQueue', 120)
convert_q.clear()
upload_q.clear()
query = ParsePy.ParseQuery("UploadObject")
fzobjects = query.fetch()
for fzobj in fzobjects:
print fzobj.objectId()
fzobj.delete()
print convert_q.count()
print upload_q.count()
print len(fzobjects)
import conf
from time import time
import json
import random
from boto.sqs.connection import SQSConnection, Message
from flask import Flask, render_template, request
VERSION = "0.1"
APP = Flask(__name__)
SQS_CONN = SQSConnection(conf.AWS_KEY, conf.AWS_SECRET)
SQS_QUEUE = SQS_CONN.create_queue(conf.SQS_QUEUE_NAME)
SURVEY = json.load(open(conf.SURVEY_FILE, 'rt'))
QUESTIONS_BY_ID = {q['id']: q for q in SURVEY['questions']}
ORDERS_BY_ID = {o['id']: o for o in SURVEY['question_orders']}
def build_sampling_map():
m = {}
i = 0
for o in SURVEY['question_orders']:
for j in range(o['frequency']):
m[i + j] = o['id']
i += o['frequency']
return m
SAMPLING_MAP = build_sampling_map()
# this script expects 2 environment variables
# 1. SQS_KEY_ID (preferably an IAM user with limited rights)
# 2. SQS_SECRET_KEY (accompanying secret key)
# 3. SQS_TASK_QUEUE (the queue to use)
import os
import sys
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
import utils
# your amazon keys
key = os.environ['SQS_KEY_ID']
access = os.environ['SQS_ACCESS_KEY']
queue = os.environ['SQS_TASK_QUEUE']
if __name__ == '__main__':
region_info = utils.get_region_info()
sqs = SQSConnection(key, access, region=region_info)
tasks = sqs.create_queue(queue)
m = Message()
m.set_body(sys.argv[1])
tasks.write(m)
