def request_images(self, params):
if len(params)==0: return
scene = params[0]['scene']
scene = scene.replace('.3dm','')
lowpriority = ""
#print "low_priority"
#print params[0]['low_priority']
if params[0]['low_priority']==True:
lowpriority = "_lowpriority"
q_name = "%s%s_%s_%s" % (self.site_name, lowpriority, scene, 'request')
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
q = conn.create_queue(q_name)
q.set_message_class(Message)
params = self.trunc_params(params)
messages = []
for i in range(len(params)):
#params[i]['params']['textParam'] = 'sunsun'
body = simplejson.dumps(params[i])
sys.stderr.write(body + "\n")
messages.append((i,base64.b64encode(body),0))
for j in range(int(math.ceil(float(len(params))/10.0))):
conn.send_message_batch(q, messages[j*10:(j+1)*10])
sys.stderr.write("\n\n\nSent messages\n\n\n")
return
def _process_message():
if not g.sitemap_sqs_queue:
return
sqs = SQSConnection()
sqs_q = sqs.get_queue(g.sitemap_sqs_queue)
messages = sqs.receive_message(sqs_q, number_messages=1)
if not messages:
return
message, = messages
js = json.loads(message.get_body())
s3path = parse_s3_path(js['location'])
# There are some error cases that allow us to get messages
# for sitemap creation that are now out of date.
timestamp = js.get('timestamp')
if timestamp is not None and _before_last_sitemap(timestamp):
sqs_q.delete_message(message)
return
g.log.info("Got import job %r", js)
subreddits = find_all_subreddits(s3path)
store_sitemaps_in_s3(subreddits)
sqs_q.delete_message(message)
def test_get_messages_attributes(self):
conn = SQSConnection()
current_timestamp = int(time.time())
queue_name = 'test%d' % int(time.time())
test = conn.create_queue(queue_name)
self.addCleanup(conn.delete_queue, test)
time.sleep(65)
# Put a message in the queue.
m1 = Message()
m1.set_body('This is a test message.')
test.write(m1)
self.assertEqual(test.count(), 1)
# Check all attributes.
msgs = test.get_messages(num_messages=1, attributes='All')
for msg in msgs:
self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1')
first_rec = msg.attributes['ApproximateFirstReceiveTimestamp']
first_rec = int(first_rec) / 1000
self.assertTrue(first_rec >= current_timestamp)
# Put another message in the queue.
m2 = Message()
m2.set_body('This is another test message.')
test.write(m2)
self.assertEqual(test.count(), 1)
# Check a specific attribute.
msgs = test.get_messages(num_messages=1,
attributes='ApproximateReceiveCount')
for msg in msgs:
self.assertEqual(msg.attributes['ApproximateReceiveCount'], '1')
with self.assertRaises(KeyError):
msg.attributes['ApproximateFirstReceiveTimestamp']
def test_sqs_longpoll(self):
c = SQSConnection()
queue_name = "test_sqs_longpoll_%s" % int(time.time())
queue = c.create_queue(queue_name)
self.addCleanup(c.delete_queue, queue, True)
messages = []
# The basic idea is to spawn a timer thread that will put something
# on the queue in 5 seconds and verify that our long polling client
# sees the message after waiting for approximately that long.
def send_message():
messages.append(queue.write(queue.new_message("this is a test message")))
t = Timer(5.0, send_message)
t.start()
self.addCleanup(t.join)
start = time.time()
response = queue.read(wait_time_seconds=10)
end = time.time()
t.join()
self.assertEqual(response.id, messages[0].id)
self.assertEqual(response.get_body(), messages[0].get_body())
# The timer thread should send the message in 5 seconds, so
# we're giving +- .5 seconds for the total time the queue
# was blocked on the read call.
self.assertTrue(4.5 <= (end - start) <= 5.5)
class SqsApi(object):
def __init__(self):
self.conn = SQSConnection(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY)
def create_queue(self, name="MuscleOpsQ"):
return self.conn.create_queue(name)
def list_queues(self):
return self.conn.get_all_queues()
def write_message(self,
q,
msg="If you're sleepy and you know it; clap your hands!"
):
m = RawMessage()
m.set_body(msg)
q.write(m)
def read_message(self,
q,
msg="If you're sleepy and you know it; clap your hands!"):
rs = q.get_messages()
m = rs[0].get_body() if len(rs) else msg
return m
def test_worker_consumes_queue(self):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
queue = sqs.create_queue('test_events')
queue.write(make_message({'test': '1'}))
self.assertEqual(queue.count(), 1)
worker = Worker()
worker.ctrl.wait_time_seconds = 0.1
worker.idle_time_seconds = 0.1
worker.add_consumer('test_events', dummy_consumer)
with spawn_worker(worker):
time.sleep(.2)
self.assertEqual(queue.count(), 0)
queue.write(make_message({'test': '2'}))
self.assertEqual(queue.count(), 1)
time.sleep(.2)
self.assertEqual(queue.count(), 0)
def commentsubmit(req):
sdb = boto.connect_sdb(AWSKey, AWSSecret)
domain = sdb.get_domain('comment')
form = req.form
imagekey = form['imagekey']
user = form['commentuser']
cmt = form['comment']
import uuid
from time import strftime
guid = str(uuid.uuid1())
item = domain.new_item(guid)
item['submituser'] = user
item['imagekey'] = imagekey
item['comment'] = cmt
item['status'] = "processing"
item['submitdate'] = strftime("%Y-%m-%dT%H:%M:%S")
item.save()
sqsconn = SQSConnection(AWSKey, AWSSecret)
q = sqsconn.get_queue('commentprocess')
request = {}
request['commentkey'] = guid
request['submitdate'] = strftime("%Y-%m-%dT%H:%M:%S")
request['comment'] = str(cmt)
request['submituser'] = str(user)
m = RawMessage()
m.set_body(json.write(request))
status = q.write(m)
response = {}
if status==m:
response['complete'] = True
response['commentkey'] = guid
else:
response['complete'] = False
return json.write(response)
def test_sqs_longpoll(self):
c = SQSConnection()
queue_name = 'test_sqs_longpoll_%s' % int(time.time())
queue = c.create_queue(queue_name)
self.addCleanup(c.delete_queue, queue, True)
messages = []
# The basic idea is to spawn a timer thread that will put something
# on the queue in 5 seconds and verify that our long polling client
# sees the message after waiting for approximately that long.
def send_message():
messages.append(
queue.write(queue.new_message('this is a test message')))
t = Timer(5.0, send_message)
t.start()
self.addCleanup(t.join)
start = time.time()
response = queue.read(wait_time_seconds=10)
end = time.time()
t.join()
self.assertEqual(response.id, messages[0].id)
self.assertEqual(response.get_body(), messages[0].get_body())
# The timer thread should send the message in 5 seconds, so
# we're giving +- .5 seconds for the total time the queue
# was blocked on the read call.
self.assertTrue(4.5 <= (end - start) <= 5.5)
def op_consume(self, args):
sqs = SQSConnection()
q = sqs.get_queue('test')
q.set_message_class(RawMessage)
for message in q.get_messages():
print message.get_body()
q.delete_message(message)
def __init__(self, *args, **kwargs):
Queue.__init__(self, *args, **kwargs)
c = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
self.queue = c.lookup(self.name)
if not self.queue:
self.queue = c.create_queue(self.name, 1)
self.connection = c
def __init__(self,num_threads):
# figure out who(m) we are
self.hostname = os.uname()[1];
# make a note of when we launched
self.start_time = datetime.datetime.now();
# create a connection to SQS
conn = SQSConnection();
# ask for the JOB_QUEUE
self.jobQueue = conn.get_queue(JOB_QUEUE);
# ask for the ERR_QUEUE
self.errQueue = conn.get_queue(ERR_QUEUE);
# setup reflection for error logging etc
self.reflection = Reflection.Reflect();
# init empty list of threads
self.threads = [];
# make note of the number of requested threads
self.num_threads = num_threads;
# empty node states timer until we get job
self.node_stats_timer = None;
def publish_node_stats(self):
try:
# create a connection to SQS
conn = SQSConnection()
# ask for the QUEUE
q = conn.get_queue(NODE_STATS_QUEUE)
# create a new message
m = Message()
# populate the message with stats
m.set_body(self.node_stats())
# publish the message to SQS
q.write(m)
# schedule another publish
self.schedule_node_stats()
except Exception as e:
# blab about the err on std err
os.sys.stderr.write(str(e) + '\n')
# log error message to the error queue
self.publish_error('publish_node_stats: ' + str(e))
def main():
global task
parser = OptionParser(
"%prog --daemon_status_id <id> --queue_name <queue_name> \
[--nice <0>] [--stdout <file_name|DEFAULT>] [--stderr <file_name>|STDOUT>] [--debug]"
)
parser.add_option(
"--daemon_status_id", action="store", type="int", help="The id of the daemon status that launched this Task"
)
parser.add_option("--queue_name", action="store", type="string", help="The name of the queue from which to read")
parser.add_option("--nice", action="store", type="int", default=0, help="nice this process. defaults to 5.")
parser.add_option(
"--stdout",
action="store",
type="string",
help="Send stdout to this file, or special value 'DEFAULT' \
sends it a the stream unique to this Task request",
)
parser.add_option(
"--stderr",
action="store",
type="string",
help="Send stderr to this file, or special value 'STDOUT' sends it to stdout",
)
parser.add_option("--debug", action="store_true", help="more messages")
(options, args) = parser.parse_args()
# option parsing
if not options.daemon_status_id or not options.queue_name:
sys.exit(parser.get_usage())
log.set_logging_debug(options.debug)
if not options.nice == 0:
os.nice(options.nice)
console_stderr = None
try:
c = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
q = c.get_queue(options.queue_name)
boto_message = q.read()
task = __get_task__(boto_message, options.queue_name)
if task == None:
log.debug("No task in queue '%s' pid:%s" % (options.queue_name, os.getpid()))
sys.exit(133)
else:
log.debug("Starting SQS Queue '%s' Task:%s pid:%s" % (options.queue_name, task.get_id(), os.getpid()))
q.delete_message(boto_message)
console_stderr = __redirect_outputs__(task, options.stdout, options.stderr)
daemon_status = __get_daemon_status__(options.daemon_status_id)
__run_task__(task, daemon_status)
ending_status = task.get_current_run_status()
if ending_status == None:
sys.exit(134)
if not ending_status.was_successful():
sys.exit(1)
except SystemExit, se:
# in python 2.4, SystemExit extends Exception, this is changed in 2.5 to
# extend BaseException, specifically so this check isn't necessary. But
# we're using 2.4; upon upgrade, this check will be unecessary but ignorable.
sys.exit(se.code)
def __init__(self, num_threads):
# figure out who(m) we are
self.hostname = os.uname()[1]
# make a note of when we launched
self.start_time = datetime.datetime.now()
# create a connection to SQS
conn = SQSConnection()
# ask for the JOB_QUEUE
self.jobQueue = conn.get_queue(JOB_QUEUE)
# ask for the ERR_QUEUE
self.errQueue = conn.get_queue(ERR_QUEUE)
# setup reflection for error logging etc
self.reflection = Reflection.Reflect()
# init empty list of threads
self.threads = []
# make note of the number of requested threads
self.num_threads = num_threads
# empty node states timer until we get job
self.node_stats_timer = None
def publish_node_stats(self):
try:
# create a connection to SQS
conn = SQSConnection();
# ask for the QUEUE
q = conn.get_queue(NODE_STATS_QUEUE);
# create a new message
m = Message();
# populate the message with stats
m.set_body(self.node_stats());
# publish the message to SQS
q.write(m);
# schedule another publish
self.schedule_node_stats();
except Exception as e:
# blab about the err on std err
os.sys.stderr.write(str(e)+'\n');
# log error message to the error queue
self.publish_error('publish_node_stats: '+str(e));
def queue(self):
if self._queue:
return self._queue
conn = SQSConnection(self._aws_access_key_id, self._aws_secret_access_key, self._is_secure, self._port, region=self._region)
self._queue = conn.create_queue(self._queue_name)
self._queue.set_message_class(self._message_class)
return self._queue
def main():
try:
opts, args = getopt.getopt(sys.argv[1:], 'hcq:o:t:',
['help', 'clear', 'queue',
'output', 'timeout'])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError, e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
def __init__(self, *args, **kwargs):
Queue.__init__(self, *args, **kwargs)
c = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
self.queue = c.lookup(self.name)
if not self.queue:
self.queue = c.create_queue(self.name, 1)
self.connection = c
def __init__(self, conf, name):
super(MessageQueue, self).__init__(conf, name)
conn = SQSConnection(conf['access_key'], conf['secret_access_key'])
self._queue = conn.create_queue(self.name)
self._queue.set_attribute('MessageRetentionPeriod', base.FOURTEEN_DAYS)
self._vtime = base.get_vtime(conf)
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:], 'hcq:o:t:',
['help', 'clear', 'queue', 'output', 'timeout'])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError, e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
def test_credentialled_SQS_access(self):
"check that the credentials can access the SQS service"
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0], aws_secret_access_key=creds[1], region=region)
q = conn.create_queue("PLEASE_KEEP_FOR_TESTING", 30)
assert q
def __init__(self, name, visibility_timeout=60):
self.visibility_timeout = visibility_timeout
self.conn = SQSConnection(region=boto.sqs.regions()[1]) # eu-west1
self.q = self.conn.create_queue(name)
if self.q is None:
raise Exception("Could not get that queue " + name)
self.name = name
def __init__(self):
self.last_msg = -1
self.buffered = []
self.msg_num = 0
self.access_key_id = ""
self.secret_access_key = ""
self.conn = SQSConnection(self.access_key_id, self.secret_access_key,
True, None, None, None, None, None)
def test_empty_worker(self):
sqs = SQSConnection(region=get_sqs_region('us-east-1'))
self.assertEqual(len(sqs.get_all_queues()), 0)
with spawn_worker(Worker()):
# Worker is empty, not registering any queues
self.assertEqual(len(sqs.get_all_queues()), 0)
def get_lowpriority_wait_count(self, scenes):
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
count = 0
for scene in scenes:
q_name = "%s%s_%s_%s" % (self.site_name, '_lowpriority', scene, 'request')
q = conn.create_queue(q_name)
count += q.count()
return count
def main():
try:
opts, args = getopt.getopt(
sys.argv[1:], 'hcq:o:t:r:',
['help', 'clear', 'queue=', 'output=', 'timeout=', 'region='])
except:
usage()
sys.exit(2)
queue_name = ''
output_file = ''
timeout = 30
region = ''
clear = False
for o, a in opts:
if o in ('-h', '--help'):
usage()
sys.exit()
if o in ('-q', '--queue'):
queue_name = a
if o in ('-o', '--output'):
output_file = a
if o in ('-c', '--clear'):
clear = True
if o in ('-t', '--timeout'):
timeout = int(a)
if o in ('-r', '--region'):
region = a
if region:
c = boto.sqs.connect_to_region(region)
if c is None:
print 'Invalid region (%s)' % region
sys.exit(1)
else:
c = SQSConnection()
if queue_name:
try:
rs = [c.create_queue(queue_name)]
except SQSError as e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
else:
try:
rs = c.get_all_queues()
except SQSError as e:
print 'An Error Occurred:'
print '%s: %s' % (e.status, e.reason)
print e.body
sys.exit()
for q in rs:
if clear:
n = q.clear()
print 'clearing %d messages from %s' % (n, q.id)
elif output_file:
q.dump(output_file)
else:
print q.id, q.count(vtimeout=timeout)
def add_feed_mailserver(mx):
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
conn = SQSConnection('ID', 'KEY')
q = conn.get_queue('mailserver_list')
m = Message()
m.set_body(mx.ipaddr)
q.write(m)
return True
def _config(self, queue_name, aws_access_key_id, aws_secret_access_key, region):
cx = SQSConnection(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region=region,
is_secure=True)
self._queue = cx.get_queue(queue_name)
if not self._queue:
raise Exception('Unable to load sqs queue %s with access_key_id %s in region %s' %
(queue_name, aws_access_key_id, region))
def finshi_task_sns(taskname,node_id):
SQS_Id='TaskFinished'
time = get_time_now()
message = time + '|' + taskname + '|' + str(node_id)
conn = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
my_queue = conn.get_queue(SQS_Id)
m = Message()
m.set_body(message)
my_queue.write(m)
def push(body, queue='pi-status'):
'''
Create a JSON-encoded boto-style Message object and write it to the queue.
'''
sqs = SQSConnection()
sqs_queue = sqs.create_queue(queue)
message = JSONMessage(body=body)
sqs_queue.write(message)
def adjust_ghx(self, file_name, scene):
scene = scene.replace('.3dm','')
q_name = "%s_%s_%s" % (self.site_name, scene, 'request')
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY)
q = conn.create_queue(q_name)
q.set_message_class(Message)
body = simplejson.dumps({'operation': 'adjust_ghx', 'gh_file': file_name})
sys.stderr.write(body + "\n")
conn.send_message_batch(q, [(0,base64.b64encode(body),0)])
def __init__(self, conf, name):
super(MessageQueue, self).__init__(conf, name)
conn = SQSConnection(
conf['access_key'], conf['secret_access_key'])
self._queue = conn.create_queue(self.name)
self._queue.set_attribute(
'MessageRetentionPeriod', base.FOURTEEN_DAYS)
self._vtime = base.get_vtime(conf)
def setup_sqs_queue(self):
conn = SQSConnection()
q = conn.create_queue('some-queue')
m = Message()
m.set_body('This is my first message.')
q.write(m)
self.assertEqual(q.count(), 1)
def setup_sqs_queue(self):
conn = SQSConnection()
q = conn.create_queue('some-queue')
m = Message()
m.set_body('This is my first message.')
q.write(m)
self.assertEqual(q.count(), 1)
class TaskSQS:
TASK_QUEUE = "tasks"
RESULT_QUEUE = "results"
VISIBILITY_TIMEOUT = 120
def __init__(self):
self.conn = None
self.taskq = None
self.resultq = None
def connect(self):
# open connection
self.conn = SQSConnection()
# initialize queues
self.taskq = self.conn.create_queue(self.TASK_QUEUE, self.VISIBILITY_TIMEOUT)
self.taskq.set_message_class(JSONMessage)
self.resultq = self.conn.create_queue(self.RESULT_QUEUE, self.VISIBILITY_TIMEOUT)
self.resultq.set_message_class(JSONMessage)
def clear(self):
self.taskq.clear()
self.resultq.clear()
# self.conn.delete_queue(self.taskq)
# self.conn.delete_queue(self.resultq)
self.taskq = None
self.resultq = None
def new_task(self, task):
return JSONMessage(self.taskq, task)
def new_result(self, result):
return JSONMessage(self.resultq, result)
def put_task(self, task):
self.taskq.write(task)
def next_task(self):
next = None
while next is None:
next = self.taskq.read()
return next
def complete(self, task, result=None):
self.taskq.delete_message(task)
if result is not None:
self.put_result(result)
def put_result(self, result):
self.resultq.write(result)
def get_result(self):
result = self.resultq.read()
if result is not None:
self.resultq.delete_message(result)
return result
def queue(q_name, msg):
conn_logger.debug('queue')
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
m = Message()
m.set_body(msg)
rs = q.write(m)
conn_logger.debug('put "%s" to %s queue: ' % (msg, q_name))
def send_task_sns(taskname,taskdata,taskino,taskitype):
SQS_Id=['NewTaskForNode1','NewTaskForNode2','NewTaskForNode3','NewTaskForNode4']
time = get_time_now()
message = time + '|' + taskname + '|' + taskdata + '|' + taskino + '|' + taskitype
conn = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
for i in range(int(taskino)):
my_queue = conn.get_queue(SQS_Id[i])
m = Message()
m.set_body(message)
my_queue.write(m)
def timeout(self):
from boto.sqs.message import Message
if not self._queue and self._connected:
try:
from boto.sqs.connection import SQSConnection
sqs_connection = SQSConnection(self._access_key, self._secret_access_key)
self._queue = sqs_connection.get_queue( self._queue_name )
except Exception, e:
LOGGER.error('Could not connect to logging queue %s'%self._queue_name, exc_info=e)
self._connected = False
def queue(q_name, msg):
conn_logger.debug('queue')
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
m = Message()
m.set_body(msg)
rs = q.write(m)
conn_logger.debug('put "%s" to %s queue: ' % (msg, q_name))
def submitimage(req):
sdb = boto.connect_sdb(AWSKey, AWSSecret)
domain = sdb.get_domain('picture')
form = req.form
tags = str(form['tags'])
user = str(form['submituser'])
description = str(form['description'])
fileitem = form['image']
import uuid
from time import strftime
guid = str(uuid.uuid1())
item = domain.new_item(guid)
try: # Windows needs stdio set for binary mode.
import msvcrt
msvcrt.setmode (0, os.O_BINARY) # stdin = 0
msvcrt.setmode (1, os.O_BINARY) # stdout = 1
except ImportError:
pass
# strip leading path from file name to avoid directory traversal attacks
fname = os.path.basename(fileitem.filename)
# build absolute path to files directory
dir_path = os.path.join(os.path.dirname(req.filename), 'files')
open(os.path.join(dir_path, fname), 'wb').write(fileitem.file.read())
from boto.s3.connection import S3Connection
conn = S3Connection(AWSKey, AWSSecret)
bucket = conn.get_bucket('theimageproject')
from boto.s3.key import Key
k = Key(bucket)
k.key = guid + ".jpg"
k.set_contents_from_filename(os.path.join(dir_path, fname))
curtime = strftime("%Y-%m-%dT%H:%M:%S")
item['description'] = description
item['submituser'] = user
item['submitdate'] = curtime
item['rating'] = 0
item['ratingcount'] = 0
item['ratesort'] = "%s%s" % (0, curtime)
item['status'] = "processing"
item['tag'] = tags.split(',')
item.save()
sqsconn = SQSConnection(AWSKey, AWSSecret)
q = sqsconn.get_queue('imageprocess')
request = {}
request['imagekey'] = guid
request['submitdate'] = curtime
m = RawMessage()
m.set_body(json.write(request))
status = q.write(m)
response = {}
if status==m:
response['success'] = True
response['imagekey'] = guid
else:
response['complete'] = False
return json.write(response)
def __init__(self, name, acc_sec_pair=None, visibility_timeout=30):
self.name = name
if acc_sec_pair is None:
acc_sec_pair = aws_credentials.get_credentials()
self.region_name = 'eu-west-1'
self.region = [r for r in regions() if r.name == self.region_name][0]
self.conn = SQSConnection(aws_access_key_id=acc_sec_pair[0],
aws_secret_access_key=acc_sec_pair[1],
region=self.region)
self.q = self.conn.create_queue(name, visibility_timeout)
self.q.set_message_class(event.SQSEvent)
def sendMessageToSQSQueue(self, message_body, queue_name):
try:
sqsConn = SQSConnection(self.access_key, self.secret_key)
queue = sqsConn.create_queue(queue_name)
new_item_message = Message()
new_item_message.set_body(message_body)
if queue.write(new_item_message) :
self.logger.info('Message added to processing queue.')
else :
self.logger.error('SQS service did not accept object into the queue.')
except Exception, e:
self.logger.error('Message could not be added to processing queue. ' + str(e))
def _create_sqs_message(message):
"""A dev only function that drops a new message on the sqs queue."""
sqs = SQSConnection()
sqs_q = sqs.get_queue(g.sitemap_sqs_queue)
# it returns None on failure
assert sqs_q, "failed to connect to queue"
sqs_message = sqs_q.new_message(body=json.dumps(message))
sqs_q.write(sqs_message)
g.log.info('Queued SQS message: %r', message)
def main():
conn = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
my_queue = conn.get_queue(SQS_Id)
while True:
print my_queue.count()
if my_queue.count()>0 :
task = my_queue.read()
process(task.get_body())
my_queue.delete_message(task)
else:
time.sleep(Pull_period)
def create_queue(name):
if settings.OPENSOCIAL_DEBUG: # ローカル環境の場合なにもしない
return None
if not name:
raise JobQueueError('Not specified queue name')
# Connect to SQS queue
sri = SQSRegionInfo(name='ap-northeast-1', endpoint='ap-northeast-1.queue.amazonaws.com')
conn = SQSConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY, region=sri)
q = conn.create_queue(name)
return q
def pop_loop(queue='pi-status', wait=5):
# like pop(), but iterate
sqs = SQSConnection()
sqs_queue = sqs.create_queue(queue)
sqs_queue.set_message_class(JSONMessage)
while True:
message = sqs_queue.read(wait_time_seconds=wait)
if message is not None:
body = message.get_body()
message.delete()
yield body
def __init__(self, queue="sqs_handler_debug", aws_key=None, secret_key=None):
''' Sends log messages to SNS. Parameters:
* queue is the SQS queue. This will be created if it does not exist.
* Optional: aws_key and secret_key. If these don't exist, it will look
at the appropriate environment variables.
'''
logging.Handler.__init__(self)
if aws_key and secret_key:
conn = SQSConnection(aws_key, secret_key)
else:
conn = SQSConnection()
self.q = conn.create_queue(queue)
def timeout(self):
from boto.sqs.message import Message
if not self._queue and self._connected:
try:
from boto.sqs.connection import SQSConnection
sqs_connection = SQSConnection(self._access_key,
self._secret_access_key)
self._queue = sqs_connection.get_queue(self._queue_name)
except Exception, e:
LOGGER.error('Could not connect to logging queue %s' %
self._queue_name,
exc_info=e)
self._connected = False
def test_queue_purge(self):
conn = SQSConnection()
test = self.create_temp_queue(conn)
time.sleep(65)
# Put some messages in the queue.
for x in range(0, 4):
self.put_queue_message(test)
self.assertEqual(test.count(), 4)
# Now purge the queue
conn.purge_queue(test)
# Now assert queue count is 0
self.assertEqual(test.count(), 0)
def dequeue(q_name, func, conn=None):
conn_logger.debug('dequeuing %s' % q_name)
sqs_conn = SQSConnection(AWS_U, AWS_K)
q = sqs_conn.create_queue(q_name)
#q.clear()
rs = q.get_messages(5)
if len(rs) == 0:
conn_logger.info('%s queue is empty' % q_name)
else:
for m in rs:
msg = m.get_body()
if func(msg, conn):
q.delete_message(m)
conn_logger.info('dequeue done: %s from %s' % (msg, q_name))
def _create_test_message():
"""A dev only function that drops a new message on the sqs queue."""
sqs = SQSConnection()
sqs_q = sqs.get_queue(g.sitemap_sqs_queue)
# it returns None on failure
assert sqs_q, "failed to connect to queue"
message = sqs_q.new_message(body=json.dumps({
'job_name': 'daily-sr-sitemap-reporting',
'location': ('s3://reddit-data-analysis/big-data/r2/prod/' +
'daily_sr_sitemap_reporting/dt=2016-06-14'),
'timestamp': _current_timestamp(),
}))
sqs_q.write(message)
def main():
# create a connection to SQS
conn = SQSConnection();
# ask for the JOB_QUEUE
q = conn.get_queue(JOB_QUEUE);
# create a new message
m = Message();
m.set_body(os.sys.stdin.read());
# publish the message to SQS
q.write(m);
def test_auth_region_name_is_automatically_updated(self):
region = SQSRegionInfo(name='us-west-2',
endpoint='us-west-2.queue.amazonaws.com')
self.service_connection = SQSConnection(
https_connection_factory=self.https_connection_factory,
aws_access_key_id='aws_access_key_id',
aws_secret_access_key='aws_secret_access_key',
region=region)
self.initialize_service_connection()
self.set_http_response(status_code=200)
self.service_connection.create_queue('my_queue')
# Note the region name below is 'us-west-2'.
self.assertIn('us-west-2/sqs/aws4_request',
self.actual_request.headers['Authorization'])
def setUp(self):
# Ensure the queue is clear before we start, or we'll lose more hair
creds = aws_credentials.get_credentials()
region_name = 'eu-west-1'
region = [r for r in regions() if r.name == region_name][0]
conn = SQSConnection(aws_access_key_id=creds[0],
aws_secret_access_key=creds[1],
region=region)
q = conn.create_queue(self.qname, 30)
cruft = q.get_messages(10)
while cruft:
for c in cruft:
print 'deleting old message %s', c.__dict__
q.delete_message(c)
cruft = q.get_messages(10)
