def setUp(self):
self.sqsc = SQSConnection()
self.snsc = SNSConnection()
def test_sqs_expiration(self):
c = SQSConnection(aws_access_key_id='aws_access_key_id',
aws_secret_access_key='aws_secret_access_key')
self.assert_is_expired(c, SQS_EXPIRED, status=401)
self.assert_is_not_expired(c, GENERIC_BAD_REQUEST, status=403)
import boto
import ParsePy
from boto.s3.key import Key
from boto.sqs.message import Message
from boto.sqs.connection import SQSConnection
from boto.s3.connection import S3Connection
AWS_ACCESS_KEY = ''
AWS_SECRET_KEY = ''
ParsePy.APPLICATION_ID = ""
ParsePy.MASTER_KEY = ""
conn = SQSConnection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
convert_q = conn.create_queue('FZconvertQueue', 120)
upload_q = conn.create_queue('FZuploadQueue', 120)
convert_q.clear()
upload_q.clear()
query = ParsePy.ParseQuery("UploadObject")
fzobjects = query.fetch()
for fzobj in fzobjects:
print fzobj.objectId()
fzobj.delete()
print convert_q.count()
def create_sqs_connection(key):
return SQSConnection(key.access_key_id, key.secret_key)
def conn(self):
return SQSConnection(self.aws_access_key_id, self.aws_secret_access_key, \
region=self.region)
if __name__ == "__main__":
config = ConfigParser.ConfigParser()
config.read(config_file)
logging.getLogger().setLevel(logging.INFO)
try:
# AWS auth details
aws_access_key = config.get("AWSauth", "aws_access_key")
aws_secret_key = config.get("AWSauth", "aws_secret_key")
# S3 configuration details
s3_bucket = config.get("S3config", "s3_bucket")
signature_timeout = int(config.get("S3config", "signature_timeout"))
# SQS job queue configuration details
sqs_enabled = bool(config.get("SQSconfig", "enabled"))
logging.info("sqs_enabled: %s" % sqs_enabled)
if sqs_enabled:
queue_name = config.get("SQSconfig", "queue_name")
conn = SQSConnection(aws_access_key, aws_secret_key)
queue = conn.create_queue(queue_name)
queue.set_message_class(MHMessage)
except Exception, e:
logging.error("Error reading config file [%s]: %s" % (config_file, e))
sys.exit(1)
# Start the application
app.run()
from django.conf import settings
from django.utils import simplejson as json
from boto.exception import BotoServerError
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
from utils.loggly import loggly
sqs_conn = SQSConnection(settings.AWS_ACCESS_KEY_ID,
settings.AWS_SECRET_ACCESS_KEY)
def get_queue(queue_name):
try:
return sqs_conn.get_queue(queue_name)
except BotoServerError:
return None
def add_to_queue(queue, data):
message = Message()
message.set_body(json.dumps(data))
try:
return queue.write(message)
except BotoServerError:
return None
# this script expects 2 environment variables
# 1. SQS_KEY_ID (preferably an IAM user with limited rights)
# 2. SQS_SECRET_KEY (accompanying secret key)
# 3. SQS_TASK_QUEUE (the queue to use)
import os
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
import utils
# your amazon keys
key = os.environ['SQS_KEY_ID']
access = os.environ['SQS_ACCESS_KEY']
queue = os.environ['SQS_TASK_QUEUE']
if __name__ == '__main__':
region_info = utils.get_region_info()
sqs = SQSConnection(key, access, region=region_info)
tasks = sqs.create_queue(queue)
m = tasks.read()
while m != None:
print( m.get_body())
tasks.delete_message(m)
m = tasks.read()
#!/usr/bin/python
import timeit
import gc
import urllib
import simplejson
from random import randrange
from time import sleep, time
from boto.sqs.connection import SQSConnection
probe_loc = 'us'
queue_name = 'helomx_to_%s' % probe_loc
conn = SQSConnection('', '')
q_incoming = conn.get_queue(queue_name)
q_return = conn.get_queue('helomx_return')
# We clear the queue just in case this probe was down, so we don't keep
# scanning the mailserver over and over.
q_incoming.clear()
def check_port(ip_parts):
now = int(time())
try:
z = """\
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1.0)
s.connect(('%s', %d))
s.close()
""" % (ip_parts[0], int(ip_parts[1]))
def register_user(request):
# extract data
data = json.loads(request.body.decode('utf-8'))
region = data.get("region")
key = data.get("key")
email = data.get("email")
password = data.get("password")
code = data.get("code")
# ensure the data is valid
if None in (region, key, email, password, code):
return HttpResponse(json.dumps(INVALID_REQUEST_FORMAT))
# ensure proper key format
key = format_key(key)
# initialize summoner object value to None
summoner_o = None
# initialize riot response summoner value to None
summoner = None
try:
# get the summoner object
summoner_o = cache.get(region + key + "summoner")
if summoner_o is None:
summoner_o = Summoner.objects.get(region=region, key=key)
cache.set(region + key + "summoner", summoner_o, None)
Summoner.objects.filter(pk=summoner_o.pk).update(
accessed=datetime.now())
# check if the user object already exists
if summoner_o.user is not None:
return HttpResponse(json.dumps(SUMMONER_ALREADY_REGISTERED))
# get the summoner id
summoner_id = summoner_o.summoner_id
except Summoner.DoesNotExist:
try:
# summoner not in database, request summoner data from riot
args = {"request": 1, "key": key}
riot_response = riot_request(region, args)
except APIError as e:
if e.error_code == 404:
return HttpResponse(json.dumps(SUMMONER_DOES_NOT_EXIST))
else:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# extract the summoner
summoner = riot_response.get(key)
# get the summoner id
summoner_id = summoner.id
except AttributeError:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# use the summoner id to get rune page information to validate ownership
args = {"request": 6, "summoner_id": summoner_id}
riot_response = riot_request(region, args)
# extract the summoners rune pages
rune_pages = riot_response.get(str(summoner_id)).pages
# iterate over the pages looking for one whose name matches the code
no_match = True
for page in rune_pages:
if page.name == code:
no_match = False
break
# return error if no match found
if no_match:
return HttpResponse(json.dumps(RUNE_PAGE_CODE_NOT_FOUND))
except (APIError, AttributeError):
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
# hash password
password = hashers.make_password(password)
# if summoner object already exists wrap up registration
if summoner_o is not None:
# create a user object for the summoner object
summoner_o.user = User.objects.create(email=email, password=password)
Summoner.objects.filter(pk=summoner_o.pk).update(user=summoner_o.user)
# return the users summoner object with the email included
return HttpResponse(summoner_serializer(summoner_o, email, False))
try:
# summoner object did not already exist, use summoner id to get league information
args = {"request": 4, "summoner_ids": summoner_id}
riot_response = riot_request(region, args)
except APIError as e:
if e.error_code == 404:
return HttpResponse(json.dumps(SUMMONER_NOT_RANKED))
else:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# extract the league data
leagues = riot_response.get(str(summoner_id))
# iterate over the leagues looking for the dynamic queue league
league = None
for item in leagues:
if item.queue == "RANKED_SOLO_5x5":
league = item
# ensure the dynamic queue league was found
if league is None:
return HttpResponse(json.dumps(SUMMONER_NOT_RANKED))
# iterate over the league entries to get more detailed information
division, lp, wins, losses, series = None, None, None, None, ""
for entry in league.entries:
if entry.playerOrTeamId == str(summoner_id):
division = entry.division
lp = entry.leaguePoints
wins = entry.wins
losses = entry.losses
if entry.miniSeries is not None:
series = entry.miniSeries.progress
except AttributeError:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
# create a new user object
user_o = User.objects.create(email=email, password=password)
try:
# use the gathered information to create a summoner object
summoner_o = Summoner.objects.create(
user=user_o,
region=region,
key=key,
name=summoner.name,
summoner_id=summoner_id,
tier=league.tier,
division=division,
lp=lp,
wins=wins,
losses=losses,
series=series,
profile_icon=summoner.profileIconId)
except IntegrityError:
return HttpResponse(json.dumps(INTERNAL_PROCESSING_ERROR))
# update the newly created summoner
conn = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
queue = conn.get_queue("portal")
message = RawMessage()
message.set_body(json.dumps({"region": region, "keys": [key]}))
queue.write(message)
# return the users summoner object with the email included
return HttpResponse(summoner_serializer(summoner_o, email, False))
from boto.sqs.connection import SQSConnection
from pysqes.task import SQSTask
conn = SQSConnection('ACCESS_KEY', 'SECRETE_KEY')
task = SQSTask(conn)
@task.task
def add(a, b):
return a + b
# this will submit a job to the queue
add.delay(1, 3)
import json
# you can also pick a different serializer
json_task = SQSTask(conn, serializer=json0
json_task.schedule_task({
'test': 'json serializer'
})
def login_user_1_1(request):
# extract data
data = json.loads(request.body.decode('utf-8'))
region = data.get("region")
key = data.get("key")
# ensure the data is valid
if None in (region, key):
return HttpResponse(json.dumps(INVALID_REQUEST_FORMAT))
# ensure proper key format
key = format_key(key)
try:
# get the summoner object
summoner_o = cache.get(region + key + "summoner")
if summoner_o is None:
summoner_o = Summoner.objects.get(region=region, key=key)
cache.set(region + key + "summoner", summoner_o, None)
Summoner.objects.filter(pk=summoner_o.pk).update(
accessed=datetime.now())
# return the users summoner object
return HttpResponse(summoner_serializer(summoner_o, None, False))
except Summoner.DoesNotExist:
pass
try:
# summoner not in database, request summoner data from riot
args = {"request": 1, "key": key}
riot_response = riot_request(region, args)
except APIError as e:
if e.error_code == 404:
return HttpResponse(json.dumps(SUMMONER_DOES_NOT_EXIST))
else:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# extract the summoner
summoner = riot_response.get(key)
# extract summoner fields
summoner_id = summoner.id
name = summoner.name
profile_icon = summoner.profileIconId
except AttributeError:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# use summoner id to get league information
args = {"request": 4, "summoner_ids": summoner_id}
riot_response = riot_request(region, args)
except APIError as e:
if e.error_code == 404:
return HttpResponse(json.dumps(SUMMONER_NOT_RANKED))
else:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# extract the league data
leagues = riot_response.get(str(summoner_id))
# iterate over the leagues looking for the dynamic queue league
league = None
for item in leagues:
if item.queue == "RANKED_SOLO_5x5":
league = item
# ensure the dynamic queue league was found
if league is None:
return HttpResponse(json.dumps(SUMMONER_NOT_RANKED))
# iterate over the league entries to get more detailed information
division, lp, wins, losses, series = None, None, None, None, ""
for entry in league.entries:
if entry.playerOrTeamId == str(summoner_id):
division = entry.division
lp = entry.leaguePoints
wins = entry.wins
losses = entry.losses
if entry.miniSeries is not None:
series = entry.miniSeries.progress
# extract the tier information
tier = league.tier
except AttributeError:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# use the gathered information to create a summoner object
summoner_o = Summoner.objects.create(region=region,
key=key,
name=name,
summoner_id=summoner_id,
tier=tier,
division=division,
lp=lp,
wins=wins,
losses=losses,
series=series,
profile_icon=profile_icon)
except IntegrityError:
return HttpResponse(json.dumps(INTERNAL_PROCESSING_ERROR))
# update the newly created summoner
conn = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
queue = conn.get_queue("portal")
message = RawMessage()
message.set_body(json.dumps({"region": region, "keys": [key]}))
queue.write(message)
# return the users summoner object
return HttpResponse(summoner_serializer(summoner_o, None, False))
def add_friend(request):
# extract data
data = json.loads(request.body.decode('utf-8'))
region = data.get("region")
user_key = data.get("user_key")
friend_key = data.get("friend_key")
# ensure the data is valid
if None in (region, user_key, friend_key):
return HttpResponse(json.dumps(INVALID_REQUEST_FORMAT))
# ensure proper key format
user_key = format_key(user_key)
friend_key = format_key(friend_key)
# make sure friend is not the user
if user_key == friend_key:
return HttpResponse(json.dumps(FRIEND_EQUALS_USER))
try:
# get the users summoner object
user_o = cache.get(region + user_key + "summoner")
if user_o is None:
user_o = Summoner.objects.get(region=region, key=user_key)
cache.set(region + user_key + "summoner", user_o, None)
Summoner.objects.filter(pk=user_o.pk).update(accessed=datetime.now())
except Summoner.DoesNotExist:
return HttpResponse(json.dumps(SUMMONER_NOT_IN_DATABASE))
# check if user is at friend limit or if friend is already listed
if user_o.friends is not None:
friends = user_o.friends.split(",")
if len(friends) >= 20:
return HttpResponse(json.dumps(FRIEND_LIMIT_REACHED))
for friend in friends:
if friend == friend_key:
return HttpResponse(json.dumps(FRIEND_ALREADY_LISTED))
try:
# get the friends summoner object
friend_o = cache.get(region + friend_key + "summoner")
if friend_o is None:
friend_o = Summoner.objects.get(region=region, key=friend_key)
cache.set(region + friend_key + "summoner", friend_o, None)
Summoner.objects.filter(pk=friend_o.pk).update(accessed=datetime.now())
except Summoner.DoesNotExist:
try:
# summoner not in database, request summoner data from riot
args = {"request": 1, "key": friend_key}
riot_response = riot_request(region, args)
except APIError as e:
if e.error_code == 404:
return HttpResponse(json.dumps(SUMMONER_DOES_NOT_EXIST))
else:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# extract the summoner
friend = riot_response.get(friend_key)
except AttributeError:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# use the summoner id to get the friends league information
args = {"request": 4, "summoner_ids": friend.id}
riot_response = riot_request(region, args)
except APIError as e:
if e.error_code == 404:
return HttpResponse(json.dumps(SUMMONER_NOT_RANKED))
else:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# extract the league data
leagues = riot_response.get(str(friend.id))
# iterate over the leagues looking for the dynamic queue league
league = None
for item in leagues:
if item.queue == "RANKED_SOLO_5x5":
league = item
# ensure the dynamic queue league was found
if league is None:
return HttpResponse(json.dumps(SUMMONER_NOT_RANKED))
# iterate over the league entries to get more detailed information
division, lp, wins, losses, series = None, None, None, None, ""
for entry in league.entries:
if entry.playerOrTeamId == str(friend.id):
division = entry.division
lp = entry.leaguePoints
wins = entry.wins
losses = entry.losses
if entry.miniSeries is not None:
series = entry.miniSeries.progress
except AttributeError:
return HttpResponse(json.dumps(INVALID_RIOT_RESPONSE))
try:
# use the gathered information to create a summoner object
friend_o = Summoner.objects.create(
region=region,
key=friend_key,
name=friend.name,
summoner_id=friend.id,
tier=league.tier,
division=division,
lp=lp,
wins=wins,
losses=losses,
series=series,
profile_icon=friend.profileIconId)
except IntegrityError:
return HttpResponse(json.dumps(INTERNAL_PROCESSING_ERROR))
# update the newly created summoner
conn = SQSConnection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
queue = conn.get_queue("portal")
message = RawMessage()
message.set_body(json.dumps({"region": region, "keys": [friend_key]}))
queue.write(message)
# add the friends key to the users friend list
if user_o.friends != "":
user_o.friends += "," + friend_key
else:
user_o.friends = friend_key
Summoner.objects.filter(pk=user_o.pk).update(friends=user_o.friends)
cache.set(region + user_key + "summoner", user_o, None)
# return the friends summoner object
return HttpResponse(summoner_serializer(friend_o, None, False))
def _get_connection(self):
access_key = settings.RESTCLIENTS_AMAZON_AWS_ACCESS_KEY
secret_key = settings.RESTCLIENTS_AMAZON_AWS_SECRET_KEY
conn = SQSConnection(access_key, secret_key)
return conn
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad_queue_name')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues + 1
assert queue.count_slow() == 0
# check the visibility timeout
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
a = q.get_attributes('ApproximateNumberOfMessages')
assert a.has_key('ApproximateNumberOfMessages')
assert not a.has_key('VisibilityTimeout')
a = q.get_attributes('VisibilityTimeout')
assert not a.has_key('ApproximateNumberOfMessages')
assert a.has_key('VisibilityTimeout')
# now change the visibility timeout
timeout = 45
queue.set_timeout(timeout)
time.sleep(60)
t = queue.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue.new_message(message_body)
queue.write(message)
time.sleep(60)
assert queue.count_slow() == 1
time.sleep(90)
# now read the message from the queue with a 10 second timeout
message = queue.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue.read()
assert message
# now delete the message
queue.delete_message(message)
time.sleep(30)
assert queue.count_slow() == 0
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue = c.create_queue(queue_name, timeout)
queue.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue.new_message()
message['foo'] = 'bar'
queue.write(message)
message_body = {'fie': 'baz', 'foo': 'bar'}
message = queue.new_message(body=message_body)
queue.write(message)
time.sleep(30)
m = queue.read()
assert m['foo'] == 'bar'
# now delete that queue and messages
c.delete_queue(queue, True)
print '--- tests completed ---'
def __init__(self, conf):
super(QueueReader, self).__init__(conf)
self._conn = SQSConnection(conf['access_key'],
conf['secret_access_key'])
def test_1_basic(self):
print '--- running SQSConnection tests ---'
c = SQSConnection()
rs = c.get_all_queues()
num_queues = 0
for q in rs:
num_queues += 1
# try illegal name
try:
queue = c.create_queue('bad*queue*name')
self.fail('queue name should have been bad')
except SQSError:
pass
# now create one that should work and should be unique (i.e. a new one)
queue_name = 'test%d' % int(time.time())
timeout = 60
queue_1 = c.create_queue(queue_name, timeout)
self.addCleanup(c.delete_queue, queue_1, True)
time.sleep(60)
rs = c.get_all_queues()
i = 0
for q in rs:
i += 1
assert i == num_queues + 1
assert queue_1.count_slow() == 0
# check the visibility timeout
t = queue_1.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now try to get queue attributes
a = q.get_attributes()
assert 'ApproximateNumberOfMessages' in a
assert 'VisibilityTimeout' in a
a = q.get_attributes('ApproximateNumberOfMessages')
assert 'ApproximateNumberOfMessages' in a
assert 'VisibilityTimeout' not in a
a = q.get_attributes('VisibilityTimeout')
assert 'ApproximateNumberOfMessages' not in a
assert 'VisibilityTimeout' in a
# now change the visibility timeout
timeout = 45
queue_1.set_timeout(timeout)
time.sleep(60)
t = queue_1.get_timeout()
assert t == timeout, '%d != %d' % (t, timeout)
# now add a message
message_body = 'This is a test\n'
message = queue_1.new_message(message_body)
queue_1.write(message)
time.sleep(60)
assert queue_1.count_slow() == 1
time.sleep(90)
# now read the message from the queue with a 10 second timeout
message = queue_1.read(visibility_timeout=10)
assert message
assert message.get_body() == message_body
# now immediately try another read, shouldn't find anything
message = queue_1.read()
assert message == None
# now wait 30 seconds and try again
time.sleep(30)
message = queue_1.read()
assert message
# now delete the message
queue_1.delete_message(message)
time.sleep(30)
assert queue_1.count_slow() == 0
# try a batch write
num_msgs = 10
msgs = [(i, 'This is message %d' % i, 0) for i in range(num_msgs)]
queue_1.write_batch(msgs)
# try to delete all of the messages using batch delete
deleted = 0
while deleted < num_msgs:
time.sleep(5)
msgs = queue_1.get_messages(num_msgs)
if msgs:
br = queue_1.delete_message_batch(msgs)
deleted += len(br.results)
# create another queue so we can test force deletion
# we will also test MHMessage with this queue
queue_name = 'test%d' % int(time.time())
timeout = 60
queue_2 = c.create_queue(queue_name, timeout)
self.addCleanup(c.delete_queue, queue_2, True)
queue_2.set_message_class(MHMessage)
time.sleep(30)
# now add a couple of messages
message = queue_2.new_message()
message['foo'] = 'bar'
queue_2.write(message)
message_body = {'fie': 'baz', 'foo': 'bar'}
message = queue_2.new_message(body=message_body)
queue_2.write(message)
time.sleep(30)
m = queue_2.read()
assert m['foo'] == 'bar'
print '--- tests completed ---'
'''
Pull data down from SQS where we have been keeping it:
python data_pull.py ../analysis/sqs_data/pull_20140131.json
'''
import conf
import json
from boto.sqs.connection import SQSConnection, Message
import os
import sys
import time
SQS_CONN = SQSConnection(conf.AWS_KEY, conf.AWS_SECRET)
SQS_QUEUE = SQS_CONN.create_queue(conf.SQS_QUEUE_NAME)
if __name__ == '__main__':
output_file = sys.argv[1]
assert (not os.path.exists(output_file))
message_bodys = []
message_objects = []
some_messages = SQS_QUEUE.get_messages(10)
while some_messages:
message_bodys.extend(json.loads(m.get_body()) for m in some_messages)
message_objects.extend(some_messages)
some_messages = SQS_QUEUE.get_messages(10)
print '.',
print('Got ', len(message_bodys), ' messages')
with open(output_file, 'wt') as f:
