def poll():
running = File('/var/run/build')
if running.exists:
return
queue_url = '{ "Ref" : "InputQueue" }'
region = '{ "Ref" : "AWS::Region" }'
aws_access_key = '{"Ref": "WorkerKeys"}'
aws_secret_key = '{"Fn::GetAtt": ["WorkerKeys", "SecretAccessKey"]}'
conn = sqsconnect(region,
aws_access_key_id=aws_access_key,
aws_secret_access_key=aws_secret_key)
q = Queue(conn, queue_url)
msg = q.read(600)
if msg:
body = msg.get_body()
data = json.loads(body)
status_url = data.get('status_url', None)
running.write('.')
try:
run(data)
q.delete_message(msg)
except HandledException:
raise
except Exception, e:
if status_url:
post_status(status_url, dict(
state='error',
message=e.message
))
raise
else:
print 'All done.'
finally:
class AmazonSqsClient(BaseClient):
def __init__(self, configHolder):
self.log = None
self.aws_access_key_id = None
self.aws_secret_access_key = None
super(AmazonSqsClient, self).__init__(configHolder)
self.amazonsqs_queue_polling_timeout =\
float(self.amazonsqs_queue_polling_timeout)
if not self.aws_access_key_id:
self.aws_access_key_id = None
else:
self.aws_access_key_id = base64.b64decode(self.aws_access_key_id)
if not self.aws_secret_access_key:
self.aws_access_key_id = base64.b64decode(self.aws_access_key_id)
else:
self.aws_secret_access_key = base64.b64decode(self.aws_secret_access_key)
self.queue = None
self.messages = []
def connect(self):
region = RegionInfo(name=self.amazonsqs_queue_region,
endpoint='eu-west-1.queue.amazonaws.com')
connection = SQSConnection(region=region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
self.queue = Queue(connection=connection,
url=self.amazonsqs_queue_url,
message_class=JSONMessage)
def get_messages(self, number_messages=10):
self.messages = self.queue.get_messages(num_messages=number_messages)
self.log.debug("Got %i message(s)." % len(self.messages))
message_bodies = []
for m in self.messages:
message_bodies.append(m.get_body())
return message_bodies
def delete_messages(self):
for m in self.messages[:]:
self._delete_message(m)
self.messages.remove(m)
def _delete_message(self, message):
if message.queue:
message.delete()
else:
self.queue.delete_message(message)
def sleep(self):
time.sleep(self.amazonsqs_queue_polling_timeout)
def test_queue_arn(self):
connection = Mock()
connection.region.name = 'us-east-1'
q = Queue(
connection=connection,
url='https://sqs.us-east-1.amazonaws.com/id/queuename')
self.assertEqual(q.arn, 'arn:aws:sqs:us-east-1:id:queuename')
def test_send_message_attributes(self):
self.set_http_response(status_code=200)
queue = Queue(
url='http://sqs.us-east-1.amazonaws.com/123456789012/testQueue/',
message_class=RawMessage)
self.service_connection.send_message(queue,
'Test message',
message_attributes={
'name1': {
'data_type': 'String',
'string_value': 'Bob'
},
'name2': {
'data_type': 'Number',
'string_value': '1'
}
})
self.assert_request_parameters({
'Action': 'SendMessage',
'MessageAttribute.1.Name': 'name2',
'MessageAttribute.1.Value.DataType': 'Number',
'MessageAttribute.1.Value.StringValue': '1',
'MessageAttribute.2.Name': 'name1',
'MessageAttribute.2.Value.DataType': 'String',
'MessageAttribute.2.Value.StringValue': 'Bob',
'MessageBody': 'Test message',
'Version': '2012-11-05'
})
def test_send_message_attributes(self):
self.set_http_response(status_code=200)
queue = Queue(
url='http://sqs.us-east-1.amazonaws.com/123456789012/testQueue/',
message_class=RawMessage)
message1 = (1, 'Message 1', 0, {'name1': {'data_type': 'String',
'string_value': 'foo'}})
message2 = (2, 'Message 2', 0, {'name2': {'data_type': 'Number',
'string_value': '1'}})
self.service_connection.send_message_batch(queue, (message1, message2))
self.assert_request_parameters({
'Action': 'SendMessageBatch',
'SendMessageBatchRequestEntry.1.DelaySeconds': 0,
'SendMessageBatchRequestEntry.1.Id': 1,
'SendMessageBatchRequestEntry.1.MessageAttribute.1.DataType': 'String',
'SendMessageBatchRequestEntry.1.MessageAttribute.1.Name': 'name1',
'SendMessageBatchRequestEntry.1.MessageAttribute.1.StringValue': 'foo',
'SendMessageBatchRequestEntry.1.MessageBody': 'Message 1',
'SendMessageBatchRequestEntry.2.DelaySeconds': 0,
'SendMessageBatchRequestEntry.2.Id': 2,
'SendMessageBatchRequestEntry.2.MessageAttribute.1.DataType': 'Number',
'SendMessageBatchRequestEntry.2.MessageAttribute.1.Name': 'name2',
'SendMessageBatchRequestEntry.2.MessageAttribute.1.StringValue': '1',
'SendMessageBatchRequestEntry.2.MessageBody': 'Message 2',
'Version': '2012-11-05'
})
def test_queue_name(self):
connection = Mock()
connection.region.name = 'us-east-1'
q = Queue(
connection=connection,
url='https://sqs.us-east-1.amazonaws.com/id/queuename')
self.assertEqual(q.name, 'queuename')
def connect(self):
region = RegionInfo(name=self.amazonsqs_queue_region,
endpoint='eu-west-1.queue.amazonaws.com')
connection = SQSConnection(region=region,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key)
self.queue = Queue(connection=connection,
url=self.amazonsqs_queue_url,
message_class=JSONMessage)
def main():
conn = boto.sqs.connect_to_region("us-east-1")
queue_url = "https://sqs.us-east-1.amazonaws.com/452649417432/awseb-e-tmrnmgpqzh-stack-NewSignupQueue-MN33H2HK79KL"
if os.path.exists("pass.txt"):
with open ("pass.txt", "r") as password_file:
password = password_file.readline()
else:
sys.exit(0)
q = Queue(conn, queue_url)
while 1:
rs = q.get_messages(1)
if len(rs)!=0:
m = rs[0]
data = json.loads(m.get_body())
if 'photoId' in data and 'url' in data:
photoid = data['photoId']
print photoid
url = data['url']
urllib.urlretrieve(url, "tmp.jpg")
hash_value = imagehash.average_hash(Image.open('tmp.jpg'))
print(hash_value)
cnx = MySQLdb.connect(user='******', passwd = password, host = 'tweetmap.crsarl5br9bw.us-east-1.rds.amazonaws.com', db='tweet')
cursor = cnx.cursor()
add_hash = ("UPDATE testPhoto SET hash = %s WHERE id = %s")
data_hash = (hash_value, photoid)
cursor.execute(add_hash, data_hash)
cnx.commit()
print "insert into db"
cursor.close()
cnx.close()
else: print "no key: photoId and url"
q.delete_message(m)
def __get_queue(self):
"""
Returns the Queue object for the current configuration.
:return: the Queue
:rtype: boto.sqs.queue.Queue
"""
return Queue(connection=self.__get_sqs_connection(),
url=settings.AWS_SQS_QUEUE_URL,
message_class=RawMessage)
def test_message_attribute_response(self):
self.set_http_response(status_code=200)
queue = Queue(
url='http://sqs.us-east-1.amazonaws.com/123456789012/testQueue/',
message_class=RawMessage)
message = self.service_connection.receive_message(queue)[0]
self.assertEqual(message.get_body(), 'This is a test')
self.assertEqual(message.id, '7049431b-e5f6-430b-93c4-ded53864d02b')
self.assertEqual(message.md5, 'ce114e4501d2f4e2dcea3e17b546f339')
self.assertEqual(message.md5_message_attributes,
'324758f82d026ac6ec5b31a3b192d1e3')
mattributes = message.message_attributes
self.assertEqual(len(mattributes.keys()), 2)
self.assertEqual(mattributes['Count']['data_type'], 'Number')
self.assertEqual(mattributes['Foo']['string_value'], 'Bar')
def create_queue(self, queue_name, visibility_timeout=None):
"""
Create a new queue.
Inputs:
queue_name - The name of the new queue
visibility_timeout - (Optional) The default visibility
timeout for the new queue.
Returns:
A new Queue object representing the newly created queue.
"""
path = '/?QueueName=%s' % queue_name
if visibility_timeout:
path = path + '&DefaultVisibilityTimeout=%d' % visibility_timeout
response = self.make_request('POST', path)
body = response.read()
if response.status >= 300:
raise SQSError(response.status, response.reason, body)
q = Queue(self)
h = handler.XmlHandler(q, self)
xml.sax.parseString(body, h)
return q
region = os.getenv("AWS_REGION") or None
aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID") or None
aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY") or None
queue_url = os.getenv("QUEUE_URL")
db_host = db_name = os.getenv("DATABASE_HOST")
db_username = db_name = os.getenv("WRITER_USERNAME")
db_password = db_name = os.getenv("WRITER_PASSWORD")
db_name = os.getenv("DATABASE_NAME")
conn = boto.sqs.connect_to_region(region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
conn = boto.sqs.connect_to_region(region)
q = Queue(conn, queue_url)
engine = create_engine("mysql://{user}:{password}@{host}:3306/{name}".format(
user=db_username,
password=db_password,
host=db_host,
name=db_name,
))
metadata = MetaData()
metadata.bind = engine
queries = Table('queries', metadata, autoload=True)
while True:
messages = q.get_messages(10)
print len(messages)
aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY")
queue_url = os.getenv("QUEUE_URL")
region = os.getenv("AWS_REGION")
flickr_key = os.getenv("FLICKR_KEY")
flickr_secret = os.getenv("FLICKR_SECRET")
db_host = db_name = os.getenv("DATABASE_HOST")
db_username = db_name = os.getenv("READER_USERNAME")
db_password = db_name = os.getenv("READER_PASSWORD")
db_name = os.getenv("DATABASE_NAME")
secrets = {'api_key': flickr_key, 'api_secret': flickr_secret}
conn = boto.sqs.connect_to_region(region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
q = Queue(conn, queue_url)
app = Flask(__name__)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/api/search/<term>')
def api(term):
flickr_api.set_keys(**secrets)
write_to_sqs(term, "api")
photos = flickr_api.Photo.search(tags=term,
sort='date-posted-desc',
per_page=10)
def predefined_queues(self):
# We are using a strict sqs setup which we are given only list of predefined queues and
url = self.transport_options.get("predefined_queue_url", None)
q = Queue(connection=self.sqs, url=url, message_class=Message)
return [q]
aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID") or None
aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY") or None
queue_url = os.getenv("QUEUE_URL")
db_host = db_name = os.getenv("DATABASE_HOST")
db_username = db_name = os.getenv("WRITER_USERNAME")
db_password = db_name = os.getenv("WRITER_PASSWORD")
db_name = os.getenv("DATABASE_NAME")
conn = boto.sqs.connect_to_region(
region,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
conn = boto.sqs.connect_to_region(region)
q = Queue(conn, queue_url)
engine = create_engine("mysql://{user}:{password}@{host}:3306/{name}".format(
user=db_username,
password=db_password,
host=db_host,
name=db_name,
))
metadata = MetaData()
metadata.bind = engine
queries = Table('queries', metadata, autoload=True)
while True:
messages = q.get_messages(10)
# Send SQS message
data = dict(
project='gitbot/test',
actions_repo='git://github.com/gitbot/test.git',
repo='gitbot/www',
branch='master',
bucket='releases.dev.gitbot.test',
keys=dict(access_key=access_key, secret=secret),
command='all'
)
conn = connect_to_region('us-east-1',
aws_access_key_id=access_key,
aws_secret_access_key=secret)
queue = Queue(conn, queue_url)
message = queue.new_message(json.dumps(data))
message = queue.write(message)
# Wait for result
b = Bucket(
data['bucket'],
aws_access_key_id=access_key,
aws_secret_access_key=secret)
b.connect()
print '\n Wating for bucket:\n'
key = None
key_path = 'result.log'
while not b.bucket: