def ec2_client(region, zone, access_key_id, secret_access_key):
"""
Establish connection to EC2 client.
:param str region: The name of the EC2 region to connect to.
:param str zone: The zone for the EC2 region to connect to.
:param str access_key_id: "aws_access_key_id" credential for EC2.
:param str secret_access_key: "aws_secret_access_key" EC2 credential.
:return: An ``_EC2`` giving information about EC2 client connection
and EC2 instance zone.
"""
# Set 2 retry knobs in Boto to BOTO_NUM_RETRIES:
# 1. ``num_retries``:
# Request automatic exponential backoff and retry
# attempts by Boto if an EC2 API call fails with
# ``RequestLimitExceeded`` due to system load.
# 2. ``metadata_service_num_attempts``:
# Request for retry attempts by Boto to
# retrieve data from Metadata Service used to retrieve
# credentials for IAM roles on EC2 instances.
if not config.has_section('Boto'):
config.add_section('Boto')
config.set('Boto', 'num_retries', BOTO_NUM_RETRIES)
config.set('Boto', 'metadata_service_num_attempts', BOTO_NUM_RETRIES)
# Get Boto EC2 connection with ``EC2ResponseError`` logged by Eliot.
connection = ec2.connect_to_region(region,
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key)
return _EC2(zone=zone,
connection=_LoggedBotoConnection(connection=connection))
def ec2_client(region, zone, access_key_id, secret_access_key):
"""
Establish connection to EC2 client.
:param str region: The name of the EC2 region to connect to.
:param str zone: The zone for the EC2 region to connect to.
:param str access_key_id: "aws_access_key_id" credential for EC2.
:param str secret_access_key: "aws_secret_access_key" EC2 credential.
:return: An ``_EC2`` giving information about EC2 client connection
and EC2 instance zone.
"""
# Set 2 retry knobs in Boto to BOTO_NUM_RETRIES:
# 1. ``num_retries``:
# Request automatic exponential backoff and retry
# attempts by Boto if an EC2 API call fails with
# ``RequestLimitExceeded`` due to system load.
# 2. ``metadata_service_num_attempts``:
# Request for retry attempts by Boto to
# retrieve data from Metadata Service used to retrieve
# credentials for IAM roles on EC2 instances.
if not config.has_section('Boto'):
config.add_section('Boto')
config.set('Boto', 'num_retries', BOTO_NUM_RETRIES)
config.set('Boto', 'metadata_service_num_attempts', BOTO_NUM_RETRIES)
# Get Boto EC2 connection with ``EC2ResponseError`` logged by Eliot.
connection = ec2.connect_to_region(region,
aws_access_key_id=access_key_id,
aws_secret_access_key=secret_access_key)
return _EC2(zone=zone,
connection=_LoggedBotoConnection(connection=connection))
def inject_default( name, default ):
section = 'Boto'
value = config.get( section, name )
if value != default:
if not config.has_section( section ):
config.add_section( section )
config.set( section, name, default )
def inject_default( name, default ):
section = 'Boto'
value = config.get( section, name )
if value != default:
if not config.has_section( section ):
config.add_section( section )
config.set( section, name, default )
def tearDown(self):
for b in self.backends:
# trick the backend into retrying connections that will fail
b._swf.host = 'localhost'
b._swf.close()
config.set('Boto', 'http_socket_timeout', '70')
Defaults.DECISION_TIMEOUT = 60
super(AmazonSWFBackendThreadTestCase, self).tearDown()
def tearDown(self):
for b in self.backends:
# trick the backend into retrying connections that will fail
b._swf.host = 'localhost'
b._swf.close()
config.set('Boto', 'http_socket_timeout', '70')
Defaults.DECISION_TIMEOUT = 60
super(AmazonSWFBackendThreadTestCase, self).tearDown()
def setupLogging(level, logfile=None, queue=False):
# this overrides any user debug setting in the boto configuration
if not boto_config.has_section('Boto'):
boto_config.add_section('Boto')
import httplib
httplib.HTTPConnection.debuglevel = level
if level == 0:
level = 'WARN'
boto_config.set('Boto', 'debug', '0')
elif level == 1:
level = 'INFO'
boto_config.set('Boto', 'debug', '0')
elif level == 2:
level = 'DEBUG'
boto_config.set('Boto', 'debug', '1')
else:
level = 'DEBUG'
boto_config.set('Boto', 'debug', '2')
if logfile is None:
logging.config.dictConfig({
'version': 1,
'formatters': {
'simple': {
'class': 'logging.Formatter',
'format': simplefmt,
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': level,
'formatter': 'simple',
},
},
'loggers': {
'boto': {
'handlers': ['console'],
},
'lacli': {
'handlers': ['console']
},
'multiprocessing': {
'handlers': ['console']
},
'requests.packages.urllib3': {
'handlers': ['console'],
'propagate': True
},
},
'root': {
'level': level,
},
})
else:
raise NotImplementedError("Log to file is not implemented yet")
def __init__(self, access_key, secret_key, s3_host='s3.amazonaws.com', secure=True, num_retries=5, socket_timeout=15):
self.access_key = access_key
self.secret_key = secret_key
self.s3_host = s3_host
self.secure = secure
self.num_retries = num_retries
self.socket_timeout = socket_timeout
for section in config.sections():
config.remove_section(section)
config.add_section('Boto')
config.setbool('Boto', 'is_secure', self.secure)
config.set('Boto', 'http_socket_timeout', str(self.socket_timeout))
config.set('Boto', 'num_retries', str(self.num_retries))
self._conn = None
self.connect()
def setUp(self):
super(AmazonSWFBackendThreadTestCase, self).setUp()
# Make long-polling connections time out really quickly
# Note: this will break in the normal case, since SWF will continue to
# assign tasks to identities that have already closed their connection
try:
config.add_section('Boto')
except:
pass
config.set('Boto', 'http_socket_timeout', '5')
Defaults.DECISION_TIMEOUT = 1
self.backends = []
backend = self.construct_backend()
for p in backend.processes():
backend.cancel_process(p)
def setUp(self):
super(AmazonSWFBackendThreadTestCase, self).setUp()
# Make long-polling connections time out really quickly
# Note: this will break in the normal case, since SWF will continue to
# assign tasks to identities that have already closed their connection
try:
config.add_section('Boto')
except:
pass
config.set('Boto', 'http_socket_timeout', '5')
Defaults.DECISION_TIMEOUT = 1
self.backends = []
backend = self.construct_backend()
for p in backend.processes():
backend.cancel_process(p)
def enable_debug(args):
if "debug" in args.__dict__ and args.debug == True:
if not config.has_section('Boto'):
config.add_section('Boto')
config.set('Boto', 'debug', '2')
# Setup a database connection to be used in the rest of the code
DB_HOST = os.environ.get('DB_HOST', 'localhost:27017')
client = MongoClient(DB_HOST)
DB_NAME = os.environ.get('DB_NAME', 'autotest')
AWS_KEY = os.environ.get('AWS_KEY',None)
AWS_SECRET = os.environ.get('AWS_SECRET', None)
db1 = client[DB_NAME]
if 'DB_USER' in os.environ:
db1.authenticate(os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD'))
on_aws = True #"ON_AWS" in os.environ
if on_aws:
if not botoconfig.has_section('Credentials'):
botoconfig.add_section('Credentials')
if not botoconfig.has_option('Credentials', 'aws_access_key_id'):
botoconfig.set('Credentials', 'aws_access_key_id',
os.environ.get('AWS_KEY'))
if not botoconfig.has_option('Credentials', 'aws_secret_access_key'):
botoconfig.set('Credentials', 'aws_secret_access_key',
os.environ.get('AWS_SECRET'))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'z%h7pr=_$j%^l54+xcjco8e+*%y)%j7q^&0w_+j8nfsh=ra_n('
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATE_DEBUG = True
ALLOWED_HOSTS = []
def enable_debug(self):
if self.debug:
if not config.has_section('Boto'):
config.add_section('Boto')
config.set('Boto', 'debug', '2')
'\nKeywords: ' + hit.Keywords
])) + '\n'
########################################################################
parser = argparse.ArgumentParser(description='Get information about a HIT from Amazon Mechanical Turk')
parser.add_argument('-successfile', required=True, help='(required) The file to which you\'d like your results saved')
parser.add_argument('-sandbox', type=bool, default=False, help='Run the command in the Mechanical Turk Sandbox (used for testing purposes) NOT IMPLEMENTED')
parser.add_argument('-p', '--profile',
help='Run commands using specific aws credentials rather the default. To set-up alternative credentials see http://boto3.readthedocs.org/en/latest/guide/configuration.html#shared-credentials-file')
args = parser.parse_args()
if args.sandbox:
if not config.has_section('MTurk'):
config.add_section('MTurk')
config.set('MTurk', 'sandbox', 'True')
hitids = None
with open(expanduser(args.successfile), 'r') as successfile:
hitids = [row['hitid'] for row in DictReader(successfile, delimiter='\t')]
mtc = MTurkConnection(is_secure=True, profile_name=args.profile)
# To get any information about status, you have to get the HIT via get_all_hits
# If you just use get_hit() it gets minimal info
all_hits = mtc.get_all_hits()
currhits = []
for h in all_hits:
if h.HITId in hitids:
currhits.append(h)
def enable_debug(args):
if "debug" in args.__dict__ and args.debug == True:
if not config.has_section('Boto'):
config.add_section('Boto')
config.set('Boto', 'debug', '2')
# -*- coding: utf-8 -*-
import json
import os
from boto import config
from boto.sqs.connection import SQSConnection
from boto.sqs.message import Message
from boto.sqs.queue import Queue as AwsQueue
if not config.has_section('Boto'):
config.add_section('Boto')
config.set('Boto', 'debug', '0')
class Queue(object):
"""docstring for Queue"""
aws_key = os.environ.get('PAYPARROT_AWS_SQS_KEY')
aws_secret = os.environ.get('PAYPARROT_AWS_SQS_SECRET')
import logging
logging.basicConfig(level=logging.ERROR)
@classmethod
def get_queue(cls, queue_name):
queue_url = os.environ.get('PAYPARROT_AWS_SQS_QUEUE_%s' % queue_name.upper())
if queue_url:
conn = SQSConnection(cls.aws_key, cls.aws_secret)
return AwsQueue(connection = conn, url = queue_url)
else:
return None
# Setup a database connection to be used in the rest of the code
DB_HOST = os.environ.get('DB_HOST', 'localhost:27017')
client = MongoClient(DB_HOST)
DB_NAME = os.environ.get('DB_NAME', 'dealsdb')
db1 = client[DB_NAME]
if 'DB_USER' in os.environ:
db1.authenticate(os.environ.get('DB_USER'), os.environ.get('DB_PASSWORD'))
on_aws = False #"ON_AWS" in os.environ
if on_aws:
if not botoconfig.has_section('Credentials'):
botoconfig.add_section('Credentials')
if not botoconfig.has_option('Credentials', 'aws_access_key_id'):
botoconfig.set('Credentials', 'aws_access_key_id',
os.environ.get('AWS_KEY'))
if not botoconfig.has_option('Credentials', 'aws_secret_access_key'):
botoconfig.set('Credentials', 'aws_secret_access_key',
os.environ.get('AWS_SECRET'))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'je+95#mqpv5%c2(sdp69q)5(o0_-5%*j-a(s0w%q9@spxaipkv'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
TEMPLATE_DEBUG = True
import os
import sys
import datetime
import json
from bottle import get, post, run, request
from bottle import jinja2_template as template
import boto
import boto.s3.connection
from boto import config as botoconfig
from boto.s3.key import Key
botoconfig.add_section('s3')
botoconfig.set('s3', 'use-sigv4', 'True')
# make sure we have AWS credentials and a S3 Bucket
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
if not AWS_ACCESS_KEY_ID:
print >> sys.stderr, 'Missing environment variable AWS_ACCESS_KEY_ID'
exit(1)
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
if not AWS_SECRET_ACCESS_KEY:
print >> sys.stderr, 'Missing environment variable AWS_SECRET_ACCESS_KEY'
exit(1)
# make sure we have AWS credentials and a S3 Bucket
S3_REGION = os.environ.get('S3_REGION')