def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
while not connected:
try:
if (configuration['global']['aws_access_key_id']
and configuration['global']['aws_secret_access_key']):
connection = dynamodb.connect_to_region(
configuration['global']['region'],
aws_access_key_id=configuration['global']['aws_access_key_id'],
aws_secret_access_key=\
configuration['global']['aws_secret_access_key'])
else:
connection = dynamodb.connect_to_region(
configuration['global']['region'])
connected = True
except Exception as err:
logger.error('Failed to connect to DynamoDB: {0}'.format(err))
if retries == 0:
logger.error(
'Please report an issue at: '
'https://github.com/sebdah/dynamic-dynamodb/issues')
raise
else:
logger.error('Retrying in 5 seconds')
retries -= 1
time.sleep(5)
logger.debug('Connected to DynamoDB')
return connection
def __get_connection_dynamodb(retries=3):
""" Ensure connection to DynamoDB
:type retries: int
:param retries: Number of times to retry to connect to DynamoDB
"""
connected = False
while not connected:
try:
if (configuration['global']['aws_access_key_id'] and
configuration['global']['aws_secret_access_key']):
connection = dynamodb.connect_to_region(
configuration['global']['region'],
aws_access_key_id=configuration['global']['aws_access_key_id'],
aws_secret_access_key=\
configuration['global']['aws_secret_access_key'])
else:
connection = dynamodb.connect_to_region(
configuration['global']['region'])
connected = True
except Exception as err:
logger.error('Failed to connect to DynamoDB: {0}'.format(err))
if retries == 0:
logger.error(
'Please report an issue at: '
'https://github.com/sebdah/dynamic-dynamodb/issues')
raise
else:
logger.error('Retrying in 5 seconds')
retries -= 1
time.sleep(5)
logger.debug('Connected to DynamoDB')
return connection
def checkauth(self):
if g.sessionid:
# There is a cookie check if session is still known
try:
ddb = connect_to_region('us-west-2')
sessiontable = ddb.get_table('sessions')
sid = sessiontable.get_item(g.sessionid)
except DynamoDBKeyNotFoundError:
# The session cookie is invalid
@after_this_request
def clearcookie(response):
response.delete_cookie('sessionid')
return response
return jsonify (retval = 'NotSignedIn')
# We found a valid session check the user
usertable = ddb.get_table('users')
userid = sid['userid']
try:
user = usertable.get_item(userid)
except DynamoDBKeyNotFoundError:
# The user is invalid
sid.delete()
@after_this_request
def clearcookie(response):
response.delete_cookie('sessionid')
return response
return jsonify (retval = 'NotSignedIn')
lastused = unicode(str(datetime.utcnow()))
sid.put_attribute('lastused',lastused)
sid.save()
return jsonify (retval = 'Success', preferredUsername = user['preferredUsername'])
return jsonify (retval = 'NotSignedIn')
def savesession(self,useridentifier):
# This basically persists the session till the user clears cookies or signout is called.
# Might want to write a script that cleans up old sessions after a while. Check the lastused value.
# Also could create a UI that invalidates sessions for a given user from a web interface.
ddb = connect_to_region('us-west-2')
sessiontable = ddb.get_table('sessions')
sessionid = unicode(str(uuid.uuid1()))
lastused = unicode(str(datetime.utcnow()))
g.sessionid = sessionid
# Paranoid: Just incase the UUID is already there we will fail
try:
sid = sessiontable.get_item(sessionid)
except DynamoDBKeyNotFoundError:
# Great this session wasn't found!
sid = sessiontable.new_item(hash_key=sessionid,attrs={'userid':useridentifier,'lastused':lastused})
sid.put()
@after_this_request
def setcookie(response):
if application.debug:
response.set_cookie('sessionid', value=g.sessionid)
else:
response.set_cookie('sessionid', value=g.sessionid, secure = True, httponly = True)
return response
return 'Success'
return None
def get_table():
# url_short is the pkey
try:
conn = connect_to_region(region_name='eu-west-1')
return conn.get_table('shorturl')
except Exception as e:
raise exc.HTTPBadRequest('Error during connection %s' % e)
def get_table():
# url_short is the pkey
try:
conn = connect_to_region(region_name='eu-west-1')
return conn.get_table('shorturl')
except Exception as e:
raise exc.HTTPBadRequest('Error during connection %s' % e)
def __init__(self, db_PUBLIC_KEY, db_PRIVATE_KEY):
"""sets up connection and updates the latest counter"""
self.db_connection = dynamodb.connect_to_region(
'us-east-1',
aws_access_key_id=db_PUBLIC_KEY,
aws_secret_access_key=db_PRIVATE_KEY)
self.imtable = self.db_connection.get_table('fzimages')
self.updatelatest()
def get_dynamodb_connection(params):
region = params['--region']
access_key, secret_key = get_aws_credentials(params)
if not (access_key and secret_key):
exit_with_error('ERROR: Invalid AWS credentials supplied.')
connection = connect_to_region(region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
if not connection:
exit_with_error('ERROR: unable to connect, check your region')
return connection
def connect_to_dynamo(self, aws_access_key_id=None, aws_secret_access_key=None):
credentials = {
'aws_access_key_id' : aws_access_key_id,
'aws_secret_access_key' : aws_secret_access_key
}
region_name = boto.config.get('dynamodb', 'region', 'us-east-1')
connection = connect_to_region(region_name, **credentials)
if connection is None:
raise Exception('Region name misconfigured in boto config. '
+ 'The value must map to a real DynamoDb region.')
print 'Connected to ' + region_name
return connection
def cli():
import boto
from boto import ec2
from boto import dynamodb
parser = setup_parser()
options = parser.parse_args()
logging.basicConfig(level=logging.DEBUG)
logging.getLogger("boto").setLevel(logging.INFO)
config_path = os.path.expandvars(os.path.expanduser(options.config))
if config_path is None or not os.path.exists(config_path):
raise ValueError("Invalid configuration path %r" % options.config)
with open(config_path) as fh:
config = yaml.load(fh.read())
access_key = config.get('access-key', os.environ.get('AWS_ACCESS_KEY_ID'))
secret_key = config.get('secret-key', os.environ.get(
'AWS_SECRET_ACCESS_KEY'))
if not access_key or not secret_key:
ec2_api = boto.connect_ec2()
db_api = boto.connect_dynamodb()
else:
ec2_api = ec2.connect_to_region(
options.region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
db_api = dynamodb.connect_to_region(
options.region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
tagged_instances = options.tag or config.get("tag")
if tagged_instances:
pass
log.debug("Setting up dynamodb tables")
instance_db = get_or_create_table(
db_api, INSTANCE_TABLE,
{'hash': 'app_id', 'range': 'instance_id', 'throughput': (50, 10)})
lock_db = get_or_create_table(
db_api, Lock.lock_table_name, Lock.lock_table_options)
lock = Lock(
db_api, subprocess.check_output(['hostname']),
lock_db, None, ttl=120, delay=10, attempts=3)
log.debug("Starting snapshot runner")
runner = SnapshotRunner(config, ec2_api, instance_db, lock)
run_method = getattr(runner, options.func)
run_method(options)
def __get_connection_dynamodb():
""" Ensure connection to DynamoDB """
try:
if (configuration['global']['aws_access_key_id'] and
configuration['global']['aws_secret_access_key']):
connection = dynamodb.connect_to_region(
configuration['global']['region'],
aws_access_key_id=configuration['global']['aws_access_key_id'],
aws_secret_access_key=\
configuration['global']['aws_secret_access_key'])
else:
connection = dynamodb.connect_to_region(
configuration['global']['region'])
except Exception as err:
logger.error('Failed connecting to DynamoDB: {0}'.format(err))
logger.error(
'Please report an issue at: '
'https://github.com/sebdah/dynamic-dynamodb/issues')
raise
logger.debug('Connected to DynamoDB')
return connection
def login(db):
request_token = sess.obtain_request_token()
print request_token
sess = connect_dropbox()
db = dynamodb.connect_to_region(DYNAMODB_REGION)
table = db.get_table(DROPBOX_REQ_TOKENS_TABLE)
token_attrs = { 'key': request_token.key, 'secret': request_token.secret }
token = table.new_item(attrs=token_attrs)
token.put()
url = sess.build_authorize_url(request_token, get_url('/success'))
redirect(url)
def dump(table_name):
t0 = time.time()
folder_name = datetime.datetime.fromtimestamp(t0).strftime('%Y%m%d')
dump_dir = os.path.join(BASE_DIR, table_name, folder_name)
logger.info('Starting dump creation for table=%s...' % table_name)
logger.info('Dumping into %s' %dump_dir)
try:
os.makedirs(dump_dir)
except OSError as e:
print 'Error during dump creation: %s' %e
logger.error(e)
logger.info('Removing directory...')
shutil.rmtree(dump_dir)
sys.exit(2)
try:
conn = connect_to_region(region_name='eu-west-1')
except Exception as e:
print 'Error during dump creation: %s' %e
logger.error(e)
logger.error('DUMP_STATUS=1')
sys.exit(1)
try:
table_desc = conn.describe_table(table_name)
save_schema(dump_dir, table_desc)
save_data(conn, table_name, dump_dir)
zip_dir(folder_name + '.zip', dump_dir, BASE_DIR + table_name)
print 'Manage retention in dir %s' %dump_dir
shutil.rmtree(dump_dir)
manage_retention(os.path.join(BASE_DIR, table_name))
except Exception as e:
tf = time.time()
toff = tf - t0
print 'Error during dump creation: %s' %e
logger.error(e)
logger.error('It took: %s seconds' %toff)
logger.error('DUMP_STATUS=1')
sys.exit(1)
tf = time.time()
toff = tf - t0
print 'Success! It took %s seconds' %toff
logger.info('Success! It took: %s seconds' %toff)
logger.info('DUMP_STATUS=0')
def success(db):
sess = connect_dropbox()
tok = str(request.query.oauth_token)
db = dynamodb.connect_to_region(DYNAMODB_REGION)
table = db.get_table(DROPBOX_REQ_TOKENS_TABLE)
item = table.get_item(tok)
if item:
token = oauth.OAuthToken(item.attrs.key, item.attrs.secret)
access_token = sess.obtain_access_token(token)
cl = client.DropboxClient(sess)
response.set_cookie('access_token', access_token, secret=COOKIE_KEY)
redirect('/doclist')
else:
return 'Something went wrong'
def signout(self):
if g.sessionid:
# Clear the session
try:
ddb = connect_to_region('us-west-2')
sessiontable = ddb.get_table('sessions')
sid = sessiontable.get_item(g.sessionid)
sid.delete()
except DynamoDBKeyNotFoundError:
pass
@after_this_request
def clearcookie(response):
response.delete_cookie('sessionid')
return response
return jsonify (retval = 'Success')
return jsonify (retval = 'NotSignedIn')
def dynamodb_connection_factory():
"""
Since SessionStore is called for every single page view, we'd be
establishing new connections so frequently that performance would be
hugely impacted. We'll lazy-load this here on a per-worker basis. Since
boto.dynamodb.layer2.Layer2 objects are state-less (aside from security
tokens), we're not too concerned about thread safety issues.
"""
global _DYNAMODB_CONN
if not _DYNAMODB_CONN:
logger.debug("Creating a DynamoDB connection.")
_DYNAMODB_CONN = connect_to_region(
AWS_REGION_NAME,
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY
)
return _DYNAMODB_CONN
def get_db(self):
"""Get the data table for the controller.
"""
if not self._dynamodb:
self._dynamodb = dynamodb.connect_to_region(
self.get_region(), **(self.get_credentials()))
if self._data_table is not None:
return self._data_table
if self._table_name is None or self._table_options is None:
raise RuntimeError(
"Db requested but table not specified %s",
self.__class__.__name__)
self._data_table = self._get_table(
self._table_name, self._table_options)
return self._data_table
def getuser(self,auth_info):
# Takes a Janrain auth_info object and gets the user from DynamoDB if they exist. Otherwise creates the user.
# Check the user['isnewuser'] = True to see if the user is new.
# Assumes you are using DynamoDB in the AWS Oregon Region (see serversetup.txt for more information)
# Authentication to DynamoDB uses IAM Roles
ddb = connect_to_region('us-west-2')
usertable = ddb.get_table('users')
profile = auth_info['profile']
# The indentifier in janrain is globaly unique and we use that as the user id in dynamodb
identifier = unicode(profile['identifier'])
try:
user = usertable.get_item(identifier)
user['isnewuser'] = False
except DynamoDBKeyNotFoundError:
#This is a new user
provider = unicode(profile['providerName'])
username = unicode(profile.get('preferredUsername'))
user = usertable.new_item(hash_key=identifier,attrs={'providerName':provider,'preferredUsername':username})
user.put()
user['isnewuser'] = True
return user
def setUpClass(cls):
cls.client = client = dynamodb.connect_to_region(cls.region)
# Defer to controller lock name else test class lock name.
table_name = cls.factory and cls.factory._lock_table_name or \
cls.lock_table_name
if table_name in client.list_tables():
cls.table = client.get_table(table_name)
return
print cls.lock_table_name
cls.table = client.create_table(
table_name,
client.create_schema(
hash_key_name='key', hash_key_proto_value=str, ),
20, 20)
log.info("%s Waiting for lock table", cls.__name__)
while True:
if cls.table.status != 'ACTIVE':
time.sleep(3)
cls.table.refresh()
continue
break
log.info("%s Lock table ready", cls.__name__)
def __init__(self, db_PUBLIC_KEY, db_PRIVATE_KEY):
"""sets up connection and updates the latest counter"""
self.db_connection = dynamodb.connect_to_region('us-east-1',aws_access_key_id=db_PUBLIC_KEY,aws_secret_access_key=db_PRIVATE_KEY)
self.imtable = self.db_connection.get_table('fzimages')
self.updatelatest()
def get_conn():
return dynamodb.connect_to_region(
'us-west-2',
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
def get_conn():
return dynamodb.connect_to_region(
'us-west-2',
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
)
def _connect_to_dynamodb(self): self.database = dynamodb.connect_to_region( self.region, aws_access_key_id="AKIAJQF4ELAIIMGJABZA", aws_secret_access_key="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" )