Ejemplo n.º 1
0
def __get_connection_dynamodb(retries=3):
    """ Ensure connection to DynamoDB

    :type retries: int
    :param retries: Number of times to retry to connect to DynamoDB
    """
    connected = False
    while not connected:
        logger.debug('Connecting to DynamoDB in {0}'.format(
            get_global_option('region')))

        if (get_global_option('aws_access_key_id') and
                get_global_option('aws_secret_access_key')):
            logger.debug(
                'Authenticating to DynamoDB using '
                'credentials in configuration file')
            connection = dynamodb2.connect_to_region(
                get_global_option('region'),
                aws_access_key_id=get_global_option('aws_access_key_id'),
                aws_secret_access_key=get_global_option(
                    'aws_secret_access_key'))
        else:
            try:
                logger.debug(
                    'Authenticating to DynamoDB using EC2 instance profile')
                metadata = get_instance_metadata(timeout=1, num_retries=1)
                connection = dynamodb2.connect_to_region(
                    metadata['placement']['availability-zone'][:-1],
                    profile_name=metadata['iam']['info'][u'InstanceProfileArn'])
            except KeyError:
                logger.debug(
                    'Authenticating to DynamoDB using '
                    'env vars / boto configuration')
                connection = dynamodb2.connect_to_region(
                    get_global_option('region'))

        if not connection:
            if retries == 0:
                logger.error('Failed to connect to DynamoDB. Giving up.')
                raise
            else:
                logger.error(
                    'Failed to connect to DynamoDB. Retrying in 5 seconds')
                retries -= 1
                time.sleep(5)
        else:
            connected = True
            logger.debug('Connected to DynamoDB in {0}'.format(
                get_global_option('region')))

    return connection
Ejemplo n.º 2
0
def __get_connection_dynamodb(retries=3):
    """ Ensure connection to DynamoDB

    :type retries: int
    :param retries: Number of times to retry to connect to DynamoDB
    """
    connected = False
    while not connected:
        logger.debug('Connecting to DynamoDB in {0}'.format(
            get_global_option('region')))

        if (get_global_option('aws_access_key_id')
                and get_global_option('aws_secret_access_key')):
            logger.debug('Authenticating to DynamoDB using '
                         'credentials in configuration file')
            connection = dynamodb2.connect_to_region(
                get_global_option('region'),
                aws_access_key_id=get_global_option('aws_access_key_id'),
                aws_secret_access_key=get_global_option(
                    'aws_secret_access_key'))
        else:
            try:
                logger.debug(
                    'Authenticating to DynamoDB using EC2 instance profile')
                metadata = get_instance_metadata(timeout=1, num_retries=1)
                connection = dynamodb2.connect_to_region(
                    metadata['placement']['availability-zone'][:-1],
                    profile_name=metadata['iam']['info']
                    [u'InstanceProfileArn'])
            except KeyError:
                logger.debug('Authenticating to DynamoDB using '
                             'env vars / boto configuration')
                connection = dynamodb2.connect_to_region(
                    get_global_option('region'))

        if not connection:
            if retries == 0:
                logger.error('Failed to connect to DynamoDB. Giving up.')
                raise
            else:
                logger.error(
                    'Failed to connect to DynamoDB. Retrying in 5 seconds')
                retries -= 1
                time.sleep(5)
        else:
            connected = True
            logger.debug('Connected to DynamoDB in {0}'.format(
                get_global_option('region')))

    return connection
Ejemplo n.º 3
0
def getData(column):
    conn = dynamodb2.connect_to_region(
        REGION,
        aws_access_key_id=AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
    )  # connect to AWS Dynamo db
    table = Table(TABLE_NAME,
                  connection=conn)  # get data database table Pilogs
    results = table.scan()  # get all the data from db and store in results

    data = {}  # create an empty dictionary

    for dynamo_items in results:
        dict1 = dict(
            dynamo_items
        )  # dynamo_items = <boto.dynamodb2.items.Item object at 0x7630d0f0>
        data[dict1['Timestamp']] = {
        }  # create a nested dictionary with Timestamp as the key

        # Insert into the nested dictionary such that it will be like: {'Timestamp': {'Pi_SN':value, 'Light':value ... } }
        for key, value in dict1.items():
            if 'Pi_SN' in key:
                data[dict1['Timestamp']][
                    key] = value  # append the value of Pi_SN into the nested dictionary
            elif column in key:
                data[dict1['Timestamp']][key] = str(
                    value
                )  # append the value of column('Light', 'Temperature', 'Humidity') into the nested dictionary

    # return the nested dictiionary
    return data
Ejemplo n.º 4
0
def shortener(request):
    url = check_url(
        request.params.get('url')
    )
    if len(url) >= 2046:
        # we only accept URL shorter or equal to 2046 characters
        # Index restriction in DynamoDB
        url_short = 'toolong'
    else:
        # DynamoDB v2 high-level abstraction
        try:
            table = Table('shorturl', connection=connect_to_region('eu-west-1'))
        except Exception as e:
            raise exc.HTTPBadRequest('Error during connection %s' % e)

        url_short = _add_item(table, url)

    # Use env specific URLs
    if request.host not in ('api.geo.admin.ch', 'api3.geo.admin.ch'):
        host_url = make_api_url(request) + '/shorten/'
    else:
        host_url = ''.join((request.scheme, '://s.geo.admin.ch/'))

    return {
        'shorturl': host_url + url_short
    }
Ejemplo n.º 5
0
    def dynamo(self):
        """ Lazy loading of the dynamo connection """
        if self._dynamo is None:
            if self.region is not None:
                self._dynamo = connect_to_region(self.region)
            else:
                if not os.path.exists(self.path):
                    tarball = urlretrieve(self.link)[0]
                    subprocess.check_call(['tar', '-zxf', tarball])
                    name = os.path.basename(self.link).split('.')[0]
                    shutil.move(name, self.path)
                    os.unlink(tarball)

                lib_path = os.path.join(self.path, 'DynamoDBLocal_lib')
                jar_path = os.path.join(self.path, 'DynamoDBLocal.jar')
                cmd = [self.java, '-Djava.library.path=' + lib_path, '-jar',
                       jar_path, '--port', str(self.port)]
                self._dynamo_local = subprocess.Popen(cmd,
                                                      stdout=subprocess.PIPE,
                                                      stderr=subprocess.STDOUT)
                self._dynamo = DynamoDBConnection(
                    host='localhost',
                    port=self.port,
                    is_secure=False,
                    aws_access_key_id='',
                    aws_secret_access_key='')
        return self._dynamo
Ejemplo n.º 6
0
 def get(self):
     if self.conn is None:
         try:
             self.conn = connect_to_region(self.region)
         except Exception as e:
             raise exc.HTTPBadRequest('DynamoDB: Error during connection init %s' % e)
     return self.conn
Ejemplo n.º 7
0
 def get(self):
     if self.conn is None:
         try:
             self.conn = connect_to_region(self.region)
         except Exception as e:
             raise exc.HTTPBadRequest('DynamoDB: Error during connection init %s' % e)
     return self.conn
Ejemplo n.º 8
0
    def connection(self):
        """
        Our DynamoDB connection.

        This will be lazily created if this is the first time this is being
        accessed.  This connection is reused for performance.
        """
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'dynamo_connection'):
                kwargs = {
                    'host': self.app.config['DYNAMO_LOCAL_HOST'] if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
                    'port': int(self.app.config['DYNAMO_LOCAL_PORT']) if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
                    'is_secure': False if self.app.config['DYNAMO_ENABLE_LOCAL'] else True,
                }

                # Only apply if manually specified: otherwise, we'll let boto
                # figure it out (boto will sniff for ec2 instance profile
                # credentials).
                if self.app.config['AWS_ACCESS_KEY_ID']:
                  kwargs['aws_access_key_id'] = self.app.config['AWS_ACCESS_KEY_ID']
                if self.app.config['AWS_SECRET_ACCESS_KEY']:
                  kwargs['aws_secret_access_key'] = self.app.config['AWS_SECRET_ACCESS_KEY']

                # If DynamoDB local is disabled, we'll remove these settings.
                if not kwargs['host']:
                    del kwargs['host']
                if not kwargs['port']:
                    del kwargs['port']

                ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'], **kwargs)

            return ctx.dynamo_connection
Ejemplo n.º 9
0
 def connect(self, **kwargs):
     if 'table_prefix' in kwargs:
         DynamoDatabase.table_prefix = kwargs['table_prefix']
         del kwargs['table_prefix']
     if DynamoDatabase._db_connection is not None:
         raise DynamoException(
             'Already connected, use disconnect() before making a '
             'new connection')
     if (
         'region_name' in kwargs and
         kwargs['region_name'] == 'localhost'
     ):
         # local dynamodb
         debug = kwargs['debug'] if 'debug' in kwargs else 0
         DynamoDatabase._db_connection = \
             boto.dynamodb2.layer1.DynamoDBConnection(
                 host=kwargs.get('DYNAMODB_HOST', 'localhost'),
                 port=kwargs.get('DYNAMODB_PORT', 8000),
                 aws_access_key_id='local',
                 aws_secret_access_key='success',
                 is_secure=False,
                 debug=debug)
         DynamoDatabase.local_dynamodb = True
     else:  # Real dynamo db
         DynamoDatabase._db_connection = connect_to_region(**kwargs)
     self.get_tables()
     return DynamoDatabase._db_connection
Ejemplo n.º 10
0
def data():
    from boto import dynamodb2
    from boto.dynamodb2.table import Table

    TABLE_NAME = "issdata"
    REGION = "us-west-1"

    conn = dynamodb2.connect_to_region(
        REGION,
        aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
        aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'],
    )
    table = Table(TABLE_NAME, connection=conn)

    absolute_junk = {
        "favorite_color": "blue",
        "quest": "seek_holy_grail",
    }

    with table.batch_write() as table_batch:
        for example_counter in xrange(10):

            required_hash_data = {
                "user_id": 11,
                "timestamp":
                datetime_to_timestamp_ms(datetime.datetime.utcnow())
            }

            final_dynamo_data = dict(absolute_junk.items() +
                                     required_hash_data.items())
            table_batch.put_item(data=final_dynamo_data)
Ejemplo n.º 11
0
def dynamo_main():
    conn = dynamodb2.connect_to_region(region_name=region,
                                       aws_access_key_id=access_key,
                                       aws_secret_access_key=secret_key)
    diag_rep_table = Table(dynamodb_diag_rep_table, connection=conn)
    table = Table(dynamodb_main_org_table, connection=conn)

    diag_rep_res = diag_rep_table.scan()

    diag_rep_ids = []

    for diag_rep in diag_rep_res:
        diag_rep_ids.append(dict(diag_rep.get_raw_keys())['diagRepId']['S'])

    with table.batch_write() as batch:
        for i in range(0, 10):
            main_org = {}
            global next_id
            main_org['orgId'] = str(next_id)
            next_id += 1
            engineers = []
            engineers.append(engineer_ids[0])
            engineers.append(engineer_ids[1])
            main_org['engineers'] = engineers
            pending = []
            pending.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
            pending.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
            resp = []
            resp.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
            resp.append(diag_rep_ids[randint(0, len(diag_rep_ids) - 1)])
            main_org['pendingDiagnosticReports'] = pending
            main_org['respondedDiagnosticReports'] = resp
Ejemplo n.º 12
0
 def getTable() :
     """
     returns the Boto Table object
     """
     if(SessionTable.isLocal) :
         return Table(SessionTable.TABLE_NAME,connection=SessionTable.getLocalConnection())
     return Table(SessionTable.TABLE_NAME,connection=connect_to_region(SessionTable.DYNAMO_REGION))
Ejemplo n.º 13
0
    def connection(self):
        """
        Our DynamoDB connection.

        This will be lazily created if this is the first time this is being
        accessed.  This connection is reused for performance.
        """
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'dynamo_connection'):
                kwargs = {
                    'host': self.app.config['DYNAMO_LOCAL_HOST'] if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
                    'port': int(self.app.config['DYNAMO_LOCAL_PORT']) if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
                    'is_secure': False if self.app.config['DYNAMO_ENABLE_LOCAL'] else True,
                }

                # Only apply if manually specified: otherwise, we'll let boto
                # figure it out (boto will sniff for ec2 instance profile
                # credentials).
                if self.app.config['AWS_ACCESS_KEY_ID']:
                  kwargs['aws_access_key_id'] = self.app.config['AWS_ACCESS_KEY_ID']
                if self.app.config['AWS_SECRET_ACCESS_KEY']:
                  kwargs['aws_secret_access_key'] = self.app.config['AWS_SECRET_ACCESS_KEY']

                # If DynamoDB local is disabled, we'll remove these settings.
                if not kwargs['host']:
                    del kwargs['host']
                if not kwargs['port']:
                    del kwargs['port']

                ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'], **kwargs)

            return ctx.dynamo_connection
Ejemplo n.º 14
0
    def connection(self):
        """
        Our DynamoDB connection.

        This will be lazily created if this is the first time this is being
        accessed.  This connection is reused for performance.
        """
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'dynamo_connection'):
                kwargs = {
                    'aws_access_key_id': self.app.config['AWS_ACCESS_KEY_ID'],
                    'aws_secret_access_key': self.app.config['AWS_SECRET_ACCESS_KEY'],
                    'host': self.app.config['DYNAMO_LOCAL_HOST'] if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
                    'port': int(self.app.config['DYNAMO_LOCAL_PORT']) if self.app.config['DYNAMO_ENABLE_LOCAL'] else None,
                    'is_secure': False if self.app.config['DYNAMO_ENABLE_LOCAL'] else True,
                }

                # If DynamoDB local is disabled, we'll remove these settings.
                if not kwargs['host']:
                    del kwargs['host']
                if not kwargs['port']:
                    del kwargs['port']

                ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'], **kwargs)

            return ctx.dynamo_connection
Ejemplo n.º 15
0
 def delete(self, resource):
     if self.dry_run:
         return
     connection = dynamodb2.connect_to_region(resource.region)
     self.logger.info("Initiating deletion sequence for {0}.".format(
         resource.wrapped["TableName"]))
     connection.delete_table(resource.wrapped["TableName"])
def dynamo_main():
    conn = dynamodb2.connect_to_region(region_name=region,
                                       aws_access_key_id=access_key,
                                       aws_secret_access_key=secret_key)
    prop_appl_table = Table(dynamodb_prop_appl_table, connection=conn)
    table = Table(dynamodb_diag_rep_table, connection=conn)

    prop_appl_res = prop_appl_table.scan()

    prop_appl_ids = []

    for prop_appl in prop_appl_res:
        prop_appl_ids.append(dict(prop_appl.get_raw_keys())['propApplId']['S'])

    with table.batch_write() as batch:
        for i in range(0, 20):
            diag_rep = {}
            global next_id
            diag_rep['diagRepId'] = str(next_id)
            next_id += 1
            diag_rep['propApplId'] = prop_appl_ids[randint(
                0,
                len(prop_appl_ids) - 1)]
            diag_rep['managerId'] = manager_ids[randint(
                0,
                len(manager_ids) - 1)]
            diag_rep['organisations'] = ["1"]
            diag_rep['timestamp'] = int(time.time())
            diag_rep['description'] = "Description"
            batch.put_item(data=diag_rep)
Ejemplo n.º 17
0
def sendtodynamo_cnn(cnnjson):
  ''' Send json to DynamoDB
  Assumes that article timestamps have been deduped to avoid collisions
  '''

  conn = connect_to_region('us-west-2', aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
  
  hashkey = "CNN" # primary key to be used for DynamoDB table

  try:
    table = Table('CNN', connection=conn)
    table.describe()
  except boto.exception.JSONResponseError:
    print "Creating table"
    table = Table.create('CNN', schema=[HashKey('source'), RangeKey('tstamp',data_type=NUMBER)], throughput={'read':25, 'write':25}, indexes=[GlobalAllIndex('showidx',parts=[HashKey('show')],throughput={'read':10,'write':5})])

  iteration = 0
  for article in cnnjson:
    # Iterate through list of articles and upload to table
    rangekey = float(article['timestamp'])
    rowdata = {'source':hashkey,'tstamp':rangekey, 'cnnShow':article['show']}
    for key in article.keys():
      rowdata[key]=article[key]
    item = table.put_item(data = rowdata)
    iteration += 1
    if iteration%100==0:
      print "Uploaded "+iteration+" articles"

  return None
Ejemplo n.º 18
0
    def __init__(self, settings):
        '''
        read/write throughtput: read/write capacity times per second
        refer: http://aws.amazon.com/dynamodb/pricing/

        create layer2 connection
        '''
        self.settings = settings
        self.conn = dynamodb2.connect_to_region(settings.get('region', 'ap-northeast-1'),
                                                aws_access_key_id=settings.get('aws_access_key_id'),
                                                aws_secret_access_key=settings.get('aws_secret_access_key')
                                                )
        self.default_throughput = {'read': settings.get('default_throughput').get('read', 100),
                                   'write': settings.get('default_throughput').get('write', 100)
                                   }
        self._tables = {}
        #
        # hash_key_name is 'key', type is 'S' for string
        # no range key
        #
        self.hash_key_name = settings.get('hash_key_name', _DEFAULT_HASH_KEY_NAME)
        self.range_key_name = settings.get('range_key_name', None)
        self.data_property = settings.get('data_property', _DEFAULT_DATA_PROPERTY)
        self.default_schema = [HashKey(self.hash_key_name)]
        if self.range_key_name:
            self.default_schema.append(RangeKey(self.range_key_name))
Ejemplo n.º 19
0
 def getTable() :
     """
     returns the Boto Table object
     """
     if(SessionTable.isLocal) :
         return Table(SessionTable.TABLE_NAME,connection=SessionTable.getLocalConnection())
     return Table(SessionTable.TABLE_NAME,connection=connect_to_region(SessionTable.DYNAMO_REGION))
Ejemplo n.º 20
0
    def __init__(self, ut=False, read_tput=None, write_tput=None):
        self.conn = None
        self.tables = {}
        self.keys = {}
        if read_tput:
            self.throughput_read = read_tput
        else:
            self.throughput_read = 10
        if write_tput:
            self.throughput_write = write_tput
        else:
            self.throughput_write = 10

        # override this for testing with dynamodb local
        local = False

        if local:        
            self.conn = DynamoDBConnection(
                            aws_access_key_id='photostore',
                            aws_secret_access_key='local',
                            host='localhost',
                            port=8000,
                            is_secure=False)
        else:
            self.conn = connect_to_region(
                            'us-west-2',
                            aws_access_key_id='YOUR_ACCOUNT',
                            aws_secret_access_key='YOUR_KEY'
                        )
Ejemplo n.º 21
0
    def GetConnection(self):
        # Environment var
        if self.region == '':
            self.region = os.getenv('AWS_REGION', '') 

        # Default
        if self.region == '':
            self.region = "us-west-2"

        aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID', '')
        aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY', '') 

        if self.endpoint:
            return DynamoDBConnection(
                host=self.endpoint_data.hostname,
                port=self.endpoint_data.port, 
                aws_access_key_id=aws_access_key_id,
                aws_secret_access_key=aws_secret_access_key,
                is_secure=False
            )
        else:
            return connect_to_region(self.region,
                aws_access_key_id=aws_access_key_id,
                aws_secret_access_key=aws_secret_access_key
            )
Ejemplo n.º 22
0
def __get_connection_dynamodb(conf_file_options, retries=3):
    """ Ensure connection to DynamoDB

    :type retries: dict
    :param conf_file_options: configuration information

    :type retries: int
    :param retries: Number of times to retry to connect to DynamoDB

    """
    connected = False
    region = conf_file_options['region']

    logger = logging.getLogger('dynamic-dynamodb')

    while not connected:
        if 'aws_access_key_id' in conf_file_options and 'aws_secret_access_key' in conf_file_options:

            logger.debug(
                'Authenticating to DynamoDB using '
                'credentials in configuration file')

            connection = dynamodb2.connect_to_region(
                region,
                aws_access_key_id=conf_file_options['aws_access_key_id'],
                aws_secret_access_key=conf_file_options['aws_secret_access_key']
            )

        else:
            logger.debug(
                'Authenticating using boto\'s authentication handler')
            connection = dynamodb2.connect_to_region(region)

        if not connection:
            if retries == 0:
                logger.error('Failed to connect to DynamoDB. Giving up.')
                raise
            else:
                logger.error(
                    'Failed to connect to DynamoDB. Retrying in 5 seconds')
                retries -= 1
                time.sleep(5)
        else:
            connected = True
            logger.debug('Connected to DynamoDB in {0}'.format(region))

    return connection
 def connect(self):
     start = time.time()
     self.con = dynamodb2.connect_to_region(
         self.args.region,
         aws_access_key_id=self.args.key_id,
         aws_secret_access_key=self.args.secret,
         is_secure=True)
     self.time2connect = time.time() - start
Ejemplo n.º 24
0
 def create_table(self, name, schema, conn=None):
     try:
         if conn is None:
             conn = connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
         table = Table.create(name, schema=schema, connection=conn)
         return table
     except Exception as ex:
         print "Exception occured while creating table in dynamo:" + str(ex)
Ejemplo n.º 25
0
    def __init__(self):

        self.conn = dynamodb2.connect_to_region(
            REGION,
            aws_access_key_id=ACCESS_KEY,
            aws_secret_access_key=SECRET_KEY,
        )
        self.table = Table(TABLE_NAME, connection=self.conn)
 def connect(self):
     start=time.time()
     self.con=dynamodb2.connect_to_region(self.args.region,
         aws_access_key_id=self.args.key_id,
         aws_secret_access_key=self.args.secret,
         is_secure=True
     )
     self.time2connect=time.time()-start
Ejemplo n.º 27
0
def connect_to_db(app, table_name, hashkey):
    dyno = Table(table_name,
                 schema=[HashKey(hashkey)],
                 connection=ddb.connect_to_region(app.config['dynamodb.region'],
                                                  aws_access_key_id=app.config['keys.key_id'],
                                                  aws_secret_access_key=app.config['keys.key_secret'],
                                                  security_token=app.config['keys.key_token']))
    return dyno
Ejemplo n.º 28
0
 def __init__(self, keyid, secretkey,region,name,endpoint=None, port=None):
     self.tweetsTable = None
     self.db = None
     
     if port is None:
         port = 8000
     
     self.db = connect_to_region(region, aws_access_key_id=keyid, aws_secret_access_key=secretkey)
     self.tweetsTable = Table(name,connection=self.db)
def setup(jep):
	global table, schema, throughput, region, initwait
	connection=dynamodb2.connect_to_region(
		region,
		aws_access_key_id=os.environ['aws_access_key_id'],
		aws_secret_access_key=os.environ['aws_secret_access_key']
	)
	table = Table(tb_name,schema=schema,throughput=throughput,connection=connection)
	time.sleep(initwait)
Ejemplo n.º 30
0
 def delete(self, resource):
     if self.dry_run:
         return
     connection = dynamodb2.connect_to_region(resource.region)
     try:
         self.logger.info("Initiating deletion sequence for {0}.".format(resource.wrapped["TableName"]))
         connection.delete_table(resource.wrapped["TableName"])
     except ResourceInUseException as exc:
         self.logger.exception(exc)
Ejemplo n.º 31
0
 def configure(self, context):
     super().configure(context)
     region_name = re.sub('_', '-', self.region().name)
     self.logger.debug("Connecting to region {}...".format(region_name))
     self._conn = connect_to_region(
         region_name,
         aws_access_key_id=self.creds().access_key(),
         aws_secret_access_key=self.creds().access_secret())
     self.logger.debug("Connection complete")
Ejemplo n.º 32
0
def setup_logging():
    if not app.debug:
        # In production mode, add log handler to sys.stderr.
        app.logger.addHandler(logging.StreamHandler())
        app.logger.setLevel(logging.INFO)

        db_connection = dynamodb2.connect_to_region(AWS_REGION)

        app.db = DatabaseManager(db_connection)
Ejemplo n.º 33
0
def dynamo_main(appliances):
    conn = dynamodb2.connect_to_region(region_name=region,
                                       aws_access_key_id=access_key,
                                       aws_secret_access_key=secret_key)
    table = Table(dynamodb_appliance_table, connection=conn)

    with table.batch_write() as batch:
        for appliance in appliances:
            batch.put_item(data=appliance)
Ejemplo n.º 34
0
def connectToDynamoService():
    """
    This will use your ~/.boto configuration file containing
    access key and access secret
    """
    print 'connecting to service dynamodb...'
    conn = connect_to_region('eu-west-1')
    print 'connected.'
    return conn
Ejemplo n.º 35
0
def connect(app):
    table_name = app.config['dynamodb.turing_users']
    hashkey    = "user_id"
    dyno = Table(table_name,
                 schema=[HashKey(hashkey)],
                 connection=ddb.connect_to_region(app.config['dynamodb.region'],
                                                  aws_access_key_id=app.config['keys.key_id'],
                                                  aws_secret_access_key=app.config['keys.key_secret'],
                                                  security_token=app.config['keys.key_token']))
    return dyno
Ejemplo n.º 36
0
def client_factory():
    """Return an AmazonDB client that provides a
      ``put_item(data=data)`` method.
    """

    return Table(
        'request_storer',
        schema=[HashKey('request_id')],
        connection=dynamodb2.connect_to_region('eu-west-1')
    )
Ejemplo n.º 37
0
def __get_connection_dynamodb(retries=3):
    """ Ensure connection to DynamoDB

    :type retries: int
    :param retries: Number of times to retry to connect to DynamoDB
    """
    connected = False
    while not connected:
        logger.debug("Connecting to DynamoDB in {0}".format(get_global_option("region")))

        if get_global_option("aws_access_key_id") and get_global_option("aws_secret_access_key"):
            logger.debug("Authenticating to DynamoDB using " "credentials in configuration file")
            connection = dynamodb2.connect_to_region(
                get_global_option("region"),
                aws_access_key_id=get_global_option("aws_access_key_id"),
                aws_secret_access_key=get_global_option("aws_secret_access_key"),
            )
        else:
            try:
                logger.debug("Authenticating to DynamoDB using EC2 instance profile")
                metadata = get_instance_metadata(timeout=1, num_retries=1)
                connection = dynamodb2.connect_to_region(
                    metadata["placement"]["availability-zone"][:-1],
                    profile_name=metadata["iam"]["info"][u"InstanceProfileArn"],
                )
            except KeyError:
                logger.debug("Authenticating to DynamoDB using " "env vars / boto configuration")
                connection = dynamodb2.connect_to_region(get_global_option("region"))

        if not connection:
            if retries == 0:
                logger.error("Failed to connect to DynamoDB. Giving up.")
                raise
            else:
                logger.error("Failed to connect to DynamoDB. Retrying in 5 seconds")
                retries -= 1
                time.sleep(5)
        else:
            connected = True
            logger.debug("Connected to DynamoDB in {0}".format(get_global_option("region")))

    return connection
Ejemplo n.º 38
0
def connect(app):
    table_name = app.config['dynamodb.turing_users']
    hashkey = "user_id"
    dyno = Table(table_name,
                 schema=[HashKey(hashkey)],
                 connection=ddb.connect_to_region(
                     app.config['dynamodb.region'],
                     aws_access_key_id=app.config['keys.key_id'],
                     aws_secret_access_key=app.config['keys.key_secret'],
                     security_token=app.config['keys.key_token']))
    return dyno
Ejemplo n.º 39
0
    def __init__(self, namespace, table_name, region=None, hash_key='id', **params):

        OpenResourceNamespaceManager.__init__(self, namespace)

        if table_name is None:
            raise MissingCacheParameter('DynamoDB table name required.')

        self._hash_key = hash_key
        self._table = ddb.table.Table(table_name, connection=ddb.connect_to_region(region))
        self._flags = None
        self._item = None
Ejemplo n.º 40
0
    def fetch_unwanted_resources(self):
        for region in self.regions:
            connection = dynamodb2.connect_to_region(region.name)
            names = connection.list_tables(limit=100) or {}
            for name in names.get("TableNames"):
                resource = connection.describe_table(name)
                resource_wrapper = Resource(resource["Table"], region.name)
                if name in self.ignored_resources:
                    self.logger.info('IGNORE ' + self.to_string(resource_wrapper))
                    continue

                yield resource_wrapper
Ejemplo n.º 41
0
 def put_item(self, tablename, data, conn=None, schema=None):
     try:
         if conn is None:
             conn = connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
         table = self.get_table(tablename, conn, schema=schema)
         ret = table.put_item(data=data)
         if ret:
             print "Put successful.\n"
         else:
             print "Put failed.\n"
     except Exception as ex:
         print "DynamoDB: Exception occured in put_item in table '"+tablename+"':" + str(ex)
Ejemplo n.º 42
0
    def connection(self):
        """
        Our DynamoDB connection.

        This will be lazily created if this is the first time this is being
        accessed.  This connection is reused for performance.
        """
        ctx = stack.top
        if ctx is not None:
            if not hasattr(ctx, 'dynamo_connection'):
                ctx.dynamo_connection = connect_to_region(self.app.config['AWS_REGION'])
            return ctx.dynamo_connection
Ejemplo n.º 43
0
def put_to_dynamodb(data):
    """Make a PUT request to dynamodb2 and insert the data"""

    requests_table = Table('request_storer',
            schema=[HashKey('request_id')],
            connection=dynamodb2.connect_to_region('eu-west-1')
    )
    try:
        requests_table.put_item(data=data)
    except UnicodeDecodeError:
        data['body'] = 'Unicode error when parsing'
        requests_table.put_item(data=data)
Ejemplo n.º 44
0
def create_boto2_table(name):
    c = connect_to_region(os.getenv("AWS_REGION"),
                          host=HOST,
                          port=PORT,
                          is_secure=False)
    Table.create(name,
                 schema=[HashKey("string")],
                 throughput={
                     "read": 1,
                     "write": 1
                 },
                 connection=c)
    print(c.list_tables())
Ejemplo n.º 45
0
 def connect(self, aws_region, accessId, secretkey):
     try:
         self.aws_region = aws_region
         self.access_id = accessId
         self.secret_key = secretkey
         self.conn = dynamodb2.connect_to_region(
             self.aws_region,
             aws_access_key_id=self.access_id,
             aws_secret_access_key=self.secret_key)
         return True
     except Exception as e:
         print 'Exception happened in dynamodb connect ' + str(e.args)
         exit(2)
Ejemplo n.º 46
0
def dynamo_main(appliance_statuses, stat_icon):
    conn = dynamodb2.connect_to_region(region_name=region,
                                       aws_access_key_id=access_key,
                                       aws_secret_access_key=secret_key)
    table = Table(dynamodb_appliance_status_table, connection=conn)

    with table.batch_write() as batch:
        for appliance_stat in appliance_statuses:
            global next_id
            appliance_stat['statusId'] = str(next_id)
            appliance_stat['icon'] = stat_icon
            next_id += 1
            batch.put_item(data=appliance_stat)
Ejemplo n.º 47
0
def setup_logging():
    if not app.debug:
        # In production mode, add log handler to sys.stderr.
        app.logger.addHandler(logging.StreamHandler())
        app.logger.setLevel(logging.INFO)
        
        db_connection = dynamodb2.connect_to_region(
            AWS_REGION,
            aws_access_key_id=AWS_ACCESS_KEY_ID,
            aws_secret_access_key=AWS_SECRET_ACCESS_KEY
        )

        app.db = DatabaseManager(db_connection)
Ejemplo n.º 48
0
def __get_connection_dynamodb(retries=3):
    """ Ensure connection to DynamoDB

    :type retries: int
    :param retries: Number of times to retry to connect to DynamoDB
    """
    connected = False
    region = get_global_option('region')

    while not connected:
        if (get_global_option('aws_access_key_id') and
                get_global_option('aws_secret_access_key')):
            logger.debug(
                'Authenticating to DynamoDB using '
                'credentials in configuration file')
            connection = dynamodb2.connect_to_region(
                region,
                aws_access_key_id=get_global_option('aws_access_key_id'),
                aws_secret_access_key=get_global_option(
                    'aws_secret_access_key'))
        else:
            logger.debug(
                'Authenticating using boto\'s authentication handler')
            connection = dynamodb2.connect_to_region(region)

        if not connection:
            if retries == 0:
                logger.error('Failed to connect to DynamoDB. Giving up.')
                raise
            else:
                logger.error(
                    'Failed to connect to DynamoDB. Retrying in 5 seconds')
                retries -= 1
                time.sleep(5)
        else:
            connected = True
            logger.debug('Connected to DynamoDB in {0}'.format(region))

    return connection
Ejemplo n.º 49
0
def __get_connection_dynamodb(retries=3):
    """ Ensure connection to DynamoDB

    :type retries: int
    :param retries: Number of times to retry to connect to DynamoDB
    """
    connected = False
    region = get_global_option('region')

    while not connected:
        if (get_global_option('aws_access_key_id') and
                get_global_option('aws_secret_access_key')):
            logger.debug(
                'Authenticating to DynamoDB using '
                'credentials in configuration file')
            connection = dynamodb2.connect_to_region(
                region,
                aws_access_key_id=get_global_option('aws_access_key_id'),
                aws_secret_access_key=get_global_option(
                    'aws_secret_access_key'))
        else:
            logger.debug(
                'Authenticating using boto\'s authentication handler')
            connection = dynamodb2.connect_to_region(region)

        if not connection:
            if retries == 0:
                logger.error('Failed to connect to DynamoDB. Giving up.')
                raise
            else:
                logger.error(
                    'Failed to connect to DynamoDB. Retrying in 5 seconds')
                retries -= 1
                time.sleep(5)
        else:
            connected = True
            logger.debug('Connected to DynamoDB in {0}'.format(region))

    return connection
Ejemplo n.º 50
0
def sendtodynamo_cnn(cnnjson):
    ''' Send json to DynamoDB
  Assumes that article timestamps have been deduped to avoid collisions
  '''

    conn = connect_to_region('us-west-2',
                             aws_access_key_id=AWS_ACCESS_KEY_ID,
                             aws_secret_access_key=AWS_SECRET_ACCESS_KEY)

    hashkey = "CNN"  # primary key to be used for DynamoDB table

    try:
        table = Table('CNN', connection=conn)
        table.describe()
    except boto.exception.JSONResponseError:
        print "Creating table"
        table = Table.create(
            'CNN',
            schema=[HashKey('source'),
                    RangeKey('tstamp', data_type=NUMBER)],
            throughput={
                'read': 25,
                'write': 25
            },
            indexes=[
                GlobalAllIndex('showidx',
                               parts=[HashKey('show')],
                               throughput={
                                   'read': 10,
                                   'write': 5
                               })
            ])

    iteration = 0
    for article in cnnjson:
        # Iterate through list of articles and upload to table
        rangekey = float(article['timestamp'])
        rowdata = {
            'source': hashkey,
            'tstamp': rangekey,
            'cnnShow': article['show']
        }
        for key in article.keys():
            rowdata[key] = article[key]
        item = table.put_item(data=rowdata)
        iteration += 1
        if iteration % 100 == 0:
            print "Uploaded " + iteration + " articles"

    return None
def dynamodb2_connection_factory():
    from django.conf import settings
    if getattr(settings, 'AWS_ACCESS_KEY_ID', '') and getattr(
            settings, 'AWS_SECRET_ACCESS_KEY', ''):
        logger.info("Creating a DynamoDB connection.")
        connection_data = {
            'region_name': settings.AWS_REGION_NAME,
            'aws_access_key_id': settings.AWS_ACCESS_KEY_ID,
            'aws_secret_access_key': settings.AWS_SECRET_ACCESS_KEY,
        }
        conn = connect_to_region(**connection_data)
        setattr(settings, 'DYNAMODB_CONNECTION', conn)
    else:
        logger.error(
            "Settings is not configured properly, missing AWS access data")
Ejemplo n.º 52
0
def inserttodb():
  while True:
    Timestamp,Pi_SN,ARM_Status,GPU_Status,Humidity,Light,Pi_Temp,Temperature = getdata()
    AWS_ACCESS_KEY_ID = 'AKIAIMPG2I3WRWDTWYQA'
    AWS_SECRET_ACCESS_KEY = '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'
    REGION = 'us-west-2'
    TABLE_NAME = 'Pilogs'

    conn = dynamodb2.connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)

    table = Table(
      TABLE_NAME,
      connection=conn
    )

    results = table.scan()

    datestring = datetime.datetime.now().date()
    timestring = time.strftime("%H:%M:%S", time.localtime())
    datetime1 = str(datestring)+' '+str(timestring)

    Pi_SN = Pi_SN
    Timestamp = Timestamp
    ARM_Status = ARM_Status+'M'
    GPU_Status = GPU_Status+'M'
    Humidity = Humidity
    Light = Light
    Pi_Temp = Pi_Temp
    Temperature = Temperature
    if Temperature == None or Humidity == None:
      print 'some information is not valid'
      # for dynamo_item in results:
    else:  
      response = table.put_item(
        {
          'Pi_SN':Pi_SN,
          'Timestamp':Timestamp,
          'ARM_Status':ARM_Status,
          'GPU_Status':GPU_Status,
          'Humidity':Humidity,
          'Light':Light,
          'Pi_Temp':Pi_Temp,
          'Temperature':Temperature,
        }
      )
      print 'all values inserted into the dynamoDB'
      conn.close()
      time.sleep(15)    
Ejemplo n.º 53
0
    def setUp(self):

        for k in ('AWS_DEFAULT_REGION', 'AWS_ACCESS_KEY_ID',
                  'AWS_SECRET_ACCESS_KEY'):
            if k not in os.environ:
                return

        conn = dynamodb2.connect_to_region(
            os.environ['AWS_DEFAULT_REGION'],
            aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
            aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'])

        # Create both dynamodb table and ObjectStore
        self.bag = MutexBag(conn, 'objectstore-test-table--delete-at-will')
        self.bag.create_table()
        self.bag._release('mutex-foobar')
Ejemplo n.º 54
0
def getLatestData():
    conn = dynamodb2.connect_to_region(
        REGION,
        aws_access_key_id=AWS_ACCESS_KEY_ID,
        aws_secret_access_key=AWS_SECRET_ACCESS_KEY
    )  # connect to AWS Dynamo db
    table = Table(TABLE_NAME,
                  connection=conn)  # get data database table Pilogs
    results = table.scan()  # get all the data from db and store in results

    latest_data = {}  # create an empty dictionary

    for dynamo_items in results:  # dynamo_items = <boto.dynamodb2.items.Item object at 0x73a97b10>
        dict1 = dict(
            dynamo_items
        )  # store dynao_items in a dictionary and store it in dict1
        latest_data[dict1['Pi_SN']] = {
        }  # create a nested dictionary with Pi_SN as the key

        # Insert into the nested dictionary such that it will be like: {'Pi_SN': {'Timestamp':value, 'Pi_Temp':value ... } }
        for key, value in dict1.items():
            if 'Timestamp' in key:
                latest_data[dict1['Pi_SN']][
                    key] = value  # append the value of timestamp into the nested dictionary
            elif 'Pi_Temp' in key:
                latest_data[dict1['Pi_SN']][
                    key] = value  # append the value of Pi_Temp into the nested dictionary
            elif 'ARM_Status' in key:
                latest_data[dict1['Pi_SN']][
                    key] = value  # append the value of ARM_Status into the nested dictionary
            elif 'GPU_Status' in key:
                latest_data[dict1['Pi_SN']][
                    key] = value  # append the value of GPU_Status into the nested dictionary
            elif 'Light' in key:
                latest_data[dict1['Pi_SN']][key] = str(
                    value
                )  # append the value of Light into the nested dictionary
            elif 'Temperature' in key:
                latest_data[dict1['Pi_SN']][key] = str(
                    value
                )  # append the value of Temperature into the nested dictionary
            elif 'Humidity' in key:
                latest_data[dict1['Pi_SN']][key] = str(
                    value
                )  # append the value of Humidity into the nested dictionary
    # return the nested dictionary
    return latest_data
Ejemplo n.º 55
0
 def get_item(self, tablename, conn=None, schema=None, **kwargs):
     try:
         if conn is None:
             conn = connect_to_region(REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
         table = self.get_table(tablename, conn, schema=schema)
         print str(kwargs)
         ret = table.get_item(**kwargs)
         if ret:
             print "Get successful.\n"
         else:
             print "Get failed.\n"
         respd = {}
         for field, value in ret.items():
             respd[field] = value
         return respd
     except Exception as ex:
         print "DynamoDB: Exception occured in get_item in table '"+tablename+"':" + str(ex)
Ejemplo n.º 56
0
def find_all_users(app):
    
    table_name = app.config['dynamodb.turing_users']
    hashkey    = "user_id"
    dyno = Table(table_name,
                 schema=[HashKey(hashkey)],
                 connection=ddb.connect_to_region(app.config['dynamodb.region'],
                                                  aws_access_key_id=app.config['keys.key_id'],
                                                  aws_secret_access_key=app.config['keys.key_secret'],
                                                  security_token=app.config['keys.key_token']))
    
    try:
        results = dyno.scan()
    except ddb.exceptions.ItemNotFound:
        return None

    return results
Ejemplo n.º 57
0
def connect_to_users_db(app):

    table_name = app.config['dynamodb.turing_users']
    hashkey = "user_id"
    #dyno = dutils.connect_to_db(app, app.config['dynamodb.turing_users'], "user_id")
    dyno = Table(table_name,
                 schema=[HashKey(hashkey)],
                 connection=ddb.connect_to_region(
                     app.config['dynamodb.region'],
                     aws_access_key_id=app.config['keys.key_id'],
                     aws_secret_access_key=app.config['keys.key_secret'],
                     security_token=app.config['keys.key_token']))

    results = dyno.scan()
    print "-" * 50
    for r in results:
        print "User: {0}  Role: {1}".format(r["user_id"], r["role"])
    return results