def add_to_db(self):
items_table = Table('items')
for product in self.viable_products:
temp_item = Item(items_table,
data={
'type': 'iphone',
'title': product[0],
'itemId': product[1],
'viewItemURL': product[2],
'sellerUserName': product[3],
'positiveFeedbackPercent': product[4],
'feedbackRatingStar': product[5],
'conditionId': product[6],
'listingType': product[7],
'currentPrice': product[8],
'bidCount': product[9],
'timeLeft': product[10],
'endTime': product[11],
'carrier': product[12],
'storage': product[13],
'model': product[14],
'color': product[15],
'pmresult': product[16],
})
temp_item.save(overwrite=True)
print 'all set'
def route_email(self, ee):
print 'bag it route_email:', ee.broadcast_dict[
'derived_to'], 'from:', ee.broadcast_dict['derived_from']
try:
item = self.get_mail_table(ee.domain).query(
derived_to__eq=ee.broadcast_dict['derived_to'],
derived_from__eq=ee.broadcast_dict['derived_from'],
limit=1).next()
item['lastConnection'] = time.time()
item['connectionsMade'] = item['connectionsMade'] + 1
item['msg'] = item['msg'] + "," + ee.broadcast_dict['file_dest']
item.save()
except Exception as e:
from boto.dynamodb2.items import Item
print 'create item:', e
try:
now = time.time()
item = Item(self.get_mail_table(ee.domain),
data={
'derived_to': ee.broadcast_dict['derived_to'],
'derived_from':
ee.broadcast_dict['derived_from'],
'firstConnection': now,
'lastConnection': now,
'connectionsMade': 1,
'msg': ee.broadcast_dict['file_dest']
})
item.save()
except Exception as e2:
print e2
def delete_tables(self, new_timestamp=None):
"""Delete the tables for this block.
"""
if not new_timestamp:
new_timestamp = self.tbase
if self.data_points_table:
# noinspection PyBroadException
try:
self.data_points_table.delete()
except:
pass
self.data_points_table = None
self.dp_writer = None
if self.index_table:
try:
self.index_table.delete()
except:
pass
self.index_table = None
try:
self.item.delete()
except:
pass
self.item = Item(self.master, data=dict(self.item.items()))
self.item['state'] = 'INITIAL'
self.item['tbase'] = base_time(new_timestamp)
self.item.save(overwrite=True)
return self.state
def flush(self):
batch_data = {
self.table.table_name: [
# We'll insert data here shortly.
],
}
for put in self._to_put:
item = Item(self.table, data=put)
batch_data[self.table.table_name].append(
{'PutRequest': {
'Item': item.prepare_full(),
}})
for delete in self._to_delete:
batch_data[self.table.table_name].append(
{'DeleteRequest': {
'Key': self.table._encode_keys(delete),
}})
resp = self.table.connection.batch_write_item(batch_data)
self.handle_unprocessed(resp)
self._to_put = []
self._to_delete = []
return True
def put_item(self, data, overwrite=False):
"""
Saves an entire item to DynamoDB.
By default, if any part of the ``Item``'s original data doesn't match
what's currently in DynamoDB, this request will fail. This prevents
other processes from updating the data in between when you read the
item & when your request to update the item's data is processed, which
would typically result in some data loss.
Requires a ``data`` parameter, which should be a dictionary of the data
you'd like to store in DynamoDB.
Optionally accepts an ``overwrite`` parameter, which should be a
boolean. If you provide ``True``, this will tell DynamoDB to blindly
overwrite whatever data is present, if any.
Returns ``True`` on success.
Example::
>>> users.put_item(data={
... 'username': '******',
... 'first_name': 'Jane',
... 'last_name': 'Doe',
... 'date_joined': 126478915,
... })
True
"""
item = Item(self, data=data)
return item.save(overwrite=overwrite)
def save_partition(part):
for record in part:
fl_xy = record[1][0]
fl_yz = record[1][1]
route = fl_xy.Origin + '-' + fl_xy.Dest + '-' + fl_yz.Dest
depdate = record[0][0]
item_new = Item(out_table,
data={
"route":
route,
"depdate":
depdate,
"flight_xy":
fl_xy.UniqueCarrier + str(fl_xy.FlightNum),
"flight_yz":
fl_yz.UniqueCarrier + str(fl_yz.FlightNum),
"total_delay":
int(fl_xy.DepDelay + fl_xy.ArrDelay +
fl_yz.DepDelay + fl_yz.ArrDelay)
})
# check old item delay
try:
item_old = out_table.get_item(route=route, depdate=depdate)
if (item_old['total_delay'] > item_new['total_delay']):
item_new.save(overwrite=True)
except:
item_new.save(overwrite=True)
def do_create(request, table, id, name, response):
try:
item = table.get_item(id=id)
if item["name"] != name:
response.status = 400
return {
"errors": [{
"id_exists": {
"status": "400", # "Bad Request"
"title": "id already exists",
"detail": {
"name": item['name']
}
}
}]
}
except ItemNotFound as inf:
p = Item(table, data={'id': id, 'name': name, 'activities': set()})
p.save()
response.status = 201 # "Created"
return {
"data": {
"type": "person",
"id": id,
"links": {
"self":
"{0}://{1}/users/{2}".format(request['urlparts']['scheme'],
request['urlparts']['netloc'], id)
}
}
}
def save_partition(part):
for record in part:
item = Item(out_table, data={
"airport": record[0][0],
"carrier": record[0][1],
"average_delay": int(record[1][0] / record[1][1])
})
item.save(overwrite=True)
def save_partition(part):
for record in part:
item = Item(out_table, data={
"origin": record[0][0],
"destination": record[0][1],
"average_delay": int(record[1][0] / record[1][1])
})
item.save(overwrite=True)
def _get_safe_data(self, dictionary, checker=None):
checker = checker or Item(self.table)
data = {}
for key in dictionary:
if isinstance(dictionary[key], dict):
data[key] = self._get_safe_data(dictionary[key], checker)
elif checker._is_storable(dictionary[key]):
data[key] = dictionary[key]
return data
def test_002_update_item(self):
emp = self.employees.get_item(etype='E', id='123456789')
emp['first_name'] = 'Jane'
emp.save()
emp = self.employees.get_item(etype='E', id='123456789')
data = self.getEmployeeData('id', '123456789')[0]
expected = Item(self.employees, data=data)
expected['first_name'] = 'Jane'
self.assertEqual(emp._data, expected._data)
def get_state(table, project):
try:
return table.get_item(project=project, consistent=True)
except ItemNotFound:
state = Item(table, data={
'project': project,
'state': 'idle',
})
state.save()
def ingest_alerts():
alerts_table = Table('mbta_alerts')
saFeed = gtfs_realtime_pb2.FeedMessage()
saResponse = requests.get('https://cdn.mbta.com/realtime/Alerts.pb')
saFeed.ParseFromString(saResponse.content)
now_ts = time.time()
alerts = []
for entity in saFeed.entity:
if entity.HasField('alert'):
include_alert = False
for informed in entity.alert.informed_entity:
if informed.route_type <= 1: # Subway/Green Line
include_alert = True
break
if include_alert:
include_alert = False
for period in entity.alert.active_period:
# Include all future and current alerts
if period.end == 0 or now_ts < period.end:
include_alert = True
break
if include_alert:
alerts.append(entity)
for entity in alerts:
id = int(entity.id)
alert = entity.alert
sorted_active_periods = sorted(entity.alert.active_period, key=lambda period: period.start)
current_period = None
for period in sorted_active_periods:
if now_ts > period.start and (now_ts < period.end or period.end == 0):
current_period = period
break
if current_period == None:
continue
alert_item = None
try:
alert_item = alerts_table.get_item(alert_id=id)
except exceptions.ItemNotFound:
pass
if not alert_item or alert_item['start'] != current_period.start:
alert_item = Item(alerts_table, data={
'alert_id': id,
'start': current_period.start,
'end': current_period.end,
'future': (current_period.start > now_ts),
})
send_and_save_event(alert_item, alert, current_period)
elif alert_item['future'] == True and alert_item['start'] < now_ts:
alert_item['future'] = False
send_and_save_event(alert_item, alert, current_period)
def get_item(self, consistent=False, attributes=None, **kwargs):
"""
Fetches an item (record) from a table in DynamoDB.
To specify the key of the item you'd like to get, you can specify the
key attributes as kwargs.
Optionally accepts a ``consistent`` parameter, which should be a
boolean. If you provide ``True``, it will perform
a consistent (but more expensive) read from DynamoDB.
(Default: ``False``)
Optionally accepts an ``attributes`` parameter, which should be a
list of fieldname to fetch. (Default: ``None``, which means all fields
should be fetched)
Returns an ``Item`` instance containing all the data for that record.
Example::
# A simple hash key.
>>> john = users.get_item(username='******')
>>> john['first_name']
'John'
# A complex hash+range key.
>>> john = users.get_item(username='******', last_name='Doe')
>>> john['first_name']
'John'
# A consistent read (assuming the data might have just changed).
>>> john = users.get_item(username='******', consistent=True)
>>> john['first_name']
'Johann'
# With a key that is an invalid variable name in Python.
# Also, assumes a different schema than previous examples.
>>> john = users.get_item(**{
... 'date-joined': 127549192,
... })
>>> john['first_name']
'John'
"""
raw_key = self._encode_keys(kwargs)
item_data = self.connection.get_item(
self.table_name,
raw_key,
attributes_to_get=attributes,
consistent_read=consistent
)
if 'Item' not in item_data:
raise exceptions.ItemNotFound("Item %s couldn't be found." % kwargs)
item = Item(self)
item.load(item_data)
return item
def makeItem():
item = Item(twitter, data={ \
'user_type': "standard_user",
'post_time': time.ctime(), \
'username': sys.argv[1][0:10], \
#messages must have 140 char
'post': sys.argv[2][0:140],
'hashtaglist': set(hashtaglist)} \
)
return item
def saveToDynamo(filename):
d = {}
try:
d = getSingleFileMetadata(filename)
except:
pass
if len(d) > 0:
newItem = Item(table_s3_metadata, data=d)
newItem.save(overwrite=True)
def add_new_isoc_cw_item(self, msp_id=None, credentials=None):
now = str(datetime.now())
item = Item(self.cm.get_msp_cw_table(),
data={
"msp_id": msp_id,
"credentials": credentials,
"created_at": str(datetime.now()),
"updated_at": str(datetime.now())
})
return item.save()
def _query(self, limit=None, index=None, reverse=False, consistent=False,
exclusive_start_key=None, select=None, attributes_to_get=None,
**filter_kwargs):
"""
The internal method that performs the actual queries. Used extensively
by ``ResultSet`` to perform each (paginated) request.
"""
kwargs = {
'limit': limit,
'index_name': index,
'consistent_read': consistent,
'select': select,
'attributes_to_get': attributes_to_get,
}
if reverse:
kwargs['scan_index_forward'] = False
if exclusive_start_key:
kwargs['exclusive_start_key'] = {}
for key, value in exclusive_start_key.items():
kwargs['exclusive_start_key'][key] = \
self._dynamizer.encode(value)
# Convert the filters into something we can actually use.
kwargs['key_conditions'] = self._build_filters(
filter_kwargs,
using=QUERY_OPERATORS
)
raw_results = self.connection.query(
self.table_name,
**kwargs
)
results = []
last_key = None
for raw_item in raw_results.get('Items', []):
item = Item(self)
item.load({
'Item': raw_item,
})
results.append(item)
if raw_results.get('LastEvaluatedKey', None):
last_key = {}
for key, value in raw_results['LastEvaluatedKey'].items():
last_key[key] = self._dynamizer.decode(value)
return {
'results': results,
'last_key': last_key,
}
def update_isoc_remote(self, msp_id, credentials):
old_msp_item = self.has_item_by_mspid_remote(msp_id=msp_id)
now = str(datetime.now())
item = Item(self.cm.get_ra_table(),
data={
"msp_id": old_msp_item['msp_id'],
"credentials": credentials,
"created_at": old_msp_item['created_at'],
"updated_at": str(now)
})
return item.save(overwrite=True)
def test_node_clear_fail(self):
r = get_router_table()
router = Router(r, SinkMetrics())
def raise_condition(*args, **kwargs):
raise ConditionalCheckFailedException(None, None)
router.table.connection.put_item = Mock()
router.table.connection.put_item.side_effect = raise_condition
data = dict(uaid=dummy_uaid, node_id="asdf", connected_at=1234)
result = router.clear_node(Item(r, data))
eq_(result, False)
def new_item(self, *args):
"""
Returns a new, blank item
This is mostly for consistency with boto.dynamodb
"""
if not self.schema:
self.describe()
data = {}
for x, arg in enumerate(args):
data[self.schema[x].name] = arg
return Item(self, data=data)
def add_entry(keycode_param):
params = request.args
try:
item = Item(table,
data={
'keycode': keycode_param,
'action': params['action'],
'name': params['name']
})
except KeyError as e:
# better error?
raise e
item.save()
def createGame(self, gameId, creator, invitee):
now = str(datetime.now())
statusDate = "PENDING_" + now
item = Item(self.cm.getGameTable(),
data={
"GameId": gameId,
"HostId": creator,
"StatusDate": statusDate,
"OUser": creator,
"Turn": invitee,
"Opponent": invitee
})
return item.save()
def test_clear_node_provision_failed(self):
r = get_router_table()
router = Router(r, SinkMetrics())
router.table.connection.put_item = Mock()
def raise_error(*args, **kwargs):
raise ProvisionedThroughputExceededException(None, None)
router.table.connection.put_item.side_effect = raise_error
with self.assertRaises(ProvisionedThroughputExceededException):
router.clear_node(
Item(r, dict(uaid="asdf", connected_at="1234",
node_id="asdf")))
def _batch_get(self, keys, consistent=False):
"""
The internal method that performs the actual batch get. Used extensively
by ``BatchGetResultSet`` to perform each (paginated) request.
"""
items = {
self.table_name: {
'Keys': [],
},
}
if consistent:
items[self.table_name]['ConsistentRead'] = True
for key_data in keys:
raw_key = {}
for key, value in key_data.items():
raw_key[key] = self._dynamizer.encode(value)
items[self.table_name]['Keys'].append(raw_key)
raw_results = self.connection.batch_get_item(request_items=items)
results = []
unprocessed_keys = []
for raw_item in raw_results['Responses'].get(self.table_name, []):
item = Item(self)
item.load({
'Item': raw_item,
})
results.append(item)
raw_unproccessed = raw_results.get('UnprocessedKeys', {})
for raw_key in raw_unproccessed.get('Keys', []):
py_key = {}
for key, value in raw_key.items():
py_key[key] = self._dynamizer.decode(value)
unprocessed_keys.append(py_key)
return {
'results': results,
# NEVER return a ``last_key``. Just in-case any part of
# ``ResultSet`` peeks through, since much of the
# original underlying implementation is based on this key.
'last_key': None,
'unprocessed_keys': unprocessed_keys,
}
def _scan(self,
limit=None,
exclusive_start_key=None,
segment=None,
total_segments=None,
attributes=None,
**filter_kwargs):
"""
The internal method that performs the actual scan. Used extensively
by ``ResultSet`` to perform each (paginated) request.
"""
kwargs = {
'limit': limit,
'segment': segment,
'total_segments': total_segments,
'attributes_to_get': attributes,
}
if exclusive_start_key:
kwargs['exclusive_start_key'] = {}
for key, value in exclusive_start_key.items():
kwargs['exclusive_start_key'][key] = \
self._dynamizer.encode(value)
# Convert the filters into something we can actually use.
kwargs['scan_filter'] = self._build_filters(filter_kwargs,
using=FILTER_OPERATORS)
raw_results = self.connection.scan(self.table_name, **kwargs)
results = []
last_key = None
for raw_item in raw_results.get('Items', []):
item = Item(self)
item.load({
'Item': raw_item,
})
results.append(item)
if raw_results.get('LastEvaluatedKey', None):
last_key = {}
for key, value in raw_results['LastEvaluatedKey'].items():
last_key[key] = self._dynamizer.decode(value)
return {
'results': results,
'last_key': last_key,
}
def get_instance_assignments(self, service):
"""Get instances and assignments for a service
:param service: Service name.
:return: Map of instances of assignments (None if unassigned).
"""
live_instances = []
dead_instances = []
dead_cutoff = time.time() - INSTANCE_EXPIRY
for instance_status in self._status.query_2(
service__eq=service,
attributes=('instance_id', 'status_time')):
instance_id = instance_status['instance_id']
if instance_status['status_time'] < dead_cutoff:
dead_instances.append(instance_id)
else:
live_instances.append(instance_id)
if dead_instances:
logger.debug('Removing %d dead instances.', len(dead_instances))
with self._status.batch_write() as status_batch:
for dead_instance in dead_instances:
status_batch.delete_item(service=service,
instance_id=dead_instance)
with self._assignments.batch_write() as assignment_batch:
for dead_instance in dead_instances:
assignment_batch.delete_item(instance_id=dead_instance)
assignments = defaultdict(list)
if not live_instances:
return assignments
unassigned = set(live_instances)
keys = [{'instance_id': i} for i in live_instances]
for assignment in self._assignments.batch_get(
keys=keys, attributes=('instance_id', 'assignment')):
assigned = assignment['assignment']
instance_id = assignment['instance_id']
unassigned.remove(instance_id)
assignments[assigned].append(assignment)
assignments[None] = [
Item(self._assignments,
data={
'instance_id': instance_id,
'service': service
}) for instance_id in unassigned
]
return assignments
def save(self, obj):
"""Required functionality."""
if not obj.id:
obj.id = uuid()
stored_data = {'id': obj.id, 'value': obj.to_data()}
index_vals = obj.indexes() or {}
for key in obj.__class__.index_names() or []:
val = index_vals.get(key, '')
stored_data[key] = DynamoMappings.map_index_val(val)
table = self.get_class_table(obj.__class__)
item = Item(table, data=stored_data)
item.save(overwrite=True)
def _get_item_for_record(self, record):
if record._item:
item = record._item
data = record.get_dict()
for key in data:
if item._is_storable(data[key]) or key in item:
# only copy storable fields or those we want to reset
# for example, if item['name']='Bob' we can set it to ''
# but if item didn't have 'name' field then we should
# not set it (dynamodb does not allow empty fields)
item[key] = data[key]
return item
else:
data = self._get_safe_data(record.get_dict())
item = Item(self.table, data=data)
return item
def add_new_isoc_remote_item(self,
msp_id=None,
remote_access_status=None,
action_type=None,
user_choice=None):
now = str(datetime.now())
item = Item(self.cm.get_ra_table(),
data={
"msp_id": msp_id,
"remote_access_status": remote_access_status,
"user_choice": user_choice,
"action_type": action_type,
"created_at": now,
"updated_at": now,
})
return item.save()
