def test_all_value_types(self):
key = Config.CLIENT.key('TestPanObject', 1234)
entity = datastore.Entity(key=key)
entity['timestamp'] = datetime.datetime(2014, 9, 9, tzinfo=UTC)
key_stored = Config.CLIENT.key('SavedKey', 'right-here')
entity['key'] = key_stored
entity['truthy'] = True
entity['float'] = 2.718281828
entity['int'] = 3735928559
entity['words'] = u'foo'
entity['blob'] = b'seekretz'
entity_stored = datastore.Entity(key=key_stored)
entity_stored['hi'] = 'bye'
entity['nested'] = entity_stored
entity['items'] = [1, 2, 3]
entity['geo'] = GeoPoint(1.0, 2.0)
entity['nothing_here'] = None
# Store the entity.
self.case_entities_to_delete.append(entity)
Config.CLIENT.put(entity)
# Check the original and retrieved are the the same.
retrieved_entity = Config.CLIENT.get(entity.key)
self.assertEqual(retrieved_entity, entity)
def transactional_update(client):
# Create the entities we're going to manipulate
account1 = datastore.Entity(client.key('Account'))
account1['balance'] = 100
account2 = datastore.Entity(client.key('Account'))
account2['balance'] = 100
client.put_multi([account1, account2])
# [START transactional_update]
def transfer_funds(client, from_key, to_key, amount):
with client.transaction():
from_account, to_account = client.get_multi([from_key, to_key])
from_account['balance'] -= amount
to_account['balance'] += amount
client.put_multi([from_account, to_account])
# [END transactional_update]
# [START transactional_retry]
for _ in range(5):
try:
transfer_funds(client, account1.key, account2.key, 50)
except gcloud.exceptions.Conflict:
continue
# [END transactional_retry]
return account1.key, account2.key
def update_item(kind, name, **kwargs):
item = get_item(kind, name)
if item is None:
m = datastore.Entity(key=ds.key(kind, name),
exclude_from_indexes=exclude_indices)
else:
m = datastore.Entity(key=ds.key(kind, name),
exclude_from_indexes=exclude_indices)
m.update(dict(item))
for key, value in kwargs.items():
if type(value) == str:
m[key] = value
elif type(value) == list:
if key in m:
m[key] += value
else:
m[key] = value
m[key] = list(set(m[key]))
# If date created of file is earlier
elif key == 'date_created' and item:
vtimestamp = time.mktime(value.timetuple())
dstimestamp = time.mktime(m['date_created'].timetuple())
if vtimestamp < dstimestamp:
m[key] = value
else:
m[key] = value
if 'fq_profile_count' in m:
m['fq_profile_count'] += 1
else:
m['fq_profile_count'] = 1
ds.put(m)
def add_outline():
"""
TOPページから練習概要の日付、開始・終了時間、時間帯、IDを追加
"""
# フォームからデータを取得
outline_id = int(datetime.strftime(datetime.now(), '%Y%m%d%H%M%S'))
date = request.form.get('date')
time_category = request.form.get('time_category')
# DataStoreに格納
if date and time_category:
key1 = client.key('Outline')
outline1 = datastore.Entity(key1)
outline1.update({
'outline_id': outline_id,
'date': date,
'time_category': time_category
})
client.put(outline1)
key2 = client.key('Outline_yacht_player')
outline2 = datastore.Entity(key2)
outline2.update({'outline_id': outline_id})
client.put(outline2)
#TOPページで、入力せずに追加ボタンを押した場合は、TOPに戻る
else:
return redirect(url_for('top'))
outline_selections = query.get_outline_selections()
target_entities = query.get_outline_entities(outline_id)
return render_template('show_outline.html', title="練習概要入力",\
target_entities=target_entities, outline_selections=outline_selections)
def test_transaction_via_explicit_begin_get_commit(self):
# See https://github.com/GoogleCloudPlatform/gcloud-python/issues/1859
# Note that this example lacks the threading which provokes the race
# condition in that issue: we are basically just exercising the
# "explict" path for using transactions.
BEFORE_1 = 100
BEFORE_2 = 0
TRANSFER_AMOUNT = 40
key1 = Config.CLIENT.key('account', '123')
account1 = datastore.Entity(key=key1)
account1['balance'] = BEFORE_1
key2 = Config.CLIENT.key('account', '234')
account2 = datastore.Entity(key=key2)
account2['balance'] = BEFORE_2
Config.CLIENT.put_multi([account1, account2])
self.case_entities_to_delete.append(account1)
self.case_entities_to_delete.append(account2)
xact = Config.CLIENT.transaction()
xact.begin()
from_account = Config.CLIENT.get(key1, transaction=xact)
to_account = Config.CLIENT.get(key2, transaction=xact)
from_account['balance'] -= TRANSFER_AMOUNT
to_account['balance'] += TRANSFER_AMOUNT
xact.put(from_account)
xact.put(to_account)
xact.commit()
after1 = Config.CLIENT.get(key1)
after2 = Config.CLIENT.get(key2)
self.assertEqual(after1['balance'], BEFORE_1 - TRANSFER_AMOUNT)
self.assertEqual(after2['balance'], BEFORE_2 + TRANSFER_AMOUNT)
def submit_post(user_id, post_id, data):
post_key = client.key('Posts', post_id)
user_key = client.key('Users', user_id)
user_entity = datastore.Entity(key=user_key)
post_entity = datastore.Entity(key=post_key)
current_posts = helpers.load_entity(client.get(user_key))
current_posts['posts'].append(post_id)
user_entity.update(helpers.make_entity(current_posts))
client.put(user_entity)
post_entity.update(data)
client.put(post_entity)
def store_item(kind, name, **kwargs):
ds = datastore.Client(project='andersen-lab')
exclude = kwargs.pop('exclude_from_indexes')
if exclude:
m = datastore.Entity(key=ds.key(kind, name), exclude_from_indexes=exclude)
else:
m = datastore.Entity(key=ds.key(kind, name))
for key, value in kwargs.items():
if isinstance(value, dict):
m[key] = 'JSON:' + json.dumps(value)
else:
m[key] = value
ds.put(m)
def GetEntities(project_id, json_lines, outer_parent_key=None, namespace=None):
"""Create gcloud entities from json string entries.
project_id: String gcloud project id that entities are for.
json_lines: File or other line-by-line iterator of json strings to turn into
entities.
outer_parent_key: Optional datastore.Key instance to act as the parent_key
of all top level entities.
namespace: Optional string namespace for entities.
"""
entity_keys = {}
for line in json_lines:
item = json.loads(line)
kind, idx = item.pop('id')
parent = item.pop('parent', None)
if (kind, idx) in entity_keys:
raise DuplicateKeyError(
'Duplicate entities with id (%s, %s)' % (kind, idx))
if parent:
parent_key = entity_keys[tuple(parent)]
else:
parent_key = outer_parent_key
key = datastore.Key(
kind, idx, project=project_id, parent=parent_key, namespace=namespace)
e = datastore.Entity(key=key)
e.update(item)
entity_keys[(kind, idx)] = key
entity_keys[idx] = key
yield e
def process_item(self, item, unused_spider):
def unix_timestamp(dt):
_EPOCH = datetime(1970, 1, 1, tzinfo=pytz.utc)
if dt.tzinfo is None:
return int(time.mktime((dt.year, dt.month, dt.day,
dt.hour, dt.minute, dt.second,
-1, -1, -1)) + dt.microsecond / 1e6)
return int((dt - _EPOCH).total_seconds())
def to_json(item):
rv = dict(item)
for key in rv:
# Google DataStore cannot display str well
if isinstance(rv[key], str):
rv[key] = unicode(rv[key])
elif (isinstance(rv[key], list) or
isinstance(rv[key], dict)):
rv[key] = unicode(json.dumps(rv[key]))
elif isinstance(rv[key], datetime):
rv[key] = unix_timestamp(rv[key])
elif isinstance(rv[key], enum.Enum):
rv[key] = rv[key].value
return rv
data = to_json(item)
gclient = datastore.Client()
key = gclient.key(config.ENTRY_ENTITY, item['link_hash'])
entity = datastore.Entity(
key=key, exclude_from_indexes=['content', 'images'])
for k, v in data.iteritems():
entity[k] = v
gclient.put(entity)
return item
def put(self, character_id):
parser = reqparse.RequestParser()
for feed in SERVICES:
parser.add_argument(feed, type=bool)
args = parser.parse_args(strict=True)
client = get_client()
character_settings = client.get(client.key(SETTINGS_KIND, character_id))
if character_settings is None:
app.logger.info('Adding new stored settings for {} with the following values: {}'.format(character_id, args))
character_settings = datastore.Entity(client.key(SETTINGS_KIND, character_id))
for feed in SERVICES:
character_settings[feed] = False
else:
app.logger.info('Updating the stored settings for {} with the following values: {}'.format(character_id, args))
for feed in SERVICES:
if feed in args:
character_settings[feed] = args[feed]
client.put(character_settings)
return {}, 204
def _setup_user(self, userid_hash):
json = {"email": "*****@*****.**"}
key = self.datastore_client.key("User", userid_hash)
entity = datastore.Entity(key=key)
util._update_entity(json, {'email'}, entity)
self.datastore_client.put(entity)
def add_device():
"""
デバイス情報を追加
Args:
deviceno: デバイスID
decicename: デバイスの機種名
datetime_now: 新規データの作成日時
Return: TOPページに戻る
"""
deviceno = request.form.get('deviceno')
devicename = request.form.get('devicename')
datetime_now = datetime.now()
if deviceno and devicename:
key = client.key('Device')
device = datastore.Entity(key)
device.update({
'device_no': deviceno,
'device_name': devicename,
'created_date': datetime_now
})
client.put(device)
return redirect(url_for('top'))
def store_item(kind, name, **kwargs):
ds = google_datastore()
try:
exclude = kwargs.pop('exclude_from_indexes')
except KeyError:
exclude = False
if exclude:
m = datastore.Entity(key=ds.key(kind, name), exclude_from_indexes=exclude)
else:
m = datastore.Entity(key=ds.key(kind, name))
for key, value in kwargs.items():
if isinstance(value, dict):
m[key] = 'JSON:' + dump_json(value)
else:
m[key] = value
ds.put(m)
def add_player():
"""
選手データの追加
Args:
playername(str):admin_player.htmlで入力した選手名
year(int):admin_player.htmlで入力した入学年
datetime_now:データを作成した日時
player: 新規で作成した、選手データのエンティティ。
Return:
TOPページに戻る
"""
playername = str(request.form.get('playername'))
year = int(request.form.get('year'))
datetime_now = datetime.now()
if playername and year:
key = client.key('Player')
player = datastore.Entity(key)
player.update({
'player_name': playername,
'admission_year': year,
'created_date': datetime_now
})
client.put(player)
return redirect(url_for('top'))
def index():
ds = datastore.Client(os.environ['GCLOUD_PROJECT'])
user_ip = request.remote_addr
# Keep only the first two octets of the IP address.
if is_ipv6(user_ip):
user_ip = ':'.join(user_ip.split(':')[:2])
else:
user_ip = '.'.join(user_ip.split('.')[:2])
entity = datastore.Entity(key=ds.key('visit'))
entity.update({
'user_ip': user_ip,
'timestamp': datetime.datetime.utcnow()
})
ds.put(entity)
query = ds.query(kind='visit', order=('-timestamp', ))
results = [
'Time: {timestamp} Addr: {user_ip}'.format(**x)
for x in query.fetch(limit=10)
]
output = 'Last 10 visits:\n{}'.format('\n'.join(results))
return output, 200, {'Content-Type': 'text/plain; charset=utf-8'}
def create_bulk_entities(self, kind, num_records, record):
entities = list()
for i in xrange(num_records):
entity = datastore.Entity(self.client.key(kind))
entity.update(record)
entities.append(entity)
self.client.put_multi(entities)
def projection_query(client):
# Create the entity that we're going to query.
task = datastore.Entity(client.key('Task'))
task.update({
'type': 'Personal',
'done': False,
'priority': 4,
'description': 'Learn Cloud Datastore',
'percent_complete': 0.5
})
client.put(task)
# [START projection_query]
query = client.query(kind='Task')
query.projection = ['priority', 'percent_complete']
# [END projection_query]
# [START run_query_projection]
priorities = []
percent_completes = []
for task in query.fetch():
priorities.append(task['priority'])
percent_completes.append(task['percent_complete'])
# [END run_query_projection]
return priorities, percent_completes
def transactional_single_entity_group_read_only(client):
client.put_multi([
datastore.Entity(key=client.key('TaskList', 'default')),
datastore.Entity(key=client.key('TaskList', 'default', 'Task', 1))
])
# [START transactional_single_entity_group_read_only]
with client.transaction():
task_list_key = client.key('TaskList', 'default')
task_list = client.get(task_list_key)
query = client.query(kind='Task', ancestor=task_list_key)
tasks_in_list = list(query.fetch())
return task_list, tasks_in_list
def test_failure_with_contention(self):
contention_key = 'baz'
# Fool the Client constructor to avoid creating a new connection.
local_client = datastore.Client(dataset_id=Config.CLIENT.dataset_id,
http=object())
local_client.connection = Config.CLIENT.connection
# Insert an entity which will be retrieved in a transaction
# and updated outside it with a contentious value.
key = local_client.key('BreakTxn', 1234)
orig_entity = datastore.Entity(key=key)
orig_entity['foo'] = u'bar'
local_client.put(orig_entity)
self.case_entities_to_delete.append(orig_entity)
with self.assertRaises(Conflict):
with local_client.transaction() as txn:
entity_in_txn = local_client.get(key)
# Update the original entity outside the transaction.
orig_entity[contention_key] = u'outside'
Config.CLIENT.put(orig_entity)
# Try to update the entity which we already updated outside the
# transaction.
entity_in_txn[contention_key] = u'inside'
txn.put(entity_in_txn)
def add_yacht():
"""
ヨットデータの追加
Args:
yachtno(int): 艇番
yachtclass: 艇種
datatime_now: データの作成日
yacht: 新規作成したヨットのエンティティ
return: TOPページに戻る
"""
yachtno = int(request.form.get('yachtno'))
yachtclass = request.form.get('yachtclass')
datetime_now = datetime.now()
if yachtno and yachtclass:
key = client.key('Yacht')
yacht = datastore.Entity(key) #
yacht.update({
'yacht_no': yachtno,
'yacht_class': yachtclass,
'created_date': datetime_now
})
client.put(yacht)
return redirect(url_for('top'))
def __write(self):
rows = []
deletes = []
for message in self.__queue:
try:
timestamp, entity, count = message.split(',')
t = datetime.strptime(timestamp[:-5],'%Y-%m-%dT%H:%M:%S')
q=self.client_ds.query()
q.add_filter('entity', '=', entity)
q.add_filter('timestamp', '=', t)
q.kind = 'TwitterEntityFreq'
r=q.fetch()
for i in r:
deletes += [i.key]
ds_entity = datastore.Entity(key=self.key)
ds_entity['entity'] = entity.decode('utf-8')
ds_entity['timestamp'] = t
try:
ds_entity['frequency'] = int(count)
except ValueError:
ds_entity['frequency'] = 0
rows += [ds_entity]
except Exception as e:
print(e)
for i in xrange(0,len(deletes),500):
retries = 10
while True:
try:
self.client_ds.delete_multi(deletes[i:i+500])
break
except Exception as e:
if retries == 0:
#bail-out
raise
delay = (11-retries)**2 * 0.25
print("Deleting: Received an exception, waiting %.2f"%delay)
time.sleep(delay)
retries -= 1
retries = 10
while True:
try:
self.client_ds.put_multi(rows)
break
except Exception as e:
if retries == 0:
#bail-out
raise
delay = (11-retries)**2 * 0.25
print("Inserting: Received an exception, waiting %.2f"%delay)
time.sleep(delay)
retries -= 1
n_msg = len(self.__queue)
n_rows = len(rows)
n_dels = len(deletes)
print "[%s] Received %s messages, deleted %s rows, inserted %s rows" % (
'OK' if n_msg==n_rows else '!!', n_msg, n_dels, n_rows)
self.__queue=[]
self.__lastwrite = time.time()
def repost(ds, username, original):
now = datetime.datetime.utcnow()
new_key = path_to_key(ds, '{0}.user/{1}.post'.format(username, now))
new = datastore.Entity(new_key)
new.update(original)
ds.put(new)
def post(self):
entity = datastore.Entity(key=datastore.Key(ENTITY_KIND, project=PROJECT))
data = json.loads(self.request.body)
data['status'] = REQUEST_RECEIVED
entity['data'] = json.dumps(data)
ds.put(entity)
publish_msg(str(entity.key.id))
self.response.write(format_entity(entity))
def put(self, user_email):
if not request.json:
abort(400)
key = db.key('User', user_email)
user = datastore.Entity(key, exclude_from_indexes=['settings'])
user['updated'] = datetime.datetime.utcnow()
user['settings'] = json.dumps(request.json)
db.put(user)
return jsonify({'success': True})
def predict(abstract, title, source): prediction = pred.predict(statement, source) entity = datastore.Entity(key=key) entity['partition'] = hostname' entity['date'] = str(time.time()) entity['title' ] = title entity['prediction'] = str(prediction) clientds.put(entity) return [prediction]
def store_item(kind, name, **kwargs):
ds = google_datastore()
m = datastore.Entity(key=ds.key(kind, name))
for key, value in kwargs.items():
if type(value) == str:
m[key] = unicode(value)
else:
m[key] = value
ds.put(m)
def add_user(self, user_data):
key = self.client.key(self.dataset)
user = datastore.Entity(key)
user.update(user_data)
client.put(user)
return user.key
def add_new_post(subject, content):
key = client.key('Blog')
blog = datastore.Entity(key)
blog.update({
'created': datetime.datetime.utcnow(),
'subject': subject,
'content': content
})
client.put(blog)
return blog.key
def add_characters():
with datastore.Transaction() as xact:
for key_path, character in zip(KEY_PATHS, CHARACTERS):
if key_path[-1] != character['name']:
raise ValueError(
('Character and key don\'t agree', key_path, character))
entity = datastore.Entity(key=datastore.Key(*key_path))
entity.update(character)
xact.put(entity)
print('Adding Character %s %s' %
(character['name'], character['family']))
def exploding_properties(client):
# [START exploding_properties]
task = datastore.Entity(client.key('Task'))
task.update({
'tags': ['fun', 'programming', 'learn'],
'collaborators': ['alice', 'bob', 'charlie'],
'created': datetime.datetime.utcnow()
})
# [END exploding_properties]
return task
