def save(self):
req = datastore.CommitRequest()
req.mode = datastore.CommitRequest.NON_TRANSACTIONAL
entity = req.mutation.upsert.add()
add_key_path(entity.key, *self.key_path)
datastore.commit(req)
return self
def wipeout(cls):
""" Deletes all entities of this type from the datastore. """
_log.debug('%s.wipeout()', cls.__name__)
batch_size = 500
cursor = None
while True:
req = googledatastore.RunQueryRequest()
query = req.gql_query
query.query_string = 'SELECT __key__ FROM %s LIMIT %d OFFSET @startCursor ' % (
cls.__name__, batch_size)
query.allow_literal = True
cursor_arg = query.name_arg.add()
cursor_arg.name = 'startCursor'
if cursor is None:
cursor_arg.value.integer_value = 0
else:
cursor_arg.cursor = cursor
resp = googledatastore.run_query(req)
req = googledatastore.CommitRequest()
req.mode = googledatastore.CommitRequest.NON_TRANSACTIONAL
req.mutation.delete.extend([result.entity.key for result in resp.batch.entity_result])
googledatastore.commit(req)
if resp.batch.more_results == googledatastore.QueryResultBatch.NO_MORE_RESULTS:
break
if len(resp.batch.entity_result) == 0:
break
cursor = resp.batch.end_cursor
def process_page(soup):
started = False
request = datastore.CommitRequest()
request.mode = datastore.CommitRequest.NON_TRANSACTIONAL
for row in soup.find('table').find_all('table')[-1].findAll('tr'):
columns = [
(''.join(y.strip() for y in x.strings)
.replace(u'\x80', u'€'))
for x in row.findAll('td')]
if 'Year' in columns:
continue
link = 'http://planecheck.com/' + row.find('a').get('href')
_, list_id = link.rsplit('=', 1)
manufacturer, model = columns[0].split(u'\xa0')
year, raw_price, country, views = columns[1:]
if not u'\xa0' in raw_price:
price = None
currency = None
else:
price, currency = raw_price.split(u'\xa0')
price = int(price.replace('.', ''))
entity = request.mutation.upsert.add()
path = entity.key.path_element.add()
path.kind = 'Manufacturer'
path.name = manufacturer
path = entity.key.path_element.add()
path.kind = 'Model'
path.name = model
path = entity.key.path_element.add()
path.kind = 'Listing'
path.name = 'planecheck.com ' + list_id
if price:
prop = entity.property.add()
prop.name = 'price'
prop.value.integer_value = price
prop = entity.property.add()
prop.name = 'currency'
prop.value.string_value = currency
prop = entity.property.add()
prop.name = 'views'
prop.value.integer_value = int(views)
prop = entity.property.add()
prop.name = 'link'
prop.value.string_value = link
prop = entity.property.add()
prop.name = 'manufacturer'
prop.value.string_value = manufacturer
prop = entity.property.add()
prop.name = 'model'
prop.value.string_value = model
prop = entity.property.add()
prop.name = 'country'
prop.value.string_value = country
prop = entity.property.add()
prop.name = 'year'
prop.value.integer_value = int(year)
datastore.commit(request)
def save(self): req = datastore.CommitRequest() req.mode = datastore.CommitRequest.NON_TRANSACTIONAL entity = req.mutation.upsert.add() add_key_path(entity.key, *self.key_path) datastore.commit(req) return self
def wipeout(cls):
""" Deletes all entities of this type from the datastore. """
_log.debug('%s.wipeout()', cls.__name__)
batch_size = 500
cursor = None
while True:
req = googledatastore.RunQueryRequest()
query = req.gql_query
query.query_string = 'SELECT __key__ FROM %s LIMIT %d OFFSET @startCursor ' % (
cls.__name__, batch_size)
query.allow_literal = True
cursor_arg = query.name_arg.add()
cursor_arg.name = 'startCursor'
if cursor is None:
cursor_arg.value.integer_value = 0
else:
cursor_arg.cursor = cursor
resp = googledatastore.run_query(req)
req = googledatastore.CommitRequest()
req.mode = googledatastore.CommitRequest.NON_TRANSACTIONAL
req.mutation.delete.extend(
[result.entity.key for result in resp.batch.entity_result])
googledatastore.commit(req)
if resp.batch.more_results == googledatastore.QueryResultBatch.NO_MORE_RESULTS:
break
if len(resp.batch.entity_result) == 0:
break
cursor = resp.batch.end_cursor
def __exit__(self, type, value, traceback):
if self.commit_req:
datastore.commit(self.commit_req)
self.commit_req = None
if self.tx:
# Simple, but we might want to log or troubleshoot one day
self.tx = None
def __exit__(self, type, value, traceback):
"""End transaction."""
if self.commit_req:
datastore.commit(self.commit_req)
self.commit_req = None
if self.tx:
# Simple, but we might want to log or troubleshoot one day
self.tx = None
def __upsert(self, transaction=None):
_log.debug('%s.__upsert()', type(self).__name__)
req = googledatastore.CommitRequest()
if transaction:
req.transaction = transaction
else:
req.mode = googledatastore.CommitRequest.NON_TRANSACTIONAL
entity = self._get_entity()
req.mutation.upsert.extend([entity])
googledatastore.commit(req)
def __upsert(self, transaction = None):
_log.debug('%s.__upsert()', type(self).__name__)
req = googledatastore.CommitRequest()
if transaction:
req.transaction = transaction
else:
req.mode = googledatastore.CommitRequest.NON_TRANSACTIONAL
entity = self._get_entity()
req.mutation.upsert.extend([entity])
googledatastore.commit(req)
def checkDB(self,gitkit_user): req = datastore.BeginTransactionRequest() resp = datastore.begin_transaction(req) tx = resp.transaction req = datastore.LookupRequest() key = datastore.Key() path = key.path_element.add() path.kind = 'PSUsers' path.name = gitkit_user.user_id req.key.extend([key]) req.read_options.transaction = tx resp = datastore.lookup(req) req = datastore.CommitRequest() req.transaction = tx if resp.missing: user = req.mutation.insert.add() user.key.CopyFrom(key) userid_property = user.property.add() userid_property.name = 'userid' userid_property.value.string_value = gitkit_user.user_id display_name_property = user.property.add() display_name_property.name = 'display_name' display_name_property.value.string_value = gitkit_user.name photo_url_property = user.property.add() photo_url_property.name = 'photo_url' if gitkit_user.photo_url: photo_url_property.value.string_value = gitkit_user.photo_url else: photo_url_property.value.string_value = "/images/home/slider/slide1/cloud1.png" email_property = user.property.add() email_property.name = 'email' email_property.value.string_value = gitkit_user.email last_login_property = user.property.add() last_login_property.name = 'last_login' last_login_property.value.timestamp_microseconds_value = long(time.time() * 1e6) elif resp.found: user = resp.found[0].entity last_login_property = datastore.Property() last_login_property.name = 'last_login' for prop in user.property: if prop.name == 'last_login': prop.value.timestamp_microseconds_value = long(time.time() * 1e6) req.mutation.update.extend([user]) resp = datastore.commit(req) return None
def archive(cls):
"""Delete all Todo items that are done."""
req = datastore.BeginTransactionRequest()
resp = datastore.begin_transaction(req)
tx = resp.transaction
req = datastore.RunQueryRequest()
req.read_options.transaction = tx
q = req.query
set_kind(q, kind='Todo')
add_projection(q, '__key__')
set_composite_filter(q.filter,
datastore.CompositeFilter.AND,
set_property_filter(
datastore.Filter(),
'done', datastore.PropertyFilter.EQUAL, True),
set_property_filter(
datastore.Filter(),
'__key__', datastore.PropertyFilter.HAS_ANCESTOR,
default_todo_list.key))
resp = datastore.run_query(req)
keys = [r.entity.key for r in resp.batch.entity_result]
req = datastore.CommitRequest()
req.transaction = tx
req.mutation.delete.extend(keys)
resp = datastore.commit(req)
return ''
def archive(cls):
"""Delete all Todo items that are done."""
req = datastore.BeginTransactionRequest()
resp = datastore.begin_transaction(req)
tx = resp.transaction
req = datastore.RunQueryRequest()
req.read_options.transaction = tx
q = req.query
set_kind(q, kind='Todo')
add_projection(q, '__key__')
set_composite_filter(
q.filter, datastore.CompositeFilter.AND,
set_property_filter(datastore.Filter(), 'done',
datastore.PropertyFilter.EQUAL, True),
set_property_filter(datastore.Filter(), '__key__',
datastore.PropertyFilter.HAS_ANCESTOR,
default_todo_list.key))
resp = datastore.run_query(req)
keys = [r.entity.key for r in resp.batch.entity_result]
req = datastore.CommitRequest()
req.transaction = tx
req.mutation.delete.extend(keys)
resp = datastore.commit(req)
return ''
def save(self):
"""Update or insert a Todo item."""
req = datastore.CommitRequest()
req.mode = datastore.CommitRequest.NON_TRANSACTIONAL
req.mutations.add().upsert.CopyFrom(self.to_proto())
resp = datastore.commit(req)
if not self.id:
self.id = resp.mutation_results[0].key.path[-1].id
return self
def clouddatastore_client():
try:
logFormatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
root = logging.getLogger()
root.setLevel(logging.INFO)
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.INFO)
ch.setFormatter(logFormatter)
root.addHandler(ch)
logging.getLogger('oauth2service.client').setLevel(logging.INFO)
logging.getLogger('apiclient.discovery').setLevel(logging.INFO)
credentials = GoogleCredentials.get_application_default()
http = httplib2.Http()
http = credentials.authorize(http)
credentials.refresh(http)
credentials.access_token = 'foo'
print credentials.access_token
datastore.set_options(project_id='p0', credentials=credentials)
req = datastore.LookupRequest()
key = datastore.Key()
path = key.path.add()
path.kind = 'Employee'
path.name = 'aguadypoogznoqofmgmy'
req.keys.extend([key])
resp = datastore.lookup(req)
if resp.found:
entity = resp.found[0].entity
print (str(entity))
for prop in entity.properties:
print 'Lookup: ' + str(prop)
else:
print 'entity not found; initialize entity and insert..'
req = datastore.CommitRequest()
req.mode = datastore.CommitRequest.NON_TRANSACTIONAL
employee = req.mutations.add().insert
path = employee.key.path.add()
path.kind = 'Employee'
path.name = 'aguadypoogznoqofmgmy'
res = datastore.commit(req)
print res
except HttpError as err:
print 'Error:', pprint.pprint(err.content)
except AccessTokenRefreshError:
print ("Credentials have been revoked or expired, please re-run"
"the application to re-authorize")
def save(self):
"""Update or insert a Todo item."""
req = datastore.CommitRequest()
req.mode = datastore.CommitRequest.NON_TRANSACTIONAL
proto = self.to_proto()
mutation = req.mutation.upsert if self.id else req.mutation.insert_auto_id
mutation.extend([proto])
resp = datastore.commit(req)
if not self.id:
keys = resp.mutation_result.insert_auto_id_key
self.id = keys[0].path_element[-1].id
return self
def save(self):
"""Update or insert a Todo item."""
req = datastore.CommitRequest()
req.mode = datastore.CommitRequest.NON_TRANSACTIONAL
proto = self.to_proto()
mutation = req.mutation.upsert if self.id else req.mutation.insert_auto_id
mutation.extend([proto])
resp = datastore.commit(req)
if not self.id:
keys = resp.mutation_result.insert_auto_id_key
self.id = keys[0].path_element[-1].id
return self
def save_batch(entities):
req = googledatastore.BeginTransactionRequest()
resp = googledatastore.begin_transaction(req)
transaction = resp.transaction
# Bulk lookup
req = googledatastore.LookupRequest()
req.key.extend([entity.__get_key() for entity in entities])
req.read_options.transaction = transaction
resp = googledatastore.lookup(req)
# Update created / modified times.
missing = set()
for result in resp.missing:
key = result.entity.key.path_element[0]
missing.add(key.id or key.name)
for entity in entities:
entity.modified_time = time
if entity._get_id() in missing:
entity.created_time = time
# Bulk upsert
req = googledatastore.CommitRequest()
req.transaction = transaction
req.mutation.upsert.extend([entity._get_entity() for entity in entities])
googledatastore.commit(req)
def save_batch(entities):
req = googledatastore.BeginTransactionRequest()
resp = googledatastore.begin_transaction(req)
transaction = resp.transaction
# Bulk lookup
req = googledatastore.LookupRequest()
req.key.extend([entity.__get_key() for entity in entities])
req.read_options.transaction = transaction
resp = googledatastore.lookup(req)
# Update created / modified times.
missing = set()
for result in resp.missing:
key = result.entity.key.path_element[0]
missing.add(key.id or key.name)
for entity in entities:
entity.modified_time = time
if entity._get_id() in missing:
entity.created_time = time
# Bulk upsert
req = googledatastore.CommitRequest()
req.transaction = transaction
req.mutation.upsert.extend(
[entity._get_entity() for entity in entities])
googledatastore.commit(req)
# AddFloatToDS(entity, 'lowPriceChange', np.array(savedPrediction[symbols[i]])[:,LOW][-1] - low[i][-1])
# AddFloatToDS(entity, 'lowPriceChangePercent', (np.array(savedPrediction[symbols[i]])[:,LOW][-1] - low[i][-1])/abs(low[i][-1])*100.0)
# AddFloatToDS(entity, 'volumeChange', np.array(savedPrediction[symbols[i]])[:,VOLUME][-1] - volume[i][-1])
# AddFloatToDS(entity, 'volumeChangePercent', (np.array(savedPrediction[symbols[i]])[:,VOLUME][-1] - volume[i][-1])/abs(volume[i][-1])*100.0)
#Market snapshot
# AddFloatToDS(entity, 'predOpen', np.array(savedPrediction[symbols[i]])[:,OPEN][-1])
# AddFloatToDS(entity, 'predClose', np.array(savedPrediction[symbols[i]])[:,CLOSE][-1])
# AddFloatToDS(entity, 'predHigh', np.array(savedPrediction[symbols[i]])[:,HIGH][-1])
# AddFloatToDS(entity, 'predLow', np.array(savedPrediction[symbols[i]])[:,LOW][-1])
# AddFloatToDS(entity, 'predVolume', np.array(savedPrediction[symbols[i]])[:,VOLUME][-1])
# Execute the Commit RPC synchronously and ignore the response:
# Apply the insert mutation if the entity was not found and close
# the transaction.
datastore.commit(req)
except datastore.RPCError as e:
# RPCError is raised if any error happened during a RPC.
# It includes the `method` called and the `reason` of the
# failure as well as the original `HTTPResponse` object.
logging.error('Error while doing datastore operation')
logging.error('RPCError: %(method)s %(reason)s',
{'method': e.method,
'reason': e.reason})
logging.error('HTTPError: %(status)s %(reason)s',
{'status': e.response.status,
'reason': e.response.reason})
if rank[symbols[i]] <= 25:
#Also commit to the stock list, for faster and cheaper dataastore queries
try:
def PushData(data, original_data={}):
'''Pushes a bunch of data into the datastore. The data should be a dict. Each
key is treated as a namespace, and each value is also a dict. A new datastore
entry is upserted for every inner key, with the value pickled into the
|pickled_value| field.
For example, if given the dictionary:
{
'fruit': {
'apple': 1234,
'banana': 'yellow',
'trolling carrot': { 'arbitrarily complex': ['value', 'goes', 'here'] }
},
'animal': {
'sheep': 'baaah',
'dog': 'woof',
'trolling cat': 'moo'
}
}
this would result in a push of 6 keys in total, with the following IDs:
Key('PersistentObjectStoreItem', 'fruit/apple')
Key('PersistentObjectStoreItem', 'fruit/banana')
Key('PersistentObjectStoreItem', 'fruit/trolling carrot')
Key('PersistentObjectStoreItem', 'animal/sheep')
Key('PersistentObjectStoreItem', 'animal/dog')
Key('PersistentObjectStoreItem', 'animal/trolling cat')
If given |original_data|, this will only push key-value pairs for entries that
are either new or have changed from their original (pickled) value.
Caveat: Pickling and unpickling a dictionary can (but does not always) change
its key order. This means that objects will often be seen as changed even when
they haven't changed.
'''
datastore.set_options(dataset=_DATASET_NAME)
def flatten(dataset):
flat = {}
for namespace, items in dataset.iteritems():
for k, v in items.iteritems():
flat['%s/%s' % (namespace, k)] = cPickle.dumps(v)
return flat
logging.info('Flattening data sets...')
data = flatten(data)
original_data = flatten(original_data)
logging.info('Culling new data...')
for k in data.keys():
if ((k in original_data and original_data[k] == data[k]) or
(len(data[k]) > _MAX_ENTITY_SIZE)):
del data[k]
for batch, n, total in _CreateBatches(data):
commit_request = datastore.CommitRequest()
commit_request.mode = datastore.CommitRequest.NON_TRANSACTIONAL
commit_request.mutation.upsert.extend(list(batch))
logging.info('Committing %s/%s entities...' % (n, total))
datastore.commit(commit_request)
def delete_batch(batch):
req = googledatastore.CommitRequest()
req.mode = googledatastore.CommitRequest.NON_TRANSACTIONAL
req.mutation.delete.extend([entity.__get_key() for entity in batch])
googledatastore.commit(req)
#print resp req = datastore.CommitRequest() req.mode = datastore.CommitRequest.NON_TRANSACTIONAL cams = req.mutation.insert.add() path = cams.key.path_element.add() path.kind = 'PSCams' path.name = 'mushak0001' camid_property = cams.property.add() camid_property.name = 'camid' camid_property.value.string_value = 'mushak0001' name_property = cams.property.add() name_property.name = 'name' name_property.value.string_value = 'mushak' master_property = cams.property.add() master_property.name = 'master' master_property.value.string_value = '03086347034341246126' password_property = cams.property.add() password_property.name = 'password' password_property.value.string_value = '0253fcafdd931e2e2ae915499ed77c1c' resp = datastore.commit(req) print resp
def main():
# Set dataset id from command line argument.
if len(sys.argv) < 2:
print 'Usage: adams.py <DATASET_ID>'
sys.exit(1)
# Set the dataset from the command line parameters.
datastore.set_options(dataset=sys.argv[1])
try:
# Create a RPC request to begin a new transaction.
req = datastore.BeginTransactionRequest()
# Execute the RPC synchronously.
resp = datastore.begin_transaction(req)
# Get the transaction handle from the response.
tx = resp.transaction
# Create a RPC request to get entities by key.
req = datastore.LookupRequest()
# Create a new entity key.
key = datastore.Key()
# Set the entity key with only one `path_element`: no parent.
path = key.path_element.add()
path.kind = 'Trivia'
path.name = 'hgtg'
# Add one key to the lookup request.
req.key.extend([key])
# Set the transaction, so we get a consistent snapshot of the
# entity at the time the transaction started.
req.read_options.transaction = tx
# Execute the RPC and get the response.
resp = datastore.lookup(req)
# Create a RPC request to commit the transaction.
req = datastore.CommitRequest()
# Set the transaction to commit.
req.transaction = tx
if resp.found:
# Get the entity from the response if found.
entity = resp.found[0].entity
else:
# If no entity was found, insert a new one in the commit request mutation.
entity = req.mutation.insert.add()
# Copy the entity key.
entity.key.CopyFrom(key)
# Add two entity properties:
# - a utf-8 string: `question`
prop = entity.property.add()
prop.name = 'question'
prop.value.string_value = 'Meaning of life?'
# - a 64bit integer: `answer`
prop = entity.property.add()
prop.name = 'answer'
prop.value.integer_value = 42
# Execute the Commit RPC synchronously and ignore the response:
# Apply the insert mutation if the entity was not found and close
# the transaction.
datastore.commit(req)
props = get_property_dict(entity)
# Get question property value.
question = props['question'].string_value
# Get answer property value.
answer = props['answer'].integer_value
# Print the question and read one line from stdin.
print question
result = raw_input('> ')
if result == str(answer):
print ('fascinating, extraordinary and, '
'when you think hard about it, completely obvious.')
else:
print "Don't Panic!"
except datastore.RPCError as e:
# RPCError is raised if any error happened during a RPC.
# It includes the `method` called and the `reason` of the
# failure as well as the original `HTTPResponse` object.
logging.error('Error while doing datastore operation')
logging.error('RPCError: %(method)s %(reason)s',
{'method': e.method,
'reason': e.reason})
logging.error('HTTPError: %(status)s %(reason)s',
{'status': e.response.status,
'reason': e.response.reason})
return
def checkDB(self, gitkit_user):
req = datastore.BeginTransactionRequest()
resp = datastore.begin_transaction(req)
tx = resp.transaction
req = datastore.LookupRequest()
key = datastore.Key()
path = key.path_element.add()
path.kind = 'PSUsers'
path.name = gitkit_user.user_id
req.key.extend([key])
req.read_options.transaction = tx
resp = datastore.lookup(req)
req = datastore.CommitRequest()
req.transaction = tx
if resp.missing:
user = req.mutation.insert.add()
user.key.CopyFrom(key)
userid_property = user.property.add()
userid_property.name = 'userid'
userid_property.value.string_value = gitkit_user.user_id
display_name_property = user.property.add()
display_name_property.name = 'display_name'
display_name_property.value.string_value = gitkit_user.name
photo_url_property = user.property.add()
photo_url_property.name = 'photo_url'
if gitkit_user.photo_url:
photo_url_property.value.string_value = gitkit_user.photo_url
else:
photo_url_property.value.string_value = "/images/home/slider/slide1/cloud1.png"
email_property = user.property.add()
email_property.name = 'email'
email_property.value.string_value = gitkit_user.email
last_login_property = user.property.add()
last_login_property.name = 'last_login'
last_login_property.value.timestamp_microseconds_value = long(
time.time() * 1e6)
elif resp.found:
user = resp.found[0].entity
last_login_property = datastore.Property()
last_login_property.name = 'last_login'
for prop in user.property:
if prop.name == 'last_login':
prop.value.timestamp_microseconds_value = long(
time.time() * 1e6)
req.mutation.update.extend([user])
resp = datastore.commit(req)
return None
def main():
# Set project id from command line argument.
if len(sys.argv) < 2:
print 'Usage: adams.py <PROJECT_ID>'
sys.exit(1)
# Set the project from the command line parameters.
datastore.set_options(project_id=sys.argv[1])
try:
# Create a RPC request to begin a new transaction.
req = datastore.BeginTransactionRequest()
# Execute the RPC synchronously.
resp = datastore.begin_transaction(req)
# Get the transaction handle from the response.
tx = resp.transaction
# Create a RPC request to get entities by key.
req = datastore.LookupRequest()
# Create a new entity key.
key = datastore.Key()
# Set the entity key with only one `path` element: no parent.
elem = key.path.add()
elem.kind = 'Trivia'
elem.name = 'hgtg'
# Add one key to the lookup request.
req.keys.extend([key])
# Set the transaction, so we get a consistent snapshot of the
# entity at the time the transaction started.
req.read_options.transaction = tx
# Execute the RPC and get the response.
resp = datastore.lookup(req)
# Create a RPC request to commit the transaction.
req = datastore.CommitRequest()
# Set the transaction to commit.
req.transaction = tx
if resp.found:
# Get the entity from the response if found.
entity = resp.found[0].entity
else:
# If no entity was found, insert a new one in the commit request mutation.
entity = req.mutations.add().insert
# Copy the entity key.
entity.key.CopyFrom(key)
# Add two entity properties:
# - a utf-8 string: `question`
entity.properties['question'].string_value ='Meaning of life?'
# - a 64bit integer: `answer`
prop = entity.properties['answer'].integer_value = 42
# Execute the Commit RPC synchronously and ignore the response:
# Apply the insert mutation if the entity was not found and close
# the transaction.
datastore.commit(req)
# Get question property value.
question = entity.properties['question'].string_value
# Get answer property value.
answer = entity.properties['answer'].integer_value
# Print the question and read one line from stdin.
print question
result = raw_input('> ')
if result == str(answer):
print ('fascinating, extraordinary and, '
'when you think hard about it, completely obvious.')
else:
print "Don't Panic!"
except datastore.RPCError as e:
# RPCError is raised if any error happened during a RPC.
# It includes the `method` called, the canonical error code `code`,
# and the `message` of the failure.
logging.error('Error while doing datastore operation')
logging.error('RPCError: %(method)s %(reason)s',
{'method': e.method,
'code': e.code,
'reason': e.message})
return
def main():
datastore.set_options(dataset='glowing-thunder-842')
try:
# Create a RPC request to begin a new transaction.
req = datastore.BeginTransactionRequest()
# Execute the RPC synchronously.
resp = datastore.begin_transaction(req)
# Get the transaction handle from the response.
tx = resp.transaction
# Create a RPC request to get entities by key.
req = datastore.LookupRequest()
# Create a new entity key.
key = datastore.Key()
# Set the entity key with only one `path_element`: no parent.
path = key.path_element.add()
path.kind = 'kindlooptest'
path.name = randstr()
# Add one key to the lookup request.
req.key.extend([key])
# Set the transaction, so we get a consistent snapshot of the
# entity at the time the transaction started.
req.read_options.transaction = tx
# Execute the RPC and get the response.
resp = datastore.lookup(req)
# Create a RPC request to commit the transaction.
req = datastore.CommitRequest()
# Set the transaction to commit.
req.transaction = tx
if resp.found:
# Get the entity from the response if found.
entity = resp.found[0].entity
else:
# If no entity was found, insert a new one in the commit request mutation.
entity = req.mutation.insert.add()
# Copy the entity key.
entity.key.CopyFrom(key)
# Add two entity properties:
# - a utf-8 string: `question`
prop = entity.property.add()
prop.name = 'prop1test'
prop.value.string_value = randstr()
# - a 64bit integer: `answer`
prop = entity.property.add()
prop.name = 'prop2test'
prop.value.integer_value = 77
# Execute the Commit RPC synchronously and ignore the response:
# Apply the insert mutation if the entity was not found and close
# the transaction.
datastore.commit(req)
# Get question property value.
question = entity.property[0].value.string_value
# Get answer property value.
answer = entity.property[1].value.integer_value
# Print the question and read one line from stdin.
print answer
except datastore.RPCError as e:
# RPCError is raised if any error happened during a RPC.
# It includes the `method` called and the `reason` of the
# failure as well as the original `HTTPResponse` object.
logging.error('Error while doing datastore operation')
logging.error('RPCError: %(method)s %(reason)s',
{'method': e.method,
'reason': e.reason})
logging.error('HTTPError: %(status)s %(reason)s',
{'status': e.response.status,
'reason': e.response.reason})
return
volume[i][-NPredPast + 1:],
np.array(savedPrediction[symbols[i]])[:, VOLUME][:-1])
AddFloatToDS(entity, 'volumePredSlope', slope)
if np.mean([1.0 - volR2, abs(1.0 - slope)]) <= 0.05:
AddIntToDS(entity, 'volumeModelAccuracy', 1)
elif np.mean(
[1.0 - volR2, abs(1.0 - slope)]) < 0.1 and np.mean(
[1.0 - volR2, abs(1.0 - slope)]) > 0.05:
AddIntToDS(entity, 'volumeModelAccuracy', 2)
else:
AddIntToDS(entity, 'volumeModelAccuracy', 3)
# Execute the Commit RPC synchronously and ignore the response:
# Apply the insert mutation if the entity was not found and close
# the transaction.
datastore.commit(req)
except datastore.RPCError as e:
# RPCError is raised if any error happened during a RPC.
# It includes the `method` called and the `reason` of the
# failure as well as the original `HTTPResponse` object.
logging.error('Error while doing datastore operation')
logging.error('RPCError: %(method)s %(reason)s', {
'method': e.method,
'reason': e.reason
})
logging.error('HTTPError: %(status)s %(reason)s', {
'status': e.response.status,
'reason': e.response.reason
})
if rank[symbols[i]] <= 25:
def delete_batch(batch):
req = googledatastore.CommitRequest()
req.mode = googledatastore.CommitRequest.NON_TRANSACTIONAL
req.mutation.delete.extend(
[entity.__get_key() for entity in batch])
googledatastore.commit(req)
