def create_multi(cls, values):
"""Creates multiple unique values at once.
:param values:
A sequence of values to be unique. See :meth:`create`.
:returns:
A tuple (bool, list_of_keys). If all values were created, bool is
True and list_of_keys is empty. If one or more values weren't
created, bool is False and the list contains all the values that
already existed in datastore during the creation attempt.
"""
keys = [cls.get_key(value) for value in values]
# Preliminary check, before going for more expensive transactions.
entities = model.get_multi(keys)
existing = [entity.key.id() for entity in entities if entity]
if existing:
return False, existing
# Create all records transactionally.
created = []
entities = [cls(key=key) for key in keys]
for entity in entities:
txn = lambda: entity.put() if not entity.key.get() else None
key = model.transaction(txn)
if key:
created.append(key)
if created != keys:
# A poor man's "rollback": delete all recently created records.
model.delete_multi(created)
return False, [k.id() for k in keys if k not in created]
return True, []
def get_by_auth_token(cls, username, token):
token_key = UserToken.get_key(username, 'auth', token)
user_key = cls.get_key(username)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
return user
def get(self):
from ndb import model, query
from mol.db.layers import Layer, LayerIndex
limit = self.request.get_range('limit', min_value=1, max_value=100, default=10)
offset = self.request.get_range('offset', min_value=0, default=0)
args = {}
for arg in self.request.arguments():
if arg in INDEXED_PROPS:
if arg in POLYGON_PROPS:
args['polygon.%s' % arg] = self.request.get(arg, None)
else:
args[arg] = self.request.get(arg, None)
qry = LayerIndex.query()
if len(args) > 0:
gql = 'SELECT * FROM LayerIndex WHERE'
for k,v in args.iteritems():
gql = "%s \"%s\" = '%s' AND " % (gql, k, v)
gql = gql[:-5] # Removes trailing AND
logging.info(gql)
qry = query.parse_gql(gql)[0]
# No keyword search yet
#for keyword in keywords:
# qry = qry.filter(RecordIndex.corpus == keyword)
logging.info('QUERY='+str(qry))
layers = model.get_multi(set([x.parent() for x in qry.fetch(limit, offset=offset, keys_only=True)]))
layers_json = [simplejson.loads(x.json) for x in layers]
self.response.headers["Content-Type"] = "application/json"
self.response.out.write(simplejson.dumps(layers_json))
def search_place_keywords(cls, place, limit=10, offset=0):
names = [name.strip().lower() for name in place.split(",")]
results = {}
for name in names:
qry = cls.query()
for token in tokenize(name.split()):
qry = qry.filter(cls.k == token)
results[name] = model.get_multi([key.parent() for key in qry.fetch(limit, offset=offset, keys_only=True)])
return results
def testGetMulti(self):
ent1 = model.Model(key=model.Key('MyModel', 1))
ent2 = model.Model(key=model.Key('MyModel', 2))
ent3 = model.Model(key=model.Key('MyModel', 3))
key1 = ent1.put()
key2 = ent2.put()
key3 = ent3.put()
res = model.get_multi((key1, key2, key3))
self.assertEqual(res, [ent1, ent2, ent3])
def get_by_auth_token(cls, auth_id, token):
token_key = UserToken.get_key(auth_id, 'auth', token)
user_key = cls.get_key(auth_id)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
timestamp = int(time.mktime(valid_token.created.timetuple()))
return user, timestamp
return None, None
def get_by_auth_token(cls, auth_id, token):
token_key = UserToken.get_key(auth_id, 'auth', token)
user_key = cls.get_key(auth_id)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
timestamp = int(time.mktime(valid_token.created.timetuple()))
return user, timestamp
return None, None
def testGetMulti(self):
ent1 = model.Model(key=model.Key('MyModel', 1))
ent2 = model.Model(key=model.Key('MyModel', 2))
ent3 = model.Model(key=model.Key('MyModel', 3))
key1 = ent1.put()
key2 = ent2.put()
key3 = ent3.put()
res = model.get_multi((key1, key2, key3))
self.assertEqual(res, [ent1, ent2, ent3])
def query_bb(cls, bb): # TODO add paging support or count constraints
url='http://eighty.iriscouch.com/%s/_design/main/_spatial/points?bbox=%s' % (COUCHDB, bb)
logging.info(url)
response = urlfetch.fetch(
url=url,
method=urlfetch.GET)
if response.status_code != 200:
logging.info('NO RESULTS')
return []
keys = [model.Key(urlsafe=row['id']) for row in simplejson.loads(response.content).get('rows')]
return model.get_multi(keys)
def search(cls, args={}, keywords=[]):
qry = RecordIndex.query()
if len(args) > 0:
gql = 'SELECT * FROM RecordIndex WHERE'
for k, v in args.iteritems():
gql = "%s %s='%s' AND " % (gql, k, v)
gql = gql[:-5] # Removes trailing AND
qry = query.parse_gql(gql)[0]
for keyword in keywords:
qry = qry.filter(RecordIndex.corpus == keyword)
logging.info('QUERY=' + str(qry))
return model.get_multi([x.parent() for x in qry.fetch(keys_only=True)])
def post(self):
name = self.request.get('n', None)
if not name:
results = [simplejson.loads(x.json) for x in Variable.query().fetch()]
else:
keys = [model.Key('Variable', x) for x in name.split(',')]
results = [simplejson.loads(x.json) for x in model.get_multi(keys) if x is not None]
if len(results) == 0:
self.error(404)
return
self.response.headers["Content-Type"] = "application/json"
self.response.out.write(simplejson.dumps(results))
def search(cls, limit=0, offset=10, name=None, keywords=[], category=None, source=None):
if len(keywords) == 0 and not category and not source and not name:
return FeatureIndex.query().fetch(limit, offset=offset)
qry = FeatureIndex.query()
for keyword in keywords:
qry = qry.filter(FeatureIndex.k == keyword)
if category:
qry = qry.filter(FeatureIndex.c == category)
if source:
qry = qry.filter(FeatureIndex.s == source)
logging.info("QUERY=" + str(qry))
return model.get_multi([x.parent() for x in qry.fetch(limit, offset=offset, keys_only=True)])
def search(cls, args={}, keywords=[]):
qry = RecordIndex.query()
if len(args) > 0:
gql = "SELECT * FROM RecordIndex WHERE"
for k, v in args.iteritems():
gql = "%s %s='%s' AND " % (gql, k, v)
gql = gql[:-5] # Removes trailing AND
qry = query.parse_gql(gql)[0]
for keyword in keywords:
qry = qry.filter(RecordIndex.corpus == keyword)
logging.info("QUERY=" + str(qry))
return model.get_multi([x.parent() for x in qry.fetch(keys_only=True)])
def range_query(cls, ranges, limit, offset, name=None, source_name=None):
variables = []
qry = "PointIndex.query(AND("
if name:
qry = "%sPointIndex.name == '%s'," % (qry, name.strip().lower())
if source_name:
qry = "%sPointIndex.source == '%s'," % (qry, source_name.strip().lower())
# Add range intervals to query
for r in ranges:
var = r[0]
variables.append(var)
gte = int(float(r[1]) * pow(10, 5))
lt = int(float(r[2]) * pow(10, 5))
if var == 'lat':
var = 'y'
var_min = -90 * pow(10, 5)
var_max = 90 * pow(10, 5)
elif var == 'lng':
var = 'x'
var_min = -180 * pow(10, 5)
var_max = 180 * pow(10, 5)
else:
logging.error('Unknown variable')
return []
intervals = interval.get_optimum_query_interval(var_min, var_max, gte, lt)
if len(intervals) == 0:
logging.info('No query possible')
return []
qry = "%sOR(" % qry
for index,value in intervals.iteritems():
if not value or not index.startswith('i'):
continue
index = index.replace('i', var)
qry = '%sPointIndex.%s == %d,' % (qry, index, value)
if len(ranges) > 1:
qry = '%s), ' % qry[:-1]
# Complete query
if len(ranges) > 1:
qry = '%s)))' % qry[:-3]
else:
qry = '%s))) ' % qry[:-1]
qry = eval(qry)
#logging.info(qry)
# Get query results
results = [index.parent() for index in qry.fetch(limit, offset=offset, keys_only=True)]
#if len(results) > 0:
# logging.info('Result count = %s' % len(results))
return model.get_multi(results)
def search(cls, params):
"""Returns (records, cursor).
Arguments
args - Dictionary with Darwin Core concept keys
keywords - list of keywords to search on
"""
ctx = tasklets.get_context()
ctx.set_memcache_policy(False)
qry = RecordIndex.query()
# Add darwin core name filters
args = params['args']
if len(args) > 0:
gql = 'SELECT * FROM RecordIndex WHERE'
for k,v in args.iteritems():
gql = "%s %s = '%s' AND " % (gql, k, v)
gql = gql[:-5] # Removes trailing AND
logging.info(gql)
qry = query.parse_gql(gql)[0]
# Add full text keyword filters
keywords = params['keywords']
for keyword in keywords:
qry = qry.filter(RecordIndex.corpus == keyword)
logging.info('QUERY='+str(qry))
# Setup query paging
limit = params['limit']
cursor = params['cursor']
if cursor:
logging.info('Cursor')
index_keys, next_cursor, more = qry.fetch_page(limit, start_cursor=cursor, keys_only=True)
record_keys = [x.parent() for x in index_keys]
else:
logging.info('No cursor')
index_keys, next_cursor, more = qry.fetch_page(limit, keys_only=True)
record_keys = [x.parent() for x in index_keys]
# Return results
return (model.get_multi(record_keys), next_cursor, more)
def fromds(cls, cell_keys):
"""Returns CouchDBCell entities from a datastore query on cell keys.
Arguments:
cell_keys - A set of cell key strings (e.g., 9-15).
Returns:
A dictionary of cell key to CouchDBCell.
"""
# test
keys = [model.Key('Cell', x) for x in cell_keys]
entities = model.get_multi(keys)
#entities = Cell.get_by_key_name(cell_keys)
cells = {}
for x in entities:
if x:
cells[x.key.id()] = x
return cells
def get_by_refresh_token(cls, user_id, token):
"""Returns a user object based on a user ID and token.
:param user_id:
The user_id of the requesting user.
:param token:
The token string to be verified.
:returns:
A tuple ``(User, timestamp)``, with a user object and
the token timestamp, or ``(None, None)`` if both were not found.
"""
token_key = cls.token_model.get_key(user_id, 'refresh', token)
user_key = model.Key(cls, user_id)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
timestamp = int(time.mktime(valid_token.created.timetuple()))
return user, timestamp
return None, None
def get_by_auth_token(cls, user_id, token):
"""Returns a user object based on a user ID and token.
:param user_id:
The user_id of the requesting user.
:param token:
The token string to be verified.
:returns:
A tuple ``(User, timestamp)``, with a user object and
the token timestamp, or ``(None, None)`` if both were not found.
"""
token_key = cls.token_model.get_key(user_id, 'auth', token)
user_key = model.Key(cls, user_id)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
timestamp = int(time.mktime(valid_token.created.timetuple()))
return user, timestamp
return None, None
def get_by_auth_token(cls, user_id, token):
"""Returns a ``User`` entity from a user ID and token.
:param user_id:
The user_id of the requesting user.
:param token:
Existing Token needing verification.
:returns:
A tuple ``(User, timestamp)``, with a :class:`User` instance and
the token timestamp, or ``(None, None)`` if both were not found.
"""
token_key = cls.token_model.get_key(user_id, 'auth', token)
user_key = model.Key(cls, user_id)
# Use get_multi() to save a RPC call.
valid_token, user = model.get_multi([token_key, user_key])
if valid_token and user:
timestamp = int(time.mktime(valid_token.created.timetuple()))
return user, timestamp
return None, None
def search(cls, place): # place = "indian rock, berkeley"
names = [x.strip().lower() for x in place.split(",")]
keys = [cls.key_from_name(name) for name in names]
return [feature for feature in model.get_multi(keys) if feature]
def get_by_name_multi(cls, names):
return model.get_multi([cls.key_from_name(name) for name in names])
def messages(self):
return model.get_multi(self.message_keys)