def build_user_entity(row):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'User', row['email'])
props = dict(row)
del props['email']
datastore_helper.add_properties(entity, props)
return entity
def create_ds_entity(element):
entity = entity_pb2.Entity()
kind = 'FileMetaData'
datastore_helper.add_key_path(entity.key, kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, element)
return entity
def testIncompleteKey(self):
key = datastore.Key()
helper.add_key_path(key, 'Foo')
self.assertEquals(1, len(key.path_element))
self.assertEquals('Foo', key.path_element[0].kind)
self.assertEquals(0, key.path_element[0].id)
self.assertEquals('', key.path_element[0].name)
def testIncompleteKey(self):
key = datastore.Key()
helper.add_key_path(key, 'Foo')
self.assertEquals(1, len(key.path_element))
self.assertEquals('Foo', key.path_element[0].kind)
self.assertEquals(0, key.path_element[0].id)
self.assertEquals('', key.path_element[0].name)
def make_entity(self, content):
"""Create entity and set properties."""
entity = entity_pb2.Entity()
# We use uuid for keys.
datastore_helper.add_key_path(entity.key, self._kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, content)
return entity
def make_entity(self, content):
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
helper.add_key_path(entity.key, self._kind, str(uuid.uuid4()))
helper.add_properties(entity, content)
return entity
def testSetKeyPath(self):
key = datastore.Key()
helper.add_key_path(key, 'Foo', 1, 'Bar', 'bar')
self.assertEquals(2, len(key.path_element))
self.assertEquals('Foo', key.path_element[0].kind)
self.assertEquals(1, key.path_element[0].id)
self.assertEquals('Bar', key.path_element[1].kind)
self.assertEquals('bar', key.path_element[1].name)
def testSetKeyPath(self):
key = datastore.Key()
helper.add_key_path(key, 'Foo', 1, 'Bar', 'bar')
self.assertEquals(2, len(key.path_element))
self.assertEquals('Foo', key.path_element[0].kind)
self.assertEquals(1, key.path_element[0].id)
self.assertEquals('Bar', key.path_element[1].kind)
self.assertEquals('bar', key.path_element[1].name)
def build_company_entity(data):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'Company', data[0])
props = {}
props['reg_date'] = data[1]
props['boxes_bought'] = 0
props['boxes_prov'] = 0
datastore_helper.add_properties(entity, props)
return entity
def make_entity(self, prefix, candidates):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, "prefix", prefix, "id",
str(uuid.uuid4()), "candidates",
unicode(candidates))
# All entities created will have the same ancestor
#datastore_helper.add_properties(entity, {"wordlist": unicode(candidates)})
#datastore_helper.add_properties(entity, {"candidates": unicode(candidates)})
return entity
def make_entity(self, content):
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, {"content": unicode(content)})
return entity
def make_entity(self, content):
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, str(uuid.uuid4()))
datastore_helper.add_properties(entity, {"content": unicode(content)})
return entity
def make_entity(self, content):
"""Create entity from given string."""
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, hashlib.sha1(content).hexdigest())
datastore_helper.add_properties(entity, {'content': str(content)})
return entity
def make_entity(self, content):
"""Create entity from given string."""
entity = entity_pb2.Entity()
if self._namespace is not None:
entity.key.partition_id.namespace_id = self._namespace
# All entities created will have the same ancestor
datastore_helper.add_key_path(entity.key, self._kind, self._ancestor,
self._kind, hashlib.sha1(content).hexdigest())
datastore_helper.add_properties(entity, {'content': str(content)})
return entity
def make_ancestor_query(kind, namespace, ancestor):
"""Creates a Cloud Datastore ancestor query."""
ancestor_key = entity_pb2.Key()
datastore_helper.add_key_path(ancestor_key, kind, ancestor)
if namespace is not None:
ancestor_key.partition_id.namespace_id = namespace
query = query_pb2.Query()
query.kind.add().name = kind
datastore_helper.set_property_filter(
query.filter, '__key__', PropertyFilter.HAS_ANCESTOR, ancestor_key)
return query
def make_ancestor_query(kind, namespace, ancestor):
"""Creates a Cloud Datastore ancestor query."""
ancestor_key = entity_pb2.Key()
datastore_helper.add_key_path(ancestor_key, kind, ancestor)
if namespace is not None:
ancestor_key.partition_id.namespace_id = namespace
query = query_pb2.Query()
query.kind.add().name = kind
datastore_helper.set_property_filter(
query.filter, '__key__', PropertyFilter.HAS_ANCESTOR, ancestor_key)
return query
def process(self, element):
entity = entity_pb2.Entity()
sku = int(element.pop('sku'))
element['regularPrice'] = float(element['regularPrice'])
element['salePrice'] = float(element['salePrice'])
element['name'] = unicode(element['name'].decode('utf-8'))
element['type'] = unicode(element['type'].decode('utf-8'))
element['url'] = unicode(element['url'].decode('utf-8'))
element['image'] = unicode(element['image'].decode('utf-8'))
element['inStoreAvailability'] = unicode(
element['inStoreAvailability'])
datastore_helper.add_key_path(entity.key, 'Productx', sku)
datastore_helper.add_properties(entity, element)
return [entity]
def build_trending_entities(company):
metric = metric_list[randint(0, 4)]
if randint(1, 3) > 2:
delta = randint(15, 30)
else:
delta = -randint(15, 30)
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'Trend',
'{}-{}'.format(company[0], metric))
props = {}
props['company'] = company[0]
props['metric'] = metric
props['delta'] = delta
props['date'] = datetime.now()
datastore_helper.add_properties(entity, props)
return entity
def make_ancestor_query(kind, namespace, ancestor):
"""Creates a Cloud Datastore ancestor query.
The returned query will fetch all the entities that have the parent key name
set to the given `ancestor`.
"""
ancestor_key = entity_pb2.Key()
datastore_helper.add_key_path(ancestor_key, kind, ancestor)
if namespace is not None:
ancestor_key.partition_id.namespace_id = namespace
query = query_pb2.Query()
query.kind.add().name = kind
datastore_helper.set_property_filter(
query.filter, '__key__', PropertyFilter.HAS_ANCESTOR, ancestor_key)
return query
def process(self, element, entityName, user, dataset):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, entityName.get(),
str(uuid.uuid4()))
datastore_helper.add_properties(
entity, {
"label": unicode(element['label']),
"user": unicode(user.get()),
"dataset": unicode(dataset.get())
})
datastore_helper.add_properties(entity,
{"text": unicode(element['text'])},
exclude_from_indexes=True)
return [entity]
def make_ancestor_query(kind, namespace, ancestor):
"""Creates a Cloud Datastore ancestor query.
The returned query will fetch all the entities that have the parent key name
set to the given `ancestor`.
"""
ancestor_key = entity_pb2.Key()
datastore_helper.add_key_path(ancestor_key, kind, ancestor)
if namespace is not None:
ancestor_key.partition_id.namespace_id = namespace
query = query_pb2.Query()
query.kind.add().name = kind
datastore_helper.set_property_filter(
query.filter, '__key__', PropertyFilter.HAS_ANCESTOR, ancestor_key)
return query
def make_entity(self, content):
"""Transform each row from input file to DS Entity.
:type content: str
:param content: one line of input file.
:rtype: `entity_pb2.Entity`
:returns: Entity created for each line of input file.
"""
sku_base, similarities = itemgetter('item', 'similarity_items')(
json.loads(content))
entity = entity_pb2.Entity()
s_sim = [e for e in sorted(similarities,
key=lambda x: x['similarity'], reverse=True)][:self.sim_cap]
add_key_path(entity.key, self._kind, sku_base)
add_properties(entity, {'items': [e['item'] for e in s_sim]},
exclude_from_indexes = True)
add_properties(entity, {'scores':
[e['similarity'] for e in s_sim]},
exclude_from_indexes=True)
return entity
def build_project_entities(company):
project = project_list[randint(0, 4)]
health_index = randint(0, 5)
if health_index < 1:
health = randint(30, 50)
elif health_index < 2:
health = randint(50, 70)
else:
health = randint(70, 100)
progress = randint(50, 95)
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'Project',
'{}-{}'.format(company[0], project))
props = {}
props['due'] = datetime.now() + timedelta(randint(30, 60))
props['name'] = project
props['health'] = health
props['progress'] = progress
props['company'] = company[0]
datastore_helper.add_properties(entity, props)
return entity
def process(self, idx):
entity = entity_pb2.Entity()
embed = entity_pb2.Entity() # embedded document
ss = {
'ss': unicode('X'), # string
'bb': False, # boolean
'll': [1, 2, 3], # list of integers
'ii': 123 # integer
}
datastore_helper.add_properties(
embed, ss) # setting properties for embedded document
element = dict()
element['s'] = unicode('String') # string
element['b'] = True # boolean
element['l'] = [unicode('s'), unicode('a'),
unicode('b')] # list of strings
element['i'] = 1 # integer
element['e'] = embed # embedded document
datastore_helper.add_key_path(entity.key, 'ds',
idx) # setting id for document
datastore_helper.add_properties(
entity, element) # setting properties for document
return [entity]
def build_renewal_entities(company_updates):
# print company_updates
amount = company_updates[1]['purchased'][0] * 2499
health_index = randint(0, 5)
if health_index < 1:
health = randint(10, 30)
elif health_index < 3:
health = randint(30, 60)
else:
health = randint(60, 100)
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, 'Renewal',
'{}-{}'.format(company_updates[0], amount))
props = {}
props['due'] = datetime.now() + timedelta(randint(30, 120))
props['amount'] = amount
props['health'] = health
props['company'] = company_updates[0]
datastore_helper.add_properties(entity, props)
return entity
def test_is_key_valid(self):
key = entity_pb2.Key()
# Complete with name, no ancestor
datastore_helper.add_key_path(key, 'kind', 'name')
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with id, no ancestor
datastore_helper.add_key_path(key, 'kind', 12)
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Incomplete, no ancestor
datastore_helper.add_key_path(key, 'kind')
self.assertFalse(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with name and ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2', 'name2')
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with id and ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2', 123)
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Incomplete with ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2')
self.assertFalse(helper.is_key_valid(key))
key = entity_pb2.Key()
self.assertFalse(helper.is_key_valid(key))
def create_entity(input_features, kind):
entity = entity_pb2.Entity()
datastore_helper.add_key_path(entity.key, kind, input_features['id'])
datastore_helper.add_properties(entity,
{'text': unicode(input_features['text'])})
return entity
def test_is_key_valid(self):
key = entity_pb2.Key()
# Complete with name, no ancestor
datastore_helper.add_key_path(key, 'kind', 'name')
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with id, no ancestor
datastore_helper.add_key_path(key, 'kind', 12)
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Incomplete, no ancestor
datastore_helper.add_key_path(key, 'kind')
self.assertFalse(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with name and ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2', 'name2')
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with id and ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2', 123)
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Incomplete with ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2')
self.assertFalse(helper.is_key_valid(key))
key = entity_pb2.Key()
self.assertFalse(helper.is_key_valid(key))
