def createTagFromTree(parent):
global n
global c
global badtags
addParent = False #Indicates whether parent entity should be added
children = parent.children
for i, child in enumerate(children):
if i % 2 == 1: #parent.children returns: None child None child ...
if len(list(child.children)) is 0:
#Invariant: Child is in deepest layer
addParent = True
cname = child.name
pname = parent.name
wordpouchchild = searchTwitter(
pname, cname) #Retrieve Twitter wordPouch
#Handle child entity
if len(wordpouchchild[0]) < 10:
#This is a bad pair of tags
badTag = pname + "," + cname
badtags.append(badTag)
else:
#Create tag entity and save it in datastore
child_key = datastore.Key("tags",
pname,
"tags",
cname,
project=str(config.PROJECT_ID))
ctag_ent = datastore.Entity(key=child_key)
ctag_ent["language"] = "en"
ctag_ent["wordPouch"] = wordpouchchild[0]
ctag_ent["wordPouchScores"] = wordpouchchild[1]
DAL.saveEntity(ctag_ent)
else:
#Invariant: Child is not in deepest layer so we need to recurse
createTagFromTree(child)
if addParent:
wordpouchparent = searchTwitter(pname, None)
#Handle parent entity
if len(wordpouchparent[0]) < 10:
#This is a bad tag
badtags.append(parent.name)
else:
#Create tag entity and save it in datastore
parent_key = datastore.Key("tags",
parent.name,
project=str(config.PROJECT_ID))
ptag_ent = datastore.Entity(key=parent_key)
ptag_ent["language"] = "en"
ptag_ent["wordPouch"] = wordpouchparent[0]
ptag_ent["wordPouchScores"] = wordpouchparent[1]
DAL.saveEntity(ptag_ent)
def put(self, key=None):
if key:
self.entity.key = key
elif self.entity.key is None:
self.entity.key = datastore.Key(self.__class__.kind(),
project=self.project)
self.client.put(self.entity)
def key_from_user_id(user_id):
"""
Create a datastore key from a user ID.
Each user has at most 1 ondemand game task.
"""
return gcloud_datastore.Key(
ONDEMAND_KIND,
user_id,
project=config.GCLOUD_PROJECT,
)
def to_datastore_key(
self, entity_key: Union[EntityKey, NoneKey,
None]) -> Optional[datastore.Key]:
if entity_key is None or isinstance(entity_key, NoneKey):
return None
project = self._gumo_config.google_cloud_project.value
datastore_key = datastore.Key(*entity_key.flat_pairs(),
project=project)
return datastore_key
def entity_builder(key, interviewer, start_time, end_time, outcome_code, status, questionnaire_name="LMS2202_TST"):
entity = datastore.Entity(
datastore.Key("CallHistory", key, project="test")
)
entity["interviewer"] = interviewer
entity["call_start_time"] = start_time
entity["call_end_time"] = end_time
entity["outcome_code"] = outcome_code
entity["status"] = status
entity["questionnaire_name"] = questionnaire_name
return entity
def test_entity_from_protobuf():
class Animal(polymodel.PolyModel):
pass
class Cat(Animal):
pass
key = datastore.Key("Cat", 123, project="testing")
datastore_entity = datastore.Entity(key=key)
protobuf = helpers.entity_to_protobuf(datastore_entity)
entity = model._entity_from_protobuf(protobuf)
assert isinstance(entity, Cat)
def test_to_entity_id_key():
datastore_key = datastore.Key('Book',
12345,
'BookComment',
67890,
project='test-project')
entity_key = key_mapper.to_entity_key(datastore_key=datastore_key)
assert isinstance(entity_key, EntityKey)
assert entity_key.flat_pairs() == ['Book', 12345, 'BookComment', 67890]
assert isinstance(entity_key.name(), int)
assert entity_key.name() == 67890
def test_mass_delete():
datastore_mock = Mock()
storage = DatastoreStorage(datastore_mock)
q_mock = Mock()
q_mock.fetch.return_value = [
Mock(key=datastore.Key('task', n, project='test')) for n in range(102)
]
datastore_mock.query.return_value = q_mock
storage.delete_tasks()
assert datastore_mock.delete_multi.call_count == 2
def test_to_entity_key():
datastore_key = datastore.Key('Book',
'name',
'BookComment',
'comment',
project='test-project')
entity_key = key_mapper.to_entity_key(datastore_key=datastore_key)
assert isinstance(entity_key, EntityKey)
assert entity_key.flat_pairs() == [
'Book', 'name', 'BookComment', 'comment'
]
assert entity_key.kind() == 'BookComment'
assert entity_key.name() == 'comment'
def _ndb_entity_to_cloud_entity(ndb_entity):
"""Convert ndb.Model to cloud entity to prepare for put()."""
if ndb_entity is None:
return None
project_id = utils.get_application_id()
unindexed_properties = _unindexed_properties(ndb_entity.__class__)
ndb_entity._prepare_for_put()
ndb_entity._pre_put_hook()
if ndb_entity.key:
# Existing key.
cloud_entity = datastore.Entity(
key=_ndb_key_to_cloud_key(ndb_entity.key),
exclude_from_indexes=unindexed_properties)
else:
# Auto-generate key.
base_key = datastore.Key(ndb_entity._get_kind(), project=project_id)
generated_key = _retry_wrap(_client().allocate_ids)(base_key, 1)[0]
cloud_entity = datastore.Entity(
key=generated_key, exclude_from_indexes=unindexed_properties)
ndb_entity.key = _cloud_key_to_ndb_key(generated_key)
for key, value in ndb_entity.to_dict().iteritems():
ndb_property = getattr(ndb_entity.__class__, key)
if type(ndb_property) in UNSUPPORTED_PROPERTY_TYPES:
raise NdbPatcherException('Unsupported property type: ' +
ndb_property.__name__)
if (isinstance(value, str)
and type(ndb_property) is not ndb.BlobProperty):
# All 'str' values are written as byte strings by Cloud Datastore, but ndb
# ndb_entitys can have 'str' values for StringProperty or TextProperty, so
# check the type of the property.
value = unicode(value)
elif type(value) is datetime.date:
value = datetime.datetime.combine(value,
datetime.datetime.min.time())
elif type(value) is datetime.time:
value = datetime.datetime.combine(datetime.date(1970, 1, 1), value)
elif isinstance(value, ndb.Key):
value = _ndb_key_to_cloud_key(value)
cloud_entity[key] = value
return cloud_entity
def from_device_id(device_id: int) -> Union[datastore.Entity, None]:
"""
Gets a DeviceDataType entity from the datastore using the device
integer ID.
:param device_id:
:return:
"""
key = datastore.Key('Device', device_id, project='modular-security-system')
device = client.get(key)
try:
type_key = device.device_type_key
except AttributeError:
return None
data_type = client.get(type_key)
return data_type
def updateArticleEnt(publisherId, articleId, content, title):
#Retrieve article from Google Cloud Datastore
article_key = datastore.Key("publishers",
publisherId,
"articles",
str(articleId),
project=str(config.PROJECT_ID))
article_ent = DAL.getArticleById(publisherId, articleId)
#Update article's content and status
article_ent["title"] = title
article_ent["content"] = content
if content != '':
article_ent["status"] = "scraped"
else:
article_ent["status"] = "scrapeError"
#Return updated article to datastore
DAL.saveEntity(article_ent)
def createCustomTag(tagName):
global n
global c
wordPouch = searchTwitter(tagName, None) #Retrieve Twitter wordPouch
if len(wordPouch) == 0 or len(wordPouch[0]) < 10:
#This is a bad tag
return None
else:
#Create tag entity and save it in datastore
#ds = datastore.Client(project= str(config.PROJECT_ID)) #Setup datastore access
#tag_key= datastore.Key("tags", tagName, project=str(config.PROJECT_ID))
tag_key = datastore.Key("tags",
'CUSTOM',
"tags",
tagName,
project=config.PROJECT_ID)
tag_ent = datastore.Entity(key=tag_key)
tag_ent["language"] = "en"
tag_ent["wordPouch"] = wordPouch[0]
tag_ent["wordPouchScores"] = wordPouch[1]
DAL.saveEntity(tag_ent)
return tag_ent
def _ndb_key_to_cloud_key(ndb_key):
"""Convert a ndb.Key to a cloud entity Key."""
return datastore.Key(ndb_key.kind(),
ndb_key.id(),
project=utils.get_application_id())
def guestbook_key(guestbook_name=DEFAULT_GUESTBOOK_NAME):
"""Constructs a Datastore key for a Guestbook entity.
We use guestbook_name as the key.
"""
return ndb.Key('Guestbook', guestbook_name)
def test_get_from_cloud_datastore_entity(self):
entity = cloud_datastore_types.Entity(
key=cloud_datastore_types.Key('BaseModel', '123', project='foo'))
self.assertEqual(job_utils.get_model_kind(entity), 'BaseModel')
def key(self, kind, uid=None):
if uid:
return datastore.Key(kind, uid, project=_TEST_PROJECT)
return datastore.Key(kind, project=_TEST_PROJECT)
def test_save_partial_key(self):
"""Tests that attempting to save an entity with a partial key throws
an exeception."""
key = datastore.Key('namespace', project='project')
entity = datastore.Entity(key)
self.assertRaises(ValueError, self.database.save, entity)
def test_get_missing_property_from_cloud_datastore_entity(self):
entity = cloud_datastore_types.Entity(
key=cloud_datastore_types.Key('FooModel', '123', project='foo'))
self.assertEqual(job_utils.get_model_property(entity, 'prop'), None)
