def make_entity(self, content):
"""Create entity from given string."""
key = Key([self._kind, hashlib.sha1(content.encode('utf-8')).hexdigest()],
parent=self._parent_key)
entity = Entity(key)
entity.set_properties({'content': str(content)})
return entity
def to_entity(line):
fields = line.split(
',') #id,president,startYear,endYear,party,homeState,dateOfBirth
id = int(fields[0])
key = Key([kind, id])
entity = Entity(key)
president = fields[1]
names = president.split(' ')
entity.set_properties({
'id':
id,
'firstName':
names[0],
'lastName':
names[1],
'startYear':
int(fields[2]),
'endYear':
int(fields[3]),
'party':
fields[4],
'homeState':
fields[5],
'dateOfBirth':
datetime.strptime(fields[6], '%Y-%m-%d')
})
return entity
def process(self, element):
key = Key(['natality-guid', element['guid']])
entity = Entity(key)
entity.set_properties({
'weight': element['weight'],
'time': element['time']
})
yield entity
def make_entity(self, content):
ancestor_key = Key([self._kind, self._ancestor], self._namespace,
self._project)
# Namespace and project are inherited from parent key.
key = Key([self._kind, str(uuid.uuid4())], parent=ancestor_key)
entity = Entity(key)
entity.set_properties({'content': content})
return entity
def testEntityToClientEntity(self):
k = Key(['kind', 1234], project=self._PROJECT)
kc = k.to_client_key()
exclude_from_indexes = ('efi1', 'efi2')
e = Entity(k, exclude_from_indexes=exclude_from_indexes)
e.set_properties({'efi1': 'value', 'property': 'value'})
ec = e.to_client_entity()
self.assertEqual(kc, ec.key)
self.assertSetEqual(set(exclude_from_indexes), ec.exclude_from_indexes)
self.assertEqual('kind', ec.kind)
self.assertEqual(1234, ec.id)
def testEmbeddedClientEntityWithoutKey(self):
client_entity = entity.Entity(key.Key('foo', project='bar'))
entity_without_key = entity.Entity()
entity_without_key['test'] = True
client_entity['embedded'] = entity_without_key
e = Entity.from_client_entity(client_entity)
self.assertIsInstance(e.properties['embedded'], dict)
def testEntityFromClientEntity(self):
k = Key(['kind', 1234], project=self._PROJECT)
exclude_from_indexes = ('efi1', 'efi2')
e = Entity(k, exclude_from_indexes=exclude_from_indexes)
e.set_properties({'efi1': 'value', 'property': 'value'})
efc = Entity.from_client_entity(e.to_client_entity())
self.assertEqual(e, efc)
def process(self, element):
if self.mapper_statements:
# __init__ で下記の処理を行って mapper_statements から関数を生成し、
# 関数の実体と local_dict をインスタンス変数に格納して、その後関数を process 関数内で実行すると、
# なぜかその関数の中から str 等の __builtins__ 関数にアクセスできないため、
# process の中で最初に初期化処理も実行している
if not self.func_name:
func, local_dict = self.make_callable(self.mapper_statements)
self.func_name = func.__name__
self.local_dict = local_dict
self.local_dict["__dsflow_element__"] = element.to_client_entity()
result = eval("{}(__dsflow_element__)".format(self.func_name),
self.local_dict)
return [Entity.from_client_entity(e) for e in result]
return [element]
def get_ndb_model_from_beam_entity(
beam_entity: beam_datastore_types.Entity) -> datastore_services.Model:
"""Returns an NDB model equivalent to the given Apache Beam entity.
Args:
beam_entity: beam_datastore_types.Entity. The Apache Beam entity.
Returns:
datastore_services.Model. The NDB model.
"""
ndb_key = get_ndb_key_from_beam_key(beam_entity.key)
# We use private _lookup_model and _entity_from_ds_entity here because it
# provides a functionality that we need and writing it ourselves would be
# too complicated.
ndb_model_class = get_model_class(ndb_key.kind()) # pylint: disable=protected-access
return ndb_model._entity_from_ds_entity( # pylint: disable=protected-access
beam_entity.to_client_entity(),
model_class=ndb_model_class)
def testEntityToClientEntity(self):
# Test conversion from Beam type to client type.
k = Key(['kind', 1234], project=self._PROJECT)
kc = k.to_client_key()
exclude_from_indexes = ('datetime', 'key')
e = Entity(k, exclude_from_indexes=exclude_from_indexes)
properties = {
'datetime': datetime.datetime.utcnow(),
'key_ref': Key(['kind2', 1235]),
'bool': True,
'float': 1.21,
'int': 1337,
'unicode': 'text',
'bytes': b'bytes',
'geopoint': GeoPoint(0.123, 0.456),
'none': None,
'list': [1, 2, 3],
'entity': Entity(Key(['kind', 111])),
'dict': {
'property': 5
},
}
e.set_properties(properties)
ec = e.to_client_entity()
self.assertEqual(kc, ec.key)
self.assertSetEqual(set(exclude_from_indexes), ec.exclude_from_indexes)
self.assertEqual('kind', ec.kind)
self.assertEqual(1234, ec.id)
for name, unconverted in properties.items():
converted = ec[name]
if name == 'key_ref':
self.assertNotIsInstance(converted, Key)
self._assert_keys_equal(unconverted, converted, self._PROJECT)
elif name == 'entity':
self.assertNotIsInstance(converted, Entity)
self.assertNotIsInstance(converted.key, Key)
self._assert_keys_equal(unconverted.key, converted.key,
self._PROJECT)
else:
self.assertEqual(unconverted, converted)
# Test reverse conversion.
entity_from_client_entity = Entity.from_client_entity(ec)
self.assertEqual(e, entity_from_client_entity)
def to_entity(content):
key = Key([kind, str(uuid.uuid4())])
entity = Entity(key)
entity.set_properties({'content': content})
return entity
