def test_it(self):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.proto.datastore.v1 import entity_pb2
index_updates = 1337
keys = [
entity_pb2.Key(
path=[
entity_pb2.Key.PathElement(
kind='Foo',
id=1234,
),
],
),
entity_pb2.Key(
path=[
entity_pb2.Key.PathElement(
kind='Bar',
name='baz',
),
],
),
]
response = datastore_pb2.CommitResponse(
mutation_results=[
datastore_pb2.MutationResult(key=key) for key in keys
],
index_updates=index_updates,
)
result = self._call_fut(response)
self.assertEqual(result, (index_updates, keys))
def _make_key(id_):
from google.cloud.proto.datastore.v1 import entity_pb2
key = entity_pb2.Key()
elem = key.path.add()
elem.id = id_
return key
def test_is_key_valid(self):
key = entity_pb2.Key()
# Complete with name, no ancestor
datastore_helper.add_key_path(key, 'kind', 'name')
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with id, no ancestor
datastore_helper.add_key_path(key, 'kind', 12)
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Incomplete, no ancestor
datastore_helper.add_key_path(key, 'kind')
self.assertFalse(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with name and ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2', 'name2')
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Complete with id and ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2', 123)
self.assertTrue(helper.is_key_valid(key))
key = entity_pb2.Key()
# Incomplete with ancestor
datastore_helper.add_key_path(key, 'kind', 'name', 'kind2')
self.assertFalse(helper.is_key_valid(key))
key = entity_pb2.Key()
self.assertFalse(helper.is_key_valid(key))
def _make_key(kind, id_, project):
from google.cloud.proto.datastore.v1 import entity_pb2
key = entity_pb2.Key()
key.partition_id.project_id = project
elem = key.path.add()
elem.kind = kind
elem.id = id_
return key
def _make_mutation(id_):
from google.cloud.proto.datastore.v1 import datastore_pb2
from google.cloud.proto.datastore.v1 import entity_pb2
key = entity_pb2.Key()
key.partition_id.project_id = 'PROJECT'
elem = key.path.add()
elem.kind = 'Kind'
elem.id = id_
return datastore_pb2.MutationResult(key=key)
def to_protobuf(self):
from google.cloud.proto.datastore.v1 import entity_pb2
key = self._key = entity_pb2.Key()
# Don't assign it, because it will just get ripped out
# key.partition_id.project_id = self.project
element = key.path.add()
element.kind = self._kind
if self._id is not None:
element.id = self._id
return key
def make_ancestor_query(kind, namespace, ancestor):
"""Creates a Cloud Datastore ancestor query."""
ancestor_key = entity_pb2.Key()
datastore_helper.add_key_path(ancestor_key, kind, ancestor)
if namespace is not None:
ancestor_key.partition_id.namespace_id = namespace
query = query_pb2.Query()
query.kind.add().name = kind
datastore_helper.set_property_filter(
query.filter, '__key__', PropertyFilter.HAS_ANCESTOR, ancestor_key)
return query
def _makePB(self, project=None, namespace=None, path=()):
from google.cloud.proto.datastore.v1 import entity_pb2
pb = entity_pb2.Key()
if project is not None:
pb.partition_id.project_id = project
if namespace is not None:
pb.partition_id.namespace_id = namespace
for elem in path:
added = pb.path.add()
added.kind = elem['kind']
if 'id' in elem:
added.id = elem['id']
if 'name' in elem:
added.name = elem['name']
return pb
def make_ancestor_query(kind, namespace, ancestor):
"""Creates a Cloud Datastore ancestor query.
The returned query will fetch all the entities that have the parent key name
set to the given `ancestor`.
"""
ancestor_key = entity_pb2.Key()
datastore_helper.add_key_path(ancestor_key, kind, ancestor)
if namespace is not None:
ancestor_key.partition_id.namespace_id = namespace
query = query_pb2.Query()
query.kind.add().name = kind
datastore_helper.set_property_filter(
query.filter, '__key__', PropertyFilter.HAS_ANCESTOR, ancestor_key)
return query
def to_protobuf(self):
"""Return a protobuf corresponding to the key.
:rtype: :class:`.entity_pb2.Key`
:returns: The protobuf representing the key.
"""
key = _entity_pb2.Key()
key.partition_id.project_id = self.project
if self.namespace:
key.partition_id.namespace_id = self.namespace
for item in self.path:
element = key.path.add()
if 'kind' in item:
element.kind = item['kind']
if 'id' in item:
element.id = item['id']
if 'name' in item:
element.name = item['name']
return key
def run_data_migration():
request_data = json.loads(request.get_data())
# Required fields
fields = [
'name',
'function_kwargs',
'user'
]
# Some basic validation
for f in fields:
if f not in request_data:
resp_data = json.dumps(
{
'error': 'The ' + f + ' field is required.'
}
)
resp = Response(resp_data, status=400, mimetype='application/json')
return resp
if request_data['name'] not in migration.choices:
resp_data = json.dumps(
{
'error': 'The migration name is not valid.'
}
)
resp = Response(resp_data, status=400, mimetype='application/json')
return resp
migration_name = request_data['name']
function_kwargs = request_data['function_kwargs'] or {}
user = request_data['user']
function_kwargs.update({'name': migration_name})
# Create a MigrationHistory entity to keep track of the migration status
# set the project
project = PROJECT or 'meridianedit-staging'
# Create entity key
partition_id = entity_pb2.PartitionId(project_id=project, namespace_id="")
migration_history_obj_id = datetime.now().strftime("%Y%m%d%H%M%S")
path_element = entity_pb2.Key.PathElement(kind="MigrationHistory", name=migration_history_obj_id)
key = entity_pb2.Key(partition_id=partition_id, path=[path_element])
# Create entity and give it properties
entity = entity_pb2.Entity(key=key)
property_dict = {
'name': migration_name,
'function_kwargs': json.dumps(function_kwargs),
'started_by': user,
'status': 'running',
'created': datetime.now()
}
datastore_helper.add_properties(entity, property_dict)
# Add entity to datastore
mutations = [Mutation(insert=entity)]
client = apache_helper.get_datastore(project)
throttler = AdaptiveThrottler(window_ms=120000, bucket_ms=1000, overload_ratio=1.25)
apache_helper.write_mutations(client, project, mutations, throttler, rpc_stats_callback)
# Call the migration with any given function kwargs
migration_kwargs = {
'migration_history_obj': migration_history_obj_id,
}
migration_kwargs.update(function_kwargs)
# Run the migration in a celery task worker to prevent it timing
# out this connection. Also monitor the task so we can update
# migration status.
run_dataflow_migration.delay(pickle.dumps(entity), **migration_kwargs)
resp_data = {
'migration_history_obj_id': migration_history_obj_id
}
# A default 500 error message is returned if any of this breaks
return Response(json.dumps(resp_data), status=200, mimetype='application/json')