def save_to_db(self) -> None:
new_entry = Entity(self.datastore_key)
new_entry.update({
"prior_start_times":
self._get_prior_start_times() + [self.started_at]
})
datastore_client.put(new_entry)
def activity_entity_for_test(activity_id, with_photo=False):
activity = Entity(ds_util.client.key('Activity', activity_id))
activity['name'] = 'Activity ' + str(activity_id)
activity['id'] = activity_id
activity['description'] = 'Description: ' + str(activity_id)
activity['distance'] = 10
activity['moving_time'] = 200
activity['elapsed_time'] = 100
activity['total_elevation_gain'] = 300
activity['start_date'] = datetime.datetime.fromtimestamp(1503517240)
activity['athlete'] = Entity(ds_util.client.key('Athlete', 111))
activity['athlete']['id'] = 111
activity['athlete']['firstname'] = 'ActivityFirstName'
activity['athlete']['lastname'] = 'ActivityLastName'
activity['map'] = Entity(ds_util.client.key('Map', 111))
activity['map']['summary_polyline'] = SUMMARY_POLYLINE
if with_photo:
activity['photos'] = {
'primary': {
'id': 1234,
'unique_id': 'a807-23q4-23',
'urls': {
'600': 'https://12341234.cloudfront.net/yw7_-ahaahahahahaha_E-576x768.jpg',
'100': 'https://12341234.cloudfront.net/yw7_-ahashdhaahashahah_E-96x128.jpg',
},
}
}
return activity
def test_variable_meanings(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.helpers import _new_value_pb
entity = Entity()
name = "quux"
entity[name] = values = [1, 20, 300]
meaning = 9
entity._meanings[name] = ([None, meaning, None], values)
entity_pb = self._call_fut(entity)
# Construct the expected protobuf.
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value0 = value_pb.array_value.values.add()
value0.integer_value = values[0]
# The only array entry with a meaning is the middle one.
value1 = value_pb.array_value.values.add()
value1.integer_value = values[1]
value1.meaning = meaning
value2 = value_pb.array_value.values.add()
value2.integer_value = values[2]
self._compare_entity_proto(entity_pb, expected_pb)
def _setup_side_effects(
self,
create_client_mock,
ds_util_client_get_mock,
ds_util_client_put_mock,
fcm_util_best_clients_mock,
fcm_util_send_mock,
measures,
):
user = Entity(ds_util.client.key('User', 'someuser'))
user['preferences'] = {'daily_weight_notif': True, 'units': 'METRIC'}
service = Entity(
ds_util.client.key('Service', 'withings', parent=user.key))
service['credentials'] = {'refresh_token': 'validrefreshtoken'}
event_entity = Entity(
ds_util.client.key('SubscriptionEvent',
'Event',
parent=service.key))
series = Entity(
ds_util.client.key('Series', 'withings', parent=service.key))
series['measures'] = measures
# There are gets we need to account for.
def get_side_effect(key):
if key.name == 'someuser':
return user
elif key.name == 'withings':
return series
ds_util_client_get_mock.side_effect = get_side_effect
return WeightTrendWorker(service, event_entity)
def test_find_bot(self, mock_client_get):
workspace_key = ds_util.client.key('SlackWorkspace',
'enterprise_id-team_id')
store = DatastoreInstallationStore(ds_util.client)
bot_entity = Entity(
ds_util.client.key('SlackBot', 'bot-latest', parent=workspace_key))
bot_entity.update(
Bot(
app_id='app_id',
bot_id='bot_id',
bot_token='bot_token',
bot_user_id='bot_user_id',
installed_at=55.00,
).__dict__)
mock_client_get.return_value = bot_entity
found_bot = store.find_bot(enterprise_id='enterprise_id',
team_id='team_id')
self.assertIsNotNone(found_bot)
self.assertIsInstance(found_bot, Bot)
# Make sure we searched for the right key.
key = mock_client_get.call_args[0][0]
self.assertEqual(
key,
ds_util.client.key('SlackBot', 'bot-latest', parent=workspace_key))
def test_find_installation(self, mock_client_get):
workspace_key = ds_util.client.key('SlackWorkspace',
'enterprise_id-team_id')
store = DatastoreInstallationStore(ds_util.client)
installation_entity = Entity(
ds_util.client.key('SlackInstaller',
'installer-latest',
parent=workspace_key))
installation_entity.update(
Installation(
app_id='app_id',
enterprise_id='enterprise_id',
team_id='team_id',
user_id='user_id',
).__dict__)
mock_client_get.return_value = installation_entity
found_installation = store.find_installation(
enterprise_id='enterprise_id', team_id='team_id')
self.assertIsNotNone(found_installation)
self.assertIsInstance(found_installation, Installation)
# Make sure we searched for the right key.
key = mock_client_get.call_args[0][0]
self.assertEqual(
key,
ds_util.client.key('SlackInstaller',
'installer-latest',
parent=workspace_key),
)
def test_process_event_task_discard_duplicate(
self,
ds_util_client_get_mock,
ds_util_client_put_mock,
ds_util_client_query_mock,
ds_util_client_delete_multi_mock,
withings_create_client_mock,
withings_tasks_weight_trend_mock,
):
user = Entity(ds_util.client.key('User', 'someuser'))
user['preferences'] = {'daily_weight_notif': True}
service = Entity(ds_util.client.key('Service', 'withings', parent=user.key))
service['credentials'] = {'refresh_token': 'validrefreshtoken'}
event_entity = Entity(
ds_util.client.key('SubscriptionEvent', 'Event', parent=service.key)
)
# There are three gets we need to account for.
def get_side_effect(key):
if key.name == 'Event':
return event_entity
elif key.name == 'withings':
return service
elif key.name == 'someuser':
return user
ds_util_client_get_mock.side_effect = get_side_effect
worker = EventsWorker(service, event_entity)
worker.sync()
ds_util_client_put_mock.assert_not_called()
withings_tasks_weight_trend_mock.assert_not_called()
def test_mock_datastore_get(self, mock_datastore_service):
entity = Entity()
entity.update({
'title': 'Example Title',
'note_text': 'Example text',
'user': '******',
'last_modified_date': '',
'created_date': ''
})
entities = list()
entities.append(entity)
mock_datastore_service.return_value.get.return_value = entities
main.datastore = datastore.DatastoreService(
) # instantiate a new DatastoreService so it is replaced with mock
r = self.app.post('/note/get',
data=json.dumps({'user': '******'}),
headers={'Content-Type': 'application/json'})
assert r.status_code == 200
assert r.json == {
'matches': [{
'title': 'Example Title',
'note_text': 'Example text',
'last_modified_date': ''
}]
}
def test_variable_meanings(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.helpers import _new_value_pb
entity = Entity()
name = 'quux'
entity[name] = values = [1, 20, 300]
meaning = 9
entity._meanings[name] = ([None, meaning, None], values)
entity_pb = self._call_fut(entity)
# Construct the expected protobuf.
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value0 = value_pb.array_value.values.add()
value0.integer_value = values[0]
# The only array entry with a meaning is the middle one.
value1 = value_pb.array_value.values.add()
value1.integer_value = values[1]
value1.meaning = meaning
value2 = value_pb.array_value.values.add()
value2.integer_value = values[2]
self._compareEntityProto(entity_pb, expected_pb)
def test_installation_bot_to_entity_to_bot(self):
installation = Installation(app_id='app_id', user_id='user_id')
bot = installation.to_bot()
entity = Entity()
entity.update(bot.__dict__)
from_entity = Bot(**entity)
self.assertEqual(bot.__dict__, from_entity.__dict__)
def to_entity(cls, track, parent=None):
props = copy.deepcopy(track)
entity = Entity(
ds_util.client.key('Track', track['url'], parent=parent),
exclude_from_indexes=cls.EXCLUDE_FROM_INDEXES,
)
entity.update(props)
return entity
def to_entity(cls, item):
attributes = {
'date':
datetime.datetime.utcfromtimestamp(item['timestampGMT'] / 1000),
'fat_ratio': item['bodyFat'],
'weight': round(item['weight'] / 1000, 4),
}
entity = Entity(ds_util.client.key('Measure'))
entity.update(attributes)
return entity
def test_measures(self):
measure_item = BODY_COMP['dateWeightList'][0]
entity = GarminConverters.Measure.to_entity(measure_item)
expected_entity = Entity(ds_util.client.key('Measure'))
expected_entity.update({
'date': datetime.datetime(2020, 3, 22, 1, 49),
'fat_ratio': None,
'weight': 59.4206,
})
self.assertEqual(entity.items(), expected_entity.items())
def test_one_year_ago(
self,
create_client_mock,
ds_util_client_get_mock,
ds_util_client_put_mock,
get_now_mock,
fcm_util_best_clients_mock,
fcm_util_send_mock,
):
now = datetime.datetime(2020,
9,
25,
7,
13,
tzinfo=datetime.timezone.utc)
get_now_mock.return_value = now
measures = []
measures.insert(0, Entity())
measures[0].update({'date': now, 'weight': 30})
measures.insert(0, Entity())
measures[0].update({
'date':
datetime.datetime(2020, 9, 24, 7, 13,
tzinfo=datetime.timezone.utc),
'weight':
35,
})
measures.insert(0, Entity())
measures[0].update({
'date':
datetime.datetime(2019, 5, 18, 8, 12,
tzinfo=datetime.timezone.utc),
'weight':
60,
})
worker = self._setup_side_effects(
create_client_mock,
ds_util_client_get_mock,
ds_util_client_put_mock,
fcm_util_best_clients_mock,
fcm_util_send_mock,
measures,
)
worker.sync()
self._assert_send(
fcm_util_send_mock,
'Down 30.0 kg from a year ago',
'You were 60.0 kg on May 18, 2019',
)
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`.entity_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`google.cloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
if isinstance(pb, entity_pb2.Entity):
pb = pb._pb
key = None
if pb.HasField("key"): # Message field (Key)
key = key_from_protobuf(pb.key)
entity_props = {}
entity_meanings = {}
exclude_from_indexes = []
for prop_name, value_pb in pb.properties.items():
value = _get_value_from_value_pb(value_pb)
entity_props[prop_name] = value
# Check if the property has an associated meaning.
is_list = isinstance(value, list)
meaning = _get_meaning(value_pb, is_list=is_list)
if meaning is not None:
entity_meanings[prop_name] = (meaning, value)
# Check if ``value_pb`` was excluded from index. Lists need to be
# special-cased and we require all ``exclude_from_indexes`` values
# in a list agree.
if is_list and len(value) > 0:
exclude_values = set(value_pb.exclude_from_indexes
for value_pb in value_pb.array_value.values)
if len(exclude_values) != 1:
raise ValueError("For an array_value, subvalues must either "
"all be indexed or all excluded from "
"indexes.")
if exclude_values.pop():
exclude_from_indexes.append(prop_name)
else:
if value_pb.exclude_from_indexes:
exclude_from_indexes.append(prop_name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
entity._meanings.update(entity_meanings)
return entity
def test_bot_to_entity_to_bot(self):
bot = Bot(
app_id='app_id',
bot_id='bot_id',
bot_token='bot_token',
bot_user_id='bot_user_id',
installed_at=55.00,
)
entity = Entity()
entity.update(bot.__dict__)
from_entity = Bot(**entity)
self.assertEqual(bot.__dict__, from_entity.__dict__)
def to_entity(cls, measure, parent=None):
date = datetime.datetime.strptime(
measure['date'] + ' ' + measure['time'], '%Y-%m-%d %H:%M:%S')
entity = Entity(
ds_util.client.key('Measure', date.strftime('%s'), parent=parent))
entity.update(
dict(
id=measure['logId'],
date=date,
weight=measure['weight'],
fat_ratio=measure['fat'],
))
return entity
def entity_from_protobuf(pb):
"""Factory method for creating an entity based on a protobuf.
The protobuf should be one returned from the Cloud Datastore
Protobuf API.
:type pb: :class:`google.cloud.datastore._generated.entity_pb2.Entity`
:param pb: The Protobuf representing the entity.
:rtype: :class:`google.cloud.datastore.entity.Entity`
:returns: The entity derived from the protobuf.
"""
key = None
if pb.HasField('key'): # Message field (Key)
key = key_from_protobuf(pb.key)
entity_props = {}
entity_meanings = {}
exclude_from_indexes = []
for prop_name, value_pb in _property_tuples(pb):
value = _get_value_from_value_pb(value_pb)
entity_props[prop_name] = value
# Check if the property has an associated meaning.
is_list = isinstance(value, list)
meaning = _get_meaning(value_pb, is_list=is_list)
if meaning is not None:
entity_meanings[prop_name] = (meaning, value)
# Check if ``value_pb`` was excluded from index. Lists need to be
# special-cased and we require all ``exclude_from_indexes`` values
# in a list agree.
if is_list:
exclude_values = set(value_pb.exclude_from_indexes
for value_pb in value_pb.array_value.values)
if len(exclude_values) != 1:
raise ValueError('For an array_value, subvalues must either '
'all be indexed or all excluded from '
'indexes.')
if exclude_values.pop():
exclude_from_indexes.append(prop_name)
else:
if value_pb.exclude_from_indexes:
exclude_from_indexes.append(prop_name)
entity = Entity(key=key, exclude_from_indexes=exclude_from_indexes)
entity.update(entity_props)
entity._meanings.update(entity_meanings)
return entity
def test_process_link_shared_called(self, slack_process_link_shared_mock):
event_entity = Entity(
ds_util.client.key('SubscriptionEvent', 'slack-E232eq2ee')
)
event_entity.update(LINK_SHARED_EVENT)
self.client.post(
'/tasks/event',
data=task_util.task_body_for_test(event=event_entity),
)
# It doesn't matter what code gets returned, since the method returns
# whatever _process_link_shared returns, which is a mock. Only test
# that _process_link_shared is called.
slack_process_link_shared_mock.assert_called_once()
def test_process_link_shared(self, mock_create_unfurls, mock_slack_client):
mock_create_unfurls.return_value = {'http://example.com': 'unfurl'}
event_entity = Entity(
ds_util.client.key('SubscriptionEvent', 'slack-E232eq2ee')
)
event_entity.update(LINK_SHARED_EVENT)
r = self.client.post(
'/tasks/event',
data=task_util.task_body_for_test(event=event_entity),
)
mock_slack_client.assert_called_once()
responses.assertResponse(self, responses.OK, r)
def track_finished_for_test() -> Entity:
track = Entity(ds_util.client.key('Track', 10285651))
track.update(
{
"info": {
"gcAvatar": "https://s3.amazonaws.com/garmin-connect-prod/profile_images/avatar.png",
"session": {
"end": "2021-04-11T20:28:36.000Z",
"position": {
"lat": 37.77,
"locationName": "San Francisco",
"lon": -122.44,
},
"publisher": {
"connectUserProfileId": 123456,
"identifier": "PUBLISHERIDPUBLISHERID",
"nickname": "Joe LaPenna",
"trackerId": "UA69B7XL",
"type": "WEARABLE",
},
"publisherState": "ACTIVE",
"sessionId": "session-session",
"sessionName": "04/11/21",
"start": "2021-04-11T15:00:09.000Z",
"subscriber": {
"identifier": "*****@*****.**",
"type": "EMAIL",
},
"subscriberState": "ACTIVE",
"token": "1618153228",
"url": "https://livetrack.garmin.com/session/session-session/token/TOKENTOKEN",
"userDisplayName": "Joe LaPenna",
"viewable": "2021-04-11T20:28:36.000Z",
},
"shortened": False,
"unitId": 3996815102,
"viewable": False,
},
"start": datetime.datetime(
2021, 4, 11, 15, 00, tzinfo=datetime.timezone.utc
),
"end": datetime.datetime(2021, 4, 11, 20, 28, tzinfo=datetime.timezone.utc),
"status": 4, # FINISHED
"url": "https://livetrack.garmin.com/session/session-session/token/TOKENTOKEN",
"url_info": {"session": "session-session", "token": "TOKENTOKEN"},
}
)
return track
def test_entity(self):
from google.cloud.datastore.entity import Entity
entity = Entity()
name, value = self._call_fut(entity)
self.assertEqual(name, 'entity_value')
self.assertIs(value, entity)
def test_empty(self):
from google.cloud.proto.datastore.v1 import entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity_pb = self._call_fut(entity)
self._compareEntityProto(entity_pb, entity_pb2.Entity())
def test_put_multi_w_single_empty_entity(self):
# https://github.com/GoogleCloudPlatform/google-cloud-python/issues/649
from google.cloud.datastore.entity import Entity
creds = object()
client = self._makeOne(credentials=creds)
self.assertRaises(ValueError, client.put_multi, Entity())
def put_attendance(meeting, attendance):
exkeys = [
'name1', 'name2', 'group1', 'group2', 'email', 'attendance1',
'attendance2', 'anonymous', 'message', 'zip1', 'zip2', 'address',
'tel', 'fax', 'office', 'officetel', 'created', 'updated'
]
key = None
if not attendance.id:
key = datastore_client.key('meeting', meeting.id, 'attendance')
else:
key = datastore_client.key('meeting', meeting.id, 'attendance',
attendance.id)
entity = Entity(key=key, exclude_from_indexes=exkeys)
entity['name1'] = attendance.name1
entity['name2'] = attendance.name2
entity['group1'] = attendance.group1
entity['group2'] = attendance.group2
entity['email'] = attendance.email
entity['attendance1'] = attendance.attendance1
entity['attendance2'] = attendance.attendance2
entity['anonymous'] = attendance.anonymous
entity['message'] = attendance.message
entity['zip1'] = attendance.zip1
entity['zip2'] = attendance.zip2
entity['address'] = attendance.address
entity['tel'] = attendance.tel
entity['fax'] = attendance.fax
entity['office'] = attendance.office
entity['officetel'] = attendance.officetel
entity['created'] = attendance.created
entity['updated'] = attendance.updated
datastore_client.put(entity)
def issue(self, *args, **kwargs) -> str:
state = str(uuid4())
entity = Entity(self.client.key('SlackStateStore', state))
entity['body'] = str(time.time())
response = self.client.put(entity)
self.logger.debug(f"DS put response: {response}")
return state
def from_uid(cls, uid):
key = ds_util.client.key('User', uid)
user = ds_util.client.get(key)
if not user:
user = Entity(key)
User._set_defaults(user)
return user
def test_empty(self):
from google.cloud.datastore_v1.types import entity as entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity_pb = self._call_fut(entity)
self._compare_entity_proto(entity_pb, entity_pb2.Entity())
def to_entity(cls, measures, parent):
series = Entity(
ds_util.client.key('Series', parent.name, parent=parent))
series['measures'] = [
Measure.to_entity(measure, parent) for measure in measures
]
return series
def test_dict_to_entity_recursive(self):
from google.cloud.datastore_v1.types import entity as entity_pb2
from google.cloud.datastore.entity import Entity
entity = Entity()
entity["a"] = {"b": {"c": {"d": 1.25}, "e": True}, "f": 10}
entity_pb = self._call_fut(entity)
b_entity_pb = entity_pb2.Entity(
properties={
"c":
entity_pb2.Value(entity_value=entity_pb2.Entity(
properties={"d": entity_pb2.Value(double_value=1.25)})),
"e":
entity_pb2.Value(boolean_value=True),
})
expected_pb = entity_pb2.Entity(
properties={
"a":
entity_pb2.Value(entity_value=entity_pb2.Entity(
properties={
"b": entity_pb2.Value(entity_value=b_entity_pb),
"f": entity_pb2.Value(integer_value=10),
}))
})
self.assertEqual(entity_pb, expected_pb)
def perform_insert(entities):
results = []
rpc = transaction._rpc(self.connection.alias)
must_handle_unique = has_active_unique_constraints(self.model)
for primary, descendents in entities:
if primary.key.is_partial:
primary.key = primary.key.completed_key(rpc._generate_id())
# thanks to cloud firestore in datastore mode strong consistency
# we can query for the relevant entities to enforce uniqueness
if must_handle_unique:
perform_unique_checks(
self.model,
rpc,
primary,
test_fn=lambda stored: stored and len(stored) > 0)
rpc.put(primary)
new_key = primary.key
if descendents:
for i, descendent in enumerate(descendents):
key = rpc.key(descendent.kind,
descendent.key.id_or_name,
parent=new_key)
descendents[i] = Entity(key)
descendents[i].update(descendent)
for descendent in descendents:
rpc.put(descendent)
results.append(new_key)
return results
def perform_insert():
"""
Inserts result, and any descendents with their ancestor
value set
"""
client = transaction._rpc(self.connection.alias)
def test_fn(stored):
return stored and len(
stored) == 1 and stored[0].key != result.key
if has_active_unique_constraints(self.model):
perform_unique_checks(self.model, client, primary, test_fn)
inserted_key = client.put(result)
self.results.append((result, None))
if descendents:
for i, descendent in enumerate(descendents):
descendents[i] = Entity(
client.key(
descendent.kind,
descendent.key.name
if descendent.key.id is None else
descendent.key.id,
parent=inserted_key,
))
descendents[i].update(descendent)
client.put_multi(descendents)
def test_meaning_with_change(self):
from google.cloud.datastore_v1.proto import entity_pb2
from google.cloud.datastore.entity import Entity
from google.cloud.datastore.helpers import _new_value_pb
entity = Entity()
name = "foo"
entity[name] = value = 42
entity._meanings[name] = (9, 1337)
entity_pb = self._call_fut(entity)
expected_pb = entity_pb2.Entity()
value_pb = _new_value_pb(expected_pb, name)
value_pb.integer_value = value
# NOTE: No meaning is used since the value differs from the
# value stored.
self._compare_entity_proto(entity_pb, expected_pb)