def delete(models, **kwargs):
"""Delete one or more Model instances.
Note: db.get returns None of the key doesn't exits
so we can replace in memcache every deleted keys by None.
TODO(sahid): Needs a better doc.
"""
memclient = memcache.Client()
for retry in xrange(DATASTORE_NB_RETRY):
try:
keys, multiple = datastore.NormalizeAndTypeCheckKeys(models)
if not any(keys): return multiple and [] or None # Nothings to do.
async = db.delete_async(models, **kwargs)
mapping = dict((unicode(k), None) for k in keys)
memclient.replace_multi(mapping,
time=MEMCACHE_TIME,
key_prefix=MEMCACHE_PREFIX)
return async.get_result()
except (db.Timeout,
db.TransactionFailedError,
apiproxy_errors.ApplicationError,
apiproxy_errors.DeadlineExceededError), e:
logging.warn("Error during the delete process, "
"retry %d in %.2fs", retry, DATASTORE_TIME_RETRY)
logging.debug(e.message)
time.sleep(DATASTORE_TIME_RETRY)
def delete(models, **kwargs):
"""Delete one or more Model instances.
Note: db.get returns None of the key doesn't exits
so we can replace in memcache every deleted keys by None.
TODO(sahid): Needs a better doc.
"""
memclient = memcache.Client()
for retry in xrange(DATASTORE_NB_RETRY):
try:
keys, multiple = datastore.NormalizeAndTypeCheckKeys(models)
if not any(keys): return multiple and [] or None # Nothings to do.
async = db.delete_async(models, **kwargs)
mapping = dict((unicode(k), None) for k in keys)
memclient.replace_multi(mapping,
time=MEMCACHE_TIME,
key_prefix=MEMCACHE_PREFIX)
return async.get_result()
except (db.Timeout,
db.TransactionFailedError,
apiproxy_errors.ApplicationError,
apiproxy_errors.DeadlineExceededError), e:
logging.warn("Error during the delete process, "
"retry %d in %.2fs", retry, DATASTORE_TIME_RETRY)
logging.debug(e.message)
time.sleep(DATASTORE_TIME_RETRY)
def delete(self):
email = self.check_authorization()
if email is None:
return
self.email = email
keys = self.query_objects(email, '__key__')
db.delete_async(keys)
self.dumps({'success': True})
def clean_up_data(self):
company_future = db.delete_async(Company.all())
employee_future = db.delete_async(Employee.all())
phone_number_future = db.delete_async(PhoneNumber.all())
company_future.get_result()
employee_future.get_result()
phone_number_future.get_result()
def delete(self):
email = self.check_authorization()
if email is None:
return
self.email = email
keys = self.query_objects(email, '__key__')
db.delete_async(keys)
self.dumps({'success': True})
def clean_up_data(self):
company_future = db.delete_async(Company.all())
employee_future = db.delete_async(Employee.all())
phone_number_future = db.delete_async(PhoneNumber.all())
company_future.get_result()
employee_future.get_result()
phone_number_future.get_result()
def delete_booking_by_id(booking_id):
b = Booking.get_by_id(booking_id)
if b:
db.delete(b.documents)
db.delete(b.sectors)
db.delete(b.passengers_rel)
db.delete_async(db.Key.from_path('LastUse', str(b.key())))
b.delete()
def trans():
mobile, ms, my_profile = db.get((mobile_key, ms_key, get_user_profile_key(current_user)))
ms.majorVersion = majorVersion
ms.minorVersion = minorVersion
ms.lastHeartBeat = now_time
if appType != MISSING:
mobile.type = my_profile.mobiles[mobile.account].type_ = appType
if simCountry != MISSING:
mobile.simCountry = simCountry
if simCountryCode != MISSING:
mobile.simCountryCode = simCountryCode
if simCarrierCode != MISSING:
mobile.simCarrierCode = simCarrierCode
if simCarrierName != MISSING:
mobile.simCarrierName = simCarrierName
if netCountry != MISSING:
mobile.netCountry = netCountry
if netCountryCode != MISSING:
mobile.netCountryCode = netCountryCode
if netCarrierCode != MISSING:
mobile.netCarrierCode = netCarrierCode
if netCarrierName != MISSING:
mobile.netCarrierName = netCarrierName
if deviceModelName != MISSING:
mobile.hardwareModel = deviceModelName
if osVersion != MISSING:
mobile.osVersion = osVersion
if localeCountry != MISSING:
mobile.localeCountry = localeCountry
if localeLanguage != MISSING:
mobile.localeLanguage = localeLanguage
if timezone != MISSING:
mobile.timezone = timezone
if timezoneDeltaGMT != MISSING:
mobile.timezoneDeltaGMT = timezoneDeltaGMT
language = mobile.localeLanguage
if language:
if '-' in language:
language = get_iso_lang(language.lower())
elif mobile.localeCountry:
language = '%s_%s' % (mobile.localeLanguage, mobile.localeCountry)
if my_profile.language != language:
my_profile.language = language
# trigger friend.update service api call
deferred.defer(update_friend_service_identity_connections, my_profile.key(), _transactional=True)
db.delete_async(get_broadcast_settings_flow_cache_keys_of_user(my_profile.user))
my_profile.country = mobile.netCountry or mobile.simCountry or mobile.localeCountry
my_profile.timezone = mobile.timezone
my_profile.timezoneDeltaGMT = mobile.timezoneDeltaGMT
put_and_invalidate_cache(ms, mobile, my_profile)
def update_article_id(cls,article_id,query=None,cursor=None):
from google.appengine.ext import deferred
if query is None:
query=cls.all()
if cursor is not None:
query.with_cursor(cursor)
if query.count(limit=1):
db.delete_async([q for q in query.run(limit=50)])
cursor=query.cursor()
deferred.defer(cls.update_article_id,article_id,query,cursor)
else:
deferred.defer(cls.set_article_id,article_id)
def trans(mobile):
stopped = False
debug_request_from_ds = db.get(StartDebuggingRequest.create_key(app_user, mobile_id))
if debug_request_from_ds:
if not debug_request or debug_request.timestamp == debug_request_from_ds.timestamp:
db.delete_async(debug_request_from_ds)
start_log_forwarding(app_user, None, mobile) # target_jid=None ==> will stop log forwarding
stopped = True
if not APPSCALE and debug_request_from_ds.target_id.startswith('dbg_'):
on_trans_committed(try_or_defer, delete_xmpp_account, debug_request_from_ds.target_id, None)
return stopped
def trans():
user_profile = get_user_profile(app_user)
if user_profile.language != language:
user_profile.language = language
db.delete_async(get_broadcast_settings_flow_cache_keys_of_user(app_user))
user_profile.name = name
if tmp_avatar_key:
_update_avatar(user_profile, tmp_avatar_key, x1, y1, x2, y2, add_trial_overlay=False)
user_profile.version += 1
user_profile.put()
update_mobiles(app_user, user_profile) # update myIdentity
update_friends(user_profile) # notify my friends
return user_profile
def get(self):
status = {'success' : True}
self.response.headers['Content-Type'] = "application/json"
try:
employee1 = Employee(name = "Raj")
employee1_future = db.put_async(employee1)
employee2 = Employee(name = "Tyler")
employee2_future = db.put_async(employee2)
employee1_future.get_result()
employee2_future.get_result()
time.sleep(1)
count1 = Employee.all().count(limit=5, deadline=60)
if count1 != 2:
raise Exception('Did not retrieve 2 Employees, got ' + str(count1))
employee3 = Employee(name = "Brian")
employee3_future = db.put_async(employee3)
employee3_future.get_result()
time.sleep(1)
count2 = Employee.all().count(limit=5, deadline=60)
if count2 != 3:
raise Exception('Did not retrieve 3 Employees, got ' + str(count2))
except Exception:
status = {'success' : False}
self.response.out.write(json.dumps(status))
raise
finally:
delete_future = db.delete_async(Employee.all())
delete_future.get_result()
def ack_all(call_ids):
if not call_ids:
return
mobile_key = users.get_current_mobile().key()
rpc_items.append(db.delete_async([db.Key.from_path(RpcAPIResult.kind(), call_id, parent=mobile_key) for call_id in call_ids]),
_ack_all_deferred, mobile_key, call_ids)
def get(self):
status = {'success' : True}
self.response.headers['Content-Type'] = "application/json"
try:
employee1 = Employee(name = "Raj")
employee1_future = db.put_async(employee1)
employee2 = Employee(name = "Tyler")
employee2_future = db.put_async(employee2)
employee1_future.get_result()
employee2_future.get_result()
time.sleep(1)
count1 = Employee.all().count(limit=5, deadline=60)
if count1 != 2:
raise Exception('Did not retrieve 2 Employees, got ' + str(count1))
employee3 = Employee(name = "Brian")
employee3_future = db.put_async(employee3)
employee3_future.get_result()
time.sleep(1)
count2 = Employee.all().count(limit=5, deadline=60)
if count2 != 3:
raise Exception('Did not retrieve 3 Employees, got ' + str(count2))
except Exception:
status = {'success' : False}
self.response.out.write(json.dumps(status))
raise
finally:
delete_future = db.delete_async(Employee.all())
delete_future.get_result()
def DelMulti(self, keys):
futures = []
for key in keys:
futures.append(db.delete_async(
PersistentObjectStoreItem.CreateKey(self._namespace, key)))
# If running the dev server, the futures don't complete until the server is
# *quitting*. This is annoying. Flush now.
if IsDevServer():
[future.wait() for future in futures]
def get(self):
r = PossessionLost()
to_delete = []
poss = []
user = users.get_current_user()
gamer_key = Key.from_path(kloombaDb.Gamer.kind(), user.user_id())
possessions = GqlQuery(
'SELECT * FROM Possession WHERE ANCESTOR IS :1 AND lost = TRUE',
gamer_key).run()
r.timestamp = int(time.time())
for i in possessions:
if int(time.time()) - int(time.mktime(i.timestamp.timetuple())
) > RULES['LOST_FLOWERBED_TIMEOUT']:
to_delete.append(i.flowerbed.key())
else:
poss.append(i)
deleted = db.delete_async(to_delete)
if poss:
keys = [
kloombaDb.Possession.flowerbed.get_value_for_datastore(i)
for i in poss
]
flowerbeds = []
for i in keys:
fb = memcache.get(str(i))
if not fb:
fb = GqlQuery('SELECT * FROM Flowerbed WHERE __key__=:1',
i).get()
if fb and not memcache.add(str(i), fb):
logging.error('Memcache set failed')
if fb:
flowerbeds.append(fb)
for (p, f) in zip(poss, flowerbeds):
if p and f:
fb = r.possession.add()
fb.timestamp = int(time.mktime(p.timestamp.timetuple()))
fb.flowerbed.timestamp = int(
time.mktime(f.timestamp.timetuple()))
fb.flowerbed.id = str(f.key())
fb.flowerbed.latitude = int(f.point.lat * 1000000)
fb.flowerbed.longitude = int(f.point.lon * 1000000)
fb.flowerbed.owner = f.owner_public_id
fb.flowerbed.flowers = f.flowers
deleted.get_result()
if self.request.get('debug', False):
self.response.out.write(r)
else:
self.response.out.write(r.SerializeToString())
def trans():
user_profile = get_user_profile(app_user)
changed_properties = []
if user_profile.language != language:
user_profile.language = language
changed_properties.append(u"language")
db.delete_async(get_broadcast_settings_flow_cache_keys_of_user(app_user))
if user_profile.name != name:
changed_properties.append(u"name")
user_profile.name = name
if image:
_update_avatar(user_profile, image, False)
changed_properties.append(u"avatar")
user_profile.version += 1
user_profile.put()
update_mobiles(app_user, user_profile) # update myIdentity
update_friends(user_profile, changed_properties) # notify my friends
return user_profile
def admin_setFileVersionTag(req):
fn = req.get('filename')
res = datamodel.DB_FileVersion.getSpecificVersion(fn, 'z', int(req.get('ver')))
if not res: return RetType.JSONFAIL, {'text':'File version not found'}
addtag = req.get('addtag')
deltag = req.get('deltag')
delversion = req.get('delversion')
if addtag:
if addtag in res.tags:
return RetType.JSONFAIL, {'text':'Tag already in file version'}
res.tags.append(addtag)
res.put()
return RetType.JSONSUCCESS
if deltag:
if deltag not in res.tags:
return RetType.JSONFAIL, {'text':'Tag not in file version'}
res.tags.remove(deltag)
res.put()
return RetType.JSONSUCCESS
if delversion and delversion == str(res.version):
# delete both datamodel.DB_FileVersion and datamodel.DB_FileContent if no one else is using it (and no tags)
if len(res.tags) > 1:
return RetType.JSONFAIL, {'text':'File is tagged, can\'t delete'}
theContentKey = res.key().parent()
theHash = theContentKey.name()
db.delete_async(res.key())
usedByOtherResult = (RetType.JSONSUCCESS, {'text':'Version deleted, but content was not deleted since it is used by another version'})
others = datamodel.DB_FileVersion.all(keys_only=True).ancestor(theContentKey).fetch(2)
if len(others) != 1: return usedByOtherResult
if others[0] != res.key(): return usedByOtherResult
db.delete_async(theContentKey)
return RetType.JSONSUCCESS
return RetType.JSONFAIL, {'text': 'Nothing to do?'}
def testAsyncPutGetDelete(self):
"""Tests asynchronously putting, getting and deleting entities."""
class Person(db.Model):
name = db.StringProperty()
person = Person(name="Arthur")
async = db.put_async(person)
key = async .get_result()
self.assertEqual(key, async .get_result())
async = db.get_async(key)
person = async .get_result()
self.assertEqual("Arthur", person.name)
async = db.delete_async(key)
async .get_result()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get,
key)
def testAsyncPutGetDelete(self):
"""Tests asynchronously putting, getting and deleting entities."""
class Person(db.Model):
name = db.StringProperty()
person = Person(name="Arthur")
async = db.put_async(person)
key = async.get_result()
self.assertEqual(key, async.get_result())
async = db.get_async(key)
person = async.get_result()
self.assertEqual("Arthur", person.name)
async = db.delete_async(key)
async.get_result()
self.assertRaises(datastore_errors.EntityNotFoundError, datastore.Get, key)
def get(self):
r = PossessionLost()
to_delete = []
poss = []
user = users.get_current_user()
gamer_key = Key.from_path(kloombaDb.Gamer.kind(), user.user_id())
possessions = GqlQuery('SELECT * FROM Possession WHERE ANCESTOR IS :1 AND lost = TRUE', gamer_key).run()
r.timestamp = int(time.time())
for i in possessions:
if int(time.time()) - int(time.mktime(i.timestamp.timetuple())) > RULES['LOST_FLOWERBED_TIMEOUT']:
to_delete.append(i.flowerbed.key())
else:
poss.append(i)
deleted = db.delete_async(to_delete)
if poss:
keys = [kloombaDb.Possession.flowerbed.get_value_for_datastore(i) for i in poss]
flowerbeds = []
for i in keys:
fb = memcache.get(str(i))
if not fb:
fb = GqlQuery('SELECT * FROM Flowerbed WHERE __key__=:1', i).get()
if fb and not memcache.add(str(i), fb):
logging.error('Memcache set failed')
if fb:
flowerbeds.append(fb)
for (p, f) in zip(poss, flowerbeds):
if p and f:
fb = r.possession.add()
fb.timestamp = int(time.mktime(p.timestamp.timetuple()))
fb.flowerbed.timestamp = int(time.mktime(f.timestamp.timetuple()))
fb.flowerbed.id = str(f.key())
fb.flowerbed.latitude = int(f.point.lat * 1000000)
fb.flowerbed.longitude = int(f.point.lon * 1000000)
fb.flowerbed.owner = f.owner_public_id
fb.flowerbed.flowers = f.flowers
deleted.get_result()
if self.request.get('debug', False):
self.response.out.write(r)
else:
self.response.out.write(r.SerializeToString())
def delete_async(cls, key): # pragma: no cover
return db.delete_async(key)
def delete(self): delete_future = db.delete_async(Module.all()) delete_future.get_result()
blob_keys.append(file_obj.blob.key())
if blob_keys and _delete_old_blobs:
blobstore.delete(blob_keys)
# Flag these files in cache as non-existent, cleanup subdir and blob caches.
file_ents = _GetFileEntities(file_objs)
files_cache.SetFileDoesNotExist(paths)
files_cache.ClearSubdirsForFiles(file_ents)
if _delete_old_blobs:
files_cache.ClearBlobsForFiles(file_ents)
if update_subdir_caches:
paths = paths if is_multiple else [paths]
deferred.defer(ListDir, _GetCommonDir(paths))
rpc = db.delete_async(file_ents)
return rpc if async else rpc.get_result()
@hooks.ProvideHook('file-touch')
def Touch(paths, meta=None, async=False):
"""Create or update File objects by updating modified times.
Supports batch and asynchronous touches.
Args:
paths: Absolute filename, iterable of absolute filenames, or File objects.
meta: A dictionary of properties to be added to the file.
async: Whether or not to do asynchronous touches.
Returns:
File object: if only one path is given and async is False.
List of File objects: if multiple paths given and async is False.
def delete_async(self):
return db.delete_async(self)
def delete(self):
children = db.query_descendants(self).fetch(100)
for c in children:
c.delete()
return db.delete_async(self)
def delete_async(self):
children = db.query_descendants(self).fetch(100)
for c in children:
c.delete()
return db.delete_async(self)
updated = db.DateTimeProperty(auto_now=True)
@classmethod
def get(cls, key, async=True):
if async:
return AsyncValue(db.get_async(key))
else:
model = cls.get_by_key_name(key)
if model is None:
raise KeyError
return model.value
@classmethod
def delete(cls, key, async=True):
if async:
db.delete_async(key)
else:
db.delete(key)
@classmethod
def set(cls, key, value, collection, async=True):
model = cls(key_name=key, value=value, collection=collection)
if async:
db.put_async(model)
else:
model.put()
class DataStore:
def __init__(self, collection, persistent=True):
self._collection = collection
def trans():
# type: () -> tuple[Mobile, UserProfile, bool]
keys = (mobile_key, ms_key, get_user_profile_key(current_user))
mobile, ms, my_profile = db.get(keys) # type: (Mobile, MobileSettings, UserProfile)
if mobile.account not in my_profile.mobiles:
logging.warn('Mobile account "%s" of user %s has been unregistered', mobile.account, current_user)
return mobile, my_profile, False
if appType != MISSING:
mobile.type = my_profile.mobiles[mobile.account].type_ = appType
if simCountry != MISSING:
mobile.simCountry = simCountry
if simCountryCode != MISSING:
mobile.simCountryCode = simCountryCode
if simCarrierCode != MISSING:
mobile.simCarrierCode = simCarrierCode
if simCarrierName != MISSING:
mobile.simCarrierName = simCarrierName
if netCountry != MISSING:
mobile.netCountry = netCountry
if netCountryCode != MISSING:
mobile.netCountryCode = netCountryCode
if netCarrierCode != MISSING:
mobile.netCarrierCode = netCarrierCode
if netCarrierName != MISSING:
mobile.netCarrierName = netCarrierName
if deviceModelName != MISSING:
mobile.hardwareModel = deviceModelName
if osVersion != MISSING:
mobile.osVersion = osVersion
if localeCountry != MISSING:
mobile.localeCountry = localeCountry
if localeLanguage != MISSING:
mobile.localeLanguage = localeLanguage
if timezone != MISSING:
mobile.timezone = timezone
if timezoneDeltaGMT != MISSING:
mobile.timezoneDeltaGMT = timezoneDeltaGMT
if deviceId != MISSING:
mobile.deviceId = deviceId
language = mobile.localeLanguage
if language:
should_update_embedded_apps = False
if '-' in language:
language = get_iso_lang(language.lower())
elif mobile.localeCountry:
language = '%s_%s' % (mobile.localeLanguage, mobile.localeCountry)
if my_profile.language != language:
my_profile.language = language
# trigger friend.update service api call
deferred.defer(update_friend_service_identity_connections, my_profile.key(), [u"language"],
_transactional=True)
db.delete_async(get_broadcast_settings_flow_cache_keys_of_user(my_profile.user))
if embeddedApps:
should_update_embedded_apps = True
deferred.defer(update_look_and_feel_for_user, current_user, _transactional=True, _queue=FAST_QUEUE)
# User updated to app version x.1.x, send custom translations
# todo: this will also trigger when user updates from 3.0.x to 3.1.x which we don't really want
should_update_embedded_apps = should_update_embedded_apps or majorVersion > 1 and ms.majorVersion == 0
# Update when embedded apps are different
should_update_embedded_apps = should_update_embedded_apps or not set(embeddedApps).issubset(
set(my_profile.embedded_apps or []))
if should_update_embedded_apps:
deferred.defer(update_embedded_app_translations_for_user, current_user, embeddedApps, language,
_transactional=True)
ms.majorVersion = majorVersion
ms.minorVersion = minorVersion
ms.lastHeartBeat = now_time
my_profile.country = mobile.netCountry or mobile.simCountry or mobile.localeCountry
my_profile.timezone = mobile.timezone
my_profile.timezoneDeltaGMT = mobile.timezoneDeltaGMT
my_profile.embedded_apps = embeddedApps
must_update_app_settings = False
if my_profile.tos_version != get_current_document_version(DOC_TERMS):
if mobile.is_android:
version = Features.ASK_TOS.android
elif mobile.is_ios:
version = Features.ASK_TOS.ios
else:
version = Version(0, 1)
must_update_app_settings = Version(majorVersion, minorVersion) >= version
put_and_invalidate_cache(ms, mobile, my_profile)
return mobile, my_profile, must_update_app_settings
def process(body, instant=False):
try:
request = json.loads(body)
except:
logging.warn(body)
raise
mobile = users.get_current_mobile()
jid = mobile.account
memcache_rpc = memcache.Client().set_multi_async(
{"last_user_heart_beat_%s" % users.get_current_user().email(): now()})
logging.info("Incoming HTTP call from %s", jid)
azzert(request[API_VERSION] == 1)
responses = list()
acks = list()
calls = list()
starttime = now()
timeToStartFlushing = lambda: now() - starttime > 15
sizes = list()
def processResult(r):
users.set_backlog_item(r)
try:
ack = cresult(r)
acks.append(ack)
sizes.append(len(ack))
finally:
users.set_backlog_item(None)
def processCall(c):
users.set_backlog_item(c)
try:
if mobile.is_phone_unregistered:
# Do not process calls coming from an unregistered app.
api_version, callid, _, _ = parse_and_validate_request(c)
timestamp = c[CALL_TIMESTAMP] if CALL_TIMESTAMP in c else now()
result = {
CALL_ID: callid,
API_VERSION: api_version,
ERROR: "Unknown function call!",
STATUS: STATUS_FAIL,
CALL_TIMESTAMP: timestamp}
else:
result = call(c)
if not result:
return
_, result = result
responses.append(result)
sizes.append(len(json.dumps(result)))
finally:
users.set_backlog_item(None)
def stream():
if not instant:
ack_all(request.get(u"a", []))
for r in request.get(u"r", []):
yield processResult, r
if timeToStartFlushing() or sum(sizes) > MAX_RPC_SIZE:
return
for c in request.get(u"c", []):
yield processCall, c
if timeToStartFlushing() or sum(sizes) > MAX_RPC_SIZE:
return
foreach(lambda (f, a): f(a), stream())
if not DEBUG and not instant:
wait_for_rpcs()
if mobile.type in Mobile.ANDROID_TYPES and mobile.pushId and not instant:
rpc_items.append(db.delete_async(OutStandingFirebaseKick.createKey(mobile.pushId)), None)
more = False
if sum(sizes) > MAX_RPC_SIZE:
more = True
elif not instant:
count = 0
deferred_kicks = list()
while True:
if mobile.is_phone_unregistered:
memcache_key = "unregistered" + str(mobile.key())
if memcache.get(memcache_key): # @UndefinedVariable
logging.debug("send empty result to give the phone the chance to finish unregistering")
break
logging.debug("%s (user: %s. status: %s) should unregister itself. "
"Close the communication channel via only allowing the Unregister Call",
mobile.account, mobile.user, mobile.status)
qry = get_filtered_backlog(mobile, "com.mobicage.capi.system.unregisterMobile")
else:
# Stream the backlog as normal
qry = get_limited_backlog(mobile, 21)
for b in qry:
count += 1
calls.append(json.loads(b.call))
sizes.append(len(b.call))
if b.deferredKick:
deferred_kicks.append(b.key().name())
if sum(sizes) > MAX_RPC_SIZE:
more = True
break
if mobile.is_phone_unregistered:
if not calls:
logging.debug("No unregister calls found in the backlog, re-add it.")
from rogerthat.bizz.system import unregister_mobile
unregister_mobile(users.get_current_user(), mobile)
time.sleep(2) # Make sure when the query runs again, we will get results
else:
memcache.set(memcache_key, "call sent", 60) # @UndefinedVariable
break
else:
break
if count == 21:
calls.pop()
more = True
memcache_stuff = dict()
for c in calls:
call_id = c["ci"]
if call_id in deferred_kicks:
memcache_stuff["capi_sent_to_phone:%s" % call_id] = True
if memcache_stuff:
memcache.set_multi(memcache_stuff, 10) # @UndefinedVariable
result = dict()
result[API_VERSION] = 1
no_api_direct_path = DEBUG or APPSCALE
endpoint = u'/json-rpc/instant' if instant else u'/json-rpc'
if no_api_direct_path:
result[API_DIRECT_PATH_KEY] = "%s%s" % (get_server_settings().baseUrl, endpoint)
else:
result[API_DIRECT_PATH_KEY] = u"https://%s.appspot.com%s" % (APPENGINE_APP_ID, endpoint)
if responses:
result[u"r"] = responses
if acks:
result[u"a"] = acks
if calls:
result[u"c"] = calls
result[u"more"] = more
result[u"t"] = now()
memcache_rpc.get_result()
offload(users.get_current_user(), OFFLOAD_TYPE_APP, request, result)
return json.dumps(result)
def delete(self): delete_future = db.delete_async(Counter.all()) delete_future.get_result()
def delete(self): delete_future = db.delete_async(Module.all()) delete_future.get_result()
def delete_async(cls, key):
return db.delete_async(key)
def remove_old_subscribers():
old_subscribers = Subscriber.all(keys_only=True).filter('time <', datetime.utcnow() - timedelta(days=1)).fetch(1000)
if old_subscribers:
db.delete_async(old_subscribers)
memcache_client.delete_multi_async(['get_subscribers'])
def delete(self): delete_future = db.delete_async(Project.all()) delete_future.get_result()
except ServiceApiException, sae:
if (sae.code >= ERROR_CODE_WARNING_THRESHOLD):
logging.warning(u"Service API exception occurred", exc_info=1)
else:
logging.exception(u"Severe Service API Exception")
error_code = sae.code
error_message = sae.message
finally:
request = json.loads(service_api_callback.call)
if log and sik:
log_service_activity(sik.user, request["id"], ServiceLog.TYPE_CALLBACK, status, request["method"],
service_api_callback.call, raw_result_unicode, error_code, error_message)
if service_api_callback.is_saved():
service_api_callback_key = service_api_callback.key()
rpc_items.append(db.delete_async(service_api_callback_key),
_delete_service_api_callback_deferred, service_api_callback_key)
if synchronous and error:
raise ServiceApiException(error_code, error_message)
def _delete_service_api_callback_deferred(service_api_callback_key):
db.delete(service_api_callback_key)
def _validate_api_callback(result_f, error_f, target, alias, f):
def raise_invalid_target():
raise ValueError(
"Target argument should be of type rogerthat.models.ServiceProfile or [rogerthat.models.ServiceProfile]")
def delete(self): delete_future = db.delete_async(Project.all()) delete_future.get_result()
def trans():
# type: () -> FriendMap
friend_map = FriendMap.get(friend_map_key) # type: FriendMap
if not friend_map:
logging.warn("FriendMap not found for key: %s" % friend_map_key)
return
email = remove_slash_default(profile_info.user).email()
if email not in friend_map.friendDetails:
logging.warn(list(friend_map.friendDetails))
logging.warn("Probably friend %s was removed while updating %s" % (email, friend_map.user.email()))
return
friend_detail = friend_map.friendDetails[email]
friend_detail.avatarId = avatar_id
if profile_info.isServiceIdentity:
target_language = get_user_profile(friend_map.user).language
friend_detail.name = translator.translate(ServiceTranslation.IDENTITY_TEXT, profile_info.name,
target_language)
else:
friend_detail.name = profile_info.name
friend_detail.type = helper.friend_type
friend_detail.relationVersion += 1
friend_map.generation += 1
to_put = [friend_map]
if profile_info.isServiceIdentity:
fsic_key = FriendServiceIdentityConnection.createKey(friend_map.user, profile_info.user)
fsic = db.get(fsic_key)
if not fsic:
logging.warn("FriendServiceIdentityConnection not found for key: %s" % fsic_key)
return
for attr in ('disabled_broadcast_types', 'enabled_broadcast_types'):
if getattr(fsic, attr) is None:
setattr(fsic, attr, [])
_, app_id = get_app_user_tuple(friend_map.user)
updated = False
enabled_broadcast_types = list(fsic.enabled_broadcast_types)
if app_id == App.APP_ID_OSA_LOYALTY:
if enabled_broadcast_types:
enabled_broadcast_types = []
updated = True
else:
# Add new broadcast types
for broadcast_type in service_profile.broadcastTypes:
if broadcast_type not in (fsic.disabled_broadcast_types + fsic.enabled_broadcast_types):
enabled_broadcast_types = list(set(enabled_broadcast_types) | {broadcast_type})
updated = True
# Remove deleted broadcast types
for broadcast_type in fsic.enabled_broadcast_types:
if broadcast_type not in service_profile.broadcastTypes:
enabled_broadcast_types.remove(broadcast_type)
updated = True
if updated:
friend_detail.relationVersion += 1
fsic.enabled_broadcast_types = enabled_broadcast_types
to_put.append(fsic)
if updated or clear_broadcast_settings_cache:
logging.info("Deleting BroadcastSettingsFlowCache from datastore")
db.delete_async(BroadcastSettingsFlowCache.create_key(friend_map.user, profile_info.user))
logging.info("updating friend to friend_map.generation: %s, friend_detail.relationVersion: %s",
friend_map.generation, friend_detail.relationVersion)
logging.debug("debugging_branding _update_friend friend_map.gen %s, friend_detail.relv %s", friend_map.generation, friend_detail.relationVersion)
to_put.extend(create_update_friend_requests(helper, helper.profile_info_user, friend_map,
UpdateFriendRequestTO.STATUS_MODIFIED))
put_and_invalidate_cache(*to_put)
def _ack_all_deferred(mobile_key, call_ids):
db.delete_async([db.Key.from_path(RpcAPIResult.kind(), call_id, parent=mobile_key) for call_id in call_ids])
def put_news_item(service_identity_user,
title,
message,
broadcast_type,
sponsored,
image,
action_button,
order_items,
news_type,
qr_code_caption,
app_ids,
scheduled_at,
news_id=None,
broadcast_on_facebook=False,
broadcast_on_twitter=False,
facebook_access_token=None,
target_audience=None,
role_ids=None,
host=None):
"""
Creates a news item first then processes the payment if necessary (not necessary for non-promoted posts).
If the payment was unsuccessful it will be retried in a deferred task.
Args:
service_identity_user (users.User)
title (unicode)
message (unicode)
broadcast_type (unicode)
sponsored (bool)
image (unicode)
action_button (NewsActionButtonTO)
order_items (list of OrderItemTO)
news_type (int)
qr_code_caption (unicode)
app_ids (list of unicode)
scheduled_at (long)
news_id (long): id of the news item to update. When not provided a new news item will be created.
broadcast_on_facebook (bool)
broadcast_on_twitter (bool)
facebook_access_token (unicode): user or page access token
target_audience (NewsTargetAudienceTO)
role_ids (list of long) the list of role ids to filter sending the news to their members
host (unicode): host of the api request (used for social media apps)
Returns:
news_item (NewsBroadcastItemTO)
"""
if not order_items or order_items is MISSING:
order_items = []
if news_type == NewsItem.TYPE_QR_CODE:
sln_settings = get_solution_settings(
get_service_user_from_service_identity_user(service_identity_user))
azzert(SolutionModule.LOYALTY in sln_settings.modules)
sponsored_until = None
should_save_coupon = news_type == NewsItem.TYPE_QR_CODE and not news_id
sponsored_app_ids = set()
extra_app_ids = []
si = get_service_identity(service_identity_user)
for order_item in reversed(order_items):
if order_item.product == Product.PRODUCT_NEWS_PROMOTION and sponsored:
azzert(order_item.app_id)
azzert(order_item.app_id not in sponsored_app_ids)
sponsored_app_ids.add(order_item.app_id)
order_item.count = get_sponsored_news_count_in_app(
service_identity_user, order_item.app_id).count
elif order_item.product == Product.PRODUCT_EXTRA_CITY:
azzert(order_item.app_id)
azzert(order_item.app_id not in extra_app_ids)
extra_app_ids.append(order_item.app_id)
if order_item.app_id in si.appIds:
order_items.remove(order_item)
else:
raise BusinessException('Invalid product %s' % order_item.product)
if not news_id and not app_ids:
raise BusinessException(
'Please select at least one app to publish this news in')
if sponsored:
sponsored_until_date = datetime.datetime.utcnow() + datetime.timedelta(
days=SPONSOR_DAYS)
sponsored_until = long(sponsored_until_date.strftime('%s'))
# for sponsored news that is free in certain apps no order item is given, so add it here
sponsored_counts = get_sponsored_news_count(service_identity_user,
app_ids)
for sponsored_count in sponsored_counts:
if sponsored_count.remaining_free != 0 and sponsored_count.app_id in app_ids:
sponsored_app_ids.add(sponsored_count.app_id)
app_ids = list(sponsored_app_ids)
service_user, identity = get_service_identity_tuple(service_identity_user)
default_app = get_app(si.defaultAppId)
if App.APP_ID_ROGERTHAT in si.appIds and App.APP_ID_ROGERTHAT not in app_ids:
app_ids.append(App.APP_ID_ROGERTHAT)
if default_app.demo and App.APP_ID_ROGERTHAT in app_ids:
app_ids.remove(App.APP_ID_ROGERTHAT)
kwargs = {
'sticky': sponsored,
'sticky_until': sponsored_until,
'message': message,
'broadcast_type': broadcast_type,
'service_identity': identity,
'news_id': news_id,
'app_ids': app_ids,
'image': image,
'scheduled_at': scheduled_at,
'target_audience': target_audience,
'role_ids': role_ids
}
if not news_id:
kwargs['news_type'] = news_type
if news_type == NewsItem.TYPE_QR_CODE:
if should_save_coupon:
def trans():
coupon = NewsCoupon(
parent=NewsCoupon.create_parent_key(service_identity_user),
content=qr_code_caption)
coupon.put()
return coupon
coupon = db.run_in_transaction(trans)
kwargs['qr_code_content'] = u'%s' % json.dumps({'c': coupon.id})
kwargs['qr_code_caption'] = qr_code_caption
elif news_type == NewsItem.TYPE_NORMAL:
kwargs.update({
'action_buttons': [action_button] if action_button else [],
'title':
title
})
else:
raise BusinessException('Invalid news type')
for key, value in kwargs.items():
if value is MISSING:
del kwargs[key]
with users.set_user(service_user):
try:
def trans():
news_item = news.publish(accept_missing=True, **kwargs)
if should_save_coupon:
_save_coupon_news_id(news_item.id, coupon)
elif news_type == NewsItem.TYPE_QR_CODE and qr_code_caption is not MISSING and qr_code_caption and news_id:
news_coupon = NewsCoupon.get_by_news_id(
service_identity_user, news_id)
if news_coupon:
news_coupon.content = qr_code_caption
news_coupon.put()
else:
logging.warn(
'Not updating qr_code_caption for non-existing coupon for news with id %d',
news_id)
if order_items:
create_and_pay_news_order(service_user, news_item.id,
order_items)
return news_item
news_item = run_in_xg_transaction(trans)
if broadcast_on_facebook or broadcast_on_twitter:
if scheduled_at is not MISSING and scheduled_at > 0:
schedule_post_to_social_media(service_user, host,
broadcast_on_facebook,
broadcast_on_twitter,
facebook_access_token,
news_item.id, scheduled_at)
else:
post_to_social_media(service_user, broadcast_on_facebook,
broadcast_on_twitter,
facebook_access_token, news_item.id)
return NewsBroadcastItemTO.from_news_item_to(
news_item, broadcast_on_facebook, broadcast_on_twitter)
except:
if should_save_coupon:
db.delete_async(coupon)
raise
def delete(self): delete_future = db.delete_async(Counter.all()) delete_future.get_result()
def delete_async(cls, key):
return db.delete_async(key)
def post(self):
if not self.set_user():
self.abort(401)
return
connect_rpc = None
try:
authorization_code = self.request.POST['authorization_code']
state = self.request.POST['state']
scopes = self.request.POST['scope'].split(' ')
logging.debug('POST params: %s', dict(self.request.POST))
app_user = users.get_current_user()
app_id = users.get_current_app_id()
app = get_app_by_id(app_id)
azzert(app and app.supports_mdp)
mdp_state = MyDigiPassState.get(MyDigiPassState.create_key(app_user))
azzert(mdp_state.state == state)
db.delete_async(mdp_state)
mdp_result, connect_rpc = self.authorize_mdp(app_user, app, authorization_code, scopes)
# mdp_result: dict with following keys: uuid, access_token, scope
mdp_profile_pointer = MyDigiPassProfilePointer.create(app_user, mdp_result['uuid'])
mdp_profile_pointer.put()
endpoints = { self.SCOPE_TO_ENDPOINT_MAPPING[s] for s in scopes }
rpcs = {endpoint: self.get_mdp_data_async(mdp_result['access_token'], endpoint)
for endpoint in endpoints}
result = MyDigiPassWidgetResultTO(initialize=True)
for endpoint, rpc in rpcs.iteritems():
response = rpc.get_result()
if response.status_code != 200:
if endpoint in (MdpEndpoint.EID_DATA, MdpEndpoint.EID_PHOTO):
try:
error_dict = json.loads(response.content)
except:
pass
else:
if error_dict.get('error') == 'insufficient_permissions':
raise BusinessException(localize(self.language(), u'mdp_missing_eid_data'))
raise Exception('Failed to get %s data for MDP user %s: %s' % (endpoint, app_user.email(), response.content))
if endpoint == MdpEndpoint.EID_PHOTO:
logging.debug('Got MDP %s for %s', endpoint, app_user.email())
photo = response.content
if not photo:
raise BusinessException(localize(self.language(), u'mdp_missing_eid_data'))
result.eid_photo = base64.b64encode(photo).decode('utf-8')
elif endpoint == MdpEndpoint.EID_DATA:
response_dict = json.loads(response.content)
logging.debug('Got MDP %s for %s:\n%s', endpoint, app_user.email(), response_dict)
if not response_dict:
raise BusinessException(localize(self.language(), u'mdp_missing_eid_data'))
if MdpScope.EID_PROFILE in scopes:
result.eid_profile = parse_complex_value(MyDigiPassEidProfile, response_dict, False)
if MdpScope.EID_ADDRESS in scopes:
result.eid_address = parse_complex_value(MyDigiPassEidAddress, response_dict, False)
elif endpoint == MdpEndpoint.USER_DATA:
response_dict = json.loads(response.content)
logging.debug('Got MDP %s for %s:\n%s', endpoint, app_user.email(), response_dict)
if MdpScope.EMAIL in scopes:
result.email = response_dict['email']
if not result.email:
raise BusinessException(localize(self.language(), u'mdp_missing_email'))
if MdpScope.PHONE in scopes:
result.phone = response_dict['phone_number']
if not result.phone:
raise BusinessException(localize(self.language(), u'mdp_missing_phone'))
if MdpScope.ADDRESS in scopes:
result.address = parse_complex_value(MyDigiPassAddress, response_dict, False)
if not all((result.address.address_1, result.address.zip, result.address.country)):
raise BusinessException(localize(self.language(), u'mdp_missing_address'))
if MdpScope.PROFILE in scopes:
result.profile = parse_complex_value(MyDigiPassProfile, response_dict, False)
if not all((result.profile.first_name, result.profile.last_name)):
raise BusinessException(localize(self.language(), u'mdp_missing_profile'))
else:
azzert(False, 'Unexpected endpoint: %s' % endpoint)
result_dict = serialize_complex_value(result, MyDigiPassWidgetResultTO, False, skip_missing=True)
except BusinessException, e:
result_dict = dict(error=e.message,
mdp_update_required=isinstance(e, MissingScopesException))
def delete_async(self):
return db.delete_async(self)
except ServiceApiException, sae:
if (sae.code >= ERROR_CODE_WARNING_THRESHOLD):
logging.warning(u"Service API exception occurred", exc_info=1)
else:
logging.exception(u"Severe Service API Exception")
error_code = sae.code
error_message = sae.message
finally:
request = json.loads(service_api_callback.call)
if log and sik:
log_service_activity(sik.user, request["id"], ServiceLog.TYPE_CALLBACK, status, request["method"],
service_api_callback.call, raw_result_unicode, error_code, error_message)
if service_api_callback.is_saved():
service_api_callback_key = service_api_callback.key()
rpc_items.append(db.delete_async(service_api_callback_key), _delete_service_api_callback_deferred, service_api_callback_key)
if synchronous and error:
raise ServiceApiException(error_code, error_message)
def _delete_service_api_callback_deferred(service_api_callback_key):
db.delete(service_api_callback_key)
def _validate_api_callback(result_f, error_f, target, alias, f):
def raise_invalid_target():
raise ValueError("Target argument should be of type rogerthat.models.ServiceProfile or [rogerthat.models.ServiceProfile]")
check_decorations(result_f)
check_decorations(error_f)
if not isinstance(target, ServiceProfile):
raise_invalid_target()