def testKnownValues(self):
# random strings of lengths 0-19
data = [
('', ''),
('\xf9', 'yF=='),
('*\xc9', '9gZ='),
('T\xe7`', 'KDSV'),
('\xd2\xe9H\x0c', 'oi_72-=='),
('K\x84\x03\xeb\xe8', 'HsF2uyV='),
('\xebl\xe5\xa3\xa3\xf8', 'uqn_cuEs'),
('\x04\x88yR\xef\xa1M', '07WtJiyWIF=='),
('h\x8c\xa2\xb8h\x8c\x19v', 'P7mXi5XB5MN='),
('\x06\xc7_M\x19$\x88v\xb4', '0gSUIGZZX6Po'),
('\x1d\xab\xefI\xf7\x7fY\xa4\r\xe8', '6PjjHUSzLPFCu-=='),
('\xa4=\xe6\x1b\x00\xb1\r\xba\xcc\xca\xf4', 'd2ra5k1l2QfBmjF='),
('\xd7\xac\xa8\x97\xc2\x14\x16)\xf5"\xc8d', 'pumc_w7J4Xbp7gWZ'),
('\xab\xb3%\xd3&I\xfd\x9cc\x91\x17\xd7\xdf', 'evB_omO8zOlYZGUMrk=='),
('O\x8dO\xf4\nd\xc4\xf5W]\xdf\xd3\xa9\xfe', 'IspEx-dZlEKMMSzIeUs='),
("\xca\xdc\x8d'\xf0\xc5a\x93b\x1c@4\xdaC\x9a", 'mhmC8z24NOCX63-oqZDP'),
('\xe1\x00\xf7\xd8p\xef\x08v\xca\x9b\x81INPvu', 'sF2rq62j16Q9as48I_0qSF=='),
('8_\xaeS\xb9\xa9\xb7\x1e\x99\x8c\x06\xc7\xa9\xa2F\xb5\x0f', 'D4yiJvadhluOY-Q6eP85hFw='),
('"\xc3)!J H\x98\ro\xe1\\\xc2a\xc9\xe2v\xe8', '7gBd7JcVH8VCQy4Rka68sbQc'),
('\xe3\xb1?_\x97a<\xc5\xf5Cj\x86\xbeB\xc3F\xcc\x1ai', 'sv3zMtSWEBMpFqe5jZA2GgkPPF=='),
]
for s, expected in data:
try:
self.assertEqual(base64hex.B64HexEncode(s), expected)
self.assertEqual(base64hex.B64HexDecode(expected), s)
self.assertEqual(base64hex.B64HexEncode(s, padding=False), expected.rstrip('='))
self.assertEqual(base64hex.B64HexDecode(expected.rstrip('='), padding=False), s)
except:
logging.info("failed on %r", (s, expected))
raise
def testSortOrder(self):
num_trials = 1000
for i in xrange(num_trials):
s1 = self._RandomString()
s2 = self._RandomString()
enc1 = base64hex.B64HexEncode(s1)
enc2 = base64hex.B64HexEncode(s2)
assert (s1 < s2) == (enc1 < enc2), 's1: %s, s2: %s, enc1: %s, enc2: %s' % (s1, s2, enc1, enc2)
def ConstructTimestampAssetId(id_prefix,
timestamp,
device_id,
uniquifier,
reverse_ts=True):
"""Constructs an asset id that has a leading 4-byte encoded timestamp,
which may be reversed. The asset id is base-64 hex encoded so that it
sorts the same way as its binary representation and can be safely
included in URLs. The "id_prefix" is appended to the resulting id.
The binary format of the asset id is as follows:
timestamp (32 bits): whole seconds since Unix epoch
device_id (var-length numeric): id of the generating device
client_id (var-length numeric): unique id generated by the device
server_id (byte str): optionally generated by the server
"""
assert IdPrefix.IsValid(id_prefix), id_prefix
# Drop fractional seconds and possibly reverse the timestamp before converting to raw bytes.
assert timestamp < 1L << 32, timestamp
if reverse_ts:
timestamp = (1L << 32) - int(timestamp) - 1
byte_str = struct.pack('>I', timestamp)
assert len(byte_str) == 4, timestamp
# Append the encoded device_id.
byte_str += util.EncodeVarLengthNumber(device_id)
# Append the encoded asset-id uniquifier.
byte_str += _EncodeUniquifier(uniquifier)
# Base64-hex encode the bytes for URL-inclusion safety.
return id_prefix + base64hex.B64HexEncode(byte_str, padding=False)
def testEncodeDecode(self):
num_trials = 1000
for i in xrange(num_trials):
s = self._RandomString()
enc = base64hex.B64HexEncode(s)
dec = base64hex.B64HexDecode(enc)
self.assertEqual(s, dec)
def Index(self, col, value):
"""Returns words as contiguous alpha numeric strings (and
apostrophes) which are of length > 1 and are also not in the stop
words list. Each term is freighted with a list of term positions
(formatted as a packed binary string).
"""
terms = {}
expansions = {} # map from term to expanded set of terms
tokens = self._Tokenize(value)
for pos, term in zip(xrange(len(tokens)), tokens):
if term == '_':
continue
if term not in expansions:
expansions[term] = self._ExpandTerm(col, term)
for exp_term in expansions[term]:
if not terms.has_key(exp_term):
terms[exp_term] = ''
if pos < 1<<16:
terms[exp_term] += struct.pack('>H', pos)
# Base64Hex Encode positions.
for k,v in terms.items():
if v:
terms[k] = base64hex.B64HexEncode(v, padding=False)
return terms
def CreateSortKeyPrefix(timestamp, randomness=True, reverse=False):
"""Returns a sort key which will sort by 'timestamp'. If
'randomness' is True, 16 bits of randomness (which would otherwise
be lost to b64-encoding padding) are added into the free bits. These
are meant to minimize the chance of collision when the sort key
prefix is meant to provide uniqueness but many keys may be created
in the same second. If 'reverse' is True, the timestamp is reversed
by subtracting from 2^32. The result is base64hex-encoded.
"""
assert timestamp < 1L << 32, timestamp
if reverse:
timestamp = (1L << 32) - int(timestamp) - 1
if randomness:
random_bits = random.getrandbits(16) & 0xffff
else:
random_bits = 0
return base64hex.B64HexEncode(
struct.pack('>IH', int(timestamp), random_bits))
def ConstructAssetId(id_prefix, device_id, uniquifier):
"""Constructs an asset id that does not have a timestamp part. The
asset id is base-64 hex encoded so that it sorts the same way as
its binary representation and can be safely included in URLs. The
"id_prefix" is appended to the resulting id. The binary format of
the asset id is as follows:
device_id (var-length numeric): id of the generating device
client_id (var-length numeric): unique id generated by the device
server_id (byte str): optionally generated by the server
"""
assert IdPrefix.IsValid(id_prefix), id_prefix
# Encode the device_id.
byte_str = util.EncodeVarLengthNumber(device_id)
# Append the encoded asset-id uniquifier.
byte_str += _EncodeUniquifier(uniquifier)
# Base64-hex encode the bytes to preserve ordering while attaining URL-inclusion safety.
return id_prefix + base64hex.B64HexEncode(byte_str, padding=False)
def CreateInvitationURL(cls, client, user_id, identity_key, viewpoint_id,
default_url):
"""Creates and returns a prospective user invitation ShortURL object. The URL is handled
by an instance of AuthProspectiveHandler, which is "listening" at "/pr/...". The ShortURL
group is partitioned by user id so that incorrect guesses only affect a single user.
"""
identity_type, identity_value = Identity.SplitKey(identity_key)
now = util.GetCurrentTimestamp()
expires = now + Identity._TIME_TO_INVITIATION_EXPIRATION
encoded_user_id = base64hex.B64HexEncode(
util.EncodeVarLengthNumber(user_id), padding=False)
short_url = yield ShortURL.Create(client,
group_id='pr/%s' % encoded_user_id,
timestamp=now,
expires=expires,
identity_key=identity_key,
viewpoint_id=viewpoint_id,
default_url=default_url,
is_sms=identity_type == 'Phone')
raise gen.Return(short_url)
def Create(cls, client, group_id, timestamp, expires, **kwargs):
"""Allocate a new ShortURL DB object by finding an unused random key within the group."""
# Try several times to generate a unique key.
for i in xrange(ShortURL._KEY_GEN_TRIES):
# Generate a random 6-byte key, using URL-safe base64 encoding.
random_key = base64hex.B64HexEncode(
os.urandom(ShortURL.KEY_LEN_IN_BYTES))
short_url = ShortURL(group_id, random_key)
short_url.timestamp = timestamp
short_url.expires = expires
short_url.json = kwargs
try:
yield short_url.Update(client, expected={'random_key': False})
except DBConditionalCheckFailedError as ex:
# Key is already in use, generate another.
continue
raise gen.Return(short_url)
logging.warning(
'cannot allocate a unique random key for group id "%s"', group_id)
raise TooManyRetriesError('Failed to allocate unique URL key.')
def PackPlacemark(placemark):
"""Converts 'placemark' named tuple into a packed,
base64-hex-encoded, comma-separated representation for storage in
DynamoDB.
"""
return ','.join([base64hex.B64HexEncode(x.encode('utf-8'), padding=False) for x in placemark])
def PackLocation(location):
"""Converts 'location' named tuple into a packed, base64-hex-encoded
string representation for storage in DynamoDB.
"""
packed = struct.pack('>ddd', *[float(x) for x in location])
return base64hex.B64HexEncode(packed)
def SendVerifyIdMessage(cls, client, action, use_short_token,
is_mobile_app, identity_key, user_id, user_name,
**kwargs):
"""Sends a verification email or SMS message to the given identity. This message may
directly contain an access code (e.g. if an SMS is sent), or it may contain a ShortURL
link to a page which reveals the access code (e.g. if email was triggered by the mobile
app). Or it may contain a link to a page which confirms the user's password and redirects
them to the web site (e.g. if email was triggered by the web site).
"""
# Ensure that identity exists.
identity = yield gen.Task(Identity.Query,
client,
identity_key,
None,
must_exist=False)
if identity is None:
identity = Identity.CreateFromKeywords(key=identity_key)
yield gen.Task(identity.Update, client)
identity_type, identity_value = Identity.SplitKey(identity.key)
message_type = 'emails' if identity_type == 'Email' else 'messages'
# Throttle the rate at which email/SMS messages can be sent to this identity. The updated
# count will be saved by CreateAccessTokenURL.
auth_throttle = identity.auth_throttle or {}
per_min_dict, is_throttled = util.ThrottleRate(
auth_throttle.get('per_min',
None), VerifyIdBaseHandler._MAX_MESSAGES_PER_MIN,
constants.SECONDS_PER_MINUTE)
if is_throttled:
# Bug 485: Silently do not send the email if throttled. We don't want to give user error
# if they exit out of confirm code screen, then re-create account, etc.
return
per_day_dict, is_throttled = util.ThrottleRate(
auth_throttle.get('per_day',
None), VerifyIdBaseHandler._MAX_MESSAGES_PER_DAY,
constants.SECONDS_PER_DAY)
if is_throttled:
raise InvalidRequestError(TOO_MANY_MESSAGES_DAY,
message_type=message_type,
identity_value=Identity.GetDescription(
identity.key))
identity.auth_throttle = {
'per_min': per_min_dict,
'per_day': per_day_dict
}
# Create a ShortURL link that will supply the access token to the user when clicked.
# Use a URL path like "idm/*" for the mobile app, and "idw/*" for the web.
encoded_user_id = base64hex.B64HexEncode(
util.EncodeVarLengthNumber(user_id), padding=False)
group_id = '%s/%s' % ('idm' if is_mobile_app else 'idw',
encoded_user_id)
short_url = yield gen.Task(identity.CreateAccessTokenURL,
client,
group_id,
use_short_token=use_short_token,
action=action,
identity_key=identity.key,
user_name=user_name,
**kwargs)
# Send email/SMS in order to verify that the user controls the identity.
if identity_type == 'Email':
args = VerifyIdBaseHandler._GetAuthEmail(client, action,
use_short_token,
user_name, identity,
short_url)
yield gen.Task(EmailManager.Instance().SendEmail,
description=action,
**args)
else:
args = VerifyIdBaseHandler._GetAccessTokenSms(identity)
yield gen.Task(SMSManager.Instance().SendSMS,
description=action,
**args)
# In dev servers, display a popup with the generated code (OS X 10.9-only).
if (options.options.localdb and platform.system() == 'Darwin'
and platform.mac_ver()[0] == '10.9'):
subprocess.call([
'osascript', '-e',
'display notification "%s" with title "Viewfinder"' %
identity.access_token
])