def testKnownValues(self): # random strings of lengths 0-19 data = [ ('', ''), ('\xf9', 'yF=='), ('*\xc9', '9gZ='), ('T\xe7`', 'KDSV'), ('\xd2\xe9H\x0c', 'oi_72-=='), ('K\x84\x03\xeb\xe8', 'HsF2uyV='), ('\xebl\xe5\xa3\xa3\xf8', 'uqn_cuEs'), ('\x04\x88yR\xef\xa1M', '07WtJiyWIF=='), ('h\x8c\xa2\xb8h\x8c\x19v', 'P7mXi5XB5MN='), ('\x06\xc7_M\x19$\x88v\xb4', '0gSUIGZZX6Po'), ('\x1d\xab\xefI\xf7\x7fY\xa4\r\xe8', '6PjjHUSzLPFCu-=='), ('\xa4=\xe6\x1b\x00\xb1\r\xba\xcc\xca\xf4', 'd2ra5k1l2QfBmjF='), ('\xd7\xac\xa8\x97\xc2\x14\x16)\xf5"\xc8d', 'pumc_w7J4Xbp7gWZ'), ('\xab\xb3%\xd3&I\xfd\x9cc\x91\x17\xd7\xdf', 'evB_omO8zOlYZGUMrk=='), ('O\x8dO\xf4\nd\xc4\xf5W]\xdf\xd3\xa9\xfe', 'IspEx-dZlEKMMSzIeUs='), ("\xca\xdc\x8d'\xf0\xc5a\x93b\x1c@4\xdaC\x9a", 'mhmC8z24NOCX63-oqZDP'), ('\xe1\x00\xf7\xd8p\xef\x08v\xca\x9b\x81INPvu', 'sF2rq62j16Q9as48I_0qSF=='), ('8_\xaeS\xb9\xa9\xb7\x1e\x99\x8c\x06\xc7\xa9\xa2F\xb5\x0f', 'D4yiJvadhluOY-Q6eP85hFw='), ('"\xc3)!J H\x98\ro\xe1\\\xc2a\xc9\xe2v\xe8', '7gBd7JcVH8VCQy4Rka68sbQc'), ('\xe3\xb1?_\x97a<\xc5\xf5Cj\x86\xbeB\xc3F\xcc\x1ai', 'sv3zMtSWEBMpFqe5jZA2GgkPPF=='), ] for s, expected in data: try: self.assertEqual(base64hex.B64HexEncode(s), expected) self.assertEqual(base64hex.B64HexDecode(expected), s) self.assertEqual(base64hex.B64HexEncode(s, padding=False), expected.rstrip('=')) self.assertEqual(base64hex.B64HexDecode(expected.rstrip('='), padding=False), s) except: logging.info("failed on %r", (s, expected)) raise
def testSortOrder(self): num_trials = 1000 for i in xrange(num_trials): s1 = self._RandomString() s2 = self._RandomString() enc1 = base64hex.B64HexEncode(s1) enc2 = base64hex.B64HexEncode(s2) assert (s1 < s2) == (enc1 < enc2), 's1: %s, s2: %s, enc1: %s, enc2: %s' % (s1, s2, enc1, enc2)
def ConstructTimestampAssetId(id_prefix, timestamp, device_id, uniquifier, reverse_ts=True): """Constructs an asset id that has a leading 4-byte encoded timestamp, which may be reversed. The asset id is base-64 hex encoded so that it sorts the same way as its binary representation and can be safely included in URLs. The "id_prefix" is appended to the resulting id. The binary format of the asset id is as follows: timestamp (32 bits): whole seconds since Unix epoch device_id (var-length numeric): id of the generating device client_id (var-length numeric): unique id generated by the device server_id (byte str): optionally generated by the server """ assert IdPrefix.IsValid(id_prefix), id_prefix # Drop fractional seconds and possibly reverse the timestamp before converting to raw bytes. assert timestamp < 1L << 32, timestamp if reverse_ts: timestamp = (1L << 32) - int(timestamp) - 1 byte_str = struct.pack('>I', timestamp) assert len(byte_str) == 4, timestamp # Append the encoded device_id. byte_str += util.EncodeVarLengthNumber(device_id) # Append the encoded asset-id uniquifier. byte_str += _EncodeUniquifier(uniquifier) # Base64-hex encode the bytes for URL-inclusion safety. return id_prefix + base64hex.B64HexEncode(byte_str, padding=False)
def testEncodeDecode(self): num_trials = 1000 for i in xrange(num_trials): s = self._RandomString() enc = base64hex.B64HexEncode(s) dec = base64hex.B64HexDecode(enc) self.assertEqual(s, dec)
def Index(self, col, value): """Returns words as contiguous alpha numeric strings (and apostrophes) which are of length > 1 and are also not in the stop words list. Each term is freighted with a list of term positions (formatted as a packed binary string). """ terms = {} expansions = {} # map from term to expanded set of terms tokens = self._Tokenize(value) for pos, term in zip(xrange(len(tokens)), tokens): if term == '_': continue if term not in expansions: expansions[term] = self._ExpandTerm(col, term) for exp_term in expansions[term]: if not terms.has_key(exp_term): terms[exp_term] = '' if pos < 1<<16: terms[exp_term] += struct.pack('>H', pos) # Base64Hex Encode positions. for k,v in terms.items(): if v: terms[k] = base64hex.B64HexEncode(v, padding=False) return terms
def CreateSortKeyPrefix(timestamp, randomness=True, reverse=False): """Returns a sort key which will sort by 'timestamp'. If 'randomness' is True, 16 bits of randomness (which would otherwise be lost to b64-encoding padding) are added into the free bits. These are meant to minimize the chance of collision when the sort key prefix is meant to provide uniqueness but many keys may be created in the same second. If 'reverse' is True, the timestamp is reversed by subtracting from 2^32. The result is base64hex-encoded. """ assert timestamp < 1L << 32, timestamp if reverse: timestamp = (1L << 32) - int(timestamp) - 1 if randomness: random_bits = random.getrandbits(16) & 0xffff else: random_bits = 0 return base64hex.B64HexEncode( struct.pack('>IH', int(timestamp), random_bits))
def ConstructAssetId(id_prefix, device_id, uniquifier): """Constructs an asset id that does not have a timestamp part. The asset id is base-64 hex encoded so that it sorts the same way as its binary representation and can be safely included in URLs. The "id_prefix" is appended to the resulting id. The binary format of the asset id is as follows: device_id (var-length numeric): id of the generating device client_id (var-length numeric): unique id generated by the device server_id (byte str): optionally generated by the server """ assert IdPrefix.IsValid(id_prefix), id_prefix # Encode the device_id. byte_str = util.EncodeVarLengthNumber(device_id) # Append the encoded asset-id uniquifier. byte_str += _EncodeUniquifier(uniquifier) # Base64-hex encode the bytes to preserve ordering while attaining URL-inclusion safety. return id_prefix + base64hex.B64HexEncode(byte_str, padding=False)
def CreateInvitationURL(cls, client, user_id, identity_key, viewpoint_id, default_url): """Creates and returns a prospective user invitation ShortURL object. The URL is handled by an instance of AuthProspectiveHandler, which is "listening" at "/pr/...". The ShortURL group is partitioned by user id so that incorrect guesses only affect a single user. """ identity_type, identity_value = Identity.SplitKey(identity_key) now = util.GetCurrentTimestamp() expires = now + Identity._TIME_TO_INVITIATION_EXPIRATION encoded_user_id = base64hex.B64HexEncode( util.EncodeVarLengthNumber(user_id), padding=False) short_url = yield ShortURL.Create(client, group_id='pr/%s' % encoded_user_id, timestamp=now, expires=expires, identity_key=identity_key, viewpoint_id=viewpoint_id, default_url=default_url, is_sms=identity_type == 'Phone') raise gen.Return(short_url)
def Create(cls, client, group_id, timestamp, expires, **kwargs): """Allocate a new ShortURL DB object by finding an unused random key within the group.""" # Try several times to generate a unique key. for i in xrange(ShortURL._KEY_GEN_TRIES): # Generate a random 6-byte key, using URL-safe base64 encoding. random_key = base64hex.B64HexEncode( os.urandom(ShortURL.KEY_LEN_IN_BYTES)) short_url = ShortURL(group_id, random_key) short_url.timestamp = timestamp short_url.expires = expires short_url.json = kwargs try: yield short_url.Update(client, expected={'random_key': False}) except DBConditionalCheckFailedError as ex: # Key is already in use, generate another. continue raise gen.Return(short_url) logging.warning( 'cannot allocate a unique random key for group id "%s"', group_id) raise TooManyRetriesError('Failed to allocate unique URL key.')
def PackPlacemark(placemark): """Converts 'placemark' named tuple into a packed, base64-hex-encoded, comma-separated representation for storage in DynamoDB. """ return ','.join([base64hex.B64HexEncode(x.encode('utf-8'), padding=False) for x in placemark])
def PackLocation(location): """Converts 'location' named tuple into a packed, base64-hex-encoded string representation for storage in DynamoDB. """ packed = struct.pack('>ddd', *[float(x) for x in location]) return base64hex.B64HexEncode(packed)
def SendVerifyIdMessage(cls, client, action, use_short_token, is_mobile_app, identity_key, user_id, user_name, **kwargs): """Sends a verification email or SMS message to the given identity. This message may directly contain an access code (e.g. if an SMS is sent), or it may contain a ShortURL link to a page which reveals the access code (e.g. if email was triggered by the mobile app). Or it may contain a link to a page which confirms the user's password and redirects them to the web site (e.g. if email was triggered by the web site). """ # Ensure that identity exists. identity = yield gen.Task(Identity.Query, client, identity_key, None, must_exist=False) if identity is None: identity = Identity.CreateFromKeywords(key=identity_key) yield gen.Task(identity.Update, client) identity_type, identity_value = Identity.SplitKey(identity.key) message_type = 'emails' if identity_type == 'Email' else 'messages' # Throttle the rate at which email/SMS messages can be sent to this identity. The updated # count will be saved by CreateAccessTokenURL. auth_throttle = identity.auth_throttle or {} per_min_dict, is_throttled = util.ThrottleRate( auth_throttle.get('per_min', None), VerifyIdBaseHandler._MAX_MESSAGES_PER_MIN, constants.SECONDS_PER_MINUTE) if is_throttled: # Bug 485: Silently do not send the email if throttled. We don't want to give user error # if they exit out of confirm code screen, then re-create account, etc. return per_day_dict, is_throttled = util.ThrottleRate( auth_throttle.get('per_day', None), VerifyIdBaseHandler._MAX_MESSAGES_PER_DAY, constants.SECONDS_PER_DAY) if is_throttled: raise InvalidRequestError(TOO_MANY_MESSAGES_DAY, message_type=message_type, identity_value=Identity.GetDescription( identity.key)) identity.auth_throttle = { 'per_min': per_min_dict, 'per_day': per_day_dict } # Create a ShortURL link that will supply the access token to the user when clicked. # Use a URL path like "idm/*" for the mobile app, and "idw/*" for the web. encoded_user_id = base64hex.B64HexEncode( util.EncodeVarLengthNumber(user_id), padding=False) group_id = '%s/%s' % ('idm' if is_mobile_app else 'idw', encoded_user_id) short_url = yield gen.Task(identity.CreateAccessTokenURL, client, group_id, use_short_token=use_short_token, action=action, identity_key=identity.key, user_name=user_name, **kwargs) # Send email/SMS in order to verify that the user controls the identity. if identity_type == 'Email': args = VerifyIdBaseHandler._GetAuthEmail(client, action, use_short_token, user_name, identity, short_url) yield gen.Task(EmailManager.Instance().SendEmail, description=action, **args) else: args = VerifyIdBaseHandler._GetAccessTokenSms(identity) yield gen.Task(SMSManager.Instance().SendSMS, description=action, **args) # In dev servers, display a popup with the generated code (OS X 10.9-only). if (options.options.localdb and platform.system() == 'Darwin' and platform.mac_ver()[0] == '10.9'): subprocess.call([ 'osascript', '-e', 'display notification "%s" with title "Viewfinder"' % identity.access_token ])