def _calc_checksum(self, secret): # NOTE: this bypasses bcrypt's _calc_checksum, # so has to take care of all it's issues, such as secret encoding. if isinstance(secret, unicode): secret = secret.encode("utf-8") # generate the mysql323 hash first (as it would be in the db MASK_32 = 0xffffffff MASK_31 = 0x7fffffff WHITE = b' \t' nr1 = 0x50305735 nr2 = 0x12345671 add = 7 for c in secret: if c in WHITE: continue tmp = byte_elem_value(c) nr1 ^= ((((nr1 & 63)+add)*tmp) + (nr1 << 8)) & MASK_32 nr2 = (nr2+((nr2 << 8) ^ nr1)) & MASK_32 add = (add+tmp) & MASK_32 mysql323_hash = u("%08x%08x") % (nr1 & MASK_31, nr2 & MASK_31) # NOTE: can't use digest directly, since bcrypt stops at first NULL. # NOTE: bcrypt doesn't fully mix entropy for bytes 55-72 of password # (XXX: citation needed), so we don't want key to be > 55 bytes. # thus, have to use base64 (44 bytes) rather than hex (64 bytes). key = b64encode(sha256(mysql323_hash).digest()) return self._calc_checksum_backend(key)
def _crypt_secret_to_key(secret): """convert secret to 64-bit DES key. this only uses the first 8 bytes of the secret, and discards the high 8th bit of each byte at that. a null parity bit is inserted after every 7th bit of the output. """ # NOTE: this would set the parity bits correctly, # but des_encrypt_int_block() would just ignore them... ##return sum(expand_7bit(byte_elem_value(c) & 0x7f) << (56-i*8) ## for i, c in enumerate(secret[:8])) return sum((byte_elem_value(c) & 0x7F) << (57 - i * 8) for i, c in enumerate(secret[:8]))
def _crypt_secret_to_key(secret): """convert secret to 64-bit DES key. this only uses the first 8 bytes of the secret, and discards the high 8th bit of each byte at that. a null parity bit is inserted after every 7th bit of the output. """ # NOTE: this would set the parity bits correctly, # but des_encrypt_int_block() would just ignore them... ##return sum(expand_7bit(byte_elem_value(c) & 0x7f) << (56-i*8) ## for i, c in enumerate(secret[:8])) return sum((byte_elem_value(c) & 0x7f) << (57 - i * 8) for i, c in enumerate(secret[:8]))
def _calc_checksum(self, secret): # FIXME: no idea if mysql has a policy about handling unicode passwords if isinstance(secret, unicode): secret = secret.encode("utf-8") MASK_32 = 0xffffffff MASK_31 = 0x7fffffff WHITE = b' \t' nr1 = 0x50305735 nr2 = 0x12345671 add = 7 for c in secret: if c in WHITE: continue tmp = byte_elem_value(c) nr1 ^= ((((nr1 & 63)+add)*tmp) + (nr1 << 8)) & MASK_32 nr2 = (nr2+((nr2 << 8) ^ nr1)) & MASK_32 add = (add+tmp) & MASK_32 return u("%08x%08x") % (nr1 & MASK_31, nr2 & MASK_31)
def _calc_checksum(self, secret): # FIXME: no idea if mysql has a policy about handling unicode passwords if isinstance(secret, unicode): secret = secret.encode("utf-8") MASK_32 = 0xffffffff MASK_31 = 0x7fffffff WHITE = b' \t' nr1 = 0x50305735 nr2 = 0x12345671 add = 7 for c in secret: if c in WHITE: continue tmp = byte_elem_value(c) nr1 ^= ((((nr1 & 63) + add) * tmp) + (nr1 << 8)) & MASK_32 nr2 = (nr2 + ((nr2 << 8) ^ nr1)) & MASK_32 add = (add + tmp) & MASK_32 return u("%08x%08x") % (nr1 & MASK_31, nr2 & MASK_31)
def _raw_sha2_crypt(pwd, salt, rounds, use_512=False): """perform raw sha256-crypt / sha512-crypt this function provides a pure-python implementation of the internals for the SHA256-Crypt and SHA512-Crypt algorithms; it doesn't handle any of the parsing/validation of the hash strings themselves. :arg pwd: password chars/bytes to hash :arg salt: salt chars to use :arg rounds: linear rounds cost :arg use_512: use sha512-crypt instead of sha256-crypt mode :returns: encoded checksum chars """ # =================================================================== # init & validate inputs # =================================================================== # NOTE: the setup portion of this algorithm scales ~linearly in time # with the size of the password, making it vulnerable to a DOS from # unreasonably large inputs. the following code has some optimizations # which would make things even worse, using O(pwd_len**2) memory # when calculating digest P. # # to mitigate these two issues: 1) this code switches to a # O(pwd_len)-memory algorithm for passwords that are much larger # than average, and 2) Passlib enforces a library-wide max limit on # the size of passwords it will allow, to prevent this algorithm and # others from being DOSed in this way (see passlib.exc.PasswordSizeError # for details). # validate secret if isinstance(pwd, unicode): # XXX: not sure what official unicode policy is, using this as default pwd = pwd.encode("utf-8") assert isinstance(pwd, bytes) if _BNULL in pwd: raise uh.exc.NullPasswordError( sha512_crypt if use_512 else sha256_crypt) pwd_len = len(pwd) # validate rounds assert 1000 <= rounds <= 999999999, "invalid rounds" # NOTE: spec says out-of-range rounds should be clipped, instead of # causing an error. this function assumes that's been taken care of # by the handler class. # validate salt assert isinstance(salt, unicode), "salt not unicode" salt = salt.encode("ascii") salt_len = len(salt) assert salt_len < 17, "salt too large" # NOTE: spec says salts larger than 16 bytes should be truncated, # instead of causing an error. this function assumes that's been # taken care of by the handler class. # load sha256/512 specific constants if use_512: hash_const = hashlib.sha512 transpose_map = _512_transpose_map else: hash_const = hashlib.sha256 transpose_map = _256_transpose_map # =================================================================== # digest B - used as subinput to digest A # =================================================================== db = hash_const(pwd + salt + pwd).digest() # =================================================================== # digest A - used to initialize first round of digest C # =================================================================== # start out with pwd + salt a_ctx = hash_const(pwd + salt) a_ctx_update = a_ctx.update # add pwd_len bytes of b, repeating b as many times as needed. a_ctx_update(repeat_string(db, pwd_len)) # for each bit in pwd_len: add b if it's 1, or pwd if it's 0 i = pwd_len while i: a_ctx_update(db if i & 1 else pwd) i >>= 1 # finish A da = a_ctx.digest() # =================================================================== # digest P from password - used instead of password itself # when calculating digest C. # =================================================================== if pwd_len < 96: # this method is faster under python, but uses O(pwd_len**2) memory; # so we don't use it for larger passwords to avoid a potential DOS. dp = repeat_string(hash_const(pwd * pwd_len).digest(), pwd_len) else: # this method is slower under python, but uses a fixed amount of memory. tmp_ctx = hash_const(pwd) tmp_ctx_update = tmp_ctx.update i = pwd_len - 1 while i: tmp_ctx_update(pwd) i -= 1 dp = repeat_string(tmp_ctx.digest(), pwd_len) assert len(dp) == pwd_len # =================================================================== # digest S - used instead of salt itself when calculating digest C # =================================================================== ds = hash_const(salt * (16 + byte_elem_value(da[0]))).digest()[:salt_len] assert len(ds) == salt_len, "salt_len somehow > hash_len!" # =================================================================== # digest C - for a variable number of rounds, combine A, S, and P # digests in various ways; in order to burn CPU time. # =================================================================== # NOTE: the original SHA256/512-Crypt specification performs the C digest # calculation using the following loop: # ##dc = da ##i = 0 # while i < rounds: ## tmp_ctx = hash_const(dp if i & 1 else dc) # if i % 3: # tmp_ctx.update(ds) # if i % 7: # tmp_ctx.update(dp) ## tmp_ctx.update(dc if i & 1 else dp) ## dc = tmp_ctx.digest() ## i += 1 # # The code Passlib uses (below) implements an equivalent algorithm, # it's just been heavily optimized to pre-calculate a large number # of things beforehand. It works off of a couple of observations # about the original algorithm: # # 1. each round is a combination of 'dc', 'ds', and 'dp'; determined # by the whether 'i' a multiple of 2,3, and/or 7. # 2. since lcm(2,3,7)==42, the series of combinations will repeat # every 42 rounds. # 3. even rounds 0-40 consist of 'hash(dc + round-specific-constant)'; # while odd rounds 1-41 consist of hash(round-specific-constant + dc) # # Using these observations, the following code... # * calculates the round-specific combination of ds & dp for each round 0-41 # * runs through as many 42-round blocks as possible # * runs through as many pairs of rounds as possible for remaining rounds # * performs once last round if the total rounds should be odd. # # this cuts out a lot of the control overhead incurred when running the # original loop 40,000+ times in python, resulting in ~20% increase in # speed under CPython (though still 2x slower than glibc crypt) # prepare the 6 combinations of ds & dp which are needed # (order of 'perms' must match how _c_digest_offsets was generated) dp_dp = dp + dp dp_ds = dp + ds perms = [dp, dp_dp, dp_ds, dp_ds + dp, ds + dp, ds + dp_dp] # build up list of even-round & odd-round constants, # and store in 21-element list as (even,odd) pairs. data = [(perms[even], perms[odd]) for even, odd in _c_digest_offsets] # perform as many full 42-round blocks as possible dc = da blocks, tail = divmod(rounds, 42) while blocks: for even, odd in data: dc = hash_const(odd + hash_const(dc + even).digest()).digest() blocks -= 1 # perform any leftover rounds if tail: # perform any pairs of rounds pairs = tail >> 1 for even, odd in data[:pairs]: dc = hash_const(odd + hash_const(dc + even).digest()).digest() # if rounds was odd, do one last round (since we started at 0, # last round will be an even-numbered round) if tail & 1: dc = hash_const(dc + data[pairs][0]).digest() # =================================================================== # encode digest using appropriate transpose map # =================================================================== return h64.encode_transposed_bytes(dc, transpose_map).decode("ascii")
def raw_sun_md5_crypt(secret, rounds, salt): "given secret & salt, return encoded sun-md5-crypt checksum" global MAGIC_HAMLET assert isinstance(secret, bytes) assert isinstance(salt, bytes) # validate rounds if rounds <= 0: rounds = 0 real_rounds = 4096 + rounds # NOTE: spec seems to imply max 'rounds' is 2**32-1 # generate initial digest to start off round 0. # NOTE: algorithm 'salt' includes full config string w/ trailing "$" result = md5(secret + salt).digest() assert len(result) == 16 # NOTE: many things in this function have been inlined (to speed up the loop # as much as possible), to the point that this code barely resembles # the algorithm as described in the docs. in particular: # # * all accesses to a given bit have been inlined using the formula # rbitval(bit) = (rval((bit>>3) & 15) >> (bit & 7)) & 1 # # * the calculation of coinflip value R has been inlined # # * the conditional division of coinflip value V has been inlined as # a shift right of 0 or 1. # # * the i, i+3, etc iterations are precalculated in lists. # # * the round-based conditional division of x & y is now performed # by choosing an appropriate precalculated list, so that it only # calculates the 7 bits which will actually be used. # X_ROUNDS_0, X_ROUNDS_1, Y_ROUNDS_0, Y_ROUNDS_1 = _XY_ROUNDS # NOTE: % appears to be *slightly* slower than &, so we prefer & if possible round = 0 while round < real_rounds: # convert last result byte string to list of byte-ints for easy access rval = [byte_elem_value(c) for c in result].__getitem__ # build up X bit by bit x = 0 xrounds = X_ROUNDS_1 if (rval((round >> 3) & 15) >> (round & 7)) & 1 else X_ROUNDS_0 for i, ia, ib in xrounds: a = rval(ia) b = rval(ib) v = rval((a >> (b % 5)) & 15) >> ((b >> (a & 7)) & 1) x |= ((rval((v >> 3) & 15) >> (v & 7)) & 1) << i # build up Y bit by bit y = 0 yrounds = Y_ROUNDS_1 if (rval(((round + 64) >> 3) & 15) >> (round & 7)) & 1 else Y_ROUNDS_0 for i, ia, ib in yrounds: a = rval(ia) b = rval(ib) v = rval((a >> (b % 5)) & 15) >> ((b >> (a & 7)) & 1) y |= ((rval((v >> 3) & 15) >> (v & 7)) & 1) << i # extract x'th and y'th bit, xoring them together to yeild "coin flip" coin = ((rval(x >> 3) >> (x & 7)) ^ (rval(y >> 3) >> (y & 7))) & 1 # construct hash for this round h = md5(result) if coin: h.update(MAGIC_HAMLET) h.update(unicode(round).encode("ascii")) result = h.digest() round += 1 # encode output return h64.encode_transposed_bytes(result, _chk_offsets)
def raw_sun_md5_crypt(secret, rounds, salt): """given secret & salt, return encoded sun-md5-crypt checksum""" global MAGIC_HAMLET assert isinstance(secret, bytes) assert isinstance(salt, bytes) # validate rounds if rounds <= 0: rounds = 0 real_rounds = 4096 + rounds # NOTE: spec seems to imply max 'rounds' is 2**32-1 # generate initial digest to start off round 0. # NOTE: algorithm 'salt' includes full config string w/ trailing "$" result = md5(secret + salt).digest() assert len(result) == 16 # NOTE: many things in this function have been inlined (to speed up the loop # as much as possible), to the point that this code barely resembles # the algorithm as described in the docs. in particular: # # * all accesses to a given bit have been inlined using the formula # rbitval(bit) = (rval((bit>>3) & 15) >> (bit & 7)) & 1 # # * the calculation of coinflip value R has been inlined # # * the conditional division of coinflip value V has been inlined as # a shift right of 0 or 1. # # * the i, i+3, etc iterations are precalculated in lists. # # * the round-based conditional division of x & y is now performed # by choosing an appropriate precalculated list, so that it only # calculates the 7 bits which will actually be used. # X_ROUNDS_0, X_ROUNDS_1, Y_ROUNDS_0, Y_ROUNDS_1 = _XY_ROUNDS # NOTE: % appears to be *slightly* slower than &, so we prefer & if possible round = 0 while round < real_rounds: # convert last result byte string to list of byte-ints for easy access rval = [ byte_elem_value(c) for c in result ].__getitem__ # build up X bit by bit x = 0 xrounds = X_ROUNDS_1 if (rval((round>>3) & 15)>>(round & 7)) & 1 else X_ROUNDS_0 for i, ia, ib in xrounds: a = rval(ia) b = rval(ib) v = rval((a >> (b % 5)) & 15) >> ((b>>(a&7)) & 1) x |= ((rval((v>>3)&15)>>(v&7))&1) << i # build up Y bit by bit y = 0 yrounds = Y_ROUNDS_1 if (rval(((round+64)>>3) & 15)>>(round & 7)) & 1 else Y_ROUNDS_0 for i, ia, ib in yrounds: a = rval(ia) b = rval(ib) v = rval((a >> (b % 5)) & 15) >> ((b>>(a&7)) & 1) y |= ((rval((v>>3)&15)>>(v&7))&1) << i # extract x'th and y'th bit, xoring them together to yeild "coin flip" coin = ((rval(x>>3) >> (x&7)) ^ (rval(y>>3) >> (y&7))) & 1 # construct hash for this round h = md5(result) if coin: h.update(MAGIC_HAMLET) h.update(unicode(round).encode("ascii")) result = h.digest() round += 1 # encode output return h64.encode_transposed_bytes(result, _chk_offsets)
def _raw_sha2_crypt(pwd, salt, rounds, use_512=False): """perform raw sha256-crypt / sha512-crypt this function provides a pure-python implementation of the internals for the SHA256-Crypt and SHA512-Crypt algorithms; it doesn't handle any of the parsing/validation of the hash strings themselves. :arg pwd: password chars/bytes to encrypt :arg salt: salt chars to use :arg rounds: linear rounds cost :arg use_512: use sha512-crypt instead of sha256-crypt mode :returns: encoded checksum chars """ #=================================================================== # init & validate inputs #=================================================================== # NOTE: the setup portion of this algorithm scales ~linearly in time # with the size of the password, making it vulnerable to a DOS from # unreasonably large inputs. the following code has some optimizations # which would make things even worse, using O(pwd_len**2) memory # when calculating digest P. # # to mitigate these two issues: 1) this code switches to a # O(pwd_len)-memory algorithm for passwords that are much larger # than average, and 2) Passlib enforces a library-wide max limit on # the size of passwords it will allow, to prevent this algorithm and # others from being DOSed in this way (see passlib.exc.PasswordSizeError # for details). # validate secret if isinstance(pwd, unicode): # XXX: not sure what official unicode policy is, using this as default pwd = pwd.encode("utf-8") assert isinstance(pwd, bytes) if _BNULL in pwd: raise uh.exc.NullPasswordError(sha512_crypt if use_512 else sha256_crypt) pwd_len = len(pwd) # validate rounds assert 1000 <= rounds <= 999999999, "invalid rounds" # NOTE: spec says out-of-range rounds should be clipped, instead of # causing an error. this function assumes that's been taken care of # by the handler class. # validate salt assert isinstance(salt, unicode), "salt not unicode" salt = salt.encode("ascii") salt_len = len(salt) assert salt_len < 17, "salt too large" # NOTE: spec says salts larger than 16 bytes should be truncated, # instead of causing an error. this function assumes that's been # taken care of by the handler class. # load sha256/512 specific constants if use_512: hash_const = hashlib.sha512 hash_len = 64 transpose_map = _512_transpose_map else: hash_const = hashlib.sha256 hash_len = 32 transpose_map = _256_transpose_map #=================================================================== # digest B - used as subinput to digest A #=================================================================== db = hash_const(pwd + salt + pwd).digest() #=================================================================== # digest A - used to initialize first round of digest C #=================================================================== # start out with pwd + salt a_ctx = hash_const(pwd + salt) a_ctx_update = a_ctx.update # add pwd_len bytes of b, repeating b as many times as needed. a_ctx_update(repeat_string(db, pwd_len)) # for each bit in pwd_len: add b if it's 1, or pwd if it's 0 i = pwd_len while i: a_ctx_update(db if i & 1 else pwd) i >>= 1 # finish A da = a_ctx.digest() #=================================================================== # digest P from password - used instead of password itself # when calculating digest C. #=================================================================== if pwd_len < 96: # this method is faster under python, but uses O(pwd_len**2) memory; # so we don't use it for larger passwords to avoid a potential DOS. dp = repeat_string(hash_const(pwd * pwd_len).digest(), pwd_len) else: # this method is slower under python, but uses a fixed amount of memory. tmp_ctx = hash_const(pwd) tmp_ctx_update = tmp_ctx.update i = pwd_len-1 while i: tmp_ctx_update(pwd) i -= 1 dp = repeat_string(tmp_ctx.digest(), pwd_len) assert len(dp) == pwd_len #=================================================================== # digest S - used instead of salt itself when calculating digest C #=================================================================== ds = hash_const(salt * (16 + byte_elem_value(da[0]))).digest()[:salt_len] assert len(ds) == salt_len, "salt_len somehow > hash_len!" #=================================================================== # digest C - for a variable number of rounds, combine A, S, and P # digests in various ways; in order to burn CPU time. #=================================================================== # NOTE: the original SHA256/512-Crypt specification performs the C digest # calculation using the following loop: # ##dc = da ##i = 0 ##while i < rounds: ## tmp_ctx = hash_const(dp if i & 1 else dc) ## if i % 3: ## tmp_ctx.update(ds) ## if i % 7: ## tmp_ctx.update(dp) ## tmp_ctx.update(dc if i & 1 else dp) ## dc = tmp_ctx.digest() ## i += 1 # # The code Passlib uses (below) implements an equivalent algorithm, # it's just been heavily optimized to pre-calculate a large number # of things beforehand. It works off of a couple of observations # about the original algorithm: # # 1. each round is a combination of 'dc', 'ds', and 'dp'; determined # by the whether 'i' a multiple of 2,3, and/or 7. # 2. since lcm(2,3,7)==42, the series of combinations will repeat # every 42 rounds. # 3. even rounds 0-40 consist of 'hash(dc + round-specific-constant)'; # while odd rounds 1-41 consist of hash(round-specific-constant + dc) # # Using these observations, the following code... # * calculates the round-specific combination of ds & dp for each round 0-41 # * runs through as many 42-round blocks as possible # * runs through as many pairs of rounds as possible for remaining rounds # * performs once last round if the total rounds should be odd. # # this cuts out a lot of the control overhead incurred when running the # original loop 40,000+ times in python, resulting in ~20% increase in # speed under CPython (though still 2x slower than glibc crypt) # prepare the 6 combinations of ds & dp which are needed # (order of 'perms' must match how _c_digest_offsets was generated) dp_dp = dp+dp dp_ds = dp+ds perms = [dp, dp_dp, dp_ds, dp_ds+dp, ds+dp, ds+dp_dp] # build up list of even-round & odd-round constants, # and store in 21-element list as (even,odd) pairs. data = [ (perms[even], perms[odd]) for even, odd in _c_digest_offsets] # perform as many full 42-round blocks as possible dc = da blocks, tail = divmod(rounds, 42) while blocks: for even, odd in data: dc = hash_const(odd + hash_const(dc + even).digest()).digest() blocks -= 1 # perform any leftover rounds if tail: # perform any pairs of rounds pairs = tail>>1 for even, odd in data[:pairs]: dc = hash_const(odd + hash_const(dc + even).digest()).digest() # if rounds was odd, do one last round (since we started at 0, # last round will be an even-numbered round) if tail & 1: dc = hash_const(dc + data[pairs][0]).digest() #=================================================================== # encode digest using appropriate transpose map #=================================================================== return h64.encode_transposed_bytes(dc, transpose_map).decode("ascii")
def utf8_truncate(source, index): """ helper to truncate UTF8 byte string to nearest character boundary ON OR AFTER <index>. returned prefix will always have length of at least <index>, and will stop on the first byte that's not a UTF8 continuation byte (128 - 191 inclusive). since utf8 should never take more than 4 bytes to encode known unicode values, we can stop after ``index+3`` is reached. :param bytes source: :param int index: :rtype: bytes """ # general approach: # # * UTF8 bytes will have high two bits (0xC0) as one of: # 00 -- ascii char # 01 -- ascii char # 10 -- continuation of multibyte char # 11 -- start of multibyte char. # thus we can cut on anything where high bits aren't "10" (0x80; continuation byte) # # * UTF8 characters SHOULD always be 1 to 4 bytes, though they may be unbounded. # so we just keep going until first non-continuation byte is encountered, or end of str. # this should work predictably even for malformed/non UTF8 inputs. if not isinstance(source, bytes): raise ExpectedTypeError(source, bytes, "source") # validate index end = len(source) if index < 0: index = max(0, index + end) if index >= end: return source # can stop search after 4 bytes, won't ever have longer utf8 sequence. end = min(index + 3, end) # loop until we find non-continuation byte while index < end: if byte_elem_value(source[index]) & 0xC0 != 0x80: # found single-char byte, or start-char byte. break # else: found continuation byte. index += 1 else: assert index == end # truncate at final index result = source[:index] def sanity_check(): # try to decode source try: text = source.decode("utf-8") except UnicodeDecodeError: # if source isn't valid utf8, byte level match is enough return True # validate that result was cut on character boundary assert text.startswith(result.decode("utf-8")) return True assert sanity_check() return result