def test_sorting(self):
with tempfile.TemporaryDirectory() as dir:
self.make_testfile(Path(dir, "test.bin"), reversed(self.test_set))
with SRP.Vector_file(Path(dir, "test.bin")) as fin:
fin.sort(Path(dir, "test2.bin"))
with SRP.Vector_file(Path(dir, "test2.bin")) as f2:
rows = [k for (k, v) in f2]
self.assertEqual(rows, [t[0] for t in self.test_set[:3]])
def test_prefix_lookup(self):
with tempfile.TemporaryDirectory() as dir:
with SRP.Vector_file(Path(dir, "out.bin"), mode="a",
dims=2) as fout:
for i in range(3):
for j in range(5):
fout.add_row(f"file_{i}-part-{j}",
np.array([i, j], '<f4'))
newview = SRP.Vector_file(Path(dir, "out.bin"), mode="a", dims=2)
for k, v in newview.find_prefix("file_1", "-"):
self.assertEqual(np.float32(1), v[0])
newview.close()
"""
def main():
parser = argparse.ArgumentParser(
description='Concatenate a number of vector files into single file')
parser.add_argument('outpath',
type=str,
help='Place to save the new Vector file.')
parser.add_argument(
'--mode',
type=str,
default='a',
help=
'Write mode for output. By default, appends if the file exists, can be switched to \'w\' to overwrite.'
)
parser.add_argument('--build-cache',
action='store_true',
help='Build a prefix cache after concatenation.')
parser.add_argument(
'--no-concat',
action='store_true',
help="Skip the concatenation, if you're hoping to *just* build cache.")
parser.add_argument('filepaths',
type=str,
nargs='*',
help='List of vector files being combines.')
args = parser.parse_args()
if len(args.filepaths) == 0 and not args.no_concat:
raise Exception("Nothing to do without input filepaths")
if args.no_concat and not args.build_cache:
raise Exception(
"If you're not concatenating and not building a cache, you're not doing anything."
)
if not args.no_concat:
with SRP.Vector_file(args.filepaths[0], mode="r") as vecf:
dims = vecf.dims
with SRP.Vector_file(args.outpath, mode=args.mode, dims=dims) as outf:
for efpath in args.filepaths:
print("Concatenating:", efpath)
outf.concatenate_file(efpath)
if args.build_cache:
with SRP.Vector_file(args.outpath, offset_cache=True) as outf:
print("Building prefix lookup cache")
outf._build_prefix_lookup(sep='-', dump_every=2000000)
def test_unicode(self):
"""
One of the goals is be able to pass *either* encoded or decoded
utf-8, because that tends to happen.
These tests are a lot easier to pass now that python2 is deprecated.
"""
hasher = SRP.SRP(6)
guten = u"Güten Tag"
gutenhash = np.array([0., 2., -2., 0., 2., 0.]).tolist()
basic = hasher.stable_transform(guten, log=False,
unit_length=False).tolist()
self.assertTrue(basic == gutenhash)
encoded = hasher.stable_transform(guten.encode("utf-8"),
log=False,
unit_length=False).tolist()
self.assertTrue(encoded == gutenhash)
decoded = hasher.stable_transform(
guten.encode("utf-8").decode("utf-8"),
log=False,
unit_length=False).tolist()
self.assertTrue(decoded == gutenhash)
def set(self, user, password):
self._read()
if not self.users.has_key(user):
raise ValueError, 'No such user'
s, v = SRP.new_passwd(user, password)
self.users[user] = {'s': s, 'v': v}
self._dump()
def add(self, user, password):
self._read()
if self.users.has_key(user):
raise ValueError, 'User exists'
s, v = SRP.new_passwd(user, password)
self.users[user] = {'s': s, 'v': v}
self._dump()
def _initialize_embeddings(self, chunk_file):
"""
Read in an embedding file at unit length, and adjust the metadata to match.
"""
input = SRP.Vector_file(chunk_file)
dataset = input.to_matrix(unit_length=True)
self.matrix = dataset['matrix']
ids = dataset['names']
self.ids = ids
sections = [a.split("-") for a in dataset['names']]
try:
htids, sections, starts, ends = zip(*sections)
except:
# The old format: deprecated.
htids, sections = zip(*sections)
sections = pd.DataFrame({
'mtid': ids,
'htid': htids,
'section': list(map(int, sections))
}).set_index('htid')
self.chunk_metadata = sections.join(self.metadata,
how='left').reset_index()
self.mtid_lookup = dict(
zip(dataset['names'], range(len(dataset['names']))))
self.htid_lookup = defaultdict(list)
for i, htid in enumerate(htids):
self.htid_lookup[htid].append(ids[i])
def add(self, user, password):
self._read()
if self.users.has_key(user):
raise ValueError, "User exists"
s, v = SRP.new_passwd(user, password)
self.users[user] = {"s": s, "v": v}
self._dump()
def set(self, user, password):
self._read()
if not self.users.has_key(user):
raise ValueError, "No such user"
s, v = SRP.new_passwd(user, password)
self.users[user] = {"s": s, "v": v}
self._dump()
def test_numeric_substitution(self):
hasher = SRP.SRP(36)
string1 = "I was born in 2001"
string2 = "I was born in 1907"
h1 = hasher.stable_transform(string1, log=False, standardize=True)
h2 = hasher.stable_transform(string1, log=False, standardize=True)
self.assertEqual(h1.tolist(), h2.tolist())
def SRPAuth(sock, user, passphrase=None):
"""Perform an SRP authentication on a socket. Return the session key
if authentication was successful, or raise an exception if it was not.
The other end of the socket must be ready to receive the SRP
commands."""
if not passphrase:
passphrase = getpass.getpass('Enter passphrase for %s: ' % user)
# Send the USER command.
sock.send('USER %s\n' % user)
# Get the client-side keys and send the public one.
keys = SRP.client_begin(user)
A = keys[0]
sock.send(encode_long(A))
# Read the response.
file = sock.makefile('rb')
line = file.readline()
if line[0:3] != 'KEY':
raise SRP.NoSuchUser, line
s = read_string(file)
B = read_long(file)
u = read_long(file)
# Now calculate the session key and send the proof.
K, m = SRP.client_key(user, passphrase, s, B, u, keys)
sock.send(encode_string(m))
line = file.readline()
if line[0:3] != 'AOK':
raise SRP.AuthFailure, line
# Authenticate the host.
m1 = SRP.host_authenticator(K, A, m)
m = read_string(file)
if m != m1:
raise SRP.AuthFailure, "Host authentication failed."
# All done, return the session key.
return K
def SRPAuth(sock, user, passphrase = None):
"""Perform an SRP authentication on a socket. Return the session key
if authentication was successful, or raise an exception if it was not.
The other end of the socket must be ready to receive the SRP
commands."""
if not passphrase:
passphrase = getpass.getpass('Enter passphrase for %s: ' % user)
# Send the USER command.
sock.send('USER %s\n' % user)
# Get the client-side keys and send the public one.
keys = SRP.client_begin(user)
A = keys[0]
sock.send(encode_long(A))
# Read the response.
file = sock.makefile('rb')
line = file.readline()
if line[0:3] != 'KEY':
raise SRP.NoSuchUser, line
s = read_string(file)
B = read_long(file)
u = read_long(file)
# Now calculate the session key and send the proof.
K, m = SRP.client_key(user, passphrase, s, B, u, keys)
sock.send(encode_string(m))
line = file.readline()
if line[0:3] != 'AOK':
raise SRP.AuthFailure, line
# Authenticate the host.
m1 = SRP.host_authenticator(K, A, m)
m = read_string(file)
if m != m1:
raise SRP.AuthFailure, "Host authentication failed."
# All done, return the session key.
return K
def _srp_auth(self, s, msg):
socket = self.socket[s]
pw = socket['pw']
srp = socket['srp'] = {}
srp['B'], srp['u'], srp['K'], srp['m'] = SRP.host_begin(msg['user'], msg['A'], pw['s'], pw['v'])
srp['A'] = msg['A']
self._send_msg(s, {'s': pw['s'], 'B': srp['B'], 'u': srp['u']})
socket['state'] = 1
def set_password(self, s, password):
socket = self.socket[s]
salt, v = SRP.new_passwd(self.co.user, password)
cypherv = crypt(long_to_string(v), socket['key'])[0]
self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv})
socket['state'] = 5
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def test_ascii(self):
hasher = SRP.SRP(6)
hello_world = hasher.stable_transform("hello world",
log=False,
unit_length=False)
self.assertEqual(hello_world.tolist(),
np.array([0., 0., 2., 0., 2., 0.]).tolist())
def set_password(self, s, password):
socket = self.socket[s]
salt, v = SRP.new_passwd(self.co.user, password)
cypherv = crypt(long_to_string(v), socket['key'])[0]
self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv})
socket['state'] = 5
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def SRP_transform(f):
global hasher
if hasher is None:
hasher = SRP.SRP(640)
return hasher.stable_transform(words=f['token'],
counts=f['count'],
log=True,
standardize=True)
def test_ascii_equals_unicode(self):
hasher = SRP.SRP(160)
hello_world = hasher.stable_transform("hello world",
log=False).tolist()
hello_world_unicode = hasher.stable_transform(u"hello world",
log=False).tolist()
self.assertEqual(hello_world, hello_world_unicode)
def srp_auth(self, s):
socket = self.socket[s]
assert socket['state'] == 0 or socket['state'] == 3
srp = socket['srp'] = {}
srp['keys'] = SRP.client_begin()
self._send_msg(s, {'op': 'srp auth', 'user': self.co.user, 'A': srp['keys'][0]})
socket['state'] = 1
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def make_testfile(self, path, array=None):
if array is None:
array = self.test_set
with SRP.Vector_file(path, dims=3, mode="w") as testfile:
for row in self.test_set:
if row[0] == "stop":
continue
testfile.add_row(*row)
def _srp_auth(self, s, msg):
socket = self.socket[s]
pw = socket['pw']
srp = socket['srp'] = {}
srp['B'], srp['u'], srp['K'], srp['m'] = SRP.host_begin(
msg['user'], msg['A'], pw['s'], pw['v'])
srp['A'] = msg['A']
self._send_msg(s, {'s': pw['s'], 'B': srp['B'], 'u': srp['u']})
socket['state'] = 1
def test_logs_are_plausible(self):
log_unit = np.log(1e05)
hasher = SRP.SRP(20)
log_srp = hasher.stable_transform("hello", log=True)
nonlog_srp = hasher.stable_transform("hello", log=False)
difference = sum(log_srp - (nonlog_srp) * log_unit)
# Forgive floating point error.
self.assertTrue(difference < 1e-05)
def test_wordcounts_unicode(self):
hasher = SRP.SRP(160)
wordcount_style = hasher.stable_transform(words=[u"Güten", u"Tag"],
counts=[1, 1],
log=False).tolist()
string_style = hasher.stable_transform(words=u"Güten Tag",
log=False).tolist()
self.assertEqual(wordcount_style, string_style)
def test_creation_and_reading(self):
with tempfile.TemporaryDirectory() as dir:
testloc = Path(dir, "test.bin")
self.make_testfile(testloc)
testfile2 = SRP.Vector_file(Path(dir, "test.bin"),
dims=3,
mode="a")
testfile2.add_row(*self.test_set[3])
testfile2.close()
self.assertTrue(testfile2.nrows == 4)
reloaded = SRP.Vector_file(Path(dir, "test.bin"), mode="r")
self.assertTrue(reloaded.vocab_size == 4)
# Test Iteration
read_in_values = dict()
for (i, (name, array)) in enumerate(reloaded):
read_in_values[name] = array
(comp_name, comp_array) = self.test_set[i]
self.assertEqual(comp_name, name)
self.assertEqual(array.tolist(), comp_array.tolist())
# Individual Vec Getter
keys = [i for i, a in self.test_set]
for i, key in enumerate(keys):
vec = reloaded[key]
np.testing.assert_array_almost_equal(vec, self.test_set[i][1])
# List Vec Getter
vecs = reloaded[keys]
self.assertEqual(vecs.shape, (len(keys), reloaded.dims))
for i in range(len(keys)):
np.testing.assert_array_almost_equal(vecs[i],
self.test_set[i][1])
reloaded.close()
self.assertEqual(read_in_values["foo"].tolist(),
read_in_values["foo2"].tolist())
self.assertFalse(read_in_values["foo2"].tolist() ==
read_in_values["fü"].tolist())
def read_long(file):
ll = []
while 1:
line = file.readline()
if not line:
raise EOFError
l = string.strip(line)
if not l:
break
ll.append(l)
val = SRP.string_to_long(base64.decodestring(string.join(ll, '')))
return val
def test_null_vectors(self):
"""
Not clear that these *should* be supported, but apparently they are.
"""
with tempfile.TemporaryDirectory() as dir:
p = Path(dir, "test.bin")
fout = SRP.Vector_file(p, dims=0, mode='a')
fout.add_row("good", np.array([], '<f4'))
fout.flush()
with p.open() as fin:
self.assertEqual(fin.read(), '000000001 00000\ngood \n')
fout.close()
def read_long(file):
ll = []
while 1:
line = file.readline()
if not line:
raise EOFError
l = string.strip(line)
if not l:
break
ll.append(l)
val = SRP.string_to_long(base64.decodestring(string.join(ll, '')))
return val
def srp_auth(self, s):
socket = self.socket[s]
assert socket['state'] == 0 or socket['state'] == 3
srp = socket['srp'] = {}
srp['keys'] = SRP.client_begin()
self._send_msg(s, {
'op': 'srp auth',
'user': self.co.user,
'A': srp['keys'][0]
})
socket['state'] = 1
self.rs.listen_forever(self.nh)
self.rs.doneflag.clear()
def test_entrance_format(self):
with tempfile.TemporaryDirectory() as dir:
self.make_testfile(Path(dir, "test.bin"))
with SRP.Vector_file(Path(dir, "test.bin"), dims=3,
mode="a") as testfile2:
testfile2.add_row(*self.test_set[3])
with SRP.Vector_file(Path(dir, "test2.bin"), dims=3,
mode="a") as a:
a.concatenate_file(Path(dir, "test.bin"))
a.concatenate_file(Path(dir, "test.bin"))
with SRP.Vector_file(Path(dir, 'test2.bin')) as f:
self.assertEqual(f.vocab_size, 8)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
# Trigger a warning.
f['foo']
# Verify some things
assert "duplicate identifiers" in str(w[-1].message)
self.assertTrue(testfile2.nrows == 4)
def test_standardization(self):
"""
standardization does case normalization,
and tokenizes by a charater regex.
"""
hasher = SRP.SRP(6)
string1 = "Gravity's rainbow"
hashed_standardized = hasher.stable_transform(string1,
log=False,
standardize=True)
manually_tokenized = ["Gravity", "s", "RAINBOW"]
hashed_manually_tokenized = hasher.stable_transform(manually_tokenized,
[1, 1, 1],
log=False,
standardize=True)
self.assertEqual(hashed_manually_tokenized.tolist(),
hashed_standardized.tolist())
def client_key(self, s, B, u, keys):
use_agent = True
if self.pwid is None:
use_agent = False
elif not self._query_agent_for_id(self.pwid):
use_agent = False
if use_agent:
func = self._private_key
arg = None
else:
self._get_password()
func = SRP.private_key
arg = self.password
K, m = SRP.client_key(self.user, arg, s, B, u, keys, func)
return K, m
def textset_to_srp(inputfile,
outputfile,
dim=640,
limit=float("Inf"),
log=True):
"""
A convenience wrapper for converting a text corpus to
an SRP collection as a block.
inputfile is the collection. The format is the same as the ingest
format used by bookworm and mallet; that is to say, a single unicode
file where each line is a document represented as a unique filename, then a tab,
and then a long string containing the text of the document.
To coerce into the this format, newlines must be removed.
inputfile can also be a **directory** of txt files.
outputfile is the SRP file to be created. Recommended suffix is `.bin`.
dims is the dimensionality of the output SRP.
"""
import SRP
output = Vector_file(outputfile, dims=dim, mode="w")
hasher = SRP.SRP(dim=dim)
if inputfile.endswith(".txt"):
yielder = textset_yielder
elif os.path.isdir(inputfile):
yielder = directory_yielder
else:
raise ValueError(
"Don't know how to process {}: must be a textfile or a directory".
format(inputfile))
for i, (id, txt) in enumerate(yielder(inputfile)):
transform = hasher.stable_transform(txt, log=True, standardize=True)
output.add_row(id, transform)
if i + 1 >= limit:
break
output.close()
def encode_long(val):
s = base64.encodestring(SRP.long_to_string(val))
return s + '\n'
def main():
parser = argparse.ArgumentParser(description="Convert Extracted Features files to vectors, and save in SRP's Vector_file format.")
parser.add_argument('threadno', type=int, help='Non-zero indexed number of thread.')
parser.add_argument('totalthreads', type=int, help='Number of threads running in total.')
parser.add_argument('idlist', type=str, help='CSV file of HTIDs to process. Needs a header, with column name \'htid\'.')
parser.add_argument('--outdir', '-o', type=str, default='data_outputs/', help='Directory to save results.')
parser.add_argument('--chunksize', '-c', type=int, default=10000, help='Size of chunks to roll pages into. -1 will make books into a single full book "chunk".'')
parser.add_argument('--no-srp', action='store_true', help='Turn off SRP saving')
parser.add_argument('--no-glove', action='store_true', help='Turn off Glove saving')
parser.add_argument('--glove-dims', '-g', type=int, default=300, help='Number of GloVe dimensions. Can be 50, 100, 200, or 300.')
parser.add_argument('--srp-dims', '-s', type=int, default=640, help='Number of SRP dimensions.')
parser.add_argument('--efdir', type=str, default=None,
help='Set an alternative base url for EF files, if not using the setting from local.yaml or ~/.htrc-config.yaml.')
args = parser.parse_args()
thread_no = args.threadno - 1 # Zero index.
assert not (args.no_srp & args.no_glove)
if args.efdir is not None:
customizable_resolver.dir = args.efdir
already_imported = already_imported_list(args.outdir)
print("There are {} files already imported".format(len(already_imported)))
filenames = pd.read_csv(args.idlist, low_memory = False)
thread_name = "{}-of-{}_".format(thread_no + 1, args.totalthreads)
already_seen_file = open(os.path.join(args.outdir, "already_completed_files{}.csv".format(thread_name)), "a")
if not args.no_srp:
hasher = SRP.SRP(args.srp_dims)
out_SRP = SRP.Vector_file(os.path.join(args.outdir, thread_name + "SRP_chunks.bin"), dims=args.srp_dims, mode="w")
def SRP_transform(f):
return hasher.stable_transform(words = f['lowercase'], counts = f['count'], log = True, standardize = True)
if not args.no_glove:
wem_model = wem_loader('glove-wiki-gigaword-{}'.format(args.glove_dims))
# Cross-ref with stoplist and drop stopped words
from spacy.lang.en.stop_words import STOP_WORDS
wem_vocab = set(wem_model.vocab.keys())
wem_vocab = wem_vocab.difference(STOP_WORDS)
out_glove = SRP.Vector_file(os.path.join(args.outdir, thread_name + "Glove_chunks.bin"), dims = args.glove_dims, mode="w")
def WEM_transform(f):
return transformations.chunk_to_wem(f, wem_model, vocab=wem_vocab, stop=False, log=True, min_ncount=10)
books = 0
last = None
start_time = time.time()
try:
gen = yielder(filenames.htid, thread_no, args.totalthreads,
chunk_size=args.chunksize, already_imported_list=already_imported)
for i, (id, chunk, start, end, group) in enumerate(gen):
# Count books too.
if last != id:
books += 1
if last is not None:
already_seen_file.write("{}\n".format(last))
if (books % 25 == 0):
rate = books/(time.time()-start_time)
print("{} books done on thread {} of {}, {:.02f} chunks per book, {:.02f} books per second".format(books, thread_no + 1, args.totalthreads, i/books, rate))
last = id
id = "{}-{:04d}-{}-{}".format(id, chunk, start, end)
if not args.no_srp:
SRP_rep = SRP_transform(group)
out_SRP.add_row(id, SRP_rep)
if not args.no_glove:
WEM_rep = WEM_transform(group)
if WEM_rep.shape[0] != args.glove_dims:
print(WEM_rep.shape, args.glove_dims)
try:
out_glove.add_row(id, WEM_rep.astype('<f4'))
except:
print(id, WEM_rep.shape, args.glove_dims, wem_model.vector_size)
raise
already_seen_file.write("{}\n".format(last))
if not args.no_srp:
out_SRP.close()
if not args.no_glove:
out_glove.close()
except KeyboardInterrupt:
if not args.no_srp:
out_SRP.close()
if not args.no_glove:
out_glove.close()
def message_came_in(self, s, data):
socket = self.socket[s]
try:
msg = bdecode(data)
except ValueError:
self._send_error(s, None, 'garbage data')
self._close(s)
return
if socket['state'] == 0:
try:
pw = socket['pw'] = self.passwd.get(msg['user'])
except KeyError:
self._send_error(s, None, 'Bad user')
self._close(s)
return
socket['user'] = msg['user']
if msg['op'] == 'get hash':
self._send_msg(s, {'hash': sha.new('public hash check' + pw['secret']).digest()})
socket['state'] = 3
elif msg['op'] == 'secret auth':
self._secret_auth(s)
elif msg['op'] == 'srp auth':
self._srp_auth(s, msg)
else:
self._close(s)
elif socket['state'] == 1:
srp = socket['srp']
if srp['m'].digest() != msg['m']:
self._send_error(s, None, 'Bad password')
socket['state'] = 3
return
auth = SRP.host_authenticator(srp['K'], srp['A'], srp['m'].digest())
self._send_msg(s, {'auth': auth.digest()})
self.nh.set_hmac(s, srp['m'], auth)
socket['state'] = 2
elif socket['state'] == 2:
srp = socket['srp']
if msg['op'] == 'get secret':
secret = socket['pw']['secret']
esecret = crypt(secret, srp['K'])[0]
self._send_msg(s, {'secret': esecret})
socket['state'] = 3
elif msg['op'] == 'set password':
if socket['user'] == 'anonymous':
self._send_error(s, None, 'operation not permitted')
self._close(s)
return
v = string_to_long(crypt(msg['v'], srp['K'])[0])
self.passwd.define(socket['user'], v, msg['s'])
self._send_msg(s, {'ok': 1})
self._close(s)
elif socket['state'] == 3:
if msg['op'] == 'secret auth':
self._secret_auth(s)
elif msg['op'] == 'srp auth':
self._srp_auth(s, msg)
else:
self._close(s)
elif socket['state'] == 4:
pw = socket['pw']
if len(msg['salt']) < 20:
self._send_error(s, None, 'Bad salt length')
self._close(s)
return
if msg['salt'] + socket['salt'] == socket['salt'] + msg['salt']:
self._send_error(s, None, 'Bad salt')
self._close(s)
return
base = 'session key' + pw['secret'] + socket['salt'] + msg['salt']
key = sha.new(base).digest()
socket['m_in'] = hmac.new(key, '', sha)
base = 'session key' + pw['secret'] + msg['salt'] + socket['salt']
key = sha.new(base).digest()
socket['m_out'] = hmac.new(key, '', sha)
if msg['auth'] != socket['m_out'].digest():
self._send_error(s, None, 'Bad password')
socket['state'] = 3
return
self._send_msg(s, {'auth': socket['m_in'].digest()})
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self._req_mode(s, 1)
self.socket[s] = [{}, {}, socket['user'], [], 1]
else:
self._close(s)
def _process_message(self, sock):
status = self.hstatus[sock.fileno()]
input = status['input']
input.seek(status['in_offset'])
data = input.read(4)
if len(data) != 4:
return
msg_len, = struct.unpack('<i', data)
if msg_len <= 0 or msg_len > 1024:
self._write_error(sock, 'Bad message lenth')
self._close_socket(sock)
return
data = input.read(msg_len)
if len(data) < msg_len:
return
status['in_offset'] += 4 + msg_len
try:
msg = bdecode(data)
except ValueError:
self._write_error(sock, 'Bad message')
self._close_socket(sock)
return
type = msg['type']
if type == 'CDV_AGENT_ADD_PASSWORD':
id = sha.new('password' + msg['password']).digest()
self.identities[id] = msg['password']
self._write_answer(sock, bencode({'id': id}))
elif type == 'CDV_AGENT_ADD_SECRET':
id = sha.new('public hash check' + msg['secret']).digest()
self.identities[id] = msg['secret']
self._write_answer(sock, bencode({'id': id}))
elif type == 'CDV_AGENT_ADD_ENCRYPTED_SECRET':
if not self.identities.has_key(msg['id']):
self._write_error(sock, 'No such identity')
return
secret = crypt(msg['secret'], self.identities[msg['id']])[0]
id = sha.new('public hash check' + secret).digest()
if id != msg['secret_id']:
self._write_error(sock, 'id does not match')
return
self.identities[id] = secret
self._write_answer(sock, bencode({}))
elif type == 'CDV_AGENT_QUERY_IDENTITY':
known = 0
if self.identities.has_key(msg['id']):
known = 1
self._write_answer(sock, bencode({'known': known}))
elif type == 'CDV_AGENT_SRP_PRIVATE_KEY':
x = SRP.private_key(msg['user'], msg['s'], self.identities[msg['id']])
self._write_answer(sock, bencode({'x': x}))
elif type == 'CDV_AGENT_SESSION_KEY':
if not self.identities.has_key(msg['id']):
self._write_error(sock, 'No such identity')
return
if len(msg['salt1']) < 20 or len(msg['salt2']) < 20:
self._write_error(sock, 'Bad salts')
return
if msg['salt1'] == msg['salt2']:
self._write_error(sock, 'Bad salts')
return
base = 'session key' + self.identities[msg['id']] + \
msg['salt1'] + msg['salt2']
key = sha.new(base).digest()
answer = {'key': key}
self._write_answer(sock, bencode(answer))
elif type == 'CDV_AGENT_SHUTDOWN' and self.allow_shutdown:
self.shutdown_flag = 1
else:
self._write_error(sock, 'Unknown command: ' + type)
return
def test_error_on_load(self):
with tempfile.TemporaryDirectory() as dir:
testfile = SRP.Vector_file(Path(dir, "test.bin"), dims=3, mode="w")
with self.assertRaises(TypeError):
testfile.add_row("this is a space", self.array1)
testfile.close()
iraf.apnorm1.gain = ")apnormalize.gain"
iraf.apnorm1.lsigma = ")apnormalize.lsigma"
iraf.apnorm1.usigma = ")apnormalize.usigma"
#clean out the idiotic yet somehow nessacary database
try:
files_database = os.listdir('./database')
for i in files_database:
os.remove('./database/' + i)
except OSError:
print "No database to delete yet"
#trim all images of bias/overscan, and fix keywords
all_img = list(flats + dark + obj + lamp)
all_img2 = SRP.outputnames(all_img, 'temp')
#mkcalib = raw_input("Create trimmed temp files (all)? (y/n): ").lower()
#if "y" in mkcalib:
#for x,i in enumerate(all_img):
#iraf.imcopy(i+'[1:2048,1:2048]',all_img2[x])
#fix DISPAXIS keyword
#iraf.hedit(all_img2[x]+'[0]',"DISPAXIS",1,verify="no",add="yes")
#elif "n" in mkcalib:
#print "Skipping temporary files"
#os.system("rm *temp*.fits")
#os.system("rm *tnrm*.fits")
print "Objects in data set: "
def message_came_in(self, s, data):
socket = self.socket[s]
try:
msg = bdecode(data)
except ValueError:
self._send_error(s, None, 'garbage data')
self._close(s)
return
if socket['state'] == 0:
try:
pw = socket['pw'] = self.passwd.get(msg['user'])
except KeyError:
self._send_error(s, None, 'Bad user')
self._close(s)
return
socket['user'] = msg['user']
if msg['op'] == 'get hash':
self._send_msg(s, {
'hash':
sha.new('public hash check' + pw['secret']).digest()
})
socket['state'] = 3
elif msg['op'] == 'secret auth':
self._secret_auth(s)
elif msg['op'] == 'srp auth':
self._srp_auth(s, msg)
else:
self._close(s)
elif socket['state'] == 1:
srp = socket['srp']
if srp['m'].digest() != msg['m']:
self._send_error(s, None, 'Bad password')
socket['state'] = 3
return
auth = SRP.host_authenticator(srp['K'], srp['A'],
srp['m'].digest())
self._send_msg(s, {'auth': auth.digest()})
self.nh.set_hmac(s, srp['m'], auth)
socket['state'] = 2
elif socket['state'] == 2:
srp = socket['srp']
if msg['op'] == 'get secret':
secret = socket['pw']['secret']
esecret = crypt(secret, srp['K'])[0]
self._send_msg(s, {'secret': esecret})
socket['state'] = 3
elif msg['op'] == 'set password':
if socket['user'] == 'anonymous':
self._send_error(s, None, 'operation not permitted')
self._close(s)
return
v = string_to_long(crypt(msg['v'], srp['K'])[0])
self.passwd.define(socket['user'], v, msg['s'])
self._send_msg(s, {'ok': 1})
self._close(s)
elif socket['state'] == 3:
if msg['op'] == 'secret auth':
self._secret_auth(s)
elif msg['op'] == 'srp auth':
self._srp_auth(s, msg)
else:
self._close(s)
elif socket['state'] == 4:
pw = socket['pw']
if len(msg['salt']) < 20:
self._send_error(s, None, 'Bad salt length')
self._close(s)
return
if msg['salt'] + socket['salt'] == socket['salt'] + msg['salt']:
self._send_error(s, None, 'Bad salt')
self._close(s)
return
base = 'session key' + pw['secret'] + socket['salt'] + msg['salt']
key = sha.new(base).digest()
socket['m_in'] = hmac.new(key, '', sha)
base = 'session key' + pw['secret'] + msg['salt'] + socket['salt']
key = sha.new(base).digest()
socket['m_out'] = hmac.new(key, '', sha)
if msg['auth'] != socket['m_out'].digest():
self._send_error(s, None, 'Bad password')
socket['state'] = 3
return
self._send_msg(s, {'auth': socket['m_in'].digest()})
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self._req_mode(s, 1)
self.socket[s] = [{}, {}, socket['user'], [], 1]
else:
self._close(s)
def message_came_in(self, s, data):
try:
msg = bdecode(data)
except ValueError:
self.close(s)
raise NetworkError, 'garbage data'
if msg.has_key('error'):
raise ServerError, msg['error']
socket = self.socket[s]
srp = socket['srp']
if socket['state'] == 1:
K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'],
srp['keys'])
socket['key'], socket['m_out'] = K, m
self._send_msg(s, {'m': socket['m_out'].digest()})
socket['state'] = 2
elif socket['state'] == 2:
socket['m_in'] = SRP.host_authenticator(socket['key'], srp['keys'][0], socket['m_out'].digest())
if socket['m_in'].digest() != msg['auth']:
raise ServerError, 'Bad host authentication'
return
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self.rs.doneflag.set()
elif socket['state'] == 3:
self.socket[s]['hash'] = msg['hash']
self.rs.doneflag.set()
elif socket['state'] == 4:
self.close(s)
secret = crypt(msg['secret'], socket['key'])[0]
self.auth.save_secret(secret)
self.rs.doneflag.set()
elif socket['state'] == 5:
self.close(s)
self.rs.doneflag.set()
elif socket['state'] == 6:
if len(msg['salt']) < 20:
self._send_error(s, None, 'Bad salt length')
self.close(s)
raise NetworkError, 'Bad salt from server'
salt = random_string(20)
key = self.auth.session_key(salt, msg['salt'])
socket['m_in'] = hmac.new(key, '', sha)
key = self.auth.session_key(msg['salt'], salt)
socket['m_out'] = hmac.new(key, '', sha)
self._send_msg(s, {'auth': socket['m_in'].digest(),
'salt': salt})
socket['state'] = 7
elif socket['state'] == 7:
if msg['auth'] != socket['m_out'].digest():
self._send_error(s, None, 'Bad auth')
self.close(s)
raise NetworkError, 'Bad server auth'
self._req_mode(s, 1)
self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
self.socket[s] = [{}, {}, {}, [], 1]
self.rs.doneflag.set()
else:
self.close(s)
def handle(self):
sock = self.request
file = sock.makefile('rb')
# Receive the username and public key A from the client.
while 1:
line = file.readline()
if not line:
return
if line[0:4] == 'USER':
l = string.split(line)
if len(l) == 2:
break
sock.send('Please specify USER.\n')
user = l[1]
try:
A = read_long(file)
except EOFError:
return
# Calculate the public and private values, and send the public stuff
# to the client.
try:
s, B, u, K, m = SRP.lookup(user, A)
except SRP.NoSuchUser:
sock.send('No such user "%s".\n' % user)
return
sock.send('KEY\n')
sock.send(encode_string(s))
sock.send(encode_long(B))
sock.send(encode_long(u))
# The client now sends us its proof.
try:
m1 = read_string(file)
except EOFError:
return
if m != m1:
sock.send('Client authentication failed.\n')
return
sock.send('AOK\n')
# Send the host authentication proof.
sock.send(encode_string(SRP.host_authenticator(K, A, m)))
# At this point (assuming the client accepts the host authentication),
# the socket has been authenticated and K is the secret session key.
if self.__dict__.has_key('auth_socket'):
self.auth_socket(file, sock, K)
else:
# Simple echo server.
while 1:
s = file.readline()
if not s:
return
sock.send(s)
def test_dtype(self):
hasher = SRP.SRP(6)
hello_world = hasher.stable_transform("hello world", log=False)
self.assertEqual(hello_world.dtype, np.float32)
