示例#1
0
 def test_sorting(self):
     with tempfile.TemporaryDirectory() as dir:
         self.make_testfile(Path(dir, "test.bin"), reversed(self.test_set))
         with SRP.Vector_file(Path(dir, "test.bin")) as fin:
             fin.sort(Path(dir, "test2.bin"))
         with SRP.Vector_file(Path(dir, "test2.bin")) as f2:
             rows = [k for (k, v) in f2]
         self.assertEqual(rows, [t[0] for t in self.test_set[:3]])
示例#2
0
    def test_prefix_lookup(self):
        with tempfile.TemporaryDirectory() as dir:
            with SRP.Vector_file(Path(dir, "out.bin"), mode="a",
                                 dims=2) as fout:
                for i in range(3):
                    for j in range(5):
                        fout.add_row(f"file_{i}-part-{j}",
                                     np.array([i, j], '<f4'))

            newview = SRP.Vector_file(Path(dir, "out.bin"), mode="a", dims=2)
            for k, v in newview.find_prefix("file_1", "-"):
                self.assertEqual(np.float32(1), v[0])
            newview.close()
            """
示例#3
0
def main():

    parser = argparse.ArgumentParser(
        description='Concatenate a number of vector files into single file')

    parser.add_argument('outpath',
                        type=str,
                        help='Place to save the new Vector file.')
    parser.add_argument(
        '--mode',
        type=str,
        default='a',
        help=
        'Write mode for output. By default, appends if the file exists, can be switched to \'w\' to overwrite.'
    )
    parser.add_argument('--build-cache',
                        action='store_true',
                        help='Build a prefix cache after concatenation.')
    parser.add_argument(
        '--no-concat',
        action='store_true',
        help="Skip the concatenation, if you're hoping to *just* build cache.")
    parser.add_argument('filepaths',
                        type=str,
                        nargs='*',
                        help='List of vector files being combines.')

    args = parser.parse_args()

    if len(args.filepaths) == 0 and not args.no_concat:
        raise Exception("Nothing to do without input filepaths")
    if args.no_concat and not args.build_cache:
        raise Exception(
            "If you're not concatenating and not building a cache, you're not doing anything."
        )

    if not args.no_concat:
        with SRP.Vector_file(args.filepaths[0], mode="r") as vecf:
            dims = vecf.dims

        with SRP.Vector_file(args.outpath, mode=args.mode, dims=dims) as outf:
            for efpath in args.filepaths:
                print("Concatenating:", efpath)
                outf.concatenate_file(efpath)

    if args.build_cache:
        with SRP.Vector_file(args.outpath, offset_cache=True) as outf:
            print("Building prefix lookup cache")
            outf._build_prefix_lookup(sep='-', dump_every=2000000)
示例#4
0
    def test_unicode(self):
        """
        One of the goals is be able to pass *either* encoded or decoded
        utf-8, because that tends to happen.

        These tests are a lot easier to pass now that python2 is deprecated.

        """
        hasher = SRP.SRP(6)
        guten = u"Güten Tag"
        gutenhash = np.array([0., 2., -2., 0., 2., 0.]).tolist()

        basic = hasher.stable_transform(guten, log=False,
                                        unit_length=False).tolist()
        self.assertTrue(basic == gutenhash)

        encoded = hasher.stable_transform(guten.encode("utf-8"),
                                          log=False,
                                          unit_length=False).tolist()
        self.assertTrue(encoded == gutenhash)

        decoded = hasher.stable_transform(
            guten.encode("utf-8").decode("utf-8"),
            log=False,
            unit_length=False).tolist()
        self.assertTrue(decoded == gutenhash)
示例#5
0
 def set(self, user, password):
     self._read()
     if not self.users.has_key(user):
         raise ValueError, 'No such user'
     s, v = SRP.new_passwd(user, password)
     self.users[user] = {'s': s, 'v': v}
     self._dump()
示例#6
0
 def add(self, user, password):
     self._read()
     if self.users.has_key(user):
         raise ValueError, 'User exists'
     s, v = SRP.new_passwd(user, password)
     self.users[user] = {'s': s, 'v': v}
     self._dump()
示例#7
0
    def _initialize_embeddings(self, chunk_file):
        """
        Read in an embedding file at unit length, and adjust the metadata to match.
        """
        input = SRP.Vector_file(chunk_file)
        dataset = input.to_matrix(unit_length=True)
        self.matrix = dataset['matrix']

        ids = dataset['names']
        self.ids = ids

        sections = [a.split("-") for a in dataset['names']]
        try:
            htids, sections, starts, ends = zip(*sections)
        except:
            # The old format: deprecated.
            htids, sections = zip(*sections)
        sections = pd.DataFrame({
            'mtid': ids,
            'htid': htids,
            'section': list(map(int, sections))
        }).set_index('htid')

        self.chunk_metadata = sections.join(self.metadata,
                                            how='left').reset_index()

        self.mtid_lookup = dict(
            zip(dataset['names'], range(len(dataset['names']))))

        self.htid_lookup = defaultdict(list)
        for i, htid in enumerate(htids):
            self.htid_lookup[htid].append(ids[i])
示例#8
0
 def add(self, user, password):
     self._read()
     if self.users.has_key(user):
         raise ValueError, "User exists"
     s, v = SRP.new_passwd(user, password)
     self.users[user] = {"s": s, "v": v}
     self._dump()
示例#9
0
 def set(self, user, password):
     self._read()
     if not self.users.has_key(user):
         raise ValueError, "No such user"
     s, v = SRP.new_passwd(user, password)
     self.users[user] = {"s": s, "v": v}
     self._dump()
示例#10
0
 def test_numeric_substitution(self):
     hasher = SRP.SRP(36)
     string1 = "I was born in 2001"
     string2 = "I was born in 1907"
     h1 = hasher.stable_transform(string1, log=False, standardize=True)
     h2 = hasher.stable_transform(string1, log=False, standardize=True)
     self.assertEqual(h1.tolist(), h2.tolist())
示例#11
0
def SRPAuth(sock, user, passphrase=None):
    """Perform an SRP authentication on a socket.  Return the session key
    if authentication was successful, or raise an exception if it was not.
    The other end of the socket must be ready to receive the SRP
    commands."""

    if not passphrase:
        passphrase = getpass.getpass('Enter passphrase for %s: ' % user)

    # Send the USER command.

    sock.send('USER %s\n' % user)

    # Get the client-side keys and send the public one.

    keys = SRP.client_begin(user)
    A = keys[0]
    sock.send(encode_long(A))

    # Read the response.

    file = sock.makefile('rb')
    line = file.readline()
    if line[0:3] != 'KEY':
        raise SRP.NoSuchUser, line
    s = read_string(file)
    B = read_long(file)
    u = read_long(file)

    # Now calculate the session key and send the proof.

    K, m = SRP.client_key(user, passphrase, s, B, u, keys)
    sock.send(encode_string(m))
    line = file.readline()
    if line[0:3] != 'AOK':
        raise SRP.AuthFailure, line

    # Authenticate the host.

    m1 = SRP.host_authenticator(K, A, m)
    m = read_string(file)
    if m != m1:
        raise SRP.AuthFailure, "Host authentication failed."

    # All done, return the session key.

    return K
示例#12
0
def SRPAuth(sock, user, passphrase = None):
    """Perform an SRP authentication on a socket.  Return the session key
    if authentication was successful, or raise an exception if it was not.
    The other end of the socket must be ready to receive the SRP
    commands."""

    if not passphrase:
	passphrase = getpass.getpass('Enter passphrase for %s: ' % user)

    # Send the USER command.

    sock.send('USER %s\n' % user)

    # Get the client-side keys and send the public one.

    keys = SRP.client_begin(user)
    A = keys[0]
    sock.send(encode_long(A))

    # Read the response.

    file = sock.makefile('rb')
    line = file.readline()
    if line[0:3] != 'KEY':
	raise SRP.NoSuchUser, line
    s = read_string(file)
    B = read_long(file)
    u = read_long(file)

    # Now calculate the session key and send the proof.

    K, m = SRP.client_key(user, passphrase, s, B, u, keys)
    sock.send(encode_string(m))
    line = file.readline()
    if line[0:3] != 'AOK':
	raise SRP.AuthFailure, line

    # Authenticate the host.

    m1 = SRP.host_authenticator(K, A, m)
    m = read_string(file)
    if m != m1:
	raise SRP.AuthFailure, "Host authentication failed."

    # All done, return the session key.

    return K
示例#13
0
 def _srp_auth(self, s, msg):
     socket = self.socket[s]
     pw = socket['pw']
     srp = socket['srp'] = {}
     srp['B'], srp['u'], srp['K'], srp['m'] = SRP.host_begin(msg['user'], msg['A'], pw['s'], pw['v'])
     srp['A'] = msg['A']
     self._send_msg(s, {'s': pw['s'], 'B': srp['B'], 'u': srp['u']})
     socket['state'] = 1
示例#14
0
 def set_password(self, s, password):
     socket = self.socket[s]
     salt, v = SRP.new_passwd(self.co.user, password)
     cypherv = crypt(long_to_string(v), socket['key'])[0]
     self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv})
     socket['state'] = 5
     self.rs.listen_forever(self.nh)
     self.rs.doneflag.clear()
示例#15
0
    def test_ascii(self):
        hasher = SRP.SRP(6)
        hello_world = hasher.stable_transform("hello world",
                                              log=False,
                                              unit_length=False)

        self.assertEqual(hello_world.tolist(),
                         np.array([0., 0., 2., 0., 2., 0.]).tolist())
示例#16
0
 def set_password(self, s, password):
     socket = self.socket[s]
     salt, v = SRP.new_passwd(self.co.user, password)
     cypherv = crypt(long_to_string(v), socket['key'])[0]
     self._send_msg(s, {'op': 'set password', 's': salt, 'v': cypherv})
     socket['state'] = 5
     self.rs.listen_forever(self.nh)
     self.rs.doneflag.clear()
示例#17
0
def SRP_transform(f):
    global hasher
    if hasher is None:
        hasher = SRP.SRP(640)
    return hasher.stable_transform(words=f['token'],
                                   counts=f['count'],
                                   log=True,
                                   standardize=True)
示例#18
0
    def test_ascii_equals_unicode(self):
        hasher = SRP.SRP(160)

        hello_world = hasher.stable_transform("hello world",
                                              log=False).tolist()
        hello_world_unicode = hasher.stable_transform(u"hello world",
                                                      log=False).tolist()

        self.assertEqual(hello_world, hello_world_unicode)
示例#19
0
 def srp_auth(self, s):
     socket = self.socket[s]
     assert socket['state'] == 0 or socket['state'] == 3
     srp = socket['srp'] = {}
     srp['keys'] = SRP.client_begin()
     self._send_msg(s, {'op': 'srp auth', 'user': self.co.user, 'A': srp['keys'][0]})
     socket['state'] = 1
     self.rs.listen_forever(self.nh)
     self.rs.doneflag.clear()
示例#20
0
    def make_testfile(self, path, array=None):
        if array is None:
            array = self.test_set

        with SRP.Vector_file(path, dims=3, mode="w") as testfile:
            for row in self.test_set:
                if row[0] == "stop":
                    continue
                testfile.add_row(*row)
示例#21
0
 def _srp_auth(self, s, msg):
     socket = self.socket[s]
     pw = socket['pw']
     srp = socket['srp'] = {}
     srp['B'], srp['u'], srp['K'], srp['m'] = SRP.host_begin(
         msg['user'], msg['A'], pw['s'], pw['v'])
     srp['A'] = msg['A']
     self._send_msg(s, {'s': pw['s'], 'B': srp['B'], 'u': srp['u']})
     socket['state'] = 1
示例#22
0
    def test_logs_are_plausible(self):
        log_unit = np.log(1e05)

        hasher = SRP.SRP(20)
        log_srp = hasher.stable_transform("hello", log=True)
        nonlog_srp = hasher.stable_transform("hello", log=False)
        difference = sum(log_srp - (nonlog_srp) * log_unit)

        # Forgive floating point error.
        self.assertTrue(difference < 1e-05)
示例#23
0
    def test_wordcounts_unicode(self):
        hasher = SRP.SRP(160)

        wordcount_style = hasher.stable_transform(words=[u"Güten", u"Tag"],
                                                  counts=[1, 1],
                                                  log=False).tolist()

        string_style = hasher.stable_transform(words=u"Güten Tag",
                                               log=False).tolist()

        self.assertEqual(wordcount_style, string_style)
示例#24
0
    def test_creation_and_reading(self):
        with tempfile.TemporaryDirectory() as dir:
            testloc = Path(dir, "test.bin")
            self.make_testfile(testloc)

            testfile2 = SRP.Vector_file(Path(dir, "test.bin"),
                                        dims=3,
                                        mode="a")
            testfile2.add_row(*self.test_set[3])
            testfile2.close()
            self.assertTrue(testfile2.nrows == 4)

            reloaded = SRP.Vector_file(Path(dir, "test.bin"), mode="r")
            self.assertTrue(reloaded.vocab_size == 4)

            # Test Iteration
            read_in_values = dict()
            for (i, (name, array)) in enumerate(reloaded):
                read_in_values[name] = array
                (comp_name, comp_array) = self.test_set[i]
                self.assertEqual(comp_name, name)
                self.assertEqual(array.tolist(), comp_array.tolist())

            # Individual Vec Getter
            keys = [i for i, a in self.test_set]
            for i, key in enumerate(keys):
                vec = reloaded[key]
                np.testing.assert_array_almost_equal(vec, self.test_set[i][1])

            # List Vec Getter
            vecs = reloaded[keys]
            self.assertEqual(vecs.shape, (len(keys), reloaded.dims))
            for i in range(len(keys)):
                np.testing.assert_array_almost_equal(vecs[i],
                                                     self.test_set[i][1])
            reloaded.close()

            self.assertEqual(read_in_values["foo"].tolist(),
                             read_in_values["foo2"].tolist())
            self.assertFalse(read_in_values["foo2"].tolist() ==
                             read_in_values["fü"].tolist())
示例#25
0
def read_long(file):
    ll = []
    while 1:
        line = file.readline()
        if not line:
            raise EOFError
        l = string.strip(line)
        if not l:
            break
        ll.append(l)
    val = SRP.string_to_long(base64.decodestring(string.join(ll, '')))
    return val
示例#26
0
 def test_null_vectors(self):
     """
     Not clear that these *should* be supported, but apparently they are.
     """
     with tempfile.TemporaryDirectory() as dir:
         p = Path(dir, "test.bin")
         fout = SRP.Vector_file(p, dims=0, mode='a')
         fout.add_row("good", np.array([], '<f4'))
         fout.flush()
         with p.open() as fin:
             self.assertEqual(fin.read(), '000000001 00000\ngood \n')
         fout.close()
示例#27
0
def read_long(file):
    ll = []
    while 1:
	line = file.readline()
	if not line:
	    raise EOFError
	l = string.strip(line)
	if not l:
	    break
	ll.append(l)
    val = SRP.string_to_long(base64.decodestring(string.join(ll, '')))
    return val
示例#28
0
 def srp_auth(self, s):
     socket = self.socket[s]
     assert socket['state'] == 0 or socket['state'] == 3
     srp = socket['srp'] = {}
     srp['keys'] = SRP.client_begin()
     self._send_msg(s, {
         'op': 'srp auth',
         'user': self.co.user,
         'A': srp['keys'][0]
     })
     socket['state'] = 1
     self.rs.listen_forever(self.nh)
     self.rs.doneflag.clear()
示例#29
0
    def test_entrance_format(self):
        with tempfile.TemporaryDirectory() as dir:
            self.make_testfile(Path(dir, "test.bin"))

            with SRP.Vector_file(Path(dir, "test.bin"), dims=3,
                                 mode="a") as testfile2:
                testfile2.add_row(*self.test_set[3])

            with SRP.Vector_file(Path(dir, "test2.bin"), dims=3,
                                 mode="a") as a:
                a.concatenate_file(Path(dir, "test.bin"))
                a.concatenate_file(Path(dir, "test.bin"))

            with SRP.Vector_file(Path(dir, 'test2.bin')) as f:
                self.assertEqual(f.vocab_size, 8)
                with warnings.catch_warnings(record=True) as w:
                    warnings.simplefilter("always")
                    # Trigger a warning.
                    f['foo']
                    # Verify some things
                    assert "duplicate identifiers" in str(w[-1].message)

            self.assertTrue(testfile2.nrows == 4)
示例#30
0
 def test_standardization(self):
     """
     standardization does case normalization,
     and tokenizes by a charater regex.
     """
     hasher = SRP.SRP(6)
     string1 = "Gravity's rainbow"
     hashed_standardized = hasher.stable_transform(string1,
                                                   log=False,
                                                   standardize=True)
     manually_tokenized = ["Gravity", "s", "RAINBOW"]
     hashed_manually_tokenized = hasher.stable_transform(manually_tokenized,
                                                         [1, 1, 1],
                                                         log=False,
                                                         standardize=True)
     self.assertEqual(hashed_manually_tokenized.tolist(),
                      hashed_standardized.tolist())
示例#31
0
    def client_key(self, s, B, u, keys):
        use_agent = True
        if   self.pwid is None:
            use_agent = False
        elif not self._query_agent_for_id(self.pwid):
            use_agent = False

        if use_agent:
            func = self._private_key
            arg = None

        else:
            self._get_password()
            func = SRP.private_key
            arg = self.password

        K, m = SRP.client_key(self.user, arg, s, B, u, keys, func)
        return K, m
示例#32
0
def textset_to_srp(inputfile,
                   outputfile,
                   dim=640,
                   limit=float("Inf"),
                   log=True):
    """
    A convenience wrapper for converting a text corpus to
    an SRP collection as a block.

    inputfile is the collection. The format is the same as the ingest
    format used by bookworm and mallet; that is to say, a single unicode
    file where each line is a document represented as a unique filename, then a tab,
    and then a long string containing the text of the document.
    To coerce into the this format, newlines must be removed.

    inputfile can also be a **directory** of txt files.

    outputfile is the SRP file to be created. Recommended suffix is `.bin`.

    dims is the dimensionality of the output SRP.

    """
    import SRP

    output = Vector_file(outputfile, dims=dim, mode="w")
    hasher = SRP.SRP(dim=dim)

    if inputfile.endswith(".txt"):
        yielder = textset_yielder
    elif os.path.isdir(inputfile):
        yielder = directory_yielder
    else:
        raise ValueError(
            "Don't know how to process {}: must be a textfile or a directory".
            format(inputfile))

    for i, (id, txt) in enumerate(yielder(inputfile)):
        transform = hasher.stable_transform(txt, log=True, standardize=True)
        output.add_row(id, transform)
        if i + 1 >= limit:
            break

    output.close()
示例#33
0
def encode_long(val):
    s = base64.encodestring(SRP.long_to_string(val))
    return s + '\n'
def main():
    parser = argparse.ArgumentParser(description="Convert Extracted Features files to vectors, and save in SRP's Vector_file format.")

    parser.add_argument('threadno', type=int, help='Non-zero indexed number of thread.')
    parser.add_argument('totalthreads', type=int, help='Number of threads running in total.')
    parser.add_argument('idlist', type=str, help='CSV file of HTIDs to process. Needs a header, with column name \'htid\'.')
    parser.add_argument('--outdir', '-o', type=str, default='data_outputs/', help='Directory to save results.')
    parser.add_argument('--chunksize', '-c', type=int, default=10000, help='Size of chunks to roll pages into. -1 will make books into a single full book "chunk".'')
    parser.add_argument('--no-srp', action='store_true', help='Turn off SRP saving')
    parser.add_argument('--no-glove', action='store_true', help='Turn off Glove saving')
    parser.add_argument('--glove-dims', '-g', type=int, default=300, help='Number of GloVe dimensions. Can be 50, 100, 200, or 300.')
    parser.add_argument('--srp-dims', '-s', type=int, default=640, help='Number of SRP dimensions.')
    parser.add_argument('--efdir', type=str, default=None,
                        help='Set an alternative base url for EF files, if not using the setting from local.yaml or ~/.htrc-config.yaml.')
    args = parser.parse_args()
    
    thread_no = args.threadno - 1 # Zero index.
    
    assert not (args.no_srp & args.no_glove)
    
    if args.efdir is not None:
        customizable_resolver.dir = args.efdir

    already_imported = already_imported_list(args.outdir)
    print("There are {} files already imported".format(len(already_imported)))
    filenames = pd.read_csv(args.idlist, low_memory = False)
    
    thread_name = "{}-of-{}_".format(thread_no + 1, args.totalthreads)

    already_seen_file = open(os.path.join(args.outdir, "already_completed_files{}.csv".format(thread_name)), "a")

    if not args.no_srp:
        hasher = SRP.SRP(args.srp_dims)
        out_SRP = SRP.Vector_file(os.path.join(args.outdir, thread_name + "SRP_chunks.bin"), dims=args.srp_dims, mode="w")
        
        def SRP_transform(f):
            return hasher.stable_transform(words = f['lowercase'], counts = f['count'], log = True, standardize = True)
    
    if not args.no_glove:
        wem_model = wem_loader('glove-wiki-gigaword-{}'.format(args.glove_dims))
        
        # Cross-ref with stoplist and drop stopped words
        from spacy.lang.en.stop_words import STOP_WORDS
        wem_vocab = set(wem_model.vocab.keys())
        wem_vocab = wem_vocab.difference(STOP_WORDS)
        
        out_glove = SRP.Vector_file(os.path.join(args.outdir, thread_name + "Glove_chunks.bin"), dims = args.glove_dims, mode="w")
        
        def WEM_transform(f):
            return transformations.chunk_to_wem(f, wem_model, vocab=wem_vocab, stop=False, log=True, min_ncount=10)

    books = 0
    last = None
    start_time = time.time()

    try:
        gen = yielder(filenames.htid, thread_no, args.totalthreads, 
                      chunk_size=args.chunksize, already_imported_list=already_imported)
        for i, (id, chunk, start, end, group) in enumerate(gen):
            # Count books too.
            if last != id:
                books += 1
                if last is not None:
                    already_seen_file.write("{}\n".format(last))
                    if (books % 25 == 0):
                        rate = books/(time.time()-start_time)
                        print("{} books done on thread {} of {}, {:.02f} chunks per book, {:.02f} books per second".format(books, thread_no + 1, args.totalthreads, i/books, rate))
            last = id

            id = "{}-{:04d}-{}-{}".format(id, chunk, start, end)

            if not args.no_srp:
                SRP_rep = SRP_transform(group)
                out_SRP.add_row(id, SRP_rep)

            if not args.no_glove:
                WEM_rep = WEM_transform(group)
                if WEM_rep.shape[0] != args.glove_dims:
                    print(WEM_rep.shape, args.glove_dims)
                try:
                    out_glove.add_row(id, WEM_rep.astype('<f4'))
                except:
                    print(id, WEM_rep.shape, args.glove_dims, wem_model.vector_size)
                    raise
                    
        already_seen_file.write("{}\n".format(last))

        if not args.no_srp:
            out_SRP.close()
        if not args.no_glove:
            out_glove.close()

    except KeyboardInterrupt:
        if not args.no_srp:
            out_SRP.close()
        if not args.no_glove:
            out_glove.close()
示例#35
0
    def message_came_in(self, s, data):
        socket = self.socket[s]
        try:
            msg = bdecode(data)
        except ValueError:
            self._send_error(s, None, 'garbage data')
            self._close(s)
            return
        if socket['state'] == 0:
            try:
                pw = socket['pw'] = self.passwd.get(msg['user'])
            except KeyError:
                self._send_error(s, None, 'Bad user')
                self._close(s)
                return
            socket['user'] = msg['user']
            if msg['op'] == 'get hash':
                self._send_msg(s, {'hash': sha.new('public hash check' + pw['secret']).digest()})
                socket['state'] = 3
            elif msg['op'] == 'secret auth':
                self._secret_auth(s)
            elif msg['op'] == 'srp auth':
                self._srp_auth(s, msg)
            else:
                self._close(s)
        elif socket['state'] == 1:
            srp = socket['srp']
            if srp['m'].digest() != msg['m']:
                self._send_error(s, None, 'Bad password')
                socket['state'] = 3
                return
            auth = SRP.host_authenticator(srp['K'], srp['A'], srp['m'].digest())
            self._send_msg(s, {'auth': auth.digest()})
            self.nh.set_hmac(s, srp['m'], auth)
            socket['state'] = 2
        elif socket['state'] == 2:
            srp = socket['srp']
            if msg['op'] == 'get secret':
                secret = socket['pw']['secret']
                esecret = crypt(secret, srp['K'])[0]
                self._send_msg(s, {'secret': esecret})
                socket['state'] = 3
            elif msg['op'] == 'set password':
                if socket['user'] == 'anonymous':
                    self._send_error(s, None, 'operation not permitted')
                    self._close(s)
                    return
                v = string_to_long(crypt(msg['v'], srp['K'])[0])
                self.passwd.define(socket['user'], v, msg['s'])
                self._send_msg(s, {'ok': 1})
                self._close(s)
        elif socket['state'] == 3:
            if msg['op'] == 'secret auth':
                self._secret_auth(s)
            elif msg['op'] == 'srp auth':
                self._srp_auth(s, msg)
            else:
                self._close(s)
        elif socket['state'] == 4:
            pw = socket['pw']

            if len(msg['salt']) < 20:
                self._send_error(s, None, 'Bad salt length')
                self._close(s)
                return

            if msg['salt'] + socket['salt'] == socket['salt'] + msg['salt']:
                self._send_error(s, None, 'Bad salt')
                self._close(s)
                return

            base = 'session key' + pw['secret'] + socket['salt'] + msg['salt']
            key = sha.new(base).digest()
            socket['m_in'] = hmac.new(key, '', sha)
            base = 'session key' + pw['secret'] + msg['salt'] + socket['salt']
            key = sha.new(base).digest()
            socket['m_out'] = hmac.new(key, '', sha)

            if msg['auth'] != socket['m_out'].digest():
                self._send_error(s, None, 'Bad password')
                socket['state'] = 3
                return

            self._send_msg(s, {'auth': socket['m_in'].digest()})
            self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
            self._req_mode(s, 1)
            self.socket[s] = [{}, {}, socket['user'], [], 1]
        else:
            self._close(s)
示例#36
0
    def _process_message(self, sock):
        status = self.hstatus[sock.fileno()]
        input = status['input']
        input.seek(status['in_offset'])
        data = input.read(4)
        if len(data) != 4:
            return

        msg_len, = struct.unpack('<i', data)
        if msg_len <= 0 or msg_len > 1024:
            self._write_error(sock, 'Bad message lenth')
            self._close_socket(sock)
            return
        data = input.read(msg_len)

        if len(data) < msg_len:
            return

        status['in_offset'] += 4 + msg_len

        try:
            msg = bdecode(data)
        except ValueError:
            self._write_error(sock, 'Bad message')
            self._close_socket(sock)
            return

        type = msg['type']

        if   type == 'CDV_AGENT_ADD_PASSWORD':
            id = sha.new('password' + msg['password']).digest()
            self.identities[id] = msg['password']
            self._write_answer(sock, bencode({'id': id}))

        elif type == 'CDV_AGENT_ADD_SECRET':
            id = sha.new('public hash check' + msg['secret']).digest()
            self.identities[id] = msg['secret']
            self._write_answer(sock, bencode({'id': id}))

        elif type == 'CDV_AGENT_ADD_ENCRYPTED_SECRET':
            if not self.identities.has_key(msg['id']):
                self._write_error(sock, 'No such identity')
                return
            secret = crypt(msg['secret'], self.identities[msg['id']])[0]

            id = sha.new('public hash check' + secret).digest()
            if id != msg['secret_id']:
                self._write_error(sock, 'id does not match')
                return

            self.identities[id] = secret
            self._write_answer(sock, bencode({}))

        elif type == 'CDV_AGENT_QUERY_IDENTITY':
            known = 0
            if self.identities.has_key(msg['id']):
                known = 1
            self._write_answer(sock, bencode({'known': known}))

        elif type == 'CDV_AGENT_SRP_PRIVATE_KEY':
            x = SRP.private_key(msg['user'], msg['s'], self.identities[msg['id']])
            self._write_answer(sock, bencode({'x': x}))

        elif type == 'CDV_AGENT_SESSION_KEY':
            if not self.identities.has_key(msg['id']):
                self._write_error(sock, 'No such identity')
                return

            if len(msg['salt1']) < 20 or len(msg['salt2']) < 20:
                self._write_error(sock, 'Bad salts')
                return

            if msg['salt1'] == msg['salt2']:
                self._write_error(sock, 'Bad salts')
                return

            base = 'session key' + self.identities[msg['id']] + \
                   msg['salt1'] + msg['salt2']
            key = sha.new(base).digest()
            answer = {'key': key}
            self._write_answer(sock, bencode(answer))

        elif type == 'CDV_AGENT_SHUTDOWN' and self.allow_shutdown:
            self.shutdown_flag = 1

        else:
            self._write_error(sock, 'Unknown command: ' + type)

        return
示例#37
0
 def test_error_on_load(self):
     with tempfile.TemporaryDirectory() as dir:
         testfile = SRP.Vector_file(Path(dir, "test.bin"), dims=3, mode="w")
         with self.assertRaises(TypeError):
             testfile.add_row("this is a space", self.array1)
         testfile.close()
示例#38
0
    iraf.apnorm1.gain = ")apnormalize.gain"
    iraf.apnorm1.lsigma = ")apnormalize.lsigma"
    iraf.apnorm1.usigma = ")apnormalize.usigma"

    #clean out the idiotic yet somehow nessacary database
    try:
        files_database = os.listdir('./database')
        for i in files_database:
            os.remove('./database/' + i)
    except OSError:
        print "No database to delete yet"

#trim all images of bias/overscan, and fix keywords

    all_img = list(flats + dark + obj + lamp)
    all_img2 = SRP.outputnames(all_img, 'temp')

    #mkcalib = raw_input("Create trimmed temp files (all)? (y/n): ").lower()

    #if "y" in mkcalib:
    #for x,i in enumerate(all_img):
    #iraf.imcopy(i+'[1:2048,1:2048]',all_img2[x])
    #fix DISPAXIS keyword
    #iraf.hedit(all_img2[x]+'[0]',"DISPAXIS",1,verify="no",add="yes")
    #elif "n" in mkcalib:
    #print "Skipping temporary files"

    #os.system("rm *temp*.fits")
    #os.system("rm *tnrm*.fits")

    print "Objects in data set:  "
示例#39
0
    def message_came_in(self, s, data):
        socket = self.socket[s]
        try:
            msg = bdecode(data)
        except ValueError:
            self._send_error(s, None, 'garbage data')
            self._close(s)
            return
        if socket['state'] == 0:
            try:
                pw = socket['pw'] = self.passwd.get(msg['user'])
            except KeyError:
                self._send_error(s, None, 'Bad user')
                self._close(s)
                return
            socket['user'] = msg['user']
            if msg['op'] == 'get hash':
                self._send_msg(s, {
                    'hash':
                    sha.new('public hash check' + pw['secret']).digest()
                })
                socket['state'] = 3
            elif msg['op'] == 'secret auth':
                self._secret_auth(s)
            elif msg['op'] == 'srp auth':
                self._srp_auth(s, msg)
            else:
                self._close(s)
        elif socket['state'] == 1:
            srp = socket['srp']
            if srp['m'].digest() != msg['m']:
                self._send_error(s, None, 'Bad password')
                socket['state'] = 3
                return
            auth = SRP.host_authenticator(srp['K'], srp['A'],
                                          srp['m'].digest())
            self._send_msg(s, {'auth': auth.digest()})
            self.nh.set_hmac(s, srp['m'], auth)
            socket['state'] = 2
        elif socket['state'] == 2:
            srp = socket['srp']
            if msg['op'] == 'get secret':
                secret = socket['pw']['secret']
                esecret = crypt(secret, srp['K'])[0]
                self._send_msg(s, {'secret': esecret})
                socket['state'] = 3
            elif msg['op'] == 'set password':
                if socket['user'] == 'anonymous':
                    self._send_error(s, None, 'operation not permitted')
                    self._close(s)
                    return
                v = string_to_long(crypt(msg['v'], srp['K'])[0])
                self.passwd.define(socket['user'], v, msg['s'])
                self._send_msg(s, {'ok': 1})
                self._close(s)
        elif socket['state'] == 3:
            if msg['op'] == 'secret auth':
                self._secret_auth(s)
            elif msg['op'] == 'srp auth':
                self._srp_auth(s, msg)
            else:
                self._close(s)
        elif socket['state'] == 4:
            pw = socket['pw']

            if len(msg['salt']) < 20:
                self._send_error(s, None, 'Bad salt length')
                self._close(s)
                return

            if msg['salt'] + socket['salt'] == socket['salt'] + msg['salt']:
                self._send_error(s, None, 'Bad salt')
                self._close(s)
                return

            base = 'session key' + pw['secret'] + socket['salt'] + msg['salt']
            key = sha.new(base).digest()
            socket['m_in'] = hmac.new(key, '', sha)
            base = 'session key' + pw['secret'] + msg['salt'] + socket['salt']
            key = sha.new(base).digest()
            socket['m_out'] = hmac.new(key, '', sha)

            if msg['auth'] != socket['m_out'].digest():
                self._send_error(s, None, 'Bad password')
                socket['state'] = 3
                return

            self._send_msg(s, {'auth': socket['m_in'].digest()})
            self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
            self._req_mode(s, 1)
            self.socket[s] = [{}, {}, socket['user'], [], 1]
        else:
            self._close(s)
示例#40
0
    def message_came_in(self, s, data):
        try:
            msg = bdecode(data)
        except ValueError:
            self.close(s)
            raise NetworkError, 'garbage data'
        if msg.has_key('error'):
            raise ServerError, msg['error']
        socket = self.socket[s]
        srp = socket['srp']
        if socket['state'] == 1:
            K, m = self.auth.client_key(msg['s'], msg['B'], msg['u'],
                                        srp['keys'])
            socket['key'], socket['m_out'] = K, m
            self._send_msg(s, {'m': socket['m_out'].digest()})
            socket['state'] = 2
        elif socket['state'] == 2:
            socket['m_in'] = SRP.host_authenticator(socket['key'], srp['keys'][0], socket['m_out'].digest())
            if socket['m_in'].digest() != msg['auth']:
                raise ServerError, 'Bad host authentication'
                return
            self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
            self.rs.doneflag.set()
        elif socket['state'] == 3:
            self.socket[s]['hash'] = msg['hash']
            self.rs.doneflag.set()
        elif socket['state'] == 4:
            self.close(s)
            secret = crypt(msg['secret'], socket['key'])[0]
            self.auth.save_secret(secret)
            self.rs.doneflag.set()
        elif socket['state'] == 5:
            self.close(s)
            self.rs.doneflag.set()
        elif socket['state'] == 6:
            if len(msg['salt']) < 20:
                self._send_error(s, None, 'Bad salt length')
                self.close(s)
                raise NetworkError, 'Bad salt from server'

            salt = random_string(20)

            key = self.auth.session_key(salt, msg['salt'])
            socket['m_in'] = hmac.new(key, '', sha)
            key = self.auth.session_key(msg['salt'], salt)
            socket['m_out'] = hmac.new(key, '', sha)

            self._send_msg(s, {'auth': socket['m_in'].digest(),
                               'salt': salt})
            socket['state'] = 7
        elif socket['state'] == 7:
            if msg['auth'] != socket['m_out'].digest():
                self._send_error(s, None, 'Bad auth')
                self.close(s)
                raise NetworkError, 'Bad server auth'
            self._req_mode(s, 1)
            self.nh.set_hmac(s, socket['m_in'], socket['m_out'])
            self.socket[s] = [{}, {}, {}, [], 1]
            self.rs.doneflag.set()
        else:
            self.close(s)
示例#41
0
    def handle(self):
	sock = self.request
	file = sock.makefile('rb')

	# Receive the username and public key A from the client.

	while 1:
	    line = file.readline()
	    if not line:
		return
	    if line[0:4] == 'USER':
		l = string.split(line)
		if len(l) == 2:
		    break
	    sock.send('Please specify USER.\n')
	user = l[1]
	try:
	    A = read_long(file)
	except EOFError:
	    return

	# Calculate the public and private values, and send the public stuff
	# to the client.

	try:
	    s, B, u, K, m = SRP.lookup(user, A)
	except SRP.NoSuchUser:
	    sock.send('No such user "%s".\n' % user)
	    return
	sock.send('KEY\n')
	sock.send(encode_string(s))
	sock.send(encode_long(B))
	sock.send(encode_long(u))

	# The client now sends us its proof.

	try:
	    m1 = read_string(file)
	except EOFError:
	    return
	if m != m1:
	    sock.send('Client authentication failed.\n')
	    return
	sock.send('AOK\n')

	# Send the host authentication proof.

	sock.send(encode_string(SRP.host_authenticator(K, A, m)))

	# At this point (assuming the client accepts the host authentication),
	# the socket has been authenticated and K is the secret session key.

	if self.__dict__.has_key('auth_socket'):
	    self.auth_socket(file, sock, K)
	else:

	    # Simple echo server.

	    while 1:
		s = file.readline()
		if not s:
		    return
		sock.send(s)
示例#42
0
 def test_dtype(self):
     hasher = SRP.SRP(6)
     hello_world = hasher.stable_transform("hello world", log=False)
     self.assertEqual(hello_world.dtype, np.float32)