Exemple #1
0
def attach_to_max(contents):
    """
    Defines commands to send to Max, establishes a connection to its commandPort,
    then sends the code to inject debugpy
    """

    global run_code
    config = contents['arguments']

    # Format the simulated attach response to send it back to the debugger
    # while we set up the debugpy in the background
    attach_code = ATTACH_TEMPLATE.format(
        debugpy_path=debugpy_path,
        hostname=config['debugpy']['host'],
        port=int(config['debugpy']['port'])
    )

    # Format RUN_TEMPLATE to point to the temporary
    # file containing the code to run
    run_code = RUN_TEMPLATE.format(
        dir=dirname(config['program']),
        file_name=split(config['program'])[1][:-3] or basename(split(config['program'])[0])[:-3]
    )

    # then send attach code
    log('Sending attach code to Max')
    send_py_code_to_max(attach_code)
    log('Successfully attached to Max')

    # Then start the max debugging threads
    run(start_debugging, ((config['debugpy']['host'], int(config['debugpy']['port'])),))
Exemple #2
0
def multiply_karatsuba(x, y):
    """
    Multiplies two numbers represented as arrays
    using the Karatsuba algorithm, falling back
    on grade school algorithm for the base case

    :param x: []int
    :param y: []int
    :rtype []int
    """
    x, y = match_padding(x, y)
    a, b = split(x)
    c, d = split(y)

    if len(x) == 1:
        return multiply_simple(x, y)

    res_1 = multiply_karatsuba(a, c)
    res_2 = multiply_karatsuba(b, d)

    partial = multiply_karatsuba(add(a, b), add(c, d))
    # res_3 is partial - res_1 - res_2.
    # To simplify, just add res_1 and res_2 then subtract that sum from partial
    res_3 = subtract(partial, add(res_1, res_2))

    res = add(pad(res_1, len(x), 'right'), res_2,
              pad(res_3, (len(x) + 1) // 2, 'right'))

    return res
Exemple #3
0
def main():
    import codecs
    import sys
    import itertools
    import math
    args = parse_args()
    ilines = [util.preprocess(x, args.lang) for x in codecs.open(args.input, 'r', 'utf-8').readlines()]
    rlines = [util.preprocess(x, args.lang) for x in codecs.open(args.ref, 'r', 'utf-8').readlines()]
    if len(ilines) != len(rlines):
        print("Error: input file has {0} lines, but reference has {1} lines.".format(len(ilines), len(rlines)))
        sys.exit(1)
    scores = []
    falign = open(args.align, 'w') if args.align is not None else None
    for lineno, (rline, iline) in enumerate(itertools.izip(ilines, rlines), start=1):
        if args.force_token_mode:
            rline, iline = rline.split(), iline.split()
        else:
            rline, iline = util.split(rline, args.lang), util.split(iline, args.lang)
        # iline, rline are list object
        score, alignment = ter(iline, rline, align=True)
        if args.align is not None:
            falign.write('%s\n' % ' '.join(alignment))
        scores.append(score)
        if args.verbose:
            print("Sentence {0}: {1:.4f}".format(lineno, score))
    if args.align is not None:
        falign.close()
    average = sum(scores) / len(scores)
    variance = sum((x - average) ** 2 for x in scores) / len(scores)
    stddev = math.sqrt(variance)
    print("Average={0:.4f}, Variance={1:.4f}, Standard Deviation={2:.4f}".format(average, variance, stddev))
def multiply_karatsuba_parallel(x, y, key=None):
    """
    Multiplies two numbers represented as arrays
    using the Karatsuba algorithm, falling back
    on grade school algorithm for the base case

    :param x: []int
    :param y: []int
    :rtype []int
    """
    x, y = match_padding(x, y)
    a, b = split(x)
    c, d = split(y)

    # for base case, go simple
    if len(x) == 1:
        return multiply_simple(x, y)

    # for big numbers, go parallel
    if len(x) > 300:
        # generate random ids for the subprocess outputs
        r1 = random.random()
        r2 = random.random()
        r3 = random.random()

        # run the sub-multiplications in parallel
        p1 = Process(target=multiply_karatsuba_parallel, args=[a, c, r1])
        p2 = Process(target=multiply_karatsuba_parallel, args=[b, d, r2])
        p3 = Process(target=multiply_karatsuba_parallel,
                     args=[add(a, b), add(c, d), r3])

        p1.start()
        p2.start()
        p3.start()
        p1.join()
        p2.join()
        p3.join()

        # get the results
        res_1 = return_dict[r1]
        res_2 = return_dict[r2]
        partial = return_dict[r3]

    # for smaller numbers, don't bother parallelizing
    else:
        res_1 = multiply_karatsuba_parallel(a, c)
        res_2 = multiply_karatsuba_parallel(b, d)
        partial = multiply_karatsuba_parallel(add(a, b), add(c, d))

    # do the karatsuba shuffle
    res_3 = subtract(partial, add(res_1, res_2))
    res = add(pad(res_1, len(x), 'right'), res_2,
              pad(res_3, (len(x) + 1) // 2, 'right'))

    # if we are in parallel mode, write the result to the global dict
    if key is not None:
        return_dict[key] = res

    return res
Exemple #5
0
def main():
    """
    main function to prepare data for Tiramisu algorithm
    """
    parser = argparse.ArgumentParser(
        description='reads image sets and augments the data for Tiramisu',
        prog='data_gen.py <args>')

    # Required arguments
    parser.add_argument("-i",
                        "--input",
                        required=True,
                        help="Path to image sets")
    parser.add_argument("-o",
                        "--output",
                        required=True,
                        help="Path to save test and train files")

    # Optional arguments
    parser.add_argument("-r",
                        "--ratio",
                        type=float,
                        default=0.2,
                        help="validation set ratio")

    # Creating required directories
    args = vars(parser.parse_args())
    if not os.path.exists(args['output'] + '/train/data/'):
        os.makedirs(args['output'] + '/train/data/')
    if not os.path.exists(args['output'] + '/validate/data/'):
        os.makedirs(args['output'] + '/validate/data/')
    if not os.path.exists(args['output'] + '/train/masks/'):
        os.makedirs(args['output'] + '/train/masks/')
    if not os.path.exists(args['output'] + '/validate/masks/'):
        os.makedirs(args['output'] + '/validate/masks/')
    if not os.path.exists(args['output'] + '/test/data/'):
        os.makedirs(args['output'] + '/test/data/')

    print("Creating an image per video...")
    combine(args['input'], args['output'])

    print("Generating a mask per video...")
    json_to_mask(args['input'], args['output'])

    print("augmenting the dataset...")
    slicer(args['output'])
    rotate(args['output'])
    transpose(args['output'])

    # Splitting the dataset into training and validation set
    split(args['output'], args['ratio'])
Exemple #6
0
    def generate(self, choices, offset):

        choices = choices.detach()
        b, n, s = choices.size()

        probs = offset[:, None, :].expand(b, n, s).clone()

        sel = 1.0 - probs.clone()[~ choices]
        probs[~ choices] = sel
        # probs[~ choices] = 1.0 - probs[~ choices].clone()
        # prob now contains the probability (under offset) of the choices made
        probs = probs.prod(dim=2, keepdim=True).expand(b, n, s).contiguous()

        # Generate indices from the chosen offset
        indices = util.split(choices, self.depth)

        if n > 1:
            dups = self.duplicates(indices)

            probs = probs.clone()
            probs[dups] = 0.0

        probs = probs / probs.sum(dim=1, keepdim=True)

        return indices, probs
Exemple #7
0
    def show_exits(self):
        exits = self.v_RS[self.v_RM]
        chars = util.split(exits)
        # print("> you can go : %s " % chars)
        ex = "+-----+\n"
        if "N" in chars:
            ex += "|  N  |\n"
        else:
            ex += "|     |\n"

        if "W" in chars:
            ex += "| W"
        else:
            ex += "|  "

        if "E" in chars:
            ex += " E |\n"
        else:
            ex += "   |\n"

        if "S" in chars:
            ex += "|  S  |\n"
        else:
            ex += "|     |\n"

        ex += "+-----+\n"
        print("> you can go :")
        print(ex)
Exemple #8
0
def sobel_filter(image):
    kernelx = np.array(([-1, 0, 1], [-2, 0, 2], [-1, 0, 1]))
    kernely = np.array(([-1, -2, -1], [0, 0, 0], [1, 2, 1]))

    if len(image.shape) == 2:
        gx = conv(image, kernelx)
        gy = conv(image, kernely)

        output = abs(gx) + abs(gy)  # np.sqrt(gx ** 2 + gy ** 2) slower
        output[np.where(output > MAX_PIXEL)] = MAX_PIXEL

        return output.astype(np.uint8)
    else:
        r, g, b = util.split(image)
        rx, ry = conv(r, kernelx), conv(r, kernely)
        gx, gy = conv(g, kernelx), conv(g, kernely)
        bx, by = conv(b, kernelx), conv(b, kernely)

        R = abs(rx) + abs(ry)
        G = abs(gx) + abs(gy)
        B = abs(bx) + abs(by)

        output = util.merge(R, G, B)
        output[np.where(output > MAX_PIXEL)] = MAX_PIXEL

        return output.astype(np.uint8)
Exemple #9
0
    def show_exits(self, playerLocation):
        exits = self.v_RS[playerLocation]
        chars = util.split(exits)
        # print("> you can go : %s " % chars)
        ex = "\t\t+-----+\n"
        if "N" in chars:
            ex += "\t\t|  N  |\n"
        else:
            ex += "\t\t|     |\n"

        if "W" in chars:
            ex += "\t\t| W"
        else:
            ex += "\t\t|  "

        if "E" in chars:
            ex += " E |\n"
        else:
            ex += "   |\n"

        if "S" in chars:
            ex += "\t\t|  S  |\n"
        else:
            ex += "\t\t|     |\n"

        ex += "\t\t+-----+\n"
        print("> you can go :")
        print(ex)
 def mkstemp(self, suffix="", prefix="tmp", dir=None, text=False):
     fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=self.join(dir), text=text)
     dname, fname = util.split(name)
     if dir:
         return fd, os.path.join(dir, fname)
     else:
         return fd, fname
Exemple #11
0
def harmonic_mean_filter_rgb(image, filter_size):
    r, g, b = util.split(image)
    R = harmonic_mean_filter(r, filter_size)
    G = harmonic_mean_filter(g, filter_size)
    B = harmonic_mean_filter(b, filter_size)
    output = util.merge(R, G, B)

    return output.astype(np.uint8)
Exemple #12
0
def median_filter_rgb(image, filter_size):
    r, g, b = util.split(image)
    R = median_filter(r, filter_size)
    G = median_filter(g, filter_size)
    B = median_filter(b, filter_size)
    output = util.merge(R, G, B)

    return output.astype(np.uint8)
Exemple #13
0
def contraharmonic_mean_filter_rgb(image, filter_size, Q=1):
    r, g, b = util.split(image)
    R = contraharmonic_mean_filter(r, filter_size, Q)
    G = contraharmonic_mean_filter(g, filter_size, Q)
    B = contraharmonic_mean_filter(b, filter_size, Q)
    output = util.merge(R, G, B)

    return output.astype(np.uint8)
Exemple #14
0
 def mkstemp(self, suffix='', prefix='tmp', dir=None, text=False):
     fd, name = tempfile.mkstemp(suffix=suffix, prefix=prefix,
                                 dir=self.join(dir), text=text)
     dname, fname = util.split(name)
     if dir:
         return fd, os.path.join(dir, fname)
     else:
         return fd, fname
Exemple #15
0
def canonpath(root, cwd, myname, auditor=None):
    '''return the canonical path of myname, given cwd and root'''
    if util.endswithsep(root):
        rootsep = root
    else:
        rootsep = root + os.sep
    name = myname
    if not os.path.isabs(name):
        name = os.path.join(root, cwd, name)
    name = os.path.normpath(name)
    if auditor is None:
        auditor = pathauditor(root)
    if name != rootsep and name.startswith(rootsep):
        name = name[len(rootsep):]
        auditor(name)
        return util.pconvert(name)
    elif name == root:
        return ''
    else:
        # Determine whether `name' is in the hierarchy at or beneath `root',
        # by iterating name=dirname(name) until that causes no change (can't
        # check name == '/', because that doesn't work on windows). The list
        # `rel' holds the reversed list of components making up the relative
        # file name we want.
        rel = []
        while True:
            try:
                s = util.samefile(name, root)
            except OSError:
                s = False
            if s:
                if not rel:
                    # name was actually the same as root (maybe a symlink)
                    return ''
                rel.reverse()
                name = os.path.join(*rel)
                auditor(name)
                return util.pconvert(name)
            dirname, basename = util.split(name)
            rel.append(basename)
            if dirname == name:
                break
            name = dirname

        # A common mistake is to use -R, but specify a file relative to the repo
        # instead of cwd.  Detect that case, and provide a hint to the user.
        hint = None
        try:
            if cwd != root:
                canonpath(root, root, myname, auditor)
                hint = (_("consider using '--cwd %s'")
                        % os.path.relpath(root, cwd))
        except util.Abort:
            pass

        raise util.Abort(_("%s not under root '%s'") % (myname, root),
                         hint=hint)
Exemple #16
0
 def delete(self, keys_to_delete):
     split_size = 1000
     keys_to_delete = util.split(keys_to_delete, split_size,
                                 self.__delete_wrapper)
     for set in keys_to_delete:
         params = dict({})
         params['Bucket'] = self.__bucket
         params['Delete'] = {"Objects": set}
         response = self.__client.delete_objects(**params)
Exemple #17
0
    def __call__(self,
                 path,
                 mode="r",
                 text=False,
                 atomictemp=False,
                 notindexed=False):
        '''Open ``path`` file, which is relative to vfs root.

        Newly created directories are marked as "not to be indexed by
        the content indexing service", if ``notindexed`` is specified
        for "write" mode access.
        '''
        if self._audit:
            r = util.checkosfilename(path)
            if r:
                raise util.Abort("%s: %r" % (r, path))
        self.audit(path)
        f = self.join(path)

        if not text and "b" not in mode:
            mode += "b"  # for that other OS

        nlink = -1
        if mode not in ('r', 'rb'):
            dirname, basename = util.split(f)
            # If basename is empty, then the path is malformed because it points
            # to a directory. Let the posixfile() call below raise IOError.
            if basename:
                if atomictemp:
                    util.ensuredirs(dirname, self.createmode, notindexed)
                    return util.atomictempfile(f, mode, self.createmode)
                try:
                    if 'w' in mode:
                        util.unlink(f)
                        nlink = 0
                    else:
                        # nlinks() may behave differently for files on Windows
                        # shares if the file is open.
                        fd = util.posixfile(f)
                        nlink = util.nlinks(f)
                        if nlink < 1:
                            nlink = 2  # force mktempcopy (issue1922)
                        fd.close()
                except (OSError, IOError) as e:
                    if e.errno != errno.ENOENT:
                        raise
                    nlink = 0
                    util.ensuredirs(dirname, self.createmode, notindexed)
                if nlink > 0:
                    if self._trustnlink is None:
                        self._trustnlink = nlink > 1 or util.checknlink(f)
                    if nlink > 1 or not self._trustnlink:
                        util.rename(util.mktempcopy(f), f)
        fp = util.posixfile(f, mode)
        if nlink == 0:
            self._fixfilemode(f)
        return fp
Exemple #18
0
 def delete_message_batch(self, metrics):
     start = time.time()      
     msgs_to_delete = set(self.__context[c.KEY_SUCCEEDED_MSG_IDS])  
     print "Total number of messages to delete are {}".format(len(msgs_to_delete))
     threadpool = self.__context[c.KEY_THREAD_POOL]        
     url = self.__queue_url    
     msgs_to_delete = util.split(msgs_to_delete, 10, self.__delete_wrapper)
     for msg_set in msgs_to_delete:            
         threadpool.add(retry.try_with_backoff, self.__context, self.__client.delete_message_batch, QueueUrl=url, Entries=msg_set)
     threadpool.wait()    
     return int(time.time() - start)
def separate_by_line():
    """At Mark's request, format phones as:

    <FILEID> phone1 phone2 phone3 (corresponding word) phone4 ...
    """
    destfile = path.join(outdir, 'phonbyline.txt')
    destgold = path.join(outdir, 'phongold.txt')
    fidmark = open(destfile, 'w')
    fidgold = open(destgold, 'w')
    s = []
    for pfile, wfile, _, _ in sorted(corpus.get_filesets()):
        plist = corpus.extract_content(pfile, 'phones')
        psplit = util.split(plist, lambda x: x[0] in SPLITS)
        wlist = corpus.extract_content(wfile, 'words')

        rtot = []
        for pspl in psplit:
            r = []
            for word, interval in wlist:
                phones = [x for x in pspl
                          if geq(x[1].start, interval.start)
                          and leq(x[1].end, interval.end)
                          and x[0] not in IGNORE]
                if phones:
                    r.append((word, interval, phones))
            if r:
                rtot.append(r)

        basename = path.splitext(path.basename(pfile))[0]

        for idx, split in enumerate(rtot):
            fidmark.write('{f}_{i}'.format(f=basename, i=idx))
            s.append(('{f}_{i}'.format(f=basename, i=idx), split))
            for word, interval, phones in split:
                fidmark.write(' ')
                fidmark.write(' '.join(zip(*phones)[0]))
                fidmark.write(' ')
                fidmark.write('({w})'.format(w=word))
            fidmark.write('\n')

            fidgold.write('{f}_{i}'.format(f=basename, i=idx))
            for word, interval, phones in split:
                fidgold.write(' ')
                fidgold.write(' '.join('{p} {i}'.format(p=p,
                                                        i=ival)
                                       for p, ival in phones))
                fidgold.write(' ')
                fidgold.write('({w} {i})'.format(w=word,
                                                 i=interval))
            fidgold.write('\n')
    fidmark.close()
    fidgold.close()
    with open(path.join(outdir, 'phongold.pkl'), 'wb') as fid:
        pickle.dump(s, fid, -1)
Exemple #20
0
def conv_filter(image, kernel):
    if len(image.shape) == 2:
        return conv(image, kernel)
    else:
        r, g, b = util.split(image)
        R = conv(r, kernel)
        G = conv(g, kernel)
        B = conv(b, kernel)
        output = util.merge(R, G, B)

        return output.astype(np.uint8)
Exemple #21
0
 def delete(self, keys_to_delete):
     split_size = 1000
     keys_to_delete = util.split(keys_to_delete, split_size,
                                 self.__delete_wrapper)
     for set in keys_to_delete:
         params = dict({})
         params['Bucket'] = self.__bucket
         params['Delete'] = {"Objects": set}
         util.debug_print("Deleting {}".format(params))
         response = self.__client.delete_objects(**params)
         util.debug_print("Delete response {}".format(response))
Exemple #22
0
def gaussian_filter(image, filter_size, sigma):
    kernel = gaussian_kernel(filter_size, sigma)
    if len(image.shape) == 2:
        return conv(image, kernel)
    else:
        r, g, b = util.split(image)
        R = conv(r, kernel)
        G = conv(g, kernel)
        B = conv(b, kernel)
        output = util.merge(R, G, B)

        return output.astype(np.uint8)
Exemple #23
0
def predict_via_iterations(xtrain, ytrain, xtest, ytest, regclass, speed, iterations, boundary):
    A, Y = csvio.load_train_data(xtrain, ytrain)
    A, Y = util.symmetric_permutation(A, Y)
    A_train, A_control = util.split(A, boundary)
    Y_train, Y_control = util.split(Y, boundary)

    regression = regclass(A_train, Y_train)
    losses, losses_control = train_via_iterations(regression, speed, iterations, A_control, Y_control)

    A_test = csvio.load_regressors(xtest)
    prediction = regression.predict(A_test)
    np.savetxt(ytest, prediction, fmt='%.5e', delimiter=',')

    plt.figure()
    plt.ylabel('loss')
    plt.xlabel('iterations')
    plt.plot(range(1, len(losses) + 1), losses, label='losses during studies')
    plt.plot(range(1, len(losses_control) + 1), losses_control, label='loss during control')
    plt.legend(loc='upper right', framealpha=0.95)
    plt.draw()
    plt.pause(0.001)
Exemple #24
0
    def __call__(self, path, mode="r", text=False, atomictemp=False,
                 notindexed=False):
        '''Open ``path`` file, which is relative to vfs root.

        Newly created directories are marked as "not to be indexed by
        the content indexing service", if ``notindexed`` is specified
        for "write" mode access.
        '''
        if self._audit:
            r = util.checkosfilename(path)
            if r:
                raise util.Abort("%s: %r" % (r, path))
        self.audit(path)
        f = self.join(path)

        if not text and "b" not in mode:
            mode += "b" # for that other OS

        nlink = -1
        if mode not in ('r', 'rb'):
            dirname, basename = util.split(f)
            # If basename is empty, then the path is malformed because it points
            # to a directory. Let the posixfile() call below raise IOError.
            if basename:
                if atomictemp:
                    util.ensuredirs(dirname, self.createmode, notindexed)
                    return util.atomictempfile(f, mode, self.createmode)
                try:
                    if 'w' in mode:
                        util.unlink(f)
                        nlink = 0
                    else:
                        # nlinks() may behave differently for files on Windows
                        # shares if the file is open.
                        fd = util.posixfile(f)
                        nlink = util.nlinks(f)
                        if nlink < 1:
                            nlink = 2 # force mktempcopy (issue1922)
                        fd.close()
                except (OSError, IOError) as e:
                    if e.errno != errno.ENOENT:
                        raise
                    nlink = 0
                    util.ensuredirs(dirname, self.createmode, notindexed)
                if nlink > 0:
                    if self._trustnlink is None:
                        self._trustnlink = nlink > 1 or util.checknlink(f)
                    if nlink > 1 or not self._trustnlink:
                        util.rename(util.mktempcopy(f), f)
        fp = util.posixfile(f, mode)
        if nlink == 0:
            self._fixfilemode(f)
        return fp
Exemple #25
0
def mean_filter(image, filter_size):
    kernel = np.ones((filter_size, filter_size)) * (1.0 / (filter_size**2))
    if len(image.shape) == 2:
        return conv(image, kernel)
    else:
        r, g, b = util.split(image)
        R = conv(r, kernel)
        G = conv(g, kernel)
        B = conv(b, kernel)
        output = util.merge(R, G, B)

        return output.astype(np.uint8)
def phongold():
    for phnfile, _, _, _ in get_filesets():
        bname = path.splitext(path.basename(phnfile))[0]
        for idx, pair in enumerate(util.split(extract_content(phnfile,
                                                              'phones'),
                                              lambda x: x[0] == '__')):
            try:
                phones, intervals = zip(*pair)
            except ValueError as e:
                print bname, pair
                raise e
            yield bname + '_{0}'.format(idx), phones, intervals
Exemple #27
0
def estimate(trainX, trainY, resample_num):
    sample_pos_means = []
    sample_pos_covs = []
    sample_neg_means = []
    sample_neg_covs = []

    for i in xrange(resample_num):
        [sampledX, sampledY] = util.resample(trainX, trainY)
        [positiveX, negativeX] = util.split(sampledX, sampledY)

        sample_pos_means.append(np.mean(positiveX, 0))
        sample_neg_means.append(np.mean(negativeX, 0))
        sample_pos_covs.append(np.cov(np.array(positiveX).T))
        sample_neg_covs.append(np.cov(np.array(negativeX).T))

    nominal_pos_mean = np.mean(sample_pos_means, 0)
    nominal_neg_mean = np.mean(sample_neg_means, 0)
    nominal_pos_cov = np.mean(sample_pos_covs, 0)
    nominal_neg_cov = np.mean(sample_neg_covs, 0)

    sample_pos_means_cov = np.cov(np.array(sample_pos_means).T)
    sample_neg_means_cov = np.cov(np.array(sample_neg_means).T)
    #log(sample_pos_means_cov)
    #log(sample_neg_means_cov)
    np.linalg.cholesky(sample_pos_means_cov +
                       np.eye(sample_pos_means_cov.shape[0]) * 1e-8)
    np.linalg.cholesky(sample_neg_means_cov +
                       np.eye(sample_neg_means_cov.shape[0]) * 1e-8)
    P_pos = np.linalg.inv(sample_pos_means_cov +
                          np.eye(sample_pos_means_cov.shape[0]) *
                          1e-8) / len(trainX)
    P_neg = np.linalg.inv(sample_neg_means_cov +
                          np.eye(sample_pos_means_cov.shape[0]) *
                          1e-8) / len(trainX)
    np.linalg.cholesky(P_pos + np.eye(sample_neg_means_cov.shape[0]) * 1e-3)
    np.linalg.cholesky(P_neg + np.eye(sample_neg_means_cov.shape[0]) * 1e-3)

    rho_pos = 0
    rho_neg = 0

    for cov_matrix in sample_pos_covs:
        dis = util.F_norm(cov_matrix - nominal_pos_cov)
        rho_pos = max(dis, rho_pos)

    for cov_matrix in sample_neg_covs:
        dis = util.F_norm(cov_matrix - nominal_neg_cov)
        rho_neg = max(dis, rho_neg)

    return [
        nominal_pos_mean, P_pos, nominal_neg_mean, P_neg, nominal_pos_cov,
        rho_pos, nominal_neg_cov, rho_neg
    ]
Exemple #28
0
 def delete_message_batch(self, metrics, queue_url=None):
     start = time.time()
     msgs_to_delete = set(metrics)
     threadpool = self.__context[c.KEY_THREAD_POOL]
     url = queue_url or self.__queue_url
     msgs_to_delete = util.split(msgs_to_delete, 10, self.__delete_wrapper)
     for msg_set in msgs_to_delete:
         threadpool.add(retry.try_with_backoff,
                        self.__context,
                        self.__client.delete_message_batch,
                        QueueUrl=url,
                        Entries=msg_set)
     threadpool.wait()
     return int(time.time() - start)
Exemple #29
0
def prepare_data(df):
    df = u.shuffle(df, 999)
    X, Y = u.xy(df)

    scaler = preprocessing.MinMaxScaler()
    scaler.fit(X)
    df_tr, df_te = u.split(df, 0.75)

    X_tr, Y_tr = u.xy(df_tr)
    X_te, Y_te = u.xy(df_te)

    X_te_norm = scaler.transform(X_te)
    X_tr_norm = scaler.transform(X_tr)
    return X_tr_norm, Y_tr, X_te_norm, Y_te
Exemple #30
0
	def on_btnGenSql_clicked(self,widget):
		''' SQL생성.'''
		tb = self.tbColEng
		mydict = util.load_file("dict.txt")
		assert isinstance(tb , gtk.TextBuffer)
		s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
		cols = util.split(s)
		#self.log(crud.mk_insert(cols))
		#self.log('\n\n')
		self.log(crud.mk_insert2(mydict, cols))
		self.log('\n\n')
		self.log(crud.mk_update(mydict, cols))
		self.log('\n\n')
		self.log(crud.mk_select(mydict, cols))
		self.log('\n\n')
Exemple #31
0
def laplacian_filter(image, diagonal=True):
    if diagonal:
        kernel = np.array(([-1, -1, -1], [-1, 8, -1], [-1, -1, -1]))
    else:
        kernel = np.array(([0, 1, 0], [1, -4, 1], [0, 1, 0]))

    if len(image.shape) == 2:
        return conv(image, kernel)
    else:
        r, g, b = util.split(image)
        R = conv(r, kernel)
        G = conv(g, kernel)
        B = conv(b, kernel)
        output = util.merge(R, G, B)

        return output.astype(np.uint8)
Exemple #32
0
def prepare_data(df):
    df = u.shuffle(df, 999)
    df_train, df_test = u.split(df, 0.75)

    X_train, Y_train = u.xy(df_train)
    X_test, Y_test = u.xy(df_test)

    X_train = preprocessing.maxabs_scale(X_train)
    X_test = preprocessing.maxabs_scale(X_test)

    ones = np.ones((X_train.shape[0], 1))
    X_train = np.hstack((X_train, ones))

    ones = np.ones((X_test.shape[0], 1))
    X_test = np.hstack((X_test, ones))

    return X_train, Y_train, X_test, Y_test
Exemple #33
0
	def on_btnTransText_clicked(self, widget):
		self.log("텍스트변환시작\n")
		tb = self.tbColHan
		s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())


		cols = util.split(s)
		msg =''
		for col in cols:
			camel = util.underscore_to_camel(col.upper())
			msg += col + '\n'
			msg += util.camel_to_underscore(col).upper() + '\n'
			msg += camel + '\n'
			msg += camel[0:1].upper() + camel[1:] + '\n'
			msg += '\n'
		self.log(msg);
		self.log('\n\n')
Exemple #34
0
def canonpath(root, cwd, myname, auditor=None):
    """return the canonical path of myname, given cwd and root"""
    if util.endswithsep(root):
        rootsep = root
    else:
        rootsep = root + os.sep
    name = myname
    if not os.path.isabs(name):
        name = os.path.join(root, cwd, name)
    name = os.path.normpath(name)
    if auditor is None:
        auditor = pathauditor(root)
    if name != rootsep and name.startswith(rootsep):
        name = name[len(rootsep) :]
        auditor(name)
        return util.pconvert(name)
    elif name == root:
        return ""
    else:
        # Determine whether `name' is in the hierarchy at or beneath `root',
        # by iterating name=dirname(name) until that causes no change (can't
        # check name == '/', because that doesn't work on windows). The list
        # `rel' holds the reversed list of components making up the relative
        # file name we want.
        rel = []
        while True:
            try:
                s = util.samefile(name, root)
            except OSError:
                s = False
            if s:
                if not rel:
                    # name was actually the same as root (maybe a symlink)
                    return ""
                rel.reverse()
                name = os.path.join(*rel)
                auditor(name)
                return util.pconvert(name)
            dirname, basename = util.split(name)
            rel.append(basename)
            if dirname == name:
                break
            name = dirname

        raise util.Abort(_("%s not under root '%s'") % (myname, root))
Exemple #35
0
	def __mkFrame(self,frameType=0x10,frameId=0x01,
					destAdress16=[0xFF,0xFF],broadCastRadio=0x00,
					option=0x00):
		destAdress = self.__destAdress
		header = [0x7E,0x00,0x00,frameType,frameId]	
		if frameType == 0x10:
			frame = header
			frame += destAdress + destAdress16
			frame.append(broadCastRadio)
			frame.append(option)
			for i in self.__outBuffer:
				if type(i) == int:
					frame.append(i)
				else: 
					frame.append(ord(i))
			frame[1],frame[2] = split(len(frame[3:]))
			frame.append(self.__chkSum(frame))
			return frame
def estimate(trainX, trainY, resample_num):
	sample_pos_means = []
	sample_pos_covs = []
	sample_neg_means = []
	sample_neg_covs = []

	for i in xrange(resample_num):
		[sampledX, sampledY] = util.resample(trainX, trainY)
		[positiveX, negativeX] = util.split(sampledX, sampledY)

		sample_pos_means.append(np.mean(positiveX, 0))
		sample_neg_means.append(np.mean(negativeX, 0))
		sample_pos_covs.append(np.cov(np.array(positiveX).T))
		sample_neg_covs.append(np.cov(np.array(negativeX).T))

	nominal_pos_mean = np.mean(sample_pos_means, 0)
	nominal_neg_mean = np.mean(sample_neg_means, 0)
	nominal_pos_cov = np.mean(sample_pos_covs, 0)
	nominal_neg_cov = np.mean(sample_neg_covs, 0)

	sample_pos_means_cov = np.cov(np.array(sample_pos_means).T)
	sample_neg_means_cov = np.cov(np.array(sample_neg_means).T)
	#log(sample_pos_means_cov)
	#log(sample_neg_means_cov)
	np.linalg.cholesky(sample_pos_means_cov+ np.eye(sample_pos_means_cov.shape[0]) * 1e-8)
	np.linalg.cholesky(sample_neg_means_cov+ np.eye(sample_neg_means_cov.shape[0]) * 1e-8)
	P_pos = np.linalg.inv(sample_pos_means_cov + np.eye(sample_pos_means_cov.shape[0]) * 1e-8) / len(trainX)
	P_neg = np.linalg.inv(sample_neg_means_cov + np.eye(sample_pos_means_cov.shape[0]) * 1e-8) / len(trainX)
	np.linalg.cholesky(P_pos+ np.eye(sample_neg_means_cov.shape[0]) * 1e-3)
	np.linalg.cholesky(P_neg+ np.eye(sample_neg_means_cov.shape[0]) * 1e-3)

	rho_pos = 0
	rho_neg = 0

	for cov_matrix in sample_pos_covs:
		dis = util.F_norm(cov_matrix - nominal_pos_cov)
		rho_pos = max(dis, rho_pos)

	for cov_matrix in sample_neg_covs:
		dis = util.F_norm(cov_matrix - nominal_neg_cov)
		rho_neg = max(dis, rho_neg)

	return [nominal_pos_mean, P_pos, nominal_neg_mean, P_neg,
		nominal_pos_cov, rho_pos, nominal_neg_cov, rho_neg]
    def __call__(self, path, mode="r", text=False, atomictemp=False):
        if self._audit:
            r = util.checkosfilename(path)
            if r:
                raise util.Abort("%s: %r" % (r, path))
        self.audit(path)
        f = self.join(path)

        if not text and "b" not in mode:
            mode += "b"  # for that other OS

        nlink = -1
        if mode not in ("r", "rb"):
            dirname, basename = util.split(f)
            # If basename is empty, then the path is malformed because it points
            # to a directory. Let the posixfile() call below raise IOError.
            if basename:
                if atomictemp:
                    util.ensuredirs(dirname, self.createmode)
                    return util.atomictempfile(f, mode, self.createmode)
                try:
                    if "w" in mode:
                        util.unlink(f)
                        nlink = 0
                    else:
                        # nlinks() may behave differently for files on Windows
                        # shares if the file is open.
                        fd = util.posixfile(f)
                        nlink = util.nlinks(f)
                        if nlink < 1:
                            nlink = 2  # force mktempcopy (issue1922)
                        fd.close()
                except (OSError, IOError), e:
                    if e.errno != errno.ENOENT:
                        raise
                    nlink = 0
                    util.ensuredirs(dirname, self.createmode)
                if nlink > 0:
                    if self._trustnlink is None:
                        self._trustnlink = nlink > 1 or util.checknlink(f)
                    if nlink > 1 or not self._trustnlink:
                        util.rename(util.mktempcopy(f), f)
Exemple #38
0
    def __call__(self, path, mode="r", text=False, atomictemp=False):
        if self._audit:
            r = util.checkosfilename(path)
            if r:
                raise util.Abort("%s: %r" % (r, path))
        self.audit(path)
        f = self.join(path)

        if not text and "b" not in mode:
            mode += "b"  # for that other OS

        nlink = -1
        if mode not in ('r', 'rb'):
            dirname, basename = util.split(f)
            # If basename is empty, then the path is malformed because it points
            # to a directory. Let the posixfile() call below raise IOError.
            if basename:
                if atomictemp:
                    util.ensuredirs(dirname, self.createmode)
                    return util.atomictempfile(f, mode, self.createmode)
                try:
                    if 'w' in mode:
                        util.unlink(f)
                        nlink = 0
                    else:
                        # nlinks() may behave differently for files on Windows
                        # shares if the file is open.
                        fd = util.posixfile(f)
                        nlink = util.nlinks(f)
                        if nlink < 1:
                            nlink = 2  # force mktempcopy (issue1922)
                        fd.close()
                except (OSError, IOError), e:
                    if e.errno != errno.ENOENT:
                        raise
                    nlink = 0
                    util.ensuredirs(dirname, self.createmode)
                if nlink > 0:
                    if self._trustnlink is None:
                        self._trustnlink = nlink > 1 or util.checknlink(f)
                    if nlink > 1 or not self._trustnlink:
                        util.rename(util.mktempcopy(f), f)
Exemple #39
0
 def __mkFrame(self,
               frameType=0x10,
               frameId=0x01,
               destAdress16=[0xFF, 0xFF],
               broadCastRadio=0x00,
               option=0x00):
     destAdress = self.__destAdress
     header = [0x7E, 0x00, 0x00, frameType, frameId]
     if frameType == 0x10:
         frame = header
         frame += destAdress + destAdress16
         frame.append(broadCastRadio)
         frame.append(option)
         for i in self.__outBuffer:
             if type(i) == int:
                 frame.append(i)
             else:
                 frame.append(ord(i))
         frame[1], frame[2] = split(len(frame[3:]))
         frame.append(self.__chkSum(frame))
         return frame
Exemple #40
0
def build_structure(session, file_path, root=None):
    if not root: root = get_root(session)
    print "path to root: " + path_to_root
    print "file_path: " + file_path
    path = util.path_relative_to(path_to_root, file_path)
    pathsplit = util.split(path)
    file_segment = pathsplit.pop()
    cumulative_path = ""
    root.update_hash()
    parent = root
    for segment in pathsplit:
        cumulative_path = os.path.join(cumulative_path, segment)
        folder_q = session.query(Folder) \
                   .filter(Folder.path == cumulative_path)
        if folder_q.count():
            folder = folder_q.one()
            folder.update_hash()
            parent = folder
        else:
            parent = Folder(cumulative_path, parent)
    f = File(path, parent)
    add_to_artist(session, f)
Exemple #41
0
def build_structure(session, file_path, root = None):
    if not root: root = get_root(session)
    print "path to root: " + path_to_root
    print "file_path: " + file_path
    path = util.path_relative_to(path_to_root, file_path)
    pathsplit = util.split(path)
    file_segment = pathsplit.pop()
    cumulative_path = ""
    root.update_hash()
    parent = root
    for segment in pathsplit:
        cumulative_path = os.path.join(cumulative_path, segment)
        folder_q = session.query(Folder) \
                   .filter(Folder.path == cumulative_path)
        if folder_q.count():
            folder = folder_q.one()
            folder.update_hash()
            parent = folder
        else:
            parent = Folder(cumulative_path, parent)
    f = File(path, parent)
    add_to_artist(session, f)
Exemple #42
0
	def on_btnDefMap_clicked(self,widget):
		mydict = util.load_file("dict.txt")
		tb = self.tbColHan
		assert isinstance(tb , gtk.TextBuffer)
		s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
		a = s.splitlines()
		if( len(a) <2 ):
			a = util.split(s)
		msg = 'SQL의 SELECT구문을 이용하기 바랍니다\n\n'
		msg += 'BufMap defValMap = new BufMap();\n'
		msg += 'defValMap\n'
		for l in a:
			l = re.sub('/\*.*\*/','',l)
			l = l.strip()
			t = self.get_token(l)
			engname = t
			t = '"' + t + '"'
			hanname = util.eng_to_han_one(mydict,engname )
			msg += '\t\t.put(%-30s,"0")\t\t// %s\n' %(t,hanname)
		msg += '\t\t;\n'
		self.log("\n\n")
		self.log(msg)
		self.log("\n\n")
Exemple #43
0
	s += "</Rows>\n\n"
	return s

if __name__ == '__main__':
	pass

	
	#s ="col1,col2,col3,col4,col5,col6,col7"
	#r = """안양농원; 정읍시 고부면 고부리 1234;1,000; 혼파;100;홍길동;정읍시 고부면 고부리"""
	
	
	#s ="col1,col2,col3,col4,col5,col6,col7,col8,col9"
	cols = 12
	r = """A등급 100"""
	
	rows = util.split(r)
	
	#s = util.get_col_str(len(rows))
	
	s= 'DISEN_SN,BSNS_YEAR,IHIDNUM,BIZRNO,TLPHON_NO,MOBLPHON_NO,EMAIL,DELNG_BANK_CODE,ACNUT_NO,ACNUT_OWNER_NM,FXNUM,RPRSNTV_ID,RPRSNTV_TLPHON_NO,RPRSNTV_MOBLPHON_NO,FARMNG_MNGMTSYS_ID'
	
	cols = s.split(',')
	
	print "BINDING INFO...\n\n";
	
	
	
	
	
	print get_cell_bind(cols)
	
Exemple #44
0
Fichier : ui.py Projet : ikn/wvoas
 def __init__ (self, game, event_handler):
     self.game = game
     # input
     event_handler.add_event_handlers({
         pg.MOUSEBUTTONDOWN: self.click,
         pg.MOUSEMOTION: self.set_current_from_mouse
     })
     event_handler.add_key_handlers([
         (conf.KEYS_BACK, self.quit, eh.MODE_ONDOWN),
         (conf.KEYS_NEXT, self.select, eh.MODE_ONDOWN),
     ] + [
         (ks, [(self.move, (i,))], eh.MODE_ONDOWN_REPEAT, 15, 7)
          for i, ks in enumerate(conf.KEYS_MOVE)
     ])
     game.linear_fade(*conf.LS_FADE_IN)
     # generate unlocked list
     unlocked = []
     n_stars = sum(len(lvl.get('stars', [])) for lvl in conf.LEVELS)
     got_stars = 0
     for ID, i in conf.STARS:
         if len(conf.LEVELS) > ID and len(conf.LEVELS[ID].get('stars', [])) > i:
             got_stars += 1
     secret = [i for i in xrange(len(conf.LEVELS)) if i not in conf.EXISTS]
     require = split(n_stars, len(secret))
     req = 0
     for ID, this_req in zip(secret, require):
         req += this_req
         if got_stars >= req:
             unlocked.append(ID)
     if conf.DEBUG:
         print '{0}/{1} stars; need {2}'.format(got_stars, n_stars, require)
     # generate level thumbnails
     ids = set(conf.EXISTS + unlocked)
     self.num_levels = n = len(ids)
     self.cols = cols = min(i for i in xrange(n + 1) if i * i >= n)
     self.rows = rows = n / cols + bool(n % cols)
     self.levels = levels = []
     self.level_ids = level_ids = {}
     w, h = conf.RES
     ws = split(w, cols)
     hs = split(ir(h * float(rows) / cols), rows)
     x = w_i = h_i = 0
     y = (h - sum(hs)) / 2
     draw_sfc = pg.Surface(conf.RES)
     vertical_order = []
     row = []
     vertical_order.append(row)
     for j, i in enumerate(ids):
         row.append(i)
         rect = pg.Rect((x, y, ws[w_i], hs[h_i])).inflate(-2, -2)
         # generate image
         sfc = pg.Surface(rect[2:])
         l = level.Level(game, None, i)
         l.draw(draw_sfc)
         sfc = pg.transform.smoothscale(draw_sfc, rect[2:])
         # dim or brighten surface
         if i in conf.COMPLETED_LEVELS:
             mod_sfc = pg.Surface(rect[2:]).convert_alpha()
             mod_sfc.fill(conf.LS_WON_OVERLAY)
             sfc.blit(mod_sfc, (0, 0))
         level_ids[i] = j
         levels.append((i, rect, sfc.convert()))
         # get next rect
         x += ws[w_i]
         w_i += 1
         if w_i == cols:
             row = []
             vertical_order.append(row)
             x = w_i = 0
             y += hs[h_i]
             h_i += 1
     self.last = 0
     self.last_current = self.current = None
     self.changed = False
     self.set_current_from_mouse()
     self.vertical_order = sum([[row[i] for row in vertical_order if len(row) > i] for i in xrange(cols)], [])
     self.finished = False
Exemple #45
0
              'REMAINING', duration_to_string(remaining_time),
               'TOTAL', duration_to_string(total_time))

        frogged_filename = util.filename_without_extension(filename, '.txt')

        with open(OUTPUT_FOLDER+frogged_filename+'.frog.out', 'w') as f:
            f.write(output)

if __name__ == '__main__':
    INPUT_FOLDER = '../data/plaintext/'
    OUTPUT_FOLDER = '../data/frogged/'

    files = util.todo_filepaths(INPUT_FOLDER, '.txt', OUTPUT_FOLDER, '.frog.out')
    files = sorted(files)[::-1]
    if os.path.exists('../data/frog_todo.p'):
        print ("USING FROG TODO!")
        with open('../data/frog_todo.p', 'rb') as f:
            files = pickle.load(f)
            files = [s.replace('\\','/') for s in files]


    n_processes = 2
    print ("N_CPU", util.CPU_COUNT, " N PROCESSES", n_processes)

    file_chunks = util.split(files, n_processes)

    pool = Pool(processes=n_processes)
    pool.map(frog_process_files, file_chunks)
    pool.join()
    pool.close()
Exemple #46
0
def to_func(smpls, mtd):
  buf = cStringIO.StringIO()
  if C.mod.GN in mtd.mods: buf.write(C.mod.GN + ' ')
  elif C.mod.HN in mtd.mods: buf.write(C.mod.HN + ' ')
  ret_ty = trans_ty(mtd.typ)
  cname = unicode(repr(mtd.clazz))
  mname = mtd.name
  arg_typs = mtd.param_typs
  buf.write(ret_ty + ' ' + trans_mname(cname, mname, arg_typs) + '(')

  @takes(tuple_of(unicode))
  @returns(unicode)
  def trans_param( (ty, nm) ):
    return ' '.join([trans_ty(ty), nm])

  # for instance methods, add "this" pointer into parameters
  if mtd.is_static:
    params = mtd.params[:]
  else:
    self_ty = trans_ty(unicode(repr(mtd.clazz)))
    params = [ (self_ty, C.SK.self) ] + mtd.params[:]

  # add "logging" flag into parameters
  # to check log conformity only if invocations cross the boundary
  if not mtd.is_init and not mtd.is_clinit:
    params.append( (C.SK.z, u"logging") )

  if len(params) > 0:
    buf.write(", ".join(map(trans_param, params)))
  buf.write(") {\n")

  # once function signature is dumped out, remove "logging" flag
  if not mtd.is_init and not mtd.is_clinit:
    params.pop()

  clss = util.flatten_classes([mtd.clazz], "subs")
  logged = (not mtd.is_init) and sample.mtd_appears(smpls, clss, mtd.name)
  mid = unicode(repr(mtd))
  m_ent = mid + "_ent()"
  m_ext = mid + "_ext()"
  if logged:
    global _mids
    _mids.add(mid)

  if logged: # logging method entry (>)
    _log_params = map(log_param, params)
    _retrievals, _hashes = util.split([(u'', m_ent)] + _log_params)
    ent_retrievals = util.ffilter(_retrievals)
    ent_hashes = util.ffilter(_hashes)
    buf.write("""{}
      int[P] __params = {{ {} }};
      if (logging) check_log@log(__params);
    """.format(u''.join(ent_retrievals), u", ".join(ent_hashes)))

  is_void = C.J.v == mtd.typ
  if mtd.body:
    if not is_void and not mtd.is_init:
      bodies = mtd.body[:-1] # exclude the last 'return' statement
    else: bodies = mtd.body
    buf.write('\n'.join(map(partial(trans_s, mtd), bodies)))

  if logged: # logging method exit (<)
    _log_params = []
    if mtd.body and not is_void and not mtd.is_init:
      ret_v = mtd.body[-1].e
      ret_u = unicode(trans_e(mtd, ret_v))
      # retrieve the return value to a temporary variable
      buf.write(u"""
        {} __ret = {};
      """.format(ret_ty, ret_u))
      # then, try to obtain a hash from that temporary variable
      _log_params.append(log_param( (ret_ty, u"__ret") ))

    _retrievals, _hashes = util.split([(u'', m_ext)] + _log_params)
    ext_retrievals = util.ffilter(_retrievals)
    ext_hashes = util.ffilter(_hashes)
    buf.write("""{}
      __params = {{ {} }};
      if (logging) check_log@log(__params);
    """.format(u''.join(ext_retrievals), u", ".join(ext_hashes)))

  if mtd.body and not is_void and not mtd.is_init:
    buf.write(os.linesep)
    if logged:
      # return the return value stored at the temporary variable
      buf.write("return __ret;")
    else:
      buf.write(trans_s(mtd, mtd.body[-1]))

  if mtd.is_init:
    evt_srcs = map(util.sanitize_ty, sample.evt_sources(smpls))
    cname = unicode(repr(mtd.clazz))
    if cname in evt_srcs:
      global _inits
      _inits.add(cname)
    buf.write("\nreturn {};".format(C.SK.self))

  buf.write("\n}\n")
  return buf.getvalue()
Exemple #47
0
	def get_cols_han(self):
		tb = self.tbColHan
		assert isinstance(tb , gtk.TextBuffer)
		s = tb.get_text(tb.get_start_iter(), tb.get_end_iter())
		cols = util.split(s)
		return cols
Exemple #48
0
	s += "</Band>\n"
	
	s += "\n"
	return s


if __name__ == '__main__':
	pass

	
	#head = "등급,생산량(kg)"
	head = "등급	자부담비율(%)	건초->사일리지 변환 가중치	톤당 지원금액"
	
	
	# 탭이 있으면, 탭으로 분리하도록 처리함. 
	ahead = util.split(head)
		
	
	body = util.get_col_str(len(ahead))
	
	
	abody = body.split(",")
	
	
	
	print get_columns_bylen(ahead)
	
	# head band 
	
	
	print get_band_head(ahead)
def mainRobustFisherLDAtest(dataset, alpha, resample_num=100, split_token=','):
	data_file = dataset + '/' + dataset + '.data'
	data_loader = load.loader(file_name = data_file, split_token = split_token)
	[dataX, dataY] = data_loader.load()
	dimension = data_loader.dimension

	[trainX, trainY, testX, testY] = util.divide(dataX, dataY, alpha)

	[pos_mean, pos_P, neg_mean, neg_P, pos_cov, pos_rho, neg_cov, neg_rho] = estimate(trainX, trainY, resample_num)

	M = pos_cov + neg_cov + np.eye(dimension) * (pos_rho + neg_rho)
	M0 = np.linalg.inv(M)
	# minus = np.concatenate((np.eye(dimension), -np.eye(dimension)), axis = 1)
	# choose_pos = np.concatenate((np.eye(dimension), np.zeros([dimension, dimension])), axis = 1)
	# choose_neg = np.concatenate((np.zeros([dimension, dimension]), np.eye(dimension)), axis = 1)

	# M0 = np.dot(minus.T, np.dot(M, minus))
	# M1 = np.dot(choose_pos.T, np.dot(pos_P, choose_pos))
	# M2 = np.dot(choose_neg.T, np.dot(neg_P, choose_neg))

	# sol = QCQP.qcqprel(P = {'P0':matrix(M0), 'b0':None, 'c0':0.0},
	# 	G = {'P':[matrix(M1), matrix(M2)], 'b':[None] * 2, 'c':[0.0,] * 2,
	# 	'Peq':[], 'beq':[], 'ceq':[]})

	# sol_array = np.array(sol['RQCQPx'])
	# x_pos_star = sol_array[:dimension]
	# x_neg_star = sol_array[dimension:]
	# w = np.dot(M, x_pos_star - x_neg_star)

	M1 = pos_P
	M2 = neg_P
	[train_pos_X, train_neg_X] = util.split(trainX, trainY)
	k1 = np.mean(train_pos_X, axis = 0).reshape(dimension, 1)
	k2 = np.mean(train_neg_X, axis = 0).reshape(dimension, 1)
	k1 = k1 / np.linalg.norm(k1)
	k2 = k2 / np.linalg.norm(k2)
	k1_norm = util.M_norm(M1, k1)
	k2_norm = util.M_norm(M2, k2)
	x1 = k1 / k1_norm
	x2 = k2 / k2_norm
	pos_mean = pos_mean.reshape(dimension, 1)
	neg_mean = neg_mean.reshape(dimension, 1)

	while True:
		tail = np.dot(M0, x1 - x2 + pos_mean - neg_mean)
		k1_head = (np.eye(dimension) * k1_norm ** 2 - np.dot(M1, np.dot(k1, k1.T))) / (k1_norm ** 3)
		k2_head = - (np.eye(dimension) * k2_norm ** 2 - np.dot(M2, np.dot(k2, k2.T))) / (k2_norm ** 3)
		k1_gradient = np.dot(k1_head, tail)
		k2_gradient = np.dot(k2_head, tail)
		k1 -= k1_gradient * 0.01
		k2 -= k2_gradient * 0.01
		#print ('%.9f\t %.9f\t %.9f\t %.9f \t%.9f')% (util.M_norm(M0, x1 + pos_mean - x2 - neg_mean), np.linalg.norm(np.concatenate((k1_gradient, k2_gradient), axis = 0)), util.M_norm(M1, x1), util.M_norm(M2, x2), util.F_norm(x1 + pos_mean - x2 - neg_mean))
		if np.linalg.norm(np.concatenate((k1_gradient, k2_gradient), axis = 0)) < 1e-5:
			break
		k1_norm = util.M_norm(M1, k1)
		k2_norm = util.M_norm(M2, k2)
		x1 = k1 / k1_norm
		x2 = k2 / k2_norm

	w = np.dot(M0, x1 - x2 + pos_mean - neg_mean).reshape(dimension)

	train_pos_mean = np.mean(train_pos_X, axis = 0)
	train_neg_mean = np.mean(train_neg_X, axis = 0)
	threshold = np.dot(w, (train_pos_mean + train_neg_mean) / 2.0)
	positive_lower = True if np.dot(train_pos_mean - train_neg_mean, w) > 0 else False

	predict = np.zeros(len(testY))
	testNum = len(testY)
	for i in xrange(testNum):
		value = np.dot(testX[i], w)
		if (value > threshold) == positive_lower:
			predict[i] = 1
		else:
			predict[i] = -1

	rightNum = 0
	for i in xrange(testNum):
		if predict[i] == testY[i]:
			rightNum += 1

	#print 'Right Radio: %.5f'% (float(rightNum)/float(testNum))
	return float(rightNum)/float(testNum)
Exemple #50
0
 def takeModifiees(index):
     isHit = lambda key: isinstance(key, Hit)
     modifiees, self.undet[index:] = util.split(self.undet[index:], isHit)
     return modifiees
Exemple #51
0
	return s


def mk_select(mydict, cols):
	s = ""
	s += "SELECT \n"
	for col in cols:
		col = col.strip()
		hanname = util.eng_to_han_one(mydict, col)
		col += ','
		s += "\t{col:25} /* {hanname} */\n".format(col=col,hanname=hanname)
	s += "\nFROM table"
	s += "\nWHERE 1=1"
	s = util.rm_last_one_char(s, ',')
	return s

if __name__ == "__main__":
	c = 'DISEN_SN,BSNS_YEAR,IHIDNUM,BIZRNO,TLPHON_NO,MOBLPHON_NO,EMAIL,DELNG_BANK_CODE,ACNUT_NO,ACNUT_OWNER_NM,FXNUM,RPRSNTV_ID,RPRSNTV_TLPHON_NO,RPRSNTV_MOBLPHON_NO,FARMNG_MNGMTSYS_ID'
	c = 'BSNS_YEAR,AGBS_CODE,VILAGE_REQST_SN,CMPTINST_CODE,CMPTNC_LWPRT_INSTT_CODE,VILAGE_NM,RPRSN_TV_NM,RPRSN_TV_ADRES_CODE,RPRSN_TV_ADRES_DETAIL,RPRSN_TV_RN_ADRES_CODE,RPRSN_TV_EMD_SN,RPRSN_TV_RN_ADRES_DETAIL,REQST_DE,REQST_STTUS_CODE,CHARGER_OPINION,USE_AT,LEGALDONG_CODE,REGISTER_ID,REGIST_DT,UPDT_DT,UPDUSR_ID'
	cols = util.split(c)
	
	mydict = util.load_file("dict.txt")
	print ""
	print mk_insert(mydict, cols)
	print ""
	print mk_insert2(mydict, cols)
	print ""
	print mk_update(mydict, cols)
	print ""
	print mk_select(mydict, cols)
Exemple #52
0
 def __init__(self, *args, **kwargs):
     for u in self.url:
         exec "cfg.html_root.%s = self" % "%s.%s" % ('.'.join(u.split("/")[1:]), self.__class__.__name__)
         cfg.log("Registered page: %s.%s" % ('.'.join(u.split("/")[1:]), self.__class__.__name__))
import matplotlib.pyplot as plt
from pandas import DataFrame

if len(sys.argv)<2:
	print 'Usage: python plotData2.py <data_file> (Adim - Bdim)'

[X, Y] = load.loader(sys.argv[1]).load()
start = 0
dimension = len(X[0])
if len(sys.argv) == 3:
	parts = sys.argv[2].split('-')
	start = int(parts[0])
	dimension = int(parts[1]) - start + 1

row = int(math.floor(math.sqrt(dimension - 1))) + 1
[posX, negX] = util.split(X, Y)
print 'posNum = %d, negNum = %d'%(len(posX), len(negX))
if len(posX) < len(negX):
	gap = len(negX) - len(posX)
	for i in xrange(gap):
		pickup = random.randint(0, len(posX)-1)
		posX.append(posX[pickup])
else:
	gap = len(posX) - len(negX)
	for i in xrange(gap):
		pickup = random.randint(0, len(negX)-1)
		negX.append(negX[pickup])


posX = np.array(posX)
negX = np.array(negX)
Exemple #54
0
def cost(word, guess, guessHistory):
    found = util.findStr(util.split(word), guess)
    if len(found) > 0 or guess in guessHistory:
        return 0
    return 1
def format_phonrec():
    # load up the phone recognizer files
    phonrecdir = path.join(os.environ['HOME'],
                           'data/output/lrec_buckeye/phonrecoutput',
                           'res_tri200_simple_S_bigr/')
    phonrecinput = path.join(phonrecdir, 'refTranscriptions.mlf')
    phonrecoutput = path.join(phonrecdir, 'modelTranscriptions.mlf')
    prinput = corpus.readmlf(phonrecinput)
    proutput = corpus.readmlf(phonrecoutput)
    # index the input and output by filename
    prinput = {x[0]: (x[1], x[2])
               for x in prinput}
    proutput = {x[0]: (x[1], x[2])
                for x in proutput}

    # load up the gold transcriptions
    # with open(path.join(outdir, 'phongold.pkl'), 'rb') as fid:
    #     gold = pickle.load(fid)

    # build dict from fname to phoneseq to start time
    phondict = defaultdict(dict)
    worddict = defaultdict(list)
    # for e in gold:
    #     fname = e[0].split('_')[0]
    #     phonseq = tuple(map(corpus.fold, reduce(lambda x, y: x+y,
    #                                             (zip(*p)[0]
    #                                              for p in zip(*e[1])[2]))))
    #     start_time = e[1][0][1].start
    #     phondict[fname][phonseq] = start_time

    #     for word, interval in zip(zip(*e[1])[0], zip(*e[1])[1]):
    #         worddict[fname].append((word, interval))
    for pfile, wfile, _, _ in corpus.get_filesets():
        fname = path.splitext(path.basename(pfile))[0]
        plist = corpus.extract_content(pfile, 'phones', True)
        for s in util.split(plist, lambda x: x[0] == '__'):
            if s == []:
                continue
            phonseq, intervals = zip(*s)
            phondict[fname][tuple(phonseq)] = intervals[0].start

        wlist = corpus.extract_content(wfile, 'words')
        for word, interval in wlist:
            worddict[fname].append((word, interval))
    ns2sec = 0.0000001

    markfile = path.join(outdir, 'phonrec.txt')
    goldfile = path.join(outdir, 'phonrecgold.txt')
    fidmark = open(markfile, 'w')
    fidgold = open(goldfile, 'w')
    # now line them up
    rtot = []
    missing = 0
    for fname in sorted(proutput.keys()):
        bname = fname.split('_')[0]
        gold_phones, _ = prinput[fname]
        pred_phones, pred_intervals = proutput[fname]
        start_time = phondict[bname][tuple(gold_phones)]
        pred_intervals = [corpus.Interval(start_time + i.start * ns2sec,
                                          start_time + i.end * ns2sec)
                          for i in pred_intervals]

        words, word_intervals = zip(*worddict[bname])
        wrd_idx = 0
        phn_idx = 0
        # find the starting word
        for i in range(0, len(words)):
            if geq(word_intervals[i].end, pred_intervals[phn_idx].start) \
               and leq(word_intervals[i].start, pred_intervals[phn_idx].end):
                wrd_idx = i
                break
        else:
            for word, interval in zip(words, word_intervals):
                print interval, word
            print 'PHN:', pred_intervals[phn_idx]
            print fname
            print start_time
            raise ValueError('no suitable word interval found')
        r = [(words[wrd_idx], word_intervals[wrd_idx], [])]
        r_idx = 0
        while phn_idx < len(pred_phones):
            if leq(word_intervals[wrd_idx].end, pred_intervals[phn_idx].end):
                wrd_idx += 1
                if wrd_idx >= len(words):
                    break
                r.append((words[wrd_idx], word_intervals[wrd_idx], []))
                r_idx += 1
            r[r_idx][2].append((pred_phones[phn_idx], pred_intervals[phn_idx]))
            phn_idx += 1
        rtot.append(r)

        fidmark.write(fname)
        for word, interval, plist in r:
            if plist == []:
                continue
            fidmark.write(' ')
            fidmark.write(' '.join(zip(*plist)[0]))
            fidmark.write(' ')
            fidmark.write('({w})'.format(w=word))
        fidmark.write('\n')

        fidgold.write(fname)
        for word, interval, plist in r:
            if plist == []:
                continue
            fidgold.write(' ')
            fidgold.write(' '.join('{p} {i}'.format(p=p,
                                                    i=ival)
                                   for p, ival in plist))
            fidgold.write(' ')
            fidgold.write('({w} {i})'.format(w=word,
                                             i=interval))
        fidgold.write('\n')
    fidmark.close()
    fidgold.close()
    with open(path.join(outdir, 'phonrecoutgold.pkl'), 'wb') as fid:
        pickle.dump(rtot, fid, -1)
    print 'MISSING:', missing
Exemple #56
0
 def tokenize_text(self, text):
     "Runs the lexer on text and returns a list of lists of tokens"
     return map( self.tokenize_line, util.split(text) )
Exemple #57
0
	def __chkSum(self,data):
		chkSum = 0
		for i in data[3:]:
			chkSum += i
		chkSum = split(0xFF-chkSum)[1]
		return chkSum