예제 #1
0
    def _write_job_status(self, job, aborted=False):
        """
        Write the current job status as set in this object to job status dir
        and symlink them if needed.

        * Job status goes to state/_all/<uuid>.
        * Symlink goes from state/_all/<uuid> to state/<job_name>/uuid, unless
          aborted == True, in which case it is removed.
        * Symlink goes from FIXME
        """
        jobdef_dir = os.path.join(self.status_dir, 'jobs', job.jobdef_name)
        mkdir_p(jobdef_dir)  # If it doesn't exist yet
        job_status_path = os.path.join(self.all_dir, job.id)
        job_status_link = os.path.join(jobdef_dir, job.id)

        # Write the job status from the _all dir to the state/_all/<uuid> file
        status = job.to_dict()
        with open(job_status_path, 'w') as f:
            json.dump(status, f)

        # Link the job status from the _all dir to the state/<job_name>/<uuid>
        # file, unless the build was aborted.
        if not aborted:
            if not os.path.islink(job_status_link):
                os.symlink(os.path.join('..', '_all', job.id), job_status_link)
        else:
            logging.info("Removing status link")
            if os.path.islink(job_status_link):
                os.unlink(job_status_link)
예제 #2
0
 def on_discard(self, _event):
     name = self.names[self.index]
     day = tools.get_day()
     tools.mkdir_p(os.path.join('discard', day))
     os.rename(os.path.join('infiles', name),
               os.path.join('discard', day, name))
     self.load_next_image()
예제 #3
0
파일: train_lstm.py 프로젝트: gkovaig/wian
def error_analysis(model, ids, notes, text_features, X, onehot_Y, label, task):
    criteria, num_docs, clf = model

    if task in ['sapsii', 'age', 'los']:
        V = {}
        labels_ = [0] + criteria
        for i in range(len(labels_) - 1):
            label = '[%d,%s)' % (labels_[i], labels_[i + 1])
            V[i] = label
    else:
        V = {v: k for k, v in criteria.items()}

    # for confidence
    #P = clf.predict(list(X))
    P = clf.predict(X)
    pred = P.argmax(axis=1)
    Y = onehot_Y.argmax(axis=1)

    # convert predictions to right vs wrong
    confidence = {}
    for i, scores in enumerate(P.tolist()):
        prediction = pred[i]
        pid = ids[i]
        ind = Y[i]
        confidence[pid] = (scores[ind], scores, prediction, ind)

    homedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    methoddir = os.path.join(homedir, 'output', task, 'lstm')
    mkdir_p(methoddir)

    # order predictions by confidence
    for pid, conf in sorted(confidence.items(), key=lambda t: t[1]):
        if conf[2] == conf[3]:
            success = ''
        else:
            success = '_'
        filename = os.path.join(methoddir, '%s%s.pred' % (success, pid))
        with open(filename, 'w') as f:
            print >> f, ''
            print >> f, '=' * 80
            print >> f, ''
            print >> f, pid
            print >> f, 'scores:', conf[1]
            print >> f, 'pred:  ', V[conf[2]]
            print >> f, 'ref:   ', V[conf[3]]
            print >> f, '#' * 20
            print >> f, '#' * 20
            print >> f, 'SCORES'
            pind = conf[2]
            rind = conf[3]
            print >> f, '#' * 20
            for dt, category, text in sorted(notes[pid]):
                print >> f, dt
                print >> f, category
                print >> f, text
                print >> f, '-' * 50
            print >> f, ''
            print >> f, '+' * 80
            print >> f, ''
예제 #4
0
def process(infile, group_name, day, timeid, copies=1):
    mask = Mask((A4_WIDTH, A4_HEIGHT))
    page = mask.apply_mask(infile)
    create_title(page, group_name, day, timeid)
    tools.mkdir_p(os.path.join('png', day))
    png_file = os.path.join(
        'png', day, '{}_{}.jpg'.format(timeid,
                                       tools.safe_filename(group_name)))
    page.save(png_file, quality=75)
    tools.print_image(png_file, copies)
예제 #5
0
    def extract_templates(self):
        template_dpath = os.path.join(var.CONFIG_DPATH, 'templates')
        tools.mkdir_p(template_dpath)
        if not os.listdir(template_dpath):  # Is dir empty?
            self.load_panel(self.panel_firstrun)
            wx.Yield()
            import bg_templates_tar_bz2

            default_templates = bg_templates_tar_bz2.get_tarfile()
            default_templates.extractall(template_dpath)
        SetInitialDirectory(self.panel_main.file_picker_bg, template_dpath)
        wx.CallAfter(self.load_panel, self.panel_main)
예제 #6
0
 def on_restore(self, _event):
     cwd = os.getcwd()
     day = tools.get_day()
     initial_dir = os.path.join(cwd, 'discard', day)
     if not os.path.exists(initial_dir):
         initial_dir = os.path.join(cwd, 'discard')
         tools.mkdir_p(initial_dir)
     dlg = wx.lib.imagebrowser.ImageDialog(self, initial_dir)
     dlg.Centre()
     if dlg.ShowModal() == wx.ID_OK:
         self.filename = dlg.GetFile()
         self.static_image.load_from_file(self.filename)
     dlg.Destroy()
예제 #7
0
def process(infile, group_name, day, timeid):
    page = Image.open(os.path.join('infiles', infile))
    photo_size = (A4_WIDTH * 3 / 4, A4_HEIGHT * 3 / 4)
    photo_left = (A4_WIDTH - photo_size[0]) / 2
    photo_top = (A4_HEIGHT - photo_size[1]) / 2 - A4_HEIGHT / 64
    photo_right = photo_left + photo_size[0]
    photo_bottom = photo_top + photo_size[1]
    photo_rect = (photo_left, photo_top, photo_right, photo_bottom)
    create_title(page, (A4_WIDTH, A4_HEIGHT), photo_rect, group_name)
    tools.mkdir_p(os.path.join('png', day))
    png_file = os.path.join('png', day, '{}_{}.jpg'.format(timeid, tools.safe_filename(group_name)))
    page.save(png_file, quality=75)
    tools.print_image(png_file)
예제 #8
0
 def on_get_photos(self, _event):
     if not tools.mount_camera():
         self.SetStatusText('Could not connect to camera. Try again.')
     tools.get_camera_files()
     if tools.umount_camera():
         self.SetStatusText('You can disconnect the camera now.')
     else:
         self.SetStatusText('Could not disconnect from camera.')
     names = os.listdir('infiles')
     names.sort()
     day = tools.get_day()
     tools.mkdir_p(os.path.join('outfiles', day))
     self.names = names
     if len(self.names) > 0:
         self.load_image(0)
     else:
         self.next_btn.Disable()
         self.prev_btn.Disable()
         self.load_blank()
예제 #9
0
def process(infile, day, timeid):
    try:
        page = Image.open(os.path.join('base', 'Calendar_2014.png'))
    except IOError:
        print "Cannot open calendar page base"
        return
    photo_size = (A4_WIDTH * 3 / 4, A4_HEIGHT * 3 / 8)
    fade = Fade()
    photo = fade.apply_mask(infile, photo_size)
    photo_left = (A4_WIDTH - photo_size[0]) / 2
    photo_top = 520
    photo_right = photo_left + photo_size[0]
    photo_bottom = photo_top + photo_size[1]
    photo_rect = (photo_left, photo_top, photo_right, photo_bottom)
    page.paste(photo, photo_rect)
    create_title(page, (A4_WIDTH, A4_HEIGHT), photo_rect, day, timeid)
    tools.mkdir_p(os.path.join('png', day))
    png_file = os.path.join('png', day, '{}.jpg'.format(timeid))
    page.save(png_file, quality=75)
    tools.print_image(png_file)
예제 #10
0
def run():
    if not tools.mount_camera():
        print 'Could not connect to camera. Try again.'
    tools.get_camera_files()
    if tools.umount_camera():
        print 'You can disconnect the camera now.'
    else:
        print 'Could not disconnect from camera.'
    names = os.listdir('infiles')
    names.sort()
    day = tools.get_day()
    tools.mkdir_p(os.path.join('outfiles', day))
    for infile in names:
        group_name = raw_input('Group name: ')
        timeid = time.strftime('%H%M%S', time.localtime())
        process(os.path.join('infiles', infile), group_name, day, timeid)
        outfile_name = '{}_{}.jpg'.format(timeid,
                                          tools.safe_filename(group_name))
        os.rename(os.path.join('infiles', infile),
                  os.path.join('outfiles', day, outfile_name))
    print 'Finished.'
예제 #11
0
파일: unpack.py 프로젝트: TouchTone/pyjsit
def unpack(archive, targetdir = None, progress = None):

    # Archive type?
    atype = archive_type(archive)
    
    if progress:
        nfiles = float(len(get_file_list(archive)))
    
    if targetdir:
        if not os.path.isdir(targetdir):
            mkdir_p(targetdir)
        
        if atype == "rar":
            p = pipe_unrar("x", "-y", archive, targetdir)
        else:
            p = pipe_7z("x", "-y", "-o%s" % targetdir, archive)
    else:        
        if atype == "rar":
            p = pipe_unrar("x", "-y", archive)
        else:
            p = pipe_7z("x", "-y", archive)
        
    if progress:
        gfiles = 0
        name = ""
        for l in p.stdout.readlines():
            l = l.strip()
            if l.startswith("Extracting"):
                gfiles += 1
                fs = l.split(None, 1)
                if len(fs) > 0:
                    name = fs[-1]
                else:
                    name = ""
            progress(gfiles / nfiles, name)                
    
    p.wait()
    if p.returncode != 0:
        raise UnpackError("Caught error unpacking %s, return code %d" % (archive, p.returncode))
예제 #12
0
isfile = os.path.isfile
pj = os.path.join
sep = os.path.sep

import tools

def rec_glob(p, files):
    for d in glob.glob(p):
        if os.path.isfile(d):
            files.append(d)
        rec_glob("%s/*" % d, files)
    

outdir = sys.argv[1]

tools.mkdir_p(outdir)

print "Merging into ", outdir, ":",
sys.stdout.flush()

dfiles = {}
for d in sys.argv[2:]:
    f = []
    rec_glob("%s/*" % d, f)
    print "%s (%d files)" % (d, len(f)),
    dfiles[d] = f

print

for dir,files in dfiles.iteritems():
    for f in files:
예제 #13
0
def error_analysis(model, ids, notes, text_features, X, Y, label, task):
    criteria, num_docs, clf = model

    if task in ['sapsii', 'age', 'los']:
        V = {}
        labels_ = [0] + criteria
        for i in range(len(labels_)-1):
            label = '[%d,%s)' % (labels_[i],labels_[i+1])
            V[i] = label
    else:
        V = {v:k for k,v in criteria.items()}
    V[len(V)] = '**wrong**'

    # for confidence
    P_ = clf.decision_function(X)

    # sklearn has stupid changes in API when doing binary classification. make it conform to 3+
    if len(criteria)==2:
        m = X.shape[0]
        P = np.zeros((m,2))
        P[:,0] = -P_
        P[:,1] =  P_
    else:
        P = P_

    pred = P.argmax(axis=1)

    # convert predictions to right vs wrong
    confidence = {}
    for i,scores in enumerate(P.tolist()):
        prediction = pred[i]
        pid = ids[i]
        ind = Y[i]
        confidence[pid] = (scores[ind], scores, prediction, ind)

    homedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    methoddir = os.path.join(homedir, 'output', task, 'embeddings')
    mkdir_p(methoddir)

    # order predictions by confidence
    for pid,conf in sorted(confidence.items(), key=lambda t:t[1]):
        if conf[2] == conf[3]:
            success = ''
        else:
            success = '_'
        filename = os.path.join(methoddir, '%s%s.pred' % (success,pid))
        with open(filename, 'w') as f:
            print >>f, ''
            print >>f, '=' * 80
            print >>f, ''
            print >>f, pid
            print >>f, 'scores:', conf[1]
            print >>f, 'pred:  ', V[conf[2]]
            print >>f, 'ref:   ', V[conf[3]]
            print >>f, '#'*20
            print >>f, '#'*20
            print >>f, 'SCORES'
            pind = conf[2]
            rind = conf[3]
            print >>f, '#'*20
            for dt,category,text in sorted(notes[pid]):
                print >>f, dt
                print >>f, category
                print >>f, text
                print >>f, '-'*50
            print >>f, ''
            print >>f, '+'*80
            print >>f, ''
예제 #14
0
파일: build_corpus.py 프로젝트: wboag/wian
import re
import random

import psycopg2
import pandas as pd
import numpy as np

from tools import mkdir_p

# organization: data/$pid.txt (order: demographics, outcome, notes)
homedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
pickled_dir = os.path.join(homedir, 'data', 'pickled')
readable_dir = os.path.join(homedir, 'data', 'readable')

# store the pickled and readable notes
mkdir_p(pickled_dir)
mkdir_p(readable_dir)


def main():

    try:
        size = sys.argv[1]
        if size not in ['all', 'small']:
            raise Exception('bad')
    except Exception, e:
        print '\n\tusage: python %s <all|small>\n' % sys.argv[0]
        exit(1)

    X, Y = gather_data(size)
    assert sorted(X.keys()) == sorted(Y.keys())
예제 #15
0
 def createDir(self, dirPath, prefix):
     if dir and not os.path.isdir(dirPath) :
         self.verbose(prefix + "- Creating directory '{0}'".format(dirPath))
         tools.mkdir_p(dirPath)
         self.okay(prefix + "> OK")
예제 #16
0
 def _make_status_dir(self):
     """
     Create required directories under the `status_dir`.
     """
     mkdir_p(self.jobs_dir)
     mkdir_p(self.all_dir)
예제 #17
0
            line_index))

        cv2.rectangle(block_img, (lx, ly+lh/2), (lx+lw, ly+lh/2), (250, 0, 0), 1)
        #cv2.putText(block_img, str(line_index), (lx, ly+lh/2), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 255, 2)

    # Line Features:
    # - mass/aria (ink_volume)
    # - height
    # - word_count/width (words_ratio)

    return block_img

    #cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 255), 2)

if __name__ == '__main__':
    mkdir_p('lines')
    mkdir_p('blocks')
    eprint('filename,ink_volume,fiber_ink_volume,height,width,area,words_ratio,line_index')
    #find_text_block(sys.argv[1], 'marked.png', 'dilated.png')
    for page_index, page_name in enumerate(sys.argv[1:]):
        print('Page %d' % page_index)
        dilated, img, contours = find_text_block(page_name)
        #dilated, img = find_text_block(cv2.imread(sys.argv[1]),
        #                               cv2.imread('marked.png'),
        #                               cv2.imread('dilated.png'))

        cv2.imwrite('dilated%d.png' % page_index, dilated)
        cv2.imwrite('blocks%d.png' % page_index, img)

        #lines = find_lines_in_block('blocks.png', contours)
        for block_index, contour in enumerate(contours):
예제 #18
0
def write_jobfile(cmd, jobname, sbatchpath, scratchpath,
                  nodes=1, ppn=1, gpus=0, mem=16, nhours=18):
    """
    Create a job file.

    Args:
        cmd : str, Command to execute.
        jobname : str, Name of the job.
        sbatchpath : str, Directory to store SBATCH file in.
        scratchpath : str, Directory to store output files in.
        nodes : int, optional, Number of compute nodes.
        ppn : int, optional, Number of cores per node.
        gpus : int, optional, Number of GPU cores.
        mem : int, optional, Amount, in GB, of memory.
        ndays : int, optional, Running time, in days.
        queue : str, optional, Queue name.

    Returns:
        jobfile : str, Path to the job file.
    """

    tools.mkdir_p(sbatchpath)
    jobfile = os.path.join(sbatchpath, jobname + '.s')
    logname = os.path.join('log', jobname)

    if gpus == 0:
        with open(jobfile, 'w') as f:
            f.write(
                '#! /bin/bash\n'
                + '\n'
                + '#SBATCH --nodes={}\n'.format(nodes)
                #+ '#SBATCH --ntasks=1\n'
                + '#SBATCH --ntasks-per-node=1\n'
                + '#SBATCH --cpus-per-task={}\n'.format(ppn)
                + '#SBATCH --mem={}GB\n'.format(mem)
                + '#SBATCH --time={}:00:00\n'.format(nhours)
                + '#SBATCH --job-name={}\n'.format(jobname[0:16])
                + '#SBATCH --output={}log/{}.o\n'.format(scratchpath, jobname[0:16])
                + '\n'
                + 'cd {}\n'.format(scratchpath)
                + 'pwd > {}.log\n'.format(logname)
                + 'date >> {}.log\n'.format(logname)
                + 'which python >> {}.log\n'.format(logname)
                + '{} >> {}.log 2>&1\n'.format(cmd, logname)
                + '\n'
                + 'exit 0;\n'
                )
    else:
        with open(jobfile, 'w') as f:
            f.write(
                '#! /bin/bash\n'
                + '\n'
                + '#SBATCH --nodes={}\n'.format(nodes)
                + '#SBATCH --ntasks-per-node=1\n'
                + '#SBATCH --cpus-per-task={}\n'.format(ppn)
                + '#SBATCH --mem={}GB\n'.format(mem)
                + '#SBATCH --partition=xwang_gpu\n'
                + '#SBATCH --gres=gpu:1\n'
                + '#SBATCH --time={}:00:00\n'.format(nhours)
                + '#SBATCH --job-name={}\n'.format(jobname[0:16])
                + '#SBATCH --output={}log/{}.o\n'.format(scratchpath, jobname[0:16])
                + '\n'
                + 'cd {}\n'.format(scratchpath)
                + 'pwd > {}.log\n'.format(logname)
                + 'date >> {}.log\n'.format(logname)
                + 'which python >> {}.log\n'.format(logname)
                + '{} >> {}.log 2>&1\n'.format(cmd, logname)
                + '\n'
                + 'exit 0;\n'
                )
    return jobfile
예제 #19
0
파일: train.py 프로젝트: eiroW/FDM
def train_sequential(
    model_dir,
    rule_trains,
    hp=None,
    max_steps=1e7,
    display_step=500,
    ruleset='mante',
    seed=0,
):
    '''Train the network sequentially.

    Args:
        model_dir: str, training directory
        rule_trains: a list of list of tasks to train sequentially
        hp: dictionary of hyperparameters
        max_steps: int, maximum number of training steps for each list of tasks
        display_step: int, display steps
        ruleset: the set of rules to train
        seed: int, random seed to be used

    Returns:
        model is stored at model_dir/model.ckpt
        training configuration is stored at model_dir/hp.json
    '''

    tools.mkdir_p(model_dir)

    # Network parameters
    default_hp = get_default_hp(ruleset)
    if hp is not None:
        default_hp.update(hp)
    hp = default_hp
    hp['seed'] = seed
    hp['rng'] = np.random.RandomState(seed)
    hp['rule_trains'] = rule_trains
    # Get all rules by flattening the list of lists
    hp['rules'] = [r for rs in rule_trains for r in rs]

    # Number of training iterations for each rule
    rule_train_iters = [len(r) * max_steps for r in rule_trains]

    tools.save_hp(hp, model_dir)
    # Display hp
    for key, val in hp.items():
        print('{:20s} = '.format(key) + str(val))

    # Using continual learning or not
    c, ksi = hp['c_intsyn'], hp['ksi_intsyn']

    # Build the model
    model = Model(model_dir, hp=hp)

    grad_unreg = tf.gradients(model.cost_lsq, model.var_list)

    # Store results
    log = defaultdict(list)
    log['model_dir'] = model_dir

    # Record time
    t_start = time.time()

    # tensorboard summaries
    placeholders = list()
    for v_name in ['Omega0', 'omega0', 'vdelta']:
        for v in model.var_list:
            placeholder = tf.placeholder(tf.float32, shape=v.shape)
            tf.summary.histogram(v_name + '/' + v.name, placeholder)
            placeholders.append(placeholder)
    merged = tf.summary.merge_all()
    test_writer = tf.summary.FileWriter(model_dir + '/tb')

    def relu(x):
        return x * (x > 0.)

    # Use customized session that launches the graph as well
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # penalty on deviation from initial weight
        if hp['l2_weight_init'] > 0:
            raise NotImplementedError()

        # Looping
        step_total = 0
        for i_rule_train, rule_train in enumerate(hp['rule_trains']):
            step = 0

            # At the beginning of new tasks
            # Only if using intelligent synapses
            v_current = sess.run(model.var_list)

            if i_rule_train == 0:
                v_anc0 = v_current
                Omega0 = [np.zeros(v.shape, dtype='float32') for v in v_anc0]
                omega0 = [np.zeros(v.shape, dtype='float32') for v in v_anc0]
                v_delta = [np.zeros(v.shape, dtype='float32') for v in v_anc0]
            elif c > 0:
                v_anc0_prev = v_anc0
                v_anc0 = v_current
                v_delta = [
                    v - v_prev for v, v_prev in zip(v_anc0, v_anc0_prev)
                ]

                # Make sure all elements in omega0 are non-negative
                # Penalty
                Omega0 = [
                    relu(O + o / (v_d**2 + ksi))
                    for O, o, v_d in zip(Omega0, omega0, v_delta)
                ]

                # Update cost
                model.cost_reg = tf.constant(0.)
                for v, w, v_val in zip(model.var_list, Omega0, v_current):
                    model.cost_reg += c * tf.reduce_sum(
                        tf.multiply(tf.constant(w),
                                    tf.square(v - tf.constant(v_val))))
                model.set_optimizer()

            # Store Omega0 to tf summary
            feed_dict = dict(zip(placeholders, Omega0 + omega0 + v_delta))
            summary = sess.run(merged, feed_dict=feed_dict)
            test_writer.add_summary(summary, i_rule_train)

            # Reset
            omega0 = [np.zeros(v.shape, dtype='float32') for v in v_anc0]

            # Keep training until reach max iterations
            while (step * hp['batch_size_train'] <=
                   rule_train_iters[i_rule_train]):
                # Validation
                if step % display_step == 0:
                    trial = step_total * hp['batch_size_train']
                    log['trials'].append(trial)
                    log['times'].append(time.time() - t_start)
                    log['rule_now'].append(rule_train)
                    log = do_eval(sess, model, log, rule_train)
                    if log['perf_avg'][-1] > model.hp['target_perf']:
                        print('Perf reached the target: {:0.2f}'.format(
                            hp['target_perf']))
                        break

                # Training
                rule_train_now = hp['rng'].choice(rule_train)
                # Generate a random batch of trials.
                # Each batch has the same trial length
                trial = generate_trials(rule_train_now,
                                        hp,
                                        'random',
                                        batch_size=hp['batch_size_train'])

                # Generating feed_dict.
                feed_dict = tools.gen_feed_dict(model, trial, hp)

                # Continual learning with intelligent synapses
                v_prev = v_current

                # This will compute the gradient BEFORE train step
                _, v_grad = sess.run([model.train_step, grad_unreg],
                                     feed_dict=feed_dict)
                # Get the weight after train step
                v_current = sess.run(model.var_list)

                # Update synaptic importance
                omega0 = [
                    o - (v_c - v_p) * v_g for o, v_c, v_p, v_g in zip(
                        omega0, v_current, v_prev, v_grad)
                ]

                step += 1
                step_total += 1

        print("Optimization Finished!")
예제 #20
0
파일: train.py 프로젝트: eiroW/FDM
def train_rule_only(
    model_dir,
    rule_trains,
    max_steps,
    hp=None,
    ruleset='all',
    seed=0,
):
    '''Customized training function.

    The network sequentially but only train rule for the second set.
    First train the network to perform tasks in group 1, then train on group 2.
    When training group 2, only rule connections are being trained.

    Args:
        model_dir: str, training directory
        rule_trains: a list of list of tasks to train sequentially
        hp: dictionary of hyperparameters
        max_steps: int, maximum number of training steps for each list of tasks
        display_step: int, display steps
        ruleset: the set of rules to train
        seed: int, random seed to be used

    Returns:
        model is stored at model_dir/model.ckpt
        training configuration is stored at model_dir/hp.json
    '''

    tools.mkdir_p(model_dir)

    # Network parameters
    default_hp = get_default_hp(ruleset)
    if hp is not None:
        default_hp.update(hp)
    hp = default_hp
    hp['seed'] = seed
    hp['rng'] = np.random.RandomState(seed)
    hp['rule_trains'] = rule_trains
    # Get all rules by flattening the list of lists
    hp['rules'] = [r for rs in rule_trains for r in rs]

    # Number of training iterations for each rule
    if hasattr(max_steps, '__iter__'):
        rule_train_iters = max_steps
    else:
        rule_train_iters = [len(r) * max_steps for r in rule_trains]

    tools.save_hp(hp, model_dir)
    # Display hp
    for key, val in hp.items():
        print('{:20s} = '.format(key) + str(val))

    # Build the model
    model = Model(model_dir, hp=hp)

    # Store results
    log = defaultdict(list)
    log['model_dir'] = model_dir

    # Record time
    t_start = time.time()

    # Use customized session that launches the graph as well
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # penalty on deviation from initial weight
        if hp['l2_weight_init'] > 0:
            raise NotImplementedError()

        # Looping
        step_total = 0
        for i_rule_train, rule_train in enumerate(hp['rule_trains']):
            step = 0

            if i_rule_train == 0:
                display_step = 200
            else:
                display_step = 50

            if i_rule_train > 0:
                # var_list = [v for v in model.var_list
                #             if ('input' in v.name) and ('rnn' not in v.name)]
                var_list = [
                    v for v in model.var_list if 'rule_input' in v.name
                ]
                model.set_optimizer(var_list=var_list)

            # Keep training until reach max iterations
            while (step * hp['batch_size_train'] <=
                   rule_train_iters[i_rule_train]):
                # Validation
                if step % display_step == 0:
                    trial = step_total * hp['batch_size_train']
                    log['trials'].append(trial)
                    log['times'].append(time.time() - t_start)
                    log['rule_now'].append(rule_train)
                    log = do_eval(sess, model, log, rule_train)
                    if log['perf_avg'][-1] > model.hp['target_perf']:
                        print('Perf reached the target: {:0.2f}'.format(
                            hp['target_perf']))
                        break

                # Training
                rule_train_now = hp['rng'].choice(rule_train)
                # Generate a random batch of trials.
                # Each batch has the same trial length
                trial = generate_trials(rule_train_now,
                                        hp,
                                        'random',
                                        batch_size=hp['batch_size_train'])

                # Generating feed_dict.
                feed_dict = tools.gen_feed_dict(model, trial, hp)

                # This will compute the gradient BEFORE train step
                _ = sess.run(model.train_step, feed_dict=feed_dict)

                step += 1
                step_total += 1

        print("Optimization Finished!")
예제 #21
0
 def createDir(self, dirPath, prefix):
     if dir and not os.path.isdir(dirPath) :
         self.verbose(prefix + "- Creating directory '{0}'".format(dirPath))
         tools.mkdir_p(dirPath)
         self.okay(prefix + "> OK")
예제 #22
0
파일: train.py 프로젝트: eiroW/FDM
def train(
    model_dir,
    hp=None,
    max_steps=1e7,
    display_step=500,
    ruleset='mante',
    rule_trains=None,
    rule_prob_map=None,
    seed=0,
    rich_output=False,
    load_dir=None,
    trainables=None,
):
    """Train the network.

    Args:
        model_dir: str, training directory
        hp: dictionary of hyperparameters
        max_steps: int, maximum number of training steps
        display_step: int, display steps
        ruleset: the set of rules to train
        rule_trains: list of rules to train, if None then all rules possible
        rule_prob_map: None or dictionary of relative rule probability
        seed: int, random seed to be used

    Returns:
        model is stored at model_dir/model.ckpt
        training configuration is stored at model_dir/hp.json
    """

    tools.mkdir_p(model_dir)

    # Network parameters
    default_hp = get_default_hp(ruleset)
    if hp is not None:
        default_hp.update(hp)
    hp = default_hp
    hp['seed'] = seed
    hp['rng'] = np.random.RandomState(seed)

    # Rules to train and test. Rules in a set are trained together
    if rule_trains is None:
        # By default, training all rules available to this ruleset
        hp['rule_trains'] = task.rules_dict[ruleset]
    else:
        hp['rule_trains'] = rule_trains
    hp['rules'] = hp['rule_trains']

    # Assign probabilities for rule_trains.
    if rule_prob_map is None:
        rule_prob_map = dict()

    # Turn into rule_trains format
    hp['rule_probs'] = None
    if hasattr(hp['rule_trains'], '__iter__'):
        # Set default as 1.
        rule_prob = np.array(
            [rule_prob_map.get(r, 1.) for r in hp['rule_trains']])
        hp['rule_probs'] = list(rule_prob / np.sum(rule_prob))
    tools.save_hp(hp, model_dir)

    # Build the model
    model = Model(model_dir, hp=hp)

    # Display hp
    for key, val in hp.items():
        print('{:20s} = '.format(key) + str(val))

    # Store results
    log = defaultdict(list)
    log['model_dir'] = model_dir

    # Record time
    t_start = time.time()

    with tf.Session() as sess:
        if load_dir is not None:
            model.restore(load_dir)  # complete restore
        else:
            # Assume everything is restored
            sess.run(tf.global_variables_initializer())

        # Set trainable parameters
        if trainables is None or trainables == 'all':
            var_list = model.var_list  # train everything
        elif trainables == 'input':
            # train all nputs
            var_list = [
                v for v in model.var_list
                if ('input' in v.name) and ('rnn' not in v.name)
            ]
        elif trainables == 'rule':
            # train rule inputs only
            var_list = [v for v in model.var_list if 'rule_input' in v.name]
        else:
            raise ValueError('Unknown trainables')
        model.set_optimizer(var_list=var_list)

        # penalty on deviation from initial weight
        if hp['l2_weight_init'] > 0:
            anchor_ws = sess.run(model.weight_list)
            for w, w_val in zip(model.weight_list, anchor_ws):
                model.cost_reg += (hp['l2_weight_init'] *
                                   tf.nn.l2_loss(w - w_val))

            model.set_optimizer(var_list=var_list)

        # partial weight training
        if ('p_weight_train' in hp and (hp['p_weight_train'] is not None)
                and hp['p_weight_train'] < 1.0):
            for w in model.weight_list:
                w_val = sess.run(w)
                w_size = sess.run(tf.size(w))
                w_mask_tmp = np.linspace(0, 1, w_size)
                hp['rng'].shuffle(w_mask_tmp)
                ind_fix = w_mask_tmp > hp['p_weight_train']
                w_mask = np.zeros(w_size, dtype=np.float32)
                w_mask[ind_fix] = 1e-1  # will be squared in l2_loss
                w_mask = tf.constant(w_mask)
                w_mask = tf.reshape(w_mask, w.shape)
                model.cost_reg += tf.nn.l2_loss((w - w_val) * w_mask)
            model.set_optimizer(var_list=var_list)

        step = 0
        while step * hp['batch_size_train'] <= max_steps:
            try:
                # Validation
                if step % display_step == 0:
                    log['trials'].append(step * hp['batch_size_train'])
                    log['times'].append(time.time() - t_start)
                    log = do_eval(sess, model, log, hp['rule_trains'])
                    #if log['perf_avg'][-1] > model.hp['target_perf']:
                    #check if minimum performance is above target
                    if log['perf_min'][-1] > model.hp['target_perf']:
                        print('Perf reached the target: {:0.2f}'.format(
                            hp['target_perf']))
                        break

                    if rich_output:
                        display_rich_output(model, sess, step, log, model_dir)

                # Training
                rule_train_now = hp['rng'].choice(hp['rule_trains'],
                                                  p=hp['rule_probs'])
                # Generate a random batch of trials.
                # Each batch has the same trial length
                trial = generate_trials(rule_train_now,
                                        hp,
                                        'random',
                                        batch_size=hp['batch_size_train'])

                # Generating feed_dict.
                feed_dict = tools.gen_feed_dict(model, trial, hp)
                sess.run(model.train_step, feed_dict=feed_dict)

                step += 1

            except KeyboardInterrupt:
                print("Optimization interrupted by user")
                break

        print("Optimization finished!")
예제 #23
0
def train_sequential_orthogonalized(model_dir,
                                    rule_trains,
                                    hp=None,
                                    max_steps=1e7,
                                    display_step=500,
                                    rich_output=False,
                                    ruleset='mante',
                                    applyProj='both',
                                    seed=0,
                                    nEpisodeBatches=100,
                                    projGrad=True,
                                    alpha=0.001,
                                    fixReadout=False):
    '''Train the network sequentially.

    Args:
        model_dir: str, training directory
        rule_trains: a list of list of tasks to train sequentially
        hp: dictionary of hyperparameters
        max_steps: int, maximum number of training steps for each list of tasks
        display_step: int, display steps
        ruleset: the set of rules to train
        seed: int, random seed to be used

    Returns:
        model is stored at model_dir/model.ckpt
        training configuration is stored at model_dir/hp.json
    '''

    tools.mkdir_p(model_dir)

    # Network parameters
    default_hp = get_default_hp(ruleset)
    if hp is not None:
        default_hp.update(hp)
    hp = default_hp
    hp['seed'] = seed
    hp['rng'] = np.random.RandomState(seed)
    hp['rule_trains'] = rule_trains
    # Get all rules by flattening the list of lists
    # hp['rules'] = [r for rs in rule_trains for r in rs]
    hp['rules'] = rule_trains

    # save some other parameters
    hp['alpha_projection'] = alpha
    hp['max_steps'] = max_steps

    # Number of training iterations for each rule
    rule_train_iters = [max_steps for _ in rule_trains]

    tools.save_hp(hp, model_dir)
    # Display hp
    for key, val in hp.items():
        print('{:20s} = '.format(key) + str(val))

    # Build the model
    model = Sequential_Model(model_dir,
                             projGrad=projGrad,
                             applyProj=applyProj,
                             hp=hp)

    # Store results
    log = defaultdict(list)
    log['model_dir'] = model_dir

    # Record time
    t_start = time.time()

    def relu(x):
        return x * (x > 0.)

    # -------------------------------------------------------

    # Use customized session that launches the graph as well
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())

        # penalty on deviation from initial weight
        if hp['l2_weight_init'] > 0:
            raise NotImplementedError()

        # Looping
        step_total = 0
        taskNumber = 0

        if fixReadout is True:
            my_var_list = [
                var for var in model.var_list
                if 'rnn/leaky_rnn_cell/kernel:0' in var.name
            ]
        else:
            my_var_list = [
                var for var in model.var_list
                if 'rnn/leaky_rnn_cell/kernel:0' in var.name
                or 'output/weights:0' in var.name
            ]

        # initialise projection matrices
        input_proj = tf.zeros(
            (hp['n_rnn'] + hp['n_input'], hp['n_rnn'] + hp['n_input']))
        activity_proj = tf.zeros((hp['n_rnn'], hp['n_rnn']))
        output_proj = tf.zeros((hp['n_output'], hp['n_output']))
        recurrent_proj = tf.zeros((hp['n_rnn'], hp['n_rnn']))

        for i_rule_train, rule_train in enumerate(hp['rule_trains']):

            step = 0

            model.set_optimizer(activity_proj=activity_proj,
                                input_proj=input_proj,
                                output_proj=output_proj,
                                recurrent_proj=recurrent_proj,
                                taskNumber=taskNumber,
                                var_list=my_var_list,
                                alpha=alpha)

            # Keep training until reach max iterations
            while (step * hp['batch_size_train'] <=
                   rule_train_iters[i_rule_train]):
                # Validation
                if step % display_step == 0:
                    trial = step_total * hp['batch_size_train']
                    log['trials'].append(trial)
                    log['times'].append(time.time() - t_start)
                    log['rule_now'].append(rule_train)
                    log = do_eval(sess, model, log, rule_train)
                    if log['perf_avg'][-1] > model.hp['target_perf']:
                        print('Perf reached the target: {:0.2f}'.format(
                            hp['target_perf']))
                        break

                # Training
                # rule_train_now = hp['rng'].choice(rule_train)

                # Generate a random batch of trials.
                # Each batch has the same trial length
                trial = generate_trials(rule_train,
                                        hp,
                                        'random',
                                        batch_size=hp['batch_size_train'],
                                        delay_fac=hp['delay_fac'])

                # Generating feed_dict.
                feed_dict = tools.gen_feed_dict(model, trial, hp)

                # update model
                sess.run(model.train_step, feed_dict=feed_dict)

                # # Get the weight after train step
                # v_current = sess.run(model.var_list)

                step += 1
                step_total += 1

                if step % display_step == 0:
                    model.save_ckpt(step_total)

            # ---------- save model after its completed training the current task ----------
            model.save_after_task(taskNumber)

            # ---------- generate task activity for continual learning -------
            trial = generate_trials(rule_train,
                                    hp,
                                    'random',
                                    batch_size=hp['batch_size_test'],
                                    delay_fac=hp['delay_fac'])

            # Generating feed_dict.
            feed_dict = tools.gen_feed_dict(model, trial, hp)
            eval_h, eval_x, eval_y, Wrec, Win = sess.run(
                [model.h, model.x, model.y, model.w_rec, model.w_in],
                feed_dict=feed_dict)
            full_state = np.concatenate([eval_x, eval_h], -1)

            # get weight matrix after current task
            Wfull = np.concatenate([Win, Wrec], 0)

            # joint covariance matrix of input and activity
            Shx_task = compute_covariance(
                np.reshape(full_state, (-1, hp['n_rnn'] + hp['n_input'])).T)

            # covariance matrix of output
            Sy_task = compute_covariance(
                np.reshape(eval_y, (-1, hp['n_output'])).T)

            # get block matrices from Shx_task
            # Sh_task = Shx_task[-hp['n_rnn']:, -hp['n_rnn']:]
            Sh_task = np.matmul(np.matmul(Wfull.T, Shx_task), Wfull)

            # ---------- update stored covariance matrices for continual learning -------
            if taskNumber == 0:
                input_cov = Shx_task
                activity_cov = Sh_task
                output_cov = Sy_task
            else:
                input_cov = taskNumber / (
                    taskNumber + 1) * input_cov + Shx_task / (taskNumber + 1)
                activity_cov = taskNumber / (
                    taskNumber + 1) * activity_cov + Sh_task / (taskNumber + 1)
                output_cov = taskNumber / (
                    taskNumber + 1) * output_cov + Sy_task / (taskNumber + 1)

            # ---------- update projection matrices for continual learning ----------
            activity_proj, input_proj, output_proj, recurrent_proj = compute_projection_matrices(
                activity_cov, input_cov, output_cov,
                input_cov[-hp['n_rnn']:, -hp['n_rnn']:], alpha)

            # update task number
            taskNumber += 1

        print("Optimization Finished!")
예제 #24
0
 def createDir(self, dirPath, prefix):
     if dir and not os.path.isdir(dirPath) :
         self.verbose(prefix + "- Creating directory '{0}'".format(dirPath))
         tools.mkdir_p(dirPath)
         print (colored(prefix + "> OK", 'green'))
예제 #25
0
        cv2.rectangle(block_img, (lx, ly + lh / 2), (lx + lw, ly + lh / 2),
                      (250, 0, 0), 1)
        #cv2.putText(block_img, str(line_index), (lx, ly+lh/2), cv2.FONT_HERSHEY_SIMPLEX, 0.8, 255, 2)

    # Line Features:
    # - mass/aria (ink_volume)
    # - height
    # - word_count/width (words_ratio)

    return block_img

    #cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 255), 2)


if __name__ == '__main__':
    mkdir_p('lines')
    mkdir_p('blocks')
    eprint(
        'filename,ink_volume,fiber_ink_volume,height,width,area,words_ratio,line_index'
    )
    #find_text_block(sys.argv[1], 'marked.png', 'dilated.png')
    for page_index, page_name in enumerate(sys.argv[1:]):
        print('Page %d' % page_index)
        dilated, img, contours = find_text_block(page_name)
        #dilated, img = find_text_block(cv2.imread(sys.argv[1]),
        #                               cv2.imread('marked.png'),
        #                               cv2.imread('dilated.png'))

        cv2.imwrite('dilated%d.png' % page_index, dilated)
        cv2.imwrite('blocks%d.png' % page_index, img)
예제 #26
0
def error_analysis(model, ids, notes, text_features, X, Y, label, task):
    criteria, vect, clf = model

    if task in ['sapsii', 'age', 'los']:
        V = {}
        labels_ = [0] + criteria
        for i in range(len(labels_) - 1):
            label = '[%d,%s)' % (labels_[i], labels_[i + 1])
            V[i] = label
    else:
        V = {v: k for k, v in criteria.items()}
    V[len(V)] = '**wrong**'

    # for confidence
    P_ = clf.decision_function(X)

    # sklearn has stupid changes in API when doing binary classification. make it conform to 3+
    if len(criteria) == 2:
        m = X.shape[0]
        P = np.zeros((m, 2))
        P[:, 0] = -P_
        P[:, 1] = P_

        n = clf.coef_.shape[1]
        coef_ = np.zeros((2, n))
        coef_[0, :] = -clf.coef_[0, :]
        coef_[1, :] = clf.coef_[0, :]
    else:
        P = P_
        coef_ = clf.coef_

    # hard predictions
    pred = clf.predict(X)

    # convert predictions to right vs wrong
    confidence = {}
    for i, scores in enumerate(P.tolist()):
        prediction = pred[i]
        pid = ids[i]
        ind = Y[i]
        confidence[pid] = (scores[ind], scores, prediction, ind)

    homedir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    methoddir = os.path.join(homedir, 'output', task, 'bow')
    mkdir_p(methoddir)

    def importance(featname, p):
        if featname in vect.vocabulary_:
            ind = vect.vocabulary_[featname]
            return coef_[p, ind]
        else:
            return float('-inf')

    # order predictions by confidence
    for pid, conf in sorted(confidence.items(), key=lambda t: t[1]):
        if conf[2] == conf[3]:
            success = ''
        else:
            success = '_'
        #if success == '_':
        if True:
            filename = os.path.join(methoddir, '%s%s.pred' % (success, pid))
            with open(filename, 'w') as f:
                print >> f, ''
                print >> f, '=' * 80
                print >> f, ''
                print >> f, pid
                print >> f, 'scores:', conf[1]
                print >> f, 'pred:  ', V[conf[2]]
                print >> f, 'ref:   ', V[conf[3]]
                print >> f, '#' * 20
                for feat, v in sorted(text_features[pid].items(),
                                      key=lambda t: importance(t[0], conf[3])):
                    #for feat,v in sorted(text_features[pid].items(), key=lambda t:t[1]):
                    imp = importance(feat, conf[2])
                    #imp = v
                    #print >>f, feat,
                    print >> f, '%.3f %s' % (imp, feat)
                print >> f, '#' * 20
                print >> f, 'SCORES'
                pind = conf[2]
                rind = conf[3]
                print >> f, 'predicted:', sum([
                    val * importance(feat, pind)
                    for feat, val in text_features[pid].items()
                    if float('-inf') < importance(feat, pind) < float('inf')
                ])
                print >> f, 'true:     ', sum([
                    val * importance(feat, rind)
                    for feat, val in text_features[pid].items()
                    if float('-inf') < importance(feat, rind) < float('inf')
                ])
                print >> f, '#' * 20
                #'''
                for dt, category, text in sorted(notes[pid]):
                    print >> f, dt
                    print >> f, category
                    print >> f, text
                    print >> f, '-' * 50
                #'''
                print >> f, ''
                print >> f, '+' * 80
                print >> f, ''
예제 #27
0
def main():
    global best_acc
    start_epoch = args.start_epoch  # start from epoch 0 or last checkpoint epoch

    if not os.path.isdir(args.checkpoint):
        mkdir_p(args.checkpoint)
    print('==> Preparing dataset %s' % args.dataset)

    if args.dataset == 'cifar100':
        training_loader = get_training_dataloader(settings.CIFAR100_TRAIN_MEAN,
                                                  settings.CIFAR100_TRAIN_STD,
                                                  num_workers=4,
                                                  batch_size=args.train_batch,
                                                  shuffle=True)

        test_loader = get_test_dataloader(settings.CIFAR100_TRAIN_MEAN,
                                          settings.CIFAR100_TRAIN_STD,
                                          num_workers=4,
                                          batch_size=args.test_batch,
                                          shuffle=False)
        num_classes = 100
    else:
        training_loader = get_training_dataloader_10(
            settings.CIFAR10_TRAIN_MEAN,
            settings.CIFAR10_TRAIN_STD,
            num_workers=4,
            batch_size=args.train_batch,
            shuffle=True)

        test_loader = get_test_dataloader_10(settings.CIFAR10_TRAIN_MEAN,
                                             settings.CIFAR10_TRAIN_STD,
                                             num_workers=4,
                                             batch_size=args.test_batch,
                                             shuffle=False)
        num_classes = 10
    #data preprocessing:
    print("==> creating model '{}'".format(args.arch))

    model = get_network(args, num_classes=num_classes)
    model = torch.nn.DataParallel(model).cuda()
    cudnn.benchmark = True
    print('    Total params: %.2fM' %
          (sum(p.numel() for p in model.parameters()) / 1000000.0))
    criterion1 = am_softmax.AMSoftmax()
    criterion = nn.CrossEntropyLoss()
    optimizer = optim.SGD(model.parameters(),
                          lr=args.lr,
                          momentum=0.9,
                          weight_decay=5e-4)
    title = 'cifar-10-' + args.arch
    if args.resume:
        # Load checkpoint.
        print('==> Resuming from checkpoint..')
        assert os.path.isfile(
            args.resume), 'Error: no checkpoint directory found!'
        args.checkpoint = os.path.dirname(args.resume)
        checkpoint = torch.load(args.resume)
        best_acc = checkpoint['best_acc']
        start_epoch = checkpoint['epoch']
        model.load_state_dict(checkpoint['state_dict'])
        optimizer.load_state_dict(checkpoint['optimizer'])
        logger = Logger(os.path.join(args.checkpoint, 'log.txt'),
                        title=title,
                        resume=True)
    else:
        logger = Logger(os.path.join(args.checkpoint, 'log.txt'), title=title)
        logger.set_names([
            'Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.',
            'Valid Acc.'
        ])

    train_scheduler = optim.lr_scheduler.MultiStepLR(
        optimizer, milestones=args.schedule, gamma=0.2)  #learning rate decay
    iter_per_epoch = len(training_loader)
    warmup_scheduler = WarmUpLR(optimizer, iter_per_epoch * args.warm)

    for epoch in range(start_epoch, args.epochs):
        if epoch > args.warm:
            train_scheduler.step(epoch)
        train_loss, train_acc = train(training_loader, model, warmup_scheduler,
                                      criterion, criterion1, optimizer, epoch,
                                      use_cuda)
        test_loss, test_acc = eval_training(test_loader, model, criterion,
                                            epoch, use_cuda)

        logger.append([
            optimizer.param_groups[0]['lr'], train_loss, test_loss, train_acc,
            test_acc
        ])

        # save model
        is_best = test_acc > best_acc
        best_acc = max(test_acc, best_acc)
        save_checkpoint(
            {
                'epoch': epoch + 1,
                'state_dict': model.state_dict(),
                'acc': test_acc,
                'best_acc': best_acc,
                'optimizer': optimizer.state_dict(),
            },
            is_best,
            checkpoint=args.checkpoint)

    logger.close()
    # logger.plot()
    # savefig(os.path.join(args.checkpoint, 'log.eps'))

    print('Best acc:')
    print(best_acc)
예제 #28
0
    def run(self):
        thumbnail_dpath = os.path.join(self.clip_dpath, 'media')
        try:
            tools.mkdir_p(thumbnail_dpath)
        except OSError as err:
            wx.CallAfter(
                self.wx_parent.error_message,
                _('Permission error creating clip directory.')
            )
            return

        tools.save_safe_image(self.clip_dpath, self.bg_input_fpath)
        tools.save_safe_image(self.clip_dpath, self.pip_input_fpath)
        tools.save_thumbnails(thumbnail_dpath, self.pip_input_fpath)

        pip_width, pip_height = tools.get_pip_size(
            self.pip_input_fpath,
            self.pip_scale
        )

        bg_fname = tools.sanitize_path(os.path.split(self.bg_input_fpath)[1])
        pip_fname = tools.sanitize_path(os.path.split(self.pip_input_fpath)[1])
        clip_config = tools.default_clip_config()
        clip_config['name'] = self.clip_name
        clip_config['data'] = bg_fname
        clip_config['pip'] = pip_fname
        clip_config['pipwidth'] = pip_width
        clip_config['pipheight'] = pip_height
        clip_config['category'] = self.clip_category

        clip_config_buf = ''
        # name, data, pip must be first 3 items in order
        ordered_config_items = ['name', 'data', 'pip']
        for line in ordered_config_items:
            clip_config_buf += line + ' = ' + clip_config[line] + os.linesep
            # Add the rest of the items whose order probably doesn't matter
        for key, value in clip_config.iteritems():
            key = unicode(key)
            value = unicode(value)
            if value and (key not in ordered_config_items):
                clip_config_buf += key + ' = ' + value + os.linesep

        clip_config_fpath = os.path.join(self.clip_dpath, 'playlist.sos')
        with open(clip_config_fpath, 'wb') as clip_config_fobj:
            clip_config_fobj.write(tools.encode(clip_config_buf))

        # # Update the SOS library
        # # Not used because updating the library doesn't automatically update
        # # the playlist panel in SOS Stream GUI, and updating the playlist
        # # panel in SOS Stream GUI the right way would be tedious.
        # # Rescanning the library automatically without refreshing the
        # # playlist panel could confuse users.
        # child = subprocess.Popen(
        #    ['/shared/sos/default/bin/scan_library'], stdout=subprocess.PIPE
        # )
        # while child.poll() is None:  # Read command output as it comes
        #    out = child.stdout.readline()
        #    print(out.strip())  # Put this in a status window later
        # # The menu can be updated, at least:
        # wish = Tkinter.Tk()
        # wish.withdraw()  # Hide Tk window. Possible with constructor?
        # try:
        #    wish.send('sos_stream_gui',
        #              'destroy',
        #              '.menubar.library')
        #    wish.send('sos_stream_gui',
        #              'mk_library_menu',
        #              '.menubar.library')
        # except Tkinter.TclError as err:
        #    pass  # SOS Stream GUI not open

        # Done
        success_message = '\n'.join(
            [
                _('Clip "{}" successfully added.'),
                _('To activate the changes, '
                  'perform the following actions in SOS Stream GUI:'),
                _('Library -> Update Library...\n'
                  'Library -> (all/{})')
            ]
        ).format(
            clip_config['name'],
            clip_config['category']
        )
        wx.CallAfter(self.wx_parent.information_message, success_message)
    def plot_tuning_feature(self, 
                            epochs,
                            rules = None, 
                            trial_list = None,
                            neuron_types = [('exh_neurons','mix_neurons'),('inh_neurons',)]):
        
        from scipy.optimize import curve_fit
        import math

        if trial_list is None:
            trial_list = self.trial_list
        if rules is None:
            rules = self.rules

        neuron_cate = OrderedDict()
        tuning_by_trial = OrderedDict() 

        for rule in rules:
            neuron_cate[rule] = OrderedDict()
            for epoch in epochs:
                neuron_cate[rule][epoch] = OrderedDict()
                tools.mkdir_p('figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/')
                if epoch not in self.epoch_info[rule].keys():
                    raise KeyError('Rule ',rule,' dose not have epoch ',epoch,'!')

        for trial_num in trial_list:
            tuning_by_trial[trial_num] = OrderedDict()

            for rule in rules:
                tuning_by_trial[trial_num][rule] = OrderedDict()

                for epoch in epochs:
                    tuning_by_trial[trial_num][rule][epoch] = dict()

                    for key, value in self.neuron_info[trial_num][rule][epoch].items():
                        if 'neurons' in key:
                            print(key,"number:",len(value))
                            if key not in neuron_cate[rule][epoch].keys():
                                neuron_cate[rule][epoch][key] = [len(value)]
                            else:
                                neuron_cate[rule][epoch][key].append(len(value))

                    for type_pair in neuron_types:
                        type_pair_folder = 'figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'_'.join(type_pair)+'/'
                        tools.mkdir_p(type_pair_folder)
                        fig_by_epoch = plt.figure()
                        tuning_by_neuron = list()

                        for n_type in type_pair:
                            for neuron in self.neuron_info[trial_num][rule][epoch][n_type]:
                                tuning_by_neuron.append(self.neuron_info[trial_num][rule][epoch]['firerate_max_central'][neuron])

                        tuning_by_neuron = np.array(tuning_by_neuron)
                        plt.plot(np.arange(len(self.neuron_info[trial_num][rule][epoch]['firerate_max_central'][neuron])),
                        tuning_by_neuron.T)
            
                        plt.title("Rule:"+rule+" Epoch:"+epoch+" Trial:"+str(trial_num)+\
                            " Perf:"+str(self.log['perf_'+rule][trial_num//self.log['trials'][1]])[:4])

                        save_name_neuron = type_pair_folder+str(trial_num)+'.png'
                        plt.tight_layout()
                        plt.savefig(save_name_neuron, transparent=False, bbox_inches='tight')
                        plt.close(fig_by_epoch)

                        tuning_by_trial[trial_num][rule][epoch][type_pair] = tuning_by_neuron.mean(axis=0)
        
        for type_pair in neuron_types:
            for rule in rules:
                for epoch in epochs:
                    fig_tuning = plt.figure()
                    for trial_num in trial_list:
                        plt.plot(np.arange(len(tuning_by_trial[trial_num][rule][epoch][type_pair])),
                        tuning_by_trial[trial_num][rule][epoch][type_pair], 
                        label = str(trial_num)+' Perf:'+str(self.log['perf_'+rule][trial_num//self.log['trials'][1]])[:4])
                    
                    type_pair_folder = 'figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'_'.join(type_pair)+'/'
                    save_name_trial = type_pair_folder+'tuning_all_'+str(trial_list[0])+'to'+str(trial_list[-1])+'.png'
                    plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                    plt.tight_layout()
                    plt.savefig(save_name_trial, transparent=False,bbox_inches='tight')
                    plt.close(fig_tuning)
        
        #tunning by growth and gaussian curve fit
        tuning_by_growth = OrderedDict()
        def gaussian(x, a,u, sig):
            return a*np.exp(-(x - u) ** 2 / (2 * sig ** 2)) / (sig * math.sqrt(2 * math.pi))
        
        for type_pair in neuron_types:
            tuning_by_growth[type_pair] = OrderedDict()

            for rule in rules:
                tuning_by_growth[type_pair][rule] = OrderedDict()

                for epoch in epochs:
                    tuning_by_growth[type_pair][rule][epoch] = OrderedDict()
                    for growth_key in ['less_than_I','I_to_Y','elder_than_Y']:
                        tuning_by_growth[type_pair][rule][epoch][growth_key]=dict()
                        for type_key in ['growth','tuning']:
                            tuning_by_growth[type_pair][rule][epoch][growth_key][type_key]=list()

                    for trial_num in trial_list:
                        growth = self.log['perf_'+rule][trial_num//self.log['trials'][1]]
                        if growth <= self.hp['infancy_target_perf']:
                            tuning_by_growth[type_pair][rule][epoch]['less_than_I']['growth'].append(growth)
                            tuning_by_growth[type_pair][rule][epoch]['less_than_I']['tuning'].\
                                append(tuning_by_trial[trial_num][rule][epoch][type_pair])
                        elif growth <= self.hp['young_target_perf']:
                            tuning_by_growth[type_pair][rule][epoch]['I_to_Y']['growth'].append(growth)
                            tuning_by_growth[type_pair][rule][epoch]['I_to_Y']['tuning'].\
                                append(tuning_by_trial[trial_num][rule][epoch][type_pair])
                        else:
                            tuning_by_growth[type_pair][rule][epoch]['elder_than_Y']['growth'].append(growth)
                            tuning_by_growth[type_pair][rule][epoch]['elder_than_Y']['tuning'].\
                                append(tuning_by_trial[trial_num][rule][epoch][type_pair])

                    for growth_key in ['less_than_I','I_to_Y','elder_than_Y']:
                        for type_key in ['growth','tuning']:
                            try:
                                tuning_by_growth[type_pair][rule][epoch][growth_key][type_key]=\
                                    np.array(tuning_by_growth[type_pair][rule][epoch][growth_key][type_key]).mean(axis=0)
                            except:
                                pass

        for type_pair in neuron_types:
            for rule in rules:
                for epoch in epochs:
                    fig_tuning_growth = plt.figure()
                    for growth_key in ['less_than_I','I_to_Y','elder_than_Y']:
                        
                        tuning_temp = tuning_by_growth[type_pair][rule][epoch][growth_key]['tuning']
                        temp_x = np.arange(len(tuning_temp))  ###############################################TODO:something wrong for anti saccade
                        gaussian_x = np.arange(-0.1,len(tuning_temp)-0.9,0.1)
                        #gaussian fit
                        paras , _ = curve_fit(gaussian,temp_x,tuning_temp+(-1)*np.min(tuning_temp),\
                            p0=[np.max(tuning_temp)+1,len(tuning_temp)//2,1])
                        gaussian_y = gaussian(gaussian_x,paras[0],paras[1],paras[2])-np.min(tuning_temp)*(-1)

                        tuning_by_growth[type_pair][rule][epoch][growth_key]['gaussian_fit']=gaussian_y

                        if growth_key == 'less_than_I':
                            color = 'green'
                        elif growth_key == 'I_to_Y':
                            color = 'blue'
                        else:
                            color = 'red'

                        plt.scatter(temp_x, tuning_temp, marker = '+',color = color, s = 70 ,\
                            label = growth_key+'_'+str(tuning_by_growth[type_pair][rule][epoch][growth_key]['growth'])[:4])
                        plt.plot(gaussian_x, gaussian_y, color=color,linestyle = '--',\
                            label = growth_key+'_'+str(tuning_by_growth[type_pair][rule][epoch][growth_key]['growth'])[:4])

                    plt.legend()
                    type_pair_folder = 'figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'_'.join(type_pair)+'/'
                    save_name_tun_growth = type_pair_folder+'tuning_growth_'+str(trial_list[0])+'to'+str(trial_list[-1])+'.png'
                    plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                    plt.tight_layout()
                    plt.savefig(save_name_tun_growth, transparent=False,bbox_inches='tight')
                    plt.close(fig_tuning_growth)
                    #plt.show()



                    
        
        #plot neuron category changes
        for rule in rules:
            for epoch in epochs:
                fig_cate = plt.figure()
                for key,value in neuron_cate[rule][epoch].items():
                    plt.plot(trial_list, value, label = key)
                plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                plt.tight_layout()
                plt.savefig('figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'neuron_cate_'+str(trial_list[0])+'to'+str(trial_list[-1])+'.png',transparent=False,bbox_inches='tight')
                plt.close(fig_cate)
    def plot_PSTH(self, 
                epochs, 
                rules=None, 
                trial_list=None, 
                neuron_types=[('exh_neurons','mix_neurons')], 
                norm = True,
                separate_plot = False,
                fuse_rules = False):
        
        if trial_list is None:
            trial_list = self.trial_list
        if rules is None:
            rules = self.rules

        psth_to_plot = OrderedDict()

        for trial_num in trial_list:
            psth_to_plot[trial_num] = OrderedDict()
            for rule in rules:
                H = tools.load_pickle(self.model_dir+'/'+str(trial_num)+'/'+'H_'+rule+'.pkl')
                psth_to_plot[trial_num][rule] = OrderedDict()
                for epoch in epochs:
                    psth_to_plot[trial_num][rule][epoch] = OrderedDict()
                    for type_pair in neuron_types:
                        psth_to_plot[trial_num][rule][epoch][type_pair] = OrderedDict()
                        psth_neuron = list()
                        anti_dir_psth = list()

                        for n_type in type_pair:
                            
                            for neuron in self.neuron_info[trial_num][rule][epoch][n_type]:

                                sel_loc = np.argmax(self.neuron_info[trial_num][rule][epoch]['firerate_loc_order'][neuron])
                                anti_loc = (sel_loc+len(self.in_loc_set[rule])//2)%len(self.in_loc_set[rule])

                                psth_temp = H[:,self.in_loc[rule] == sel_loc, neuron].mean(axis=1)
                                fix_level = H[self.epoch_info[rule]['fix1'][0]:self.epoch_info[rule]['fix1'][1], \
                                    self.in_loc[rule] == sel_loc, neuron].mean(axis=1).mean(axis=0)
                                if len(self.in_loc_set[rule])%2:
                                    anti_dir_psth_temp = (H[:,self.in_loc[rule] == anti_loc, neuron].mean(axis=1)+\
                                        H[:,self.in_loc[rule] == (anti_loc+1), neuron].mean(axis=1))/2.0
                                else:
                                    anti_dir_psth_temp = H[:,self.in_loc[rule] == anti_loc, neuron].mean(axis=1)

                                anti_dir_psth_norm = anti_dir_psth_temp/fix_level-1
                                psth_norm = psth_temp/fix_level-1
                                if norm:
                                    psth_neuron.append(psth_norm)
                                    anti_dir_psth.append(anti_dir_psth_norm)
                                else:
                                    psth_neuron.append(psth_temp)
                                    anti_dir_psth.append(anti_dir_psth_temp)

                        try:
                            psth_to_plot[trial_num][rule][epoch][type_pair]['sel_dir'] = np.array(psth_neuron).mean(axis=0)
                        except:
                            pass
                        try:
                            psth_to_plot[trial_num][rule][epoch][type_pair]['anti_sel_dir'] = np.array(anti_dir_psth).mean(axis=0)
                        except:
                            pass

        for rule in rules:
            for epoch in epochs:
                for type_pair in neuron_types:
                    if not separate_plot:
                        fig_psth = plt.figure()
                    for trial_num in trial_list:

                        if separate_plot:
                            fig_psth = plt.figure()
                            color = None
                        else:
                            color = kelly_colors[(trial_list.index(trial_num)+1)%len(kelly_colors)]

                        try:
                            plt.plot(np.arange(len(psth_to_plot[trial_num][rule][epoch][type_pair]['sel_dir']))*self.hp['dt']/1000,
                            psth_to_plot[trial_num][rule][epoch][type_pair]['sel_dir'],label=str(trial_num)+'sel',color=color)
                        except:
                            pass

                        try:
                            plt.plot(np.arange(len(psth_to_plot[trial_num][rule][epoch][type_pair]['anti_sel_dir']))*self.hp['dt']/1000,
                            psth_to_plot[trial_num][rule][epoch][type_pair]['anti_sel_dir'],linestyle = '--',label=str(trial_num)+'anti',color=color)
                        except:
                            pass
                        if separate_plot:
                            plt.title("Rule:"+rule+" Epoch:"+epoch+" Neuron_type:"+"_".join(type_pair))
                            plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                            type_pair_folder = 'figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'_'.join(type_pair)+'/'
                            tools.mkdir_p(type_pair_folder)
                            plt.tight_layout()
                            plt.savefig(type_pair_folder+'PSTH-'+str(trial_num)+'.png',transparent=False,bbox_inches='tight')
                            plt.close(fig_psth)

                    if not separate_plot:
                        plt.title("Rule:"+rule+" Epoch:"+epoch+" Neuron_type:"+"_".join(type_pair))
                        plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                        type_pair_folder = 'figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'_'.join(type_pair)+'/'
                        tools.mkdir_p(type_pair_folder)
                        plt.tight_layout()
                        plt.savefig(type_pair_folder+'PSTH_all_'+str(trial_list[0])+'to'+str(trial_list[-1])+'.png',
                        transparent=False,bbox_inches='tight')
                        plt.close(fig_psth)

        plot_by_growth = dict()
        for rule in rules:
            plot_by_growth[rule] = dict()

            for epoch in epochs:
                plot_by_growth[rule][epoch] = dict()

                for type_pair in neuron_types:
                    plot_by_growth[rule][epoch][type_pair] = dict()
                    plot_by_growth[rule][epoch][type_pair]['less_than_I'] = dict()
                    plot_by_growth[rule][epoch][type_pair]['I_to_Y'] = dict()
                    plot_by_growth[rule][epoch][type_pair]['elder_than_Y'] = dict()

                    for key in plot_by_growth[rule][epoch][type_pair].keys():
                        plot_by_growth[rule][epoch][type_pair][key]['sel'] = list()
                        plot_by_growth[rule][epoch][type_pair][key]['anti'] = list()
                        plot_by_growth[rule][epoch][type_pair][key]['growth'] = list()

                    for trial_num in trial_list:
                        growth = self.log['perf_'+rule][trial_num//self.log['trials'][1]]
                        if growth <= self.hp['infancy_target_perf']:
                            plot_by_growth[rule][epoch][type_pair]['less_than_I']['sel'].append(\
                                psth_to_plot[trial_num][rule][epoch][type_pair]['sel_dir'])
                            plot_by_growth[rule][epoch][type_pair]['less_than_I']['anti'].append(\
                                psth_to_plot[trial_num][rule][epoch][type_pair]['anti_sel_dir'])
                            plot_by_growth[rule][epoch][type_pair]['less_than_I']['growth'].append(growth)
                        elif growth <= self.hp['young_target_perf']:
                            plot_by_growth[rule][epoch][type_pair]['I_to_Y']['sel'].append(\
                                psth_to_plot[trial_num][rule][epoch][type_pair]['sel_dir'])
                            plot_by_growth[rule][epoch][type_pair]['I_to_Y']['anti'].append(\
                                psth_to_plot[trial_num][rule][epoch][type_pair]['anti_sel_dir'])
                            plot_by_growth[rule][epoch][type_pair]['I_to_Y']['growth'].append(growth)
                        else:
                            plot_by_growth[rule][epoch][type_pair]['elder_than_Y']['sel'].append(\
                                psth_to_plot[trial_num][rule][epoch][type_pair]['sel_dir'])
                            plot_by_growth[rule][epoch][type_pair]['elder_than_Y']['anti'].append(\
                                psth_to_plot[trial_num][rule][epoch][type_pair]['anti_sel_dir'])
                            plot_by_growth[rule][epoch][type_pair]['elder_than_Y']['growth'].append(growth)

                    for growth_key in plot_by_growth[rule][epoch][type_pair].keys():
                        for type_key, value in plot_by_growth[rule][epoch][type_pair][growth_key].items():
                            try:
                                plot_by_growth[rule][epoch][type_pair][growth_key][type_key] = np.array(value).mean(axis=0)
                            except:
                                pass

        for rule in rules:
            for epoch in epochs:
                for type_pair in neuron_types:
                    fig_psth = plt.figure()
                    for growth_key,value in plot_by_growth[rule][epoch][type_pair].items():
                        if growth_key == 'less_than_I':
                            color = 'green'
                        elif growth_key == 'I_to_Y':
                            color = 'blue'
                        else:
                            color = 'red'
                        
                        try:
                            plt.plot(np.arange(len(value['sel']))*self.hp['dt']/1000,value['sel'],
                            label = growth_key+'_'+str(value['growth'])[:4], color = color)
                        except:
                            pass

                        try:
                            plt.plot(np.arange(len(value['anti']))*self.hp['dt']/1000,value['anti'],
                            linestyle = '--',label = growth_key+'_'+str(value['growth'])[:4], color = color)
                        except:
                            pass
                    plt.title("Rule:"+rule+" Epoch:"+epoch+" Neuron_type:"+"_".join(type_pair))
                    plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                    type_pair_folder = 'figure/figure_'+self.model_dir+'/'+rule+'/'+epoch+'/'+'_'.join(type_pair)+'/'
                    tools.mkdir_p(type_pair_folder)
                    plt.tight_layout()
                    plt.savefig(type_pair_folder+'PSTH_bygrowth_'+str(trial_list[0])+'to'+str(trial_list[-1])+'.png',
                    transparent=False,bbox_inches='tight')
                    plt.close(fig_psth)

        if fuse_rules:
            ls_list = ['-','--','-.',':']
            for type_pair in neuron_types:
                for epoch in epochs:
                    fig_psth_fr = plt.figure()
                    for rule, linestyle in zip(rules,ls_list[0:len(rules)]):
                        for key,value in plot_by_growth[rule][epoch][type_pair].items():
                            if key == 'less_than_I':
                                color = 'green'
                            elif key == 'I_to_Y':
                                color = 'blue'
                            else:
                                color = 'red' 

                            try:
                                plt.plot(np.arange(len(value['sel']))*self.hp['dt']/1000,value['sel'],
                                label = rule+'_'+key+'_'+str(value['growth'])[:4], color = color, linestyle=linestyle)
                            except:
                                pass

                    plt.legend()
                    plt.title("Rule:"+rule+" Epoch:"+epoch+" Neuron_type:"+"_".join(type_pair))
                    plt.legend(bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
                    plt.tight_layout()
                    plt.savefig('figure/figure_'+self.model_dir+'/'+'-'.join(rules)+'_'+epoch+'_'+'-'.join(type_pair)\
                        +str(trial_list[0])+'to'+str(trial_list[-1])+'.png',
                    transparent=False,bbox_inches='tight')
                    plt.close(fig_psth_fr)
예제 #31
0
def train(
    model_dir,
    hp=None,
    max_steps=1e7,
    display_step=500,
    ruleset='mante',
    rule_trains=None,
    rule_prob_map=None,
    seed=0,
    rich_output=True,
    load_dir=None,
    trainables=None,
    fixReadoutandBias=False,
    fixBias=False,
):
    """Train the network.

    Args:
        model_dir: str, training directory
        hp: dictionary of hyperparameters
        max_steps: int, maximum number of training steps
        display_step: int, display steps
        ruleset: the set of rules to train
        rule_trains: list of rules to train, if None then all rules possible
        rule_prob_map: None or dictionary of relative rule probability
        seed: int, random seed to be used

    Returns:
        model is stored at model_dir/model.ckpt
        training configuration is stored at model_dir/hp.json
    """

    tools.mkdir_p(model_dir)

    # Network parameters
    default_hp = get_default_hp(ruleset)
    if hp is not None:
        default_hp.update(hp)
    hp = default_hp
    hp['seed'] = seed
    hp['rng'] = np.random.RandomState(seed)

    # Rules to train and test. Rules in a set are trained together
    if rule_trains is None:
        # By default, training all rules available to this ruleset
        hp['rule_trains'] = task.rules_dict[ruleset]
    else:
        hp['rule_trains'] = rule_trains
    hp['rules'] = hp['rule_trains']

    # Assign probabilities for rule_trains.
    if rule_prob_map is None:
        rule_prob_map = dict()

    # Turn into rule_trains format
    hp['rule_probs'] = None
    if hasattr(hp['rule_trains'], '__iter__'):
        # Set default as 1.

        rule_prob = np.array(
            [rule_prob_map.get(r, 1.) for r in hp['rule_trains']])
        hp['rule_probs'] = list(rule_prob / np.sum(rule_prob))

    tools.save_hp(hp, model_dir)

    # Build the model
    with tf.device('gpu:0'):
        model = Model(model_dir, hp=hp)

        # Display hp
        for key, val in hp.items():
            print('{:20s} = '.format(key) + str(val))

        if fixReadoutandBias is True:
            my_var_list = [
                var for var in model.var_list
                if 'rnn/leaky_rnn_cell/kernel:0' in var.name
            ]
            print(my_var_list)
        elif fixBias is True:
            my_var_list = [
                var for var in model.var_list
                if 'rnn/leaky_rnn_cell/kernel:0' in var.name
                or 'output/weights:0' in var.name
            ]
        else:
            my_var_list = model.var_list

        model.set_optimizer(var_list=my_var_list)

        # Store results
        log = defaultdict(list)
        log['model_dir'] = model_dir

        # Record time
        t_start = time.time()

        # Use customized session that launches the graph as well
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())

            # penalty on deviation from initial weight
            if hp['l2_weight_init'] > 0:
                anchor_ws = sess.run(model.weight_list)
                for w, w_val in zip(model.weight_list, anchor_ws):
                    model.cost_reg += (hp['l2_weight_init'] *
                                       tf.nn.l2_loss(w - w_val))

                model.set_optimizer(var_list=my_var_list)

            # partial weight training
            if ('p_weight_train' in hp and (hp['p_weight_train'] is not None)
                    and hp['p_weight_train'] < 1.0):
                for w in model.weight_list:
                    w_val = sess.run(w)
                    w_size = sess.run(tf.size(w))
                    w_mask_tmp = np.linspace(0, 1, w_size)
                    hp['rng'].shuffle(w_mask_tmp)
                    ind_fix = w_mask_tmp > hp['p_weight_train']
                    w_mask = np.zeros(w_size, dtype=np.float32)
                    w_mask[ind_fix] = 1e-1  # will be squared in l2_loss
                    w_mask = tf.constant(w_mask)
                    w_mask = tf.reshape(w_mask, w.shape)
                    model.cost_reg += tf.nn.l2_loss((w - w_val) * w_mask)
                model.set_optimizer(var_list=my_var_list)

            step = 0
            run_ave_time = []
            while step * hp['batch_size_train'] <= max_steps:
                try:
                    # Validation
                    if step % display_step == 0:
                        grad_norm = tf.global_norm(model.clipped_gs)
                        grad_norm_np = sess.run(grad_norm)
                        # import pdb
                        # pdb.set_trace()
                        log['grad_norm'].append(grad_norm_np.item())
                        log['trials'].append(step * hp['batch_size_train'])
                        log['times'].append(time.time() - t_start)
                        log = do_eval(sess, model, log, hp['rule_trains'])
                        # if log['perf_avg'][-1] > model.hp['target_perf']:
                        # check if minimum performance is above target
                        if log['perf_min'][-1] > model.hp['target_perf']:
                            print('Perf reached the target: {:0.2f}'.format(
                                hp['target_perf']))
                            break

                        if rich_output:
                            display_rich_output(model, sess, step, log,
                                                model_dir)

                    # Training

                    dtStart = datetime.now()
                    sess.run(model.train_step)
                    dtEnd = datetime.now()

                    if len(run_ave_time) is 0:
                        run_ave_time = np.expand_dims(
                            (dtEnd - dtStart).total_seconds(), axis=0)
                    else:
                        run_ave_time = np.concatenate(
                            (run_ave_time,
                             np.expand_dims((dtEnd - dtStart).total_seconds(),
                                            axis=0)))

                    # print(np.mean(run_ave_time))
                    # print((dtEnd-dtStart).total_seconds())

                    step += 1

                    if step < 10:
                        model.save_ckpt(step)

                    if step < 1000:
                        if step % display_step / 10 == 0:
                            model.save_ckpt(step)

                    if step % display_step == 0:
                        model.save_ckpt(step)

                except KeyboardInterrupt:
                    print("Optimization interrupted by user")
                    break

            print("Optimization finished!")