def run(self):

        s = open("log_batch.txt","a")
        s.write("acquire\n")
        s.close()
        threadLimiter.acquire()

        try:

            self.output_dir = output_dir_prefix
            self.output_dir = self.output_dir + '_a' + str(self.alg_type)
            self.output_dir = self.output_dir + '_s' + str(self.eta0)
            self.output_dir = self.output_dir + '_t' + str(self.t0)
            self.output_dir = self.output_dir + '_l' + str(self.lambda0)

            # sampling methods
            self.output_dir = self.output_dir + '_p' + str(self.svrg_pSamples)

            self.output_dir = self.output_dir + '_o' + str(self.svrg_outer_pSamples)

            self.output_dir = self.output_dir + '_e' + str(self.npasses)

            self.output_dir = self.output_dir + '_n' + str(self.nsamplesperpass)

            self.output_dir = self.output_dir + '_h' + str(self.hessian_pSamples)

            #self.output_dir = self.output_dir + '_b1000'

            print self.output_dir

            overwrite_dir = False
            existing_dir = os.access(self.output_dir, os.F_OK)
            self.cmd = ''

            # Create output directory if it does not exist. 
            if not existing_dir or overwrite_dir:

                if not existing_dir:
                    print 'mkdir ' + self.output_dir
                    if exec_cmd:
                        os.mkdir(self.output_dir)

                # Copy config file
                self.cmd = 'cp ' + input_file_fullpath + ' ' + self.output_dir
                print self.cmd
                if exec_cmd:
                    os.system(self.cmd)

                # Change directory
                print 'cd ' + self.output_dir
                if exec_cmd:
                    os.chdir(self.output_dir)

                    configIO.write(input_file,'alg_type', self.alg_type)
                    configIO.write(input_file,'eta0', self.eta0)
                    configIO.write(input_file,'T0', self.t0)
                    configIO.write(input_file,'lambda', self.lambda0)
                    configIO.write(input_file,'svrg_pSamples', self.svrg_pSamples)
                    configIO.write(input_file,'svrg_outer_pSamples', self.svrg_outer_pSamples)
                    configIO.write(input_file,'npasses', self.npasses)
                    configIO.write(input_file,'nsamplesperpass', self.nsamplesperpass)
                    configIO.write(input_file,'hessian_psamples', self.hessian_pSamples)

                    datapath_rel = configIO.read(input_file,'datapath')
                    configIO.write(input_file,'datapath', data_path + datapath_rel)

                    configIO.write(input_file,'logdir', '')

                    # bsub -We 100:00 java
                    #self.cmd = 'bsub -We 100:00 -R rusage[mem=10000] java -cp .:' + lib_path + ':' + data_path + 'src/ optimize ' + input_file
                    self.cmd = 'java -cp .:' + lib_path + ':' + data_path + 'src/ optimize ' + input_file
                    if redirect_output:
                        self.cmd = self.cmd + ' > out.txt 2>&1'
                    print self.cmd
                    if exec_cmd:
                        os.system(self.cmd)
            
        finally:
            s = open("log_batch.txt","a")
            s.write("release " + self.cmd + "\n")
            s.close()
            threadLimiter.release()
Beispiel #2
0
    filename_max_score = ''
    scores = [0]
    idx_score = 0

    # Check if configuration file exists
    config_file_pattern = _dir[0:10]
    config_filename = ''
    for i in os.listdir(_dir):
        if i[0:len(config_file_pattern
                   )] != config_file_pattern or i[len(i) - 3:len(i)] != 'txt':
            continue
        config_filename = _dir + '/' + i

    stepForOutputFiles = 1
    if config_filename != '':
        stepForOutputFiles_str = configIO.read(config_filename,
                                               'stepForOutputFiles')
        stepForOutputFiles = int(stepForOutputFiles_str)

    # Check if score file exists
    filename = ''
    for i in os.listdir(output_dir):
        if i[0:len(score_filename)] != score_filename:
            continue
        filename = output_dir + i

    if filename != '':
        f = open(filename)
        lines = f.readlines()

        if sorting_index == -1:
            if _dir.find('MSRC') != -1:
def plotWeights(dir_names, output_filename, max_delta_time = -1, dir_name_to_exclude = ''):

    current_time = time.time()

    ldirs = os.listdir('.')
    ndirs = len(ldirs)

    fig_idx = 0
    list_objs = []

    for dir_name in dir_names:

        for _dir in ldirs:

            if not os.path.isdir(_dir):
                continue
            #if _dir[0:len(dir_name)] != dir_name:
            #    continue
            m = re.search(dir_name, _dir)
            if m is None:
                continue
            #else:
            #    print 'Match: ' + _dir

            if dir_name_to_exclude != '':
                m = re.search(dir_name_to_exclude, _dir)
                if m is not None:
                    continue

            if max_delta_time > 0:
                delta_time = current_time - os.path.getmtime(_dir)
                if delta_time > max_delta_time:
                    continue

            fullpath = _dir + '/parameter_vector0/'
            lfiles = os.listdir(fullpath)
            nfiles = len(lfiles)

            # Check if configuration file exists
            config_file_pattern = _dir[0:10]
            config_filename = ''
            for i in os.listdir(_dir):
                if i[0:len(config_file_pattern)] != config_file_pattern or i[len(i)-3:len(i)] != 'txt':
                    continue
                config_filename = _dir + '/' + i

            stepForParameterFiles = int(configIO.read(config_filename, 'stepForParameterFiles'))

            # plot last one only
            for i in range(nfiles-1, nfiles):
                fileId = i * stepForParameterFiles
                weight_filename = fullpath + 'iteration_' + str(fileId) + '.txt'
                #print 'weight ' + weight_filename
                if os.access(weight_filename, os.F_OK):
                    w = readColumnFromFile(0, weight_filename)
                    w = w[8:len(w)]

                    #fig = figure()
                    #fig = figure(num=None, figsize=(16, 12), dpi=80, facecolor='w', edgecolor='k')
                    fig = figure(num=None, figsize=(30, 14), dpi=160, facecolor='w', edgecolor=None)
                    ax = fig.add_subplot(1,1,1)
                    ind = range(0,len(w))
                    ax.bar(ind, w,align='center', edgecolor="none",linewidth=0.0)
                    #ax.bar(ind, w,align='center')
                    #ax.set_xticks(ind)
                    savefig(output_filename)
                    clf
    max_score = 0
    filename_max_score = ''
    scores = [0]
    idx_score = 0

    # Check if configuration file exists
    config_file_pattern = _dir[0:10]
    config_filename = ''
    for i in os.listdir(_dir):
        if i[0:len(config_file_pattern)] != config_file_pattern or i[len(i)-3:len(i)] != 'txt':
            continue
        config_filename = _dir + '/' + i

    stepForOutputFiles = 1
    if config_filename != '':
        stepForOutputFiles_str = configIO.read(config_filename, 'stepForOutputFiles')
        stepForOutputFiles = int(stepForOutputFiles_str)

    # Check if score file exists 
    filename = ''
    for i in os.listdir(output_dir):
        if i[0:len(score_filename)] != score_filename:
            continue
        filename = output_dir + i

    if filename != '':
        f = open(filename)
        lines = f.readlines()

        if sorting_index == -1:
            if _dir.find('MSRC') != -1:
Beispiel #5
0
        best_training_loss = tokens[score_col_idx]

    cmd = pathname + '/sortScore.py -e 1 -s 0 -o 1 -a svgr_cl -d ' + _dir
    out = commands.getoutput(cmd)
    tokens = out.split('\t\t')
    if len(tokens) > 2:
        best_training_loss = tokens[score_col_idx]

    cmd = pathname + '/sortScore.py -e 1 -s 0 -o 1 -a saga -d ' + _dir
    out = commands.getoutput(cmd)
    tokens = out.split('\t\t')
    if len(tokens) > 2:
        best_training_loss = tokens[score_col_idx]

    # Read attributes from config file
    eta0 = configIO.read(config_filename,'eta0')
    T0 = configIO.read(config_filename,'T0')
    nsamplesperpass = configIO.read(config_filename,'nsamplesperpass')
    svgr_pSamples = configIO.read(config_filename,'svgr_pSamples')
    svgr_outer_pSamples = configIO.read(config_filename,'svgr_outer_pSamples')
    hessian_psamples = configIO.read(config_filename,'hessian_psamples')
    alg_type = configIO.read(config_filename,'alg_type')
    alambda = configIO.read(config_filename,'lambda')

    if idx != 0:
        fdata.write(',')

    fdata.write('{')
    fdata.write('"name" : "' + _dir + '", ')
    fdata.write('"link" : "' + options.output_name + str(idx) + '", ')
    fdata.write('"alg_type" : "' + alg_type + '", ')
Beispiel #6
0
def plotWeights(dir_names,
                output_filename,
                max_delta_time=-1,
                dir_name_to_exclude=''):

    current_time = time.time()

    ldirs = os.listdir('.')
    ndirs = len(ldirs)

    fig_idx = 0
    list_objs = []

    for dir_name in dir_names:

        for _dir in ldirs:

            if not os.path.isdir(_dir):
                continue
            #if _dir[0:len(dir_name)] != dir_name:
            #    continue
            m = re.search(dir_name, _dir)
            if m is None:
                continue
            #else:
            #    print 'Match: ' + _dir

            if dir_name_to_exclude != '':
                m = re.search(dir_name_to_exclude, _dir)
                if m is not None:
                    continue

            if max_delta_time > 0:
                delta_time = current_time - os.path.getmtime(_dir)
                if delta_time > max_delta_time:
                    continue

            fullpath = _dir + '/parameter_vector0/'
            lfiles = os.listdir(fullpath)
            nfiles = len(lfiles)

            # Check if configuration file exists
            config_file_pattern = _dir[0:10]
            config_filename = ''
            for i in os.listdir(_dir):
                if i[0:len(config_file_pattern)] != config_file_pattern or i[
                        len(i) - 3:len(i)] != 'txt':
                    continue
                config_filename = _dir + '/' + i

            stepForParameterFiles = int(
                configIO.read(config_filename, 'stepForParameterFiles'))

            # plot last one only
            for i in range(nfiles - 1, nfiles):
                fileId = i * stepForParameterFiles
                weight_filename = fullpath + 'iteration_' + str(
                    fileId) + '.txt'
                #print 'weight ' + weight_filename
                if os.access(weight_filename, os.F_OK):
                    w = readColumnFromFile(0, weight_filename)
                    w = w[8:len(w)]

                    #fig = figure()
                    #fig = figure(num=None, figsize=(16, 12), dpi=80, facecolor='w', edgecolor='k')
                    fig = figure(num=None,
                                 figsize=(30, 14),
                                 dpi=160,
                                 facecolor='w',
                                 edgecolor=None)
                    ax = fig.add_subplot(1, 1, 1)
                    ind = range(0, len(w))
                    ax.bar(ind,
                           w,
                           align='center',
                           edgecolor="none",
                           linewidth=0.0)
                    #ax.bar(ind, w,align='center')
                    #ax.set_xticks(ind)
                    savefig(output_filename)
                    clf
Beispiel #7
0
        best_training_loss = tokens[score_col_idx]

    cmd = pathname + '/sortScore.py -e 1 -s 0 -o 1 -a svgr_cl -d ' + _dir
    out = commands.getoutput(cmd)
    tokens = out.split('\t\t')
    if len(tokens) > 2:
        best_training_loss = tokens[score_col_idx]

    cmd = pathname + '/sortScore.py -e 1 -s 0 -o 1 -a saga -d ' + _dir
    out = commands.getoutput(cmd)
    tokens = out.split('\t\t')
    if len(tokens) > 2:
        best_training_loss = tokens[score_col_idx]

    # Read attributes from config file
    eta0 = configIO.read(config_filename, 'eta0')
    T0 = configIO.read(config_filename, 'T0')
    nsamplesperpass = configIO.read(config_filename, 'nsamplesperpass')
    svgr_pSamples = configIO.read(config_filename, 'svgr_pSamples')
    svgr_outer_pSamples = configIO.read(config_filename, 'svgr_outer_pSamples')
    hessian_psamples = configIO.read(config_filename, 'hessian_psamples')
    alg_type = configIO.read(config_filename, 'alg_type')
    alambda = configIO.read(config_filename, 'lambda')

    if idx != 0:
        fdata.write(',')

    fdata.write('{')
    fdata.write('"name" : "' + _dir + '", ')
    fdata.write('"link" : "' + options.output_name + str(idx) + '", ')
    fdata.write('"alg_type" : "' + alg_type + '", ')