Esempio n. 1
0
def script_saveCommands(in_dir_meta, out_dir_meta, out_file_text):
    cats = [
        dir_curr for dir_curr in os.listdir(in_dir_meta)
        if os.path.isdir(os.path.join(in_dir_meta, dir_curr))
    ]
    # print len(cats);
    commands = []
    command_text = 'ffmpeg -i '
    for cat in cats:
        dir_curr = os.path.join(in_dir_meta, cat)
        dir_out_curr = os.path.join(out_dir_meta, cat)
        print dir_out_curr

        if not os.path.exists(dir_out_curr):
            os.mkdir(dir_out_curr)
        videos = [
            file_curr for file_curr in os.listdir(dir_curr)
            if file_curr.endswith('.avi')
        ]
        for video in videos:
            out_dir_video_org = os.path.join(dir_out_curr, video[:-4])
            out_dir_video = util.escapeString(out_dir_video_org)
            command_curr = 'mkdir ' + out_dir_video + ';' + command_text + util.escapeString(
                os.path.join(dir_curr,
                             video)) + ' ' + out_dir_video + '/image_%d.ppm'
            commands.append(command_curr)
    with open(out_file_text, 'wb') as f:
        for command in commands:
            f.write(command + '\n')
Esempio n. 2
0
def script_writeFlownetCommands(params):
    video_list_file = params.video_list_file
    path_to_video_meta = params.path_to_video_meta
    in_dir_meta = params.in_dir_meta
    out_dir_meta = params.out_dir_meta
    path_to_deploy = params.path_to_deploy
    out_file_commands = params.out_file_commands
    dir_flownet_meta = params.dir_flownet_meta
    path_to_sizer = params.path_to_sizer
    caffe_bin = params.caffe_bin
    path_to_model = params.path_to_model
    text_1_org = params.text_1
    text_2_org = params.text_2
    deploy_file = params.deploy_file
    gpu = params.gpu

    im_dirs = util.readLinesFromFile(video_list_file)
    im_dirs = [
        im_dir.replace(path_to_video_meta, in_dir_meta)[:-4]
        for im_dir in im_dirs
    ]

    commands = []
    # im_dirs=['/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data/hmdb/pick/THE_WALLET_TRICK!!!_pick_f_cm_np2_ba_med_1'];
    for idx_im_dir, im_dir in enumerate(im_dirs):
        print idx_im_dir, len(im_dirs)
        out_dir_curr = im_dir.replace(in_dir_meta, out_dir_meta)
        text_1 = os.path.join(out_dir_curr, text_1_org)
        text_2 = os.path.join(out_dir_curr, text_2_org)
        out_deploy = os.path.join(out_dir_curr, deploy_file)

        subprocess.call('mkdir -p ' + util.escapeString(out_dir_curr),
                        shell=True)

        list_1, list_2 = getImageListForFlow(im_dir)
        util.writeFile(text_1, list_1)
        util.writeFile(text_2, list_2)

        # im_test=util.escapeString(list_1[0]);
        dim_list = [
            int(dimstr) for dimstr in str(
                subprocess.check_output([path_to_sizer, list_1[0]])).split(',')
        ]
        replaceProto(path_to_deploy, out_deploy, dim_list, text_1, text_2,
                     len(list_1), out_dir_curr)

        args = [
            caffe_bin, 'test', '-model',
            util.escapeString(out_deploy), '-weights', path_to_model,
            '-iterations', '1', '-gpu',
            str(gpu)
        ]

        cmd = str.join(' ', args)
        commands.append(cmd)

    # print('Executing %s' % cmd)
    util.writeFile(out_file_commands, commands)
def script_writeFlownetCommands(params):
    video_list_file= params.video_list_file;
    path_to_video_meta= params.path_to_video_meta;
    in_dir_meta= params.in_dir_meta;
    out_dir_meta= params.out_dir_meta;
    path_to_deploy= params.path_to_deploy;
    out_file_commands= params.out_file_commands;
    dir_flownet_meta= params.dir_flownet_meta;
    path_to_sizer= params.path_to_sizer;
    caffe_bin = params.caffe_bin;
    path_to_model = params.path_to_model;
    text_1_org= params.text_1;
    text_2_org= params.text_2;
    deploy_file= params.deploy_file;
    gpu= params.gpu;

    im_dirs=util.readLinesFromFile(video_list_file);
    im_dirs=[im_dir.replace(path_to_video_meta,in_dir_meta)[:-4] for im_dir in im_dirs];
    
    commands=[];
    # im_dirs=['/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data/hmdb/pick/THE_WALLET_TRICK!!!_pick_f_cm_np2_ba_med_1'];
    for idx_im_dir,im_dir in enumerate(im_dirs):
        print idx_im_dir,len(im_dirs);
        out_dir_curr=im_dir.replace(in_dir_meta,out_dir_meta);
        text_1=os.path.join(out_dir_curr,text_1_org);
        text_2=os.path.join(out_dir_curr,text_2_org);
        out_deploy=os.path.join(out_dir_curr,deploy_file);

        subprocess.call('mkdir -p '+util.escapeString(out_dir_curr), shell=True)
        
        list_1,list_2 = getImageListForFlow(im_dir)
        util.writeFile(text_1,list_1);
        util.writeFile(text_2,list_2);
        
        # im_test=util.escapeString(list_1[0]);
        dim_list = [int(dimstr) for dimstr in str(subprocess.check_output([path_to_sizer, list_1[0]])).split(',')]
        replaceProto(path_to_deploy,out_deploy,dim_list,text_1,text_2,len(list_1),out_dir_curr)
        
        args = [caffe_bin, 'test', '-model', util.escapeString(out_deploy),
            '-weights', path_to_model,
            '-iterations', '1',
            '-gpu', str(gpu)]

        cmd = str.join(' ', args)
        commands.append(cmd);

    # print('Executing %s' % cmd)
    util.writeFile(out_file_commands,commands);
def script_writeCommandsForPreprocessing(all_dirs_file,command_file_pre,num_proc,check_file=None):
    all_dirs=util.readLinesFromFile(all_dirs_file);
    all_dirs=[dir_curr[:-1] for dir_curr in all_dirs];
    
    if check_file is not None:
        all_dirs=getRemainingDirs(all_dirs,check_file);

    command_pre='echo '
    command_middle_1=';cd ~/Downloads/opticalflow; matlab -nojvm -nodisplay -nosplash -r "out_folder=\''
    command_middle='\';saveTrainingData" > '
    command_end=' 2>&1';

    commands=[];
    for dir_curr in all_dirs:
        dir_curr=util.escapeString(dir_curr);
        log_file=os.path.join(dir_curr,'log.txt');
        command=command_pre+dir_curr+command_middle_1+dir_curr+command_middle+log_file+command_end;
        commands.append(command);
    
    idx_range=util.getIdxRange(len(commands),len(commands)/num_proc)
    command_files=[];
    for i,start_idx in enumerate(idx_range[:-1]):
        command_file_curr=command_file_pre+str(i)+'.txt'
        end_idx=idx_range[i+1]
        commands_rel=commands[start_idx:end_idx];
        util.writeFile(command_file_curr,commands_rel);
        command_files.append(command_file_curr);
    return command_files;
Esempio n. 5
0
def script_writeCommandsForPreprocessing(all_dirs_file,
                                         command_file_pre,
                                         num_proc,
                                         check_file=None):
    all_dirs = util.readLinesFromFile(all_dirs_file)
    all_dirs = [dir_curr[:-1] for dir_curr in all_dirs]

    if check_file is not None:
        all_dirs = getRemainingDirs(all_dirs, check_file)

    command_pre = 'echo '
    command_middle_1 = ';cd ~/Downloads/opticalflow; matlab -nojvm -nodisplay -nosplash -r "out_folder=\''
    command_middle = '\';saveTrainingData" > '
    command_end = ' 2>&1'

    commands = []
    for dir_curr in all_dirs:
        dir_curr = util.escapeString(dir_curr)
        log_file = os.path.join(dir_curr, 'log.txt')
        command = command_pre + dir_curr + command_middle_1 + dir_curr + command_middle + log_file + command_end
        commands.append(command)

    idx_range = util.getIdxRange(len(commands), len(commands) / num_proc)
    command_files = []
    for i, start_idx in enumerate(idx_range[:-1]):
        command_file_curr = command_file_pre + str(i) + '.txt'
        end_idx = idx_range[i + 1]
        commands_rel = commands[start_idx:end_idx]
        util.writeFile(command_file_curr, commands_rel)
        command_files.append(command_file_curr)
    return command_files
def script_saveCommands(in_dir_meta,out_dir_meta,out_file_text):
    cats=[dir_curr for dir_curr in os.listdir(in_dir_meta) if os.path.isdir(os.path.join(in_dir_meta,dir_curr))];
    # print len(cats);
    commands=[];
    command_text='ffmpeg -i '
    for cat in cats:
        dir_curr=os.path.join(in_dir_meta,cat);
        dir_out_curr=os.path.join(out_dir_meta,cat);
        print dir_out_curr;
        
        if not os.path.exists(dir_out_curr):
            os.mkdir(dir_out_curr);
        videos=[file_curr for file_curr in os.listdir(dir_curr) if file_curr.endswith('.avi')];
        for video in videos:
            out_dir_video_org=os.path.join(dir_out_curr,video[:-4])
            out_dir_video = util.escapeString(out_dir_video_org)
            command_curr='mkdir '+out_dir_video+';'+command_text+util.escapeString(os.path.join(dir_curr,video))+' '+out_dir_video+'/image_%d.ppm'
            commands.append(command_curr);
    with open(out_file_text,'wb') as f:
        for command in commands:
            f.write(command+'\n');
def writeCommands_hacky(out_file_commands,dirs,caffe_bin,deploy_name,path_to_model,gpu):
    commands=[];
    for dir_curr in dirs:
        out_deploy=os.path.join(dir_curr,deploy_name);
        args = [caffe_bin, 'test', '-model', util.escapeString(out_deploy),
            '-weights', path_to_model,
            '-iterations', '1',
            '-gpu', str(gpu)]

        cmd = str.join(' ', args)
        commands.append(cmd)
    util.writeFile(out_file_commands,commands);
Esempio n. 8
0
def writeCommands_hacky(out_file_commands, dirs, caffe_bin, deploy_name,
                        path_to_model, gpu):
    commands = []
    for dir_curr in dirs:
        out_deploy = os.path.join(dir_curr, deploy_name)
        args = [
            caffe_bin, 'test', '-model',
            util.escapeString(out_deploy), '-weights', path_to_model,
            '-iterations', '1', '-gpu',
            str(gpu)
        ]

        cmd = str.join(' ', args)
        commands.append(cmd)
    util.writeFile(out_file_commands, commands)
Esempio n. 9
0
def main():
    # dir_meta='/disk2/flow_data';
    # dir_meta_old='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    # deploy_file='deploy.prototxt';
    # dir_mids=[os.path.join(dir_meta,dir_mid) for dir_mid in os.listdir(dir_meta) if os.path.isdir(os.path.join(dir_meta,dir_mid))]
    # dirs_left=[os.path.join(dir_mid,dir_curr) for dir_mid in dir_mids for dir_curr in os.listdir(dir_mid) if os.path.isdir(os.path.join(dir_mid,dir_curr))]

    # dirs_left=[os.path.join(dir_mid,dir_curr) for dir_mid in dirs_left for dir_curr in os.listdir(dir_mid) if os.path.isdir(os.path.join(dir_mid,dir_curr))]

    # print len(dirs_left);
    # print dirs_left[0];

    # for dir_curr in dirs_left:
    #     deploy_curr=os.path.join(dir_curr,deploy_file);
    #     print deploy_curr
    #     data=[];
    #     with open(deploy_curr,'r') as f:
    #         data = f.read()

    #     with open(deploy_curr+'_backup','w') as f:
    #         f.write(data);

    #     data = data.replace(dir_meta_old, dir_meta)
    #     with open(deploy_curr, "w") as f:
    #         f.write(data);

    # return
    # video_list_file='/disk2/video_data/video_list.txt'
    # path_to_video_meta='/disk2/video_data';

    # path_to_flo_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    # path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';

    # video_files=util.readLinesFromFile(video_list_file);
    # # image_dirs=[dir_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for dir_curr in video_files];
    # # flo_dirs=[dir_curr.replace(path_to_video_meta,path_to_flo_meta)[:-4] for dir_curr in video_files];
    # flo_dirs=pickle.load(open('/disk2/temp/dirs_done.p','rb'));
    # image_dirs=[dir_curr.replace(path_to_flo_meta,path_to_im_meta) for dir_curr in flo_dirs];
    # print len(image_dirs)
    # out_dir='/disk2/image_data_moved';

    # out_file='/disk2/image_data_moved/mv_commands_2.txt'
    # commands=[];
    # image_dirs_to_move=image_dirs[5000:7000];
    # for image_dir in image_dirs_to_move:
    #     image_dir=util.escapeString(image_dir);
    #     new_dir=image_dir.replace(path_to_im_meta,out_dir);
    #     command='mkdir -p '+new_dir+';';
    #     command=command+'mv '+image_dir+'/* '+new_dir;
    #     commands.append(command);
    # util.writeFile('/disk2/image_data_moved/dirs_moved_2.txt',image_dirs_to_move);
    # util.writeFile(out_file,commands);

    # return
    video_list_file = '/disk2/video_data/video_list.txt'
    path_to_video_meta = '/disk2/video_data'

    # path_to_flo_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    path_to_flo_meta = '/disk2/flow_data'
    path_to_im_meta = '/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data'

    video_files = util.readLinesFromFile(video_list_file)
    # image_dirs=[dir_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for dir_curr in video_files];
    # flo_dirs=[dir_curr.replace(path_to_video_meta,path_to_flo_meta)[:-4] for dir_curr in video_files];
    flo_dirs = pickle.load(open('/disk2/temp/dirs_done_disk2.p', 'rb'))
    image_dirs = [
        dir_curr.replace(path_to_flo_meta, path_to_im_meta)
        for dir_curr in flo_dirs
    ]
    print len(image_dirs)
    finished = []
    i = 0
    for image_dir, flo_dir in zip(image_dirs, flo_dirs):
        print i
        count_im_command = 'ls ' + os.path.join(util.escapeString(image_dir),
                                                '*.ppm') + '| wc -l'
        count_flo_command = 'ls ' + os.path.join(util.escapeString(flo_dir),
                                                 '*.flo') + '| wc -l'

        # im_count=int(subprocess.check_output(count_im_command,shell=True));
        # flo_count=int(subprocess.check_output(count_flo_command,shell=True));
        im_count = len([
            file_curr for file_curr in os.listdir(image_dir)
            if file_curr.endswith('.ppm')
        ])
        flo_count = len([
            file_curr for file_curr in os.listdir(flo_dir)
            if file_curr.endswith('.flo')
        ])
        print i, flo_count, im_count
        if flo_count + 1 == im_count:
            finished.append(1)
        else:
            finished.append(0)

        i += 1

    finished = np.array(finished)
    print 'done', sum(finished == 1)
    print 'not done', sum(finished == 0)

    pickle.dump([finished, image_dirs], open('/disk2/temp/to_rerun.p', 'wb'))

    return
    dir_flownet_meta = '/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet'
    caffe_bin = os.path.join(dir_flownet_meta, 'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,
                                 'model/flownet_official.caffemodel')

    video_list_file = '/disk2/video_data/video_list.txt'
    path_to_video_meta = '/disk2/video_data'

    in_dir_meta = '/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data'
    in_dir_meta = '/disk2/flow_data'
    # if not os.path.exists(new_in_dir_meta):
    #     os.mkdir(new_in_dir_meta);

    deploy_name = 'deploy.prototxt'
    gpu = 0

    dirs = [
        dir_curr.replace(path_to_video_meta, in_dir_meta)[:-4]
        for dir_curr in util.readLinesFromFile(video_list_file)
    ]
    dirs = [dir_curr for dir_curr in dirs if os.path.exists(dir_curr)]
    counts = [
        len(os.listdir(dir_curr)) for dir_curr in dirs
        if os.path.exists(dir_curr)
    ]
    dirs_left = []
    dirs_done = []
    for idx_count, count in enumerate(counts):
        if count == 4:
            dirs_left.append(dirs[idx_count])
            # dir_curr=dirs[idx_count]
            # deploy_curr=os.path.join(dir_curr,deploy_name);
            # im_file=os.path.join(dir_curr,'im_1.txt');
            # batch_size = sum(1 for line in open(im_file))

            # old_str='batch_size: '+str(int(ceil(batch_size/5)));
            # print old_str,

            # batch_size = int(ceil(batch_size/8));
            # new_str='batch_size: '+str(batch_size);
            # print new_str

            # data=[];
            # with open(deploy_curr,'r') as f:
            #     data = f.read()
            # # print data[:300];
            # assert old_str in data;
            # data = data.replace(old_str, new_str)
            # # print data[:300];
            # with open(deploy_curr, "w") as f:
            #     f.write(data);

            # out_dir_curr=dir_curr.replace(in_dir_meta,new_in_dir_meta);
            #mkdir of new location
            # mkdir_command='mkdir -p '+util.escapeString(out_dir_curr)
            # print mkdir_command
            # subprocess.call(mkdir_command, shell=True)

            #mv contents from old to new
            # mv_command='mv '+util.escapeString(dir_curr)+'/* '+util.escapeString(out_dir_curr);
            # print mv_command
            # subprocess.call(mv_command, shell=True)
            #append new to dirs_left
            # dirs_left.append(out_dir_curr);
            # raw_input();
        else:
            dirs_done.append(dirs[idx_count])

    print min(counts)
    counts = np.array(counts)
    print sum(counts == 4)
    print len(dirs_left)

    mid_point = len(dirs_left) / 2

    print mid_point, len(dirs_left) - mid_point
    out_file_commands = '/disk2/januaryExperiments/gettingFlows/flownet_commands_left_0.txt'
    gpu = 0
    # writeCommands_hacky(out_file_commands,dirs_left[:mid_point],caffe_bin,deploy_name,path_to_model,gpu)

    out_file_commands = '/disk2/januaryExperiments/gettingFlows/flownet_commands_left_1.txt'
    gpu = 1
    # writeCommands_hacky(out_file_commands,dirs_left[mid_point:],caffe_bin,deploy_name,path_to_model,gpu)

    print len(dirs_done)
    pickle.dump(dirs_done, open('/disk2/temp/dirs_done_disk2.p', 'wb'))

    return
    video_list_file = '/disk2/video_data/video_list.txt'
    path_to_video_meta = '/disk2/video_data'

    in_dir_meta = '/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data'
    out_dir_meta = '/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data'
    path_to_deploy = '/disk2/januaryExperiments/gettingFlows/deploy_template.prototxt'
    out_file_commands = '/disk2/januaryExperiments/gettingFlows/flownet_commands.txt'

    dir_flownet_meta = '/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet'
    path_to_sizer = os.path.join(dir_flownet_meta, 'bin/get_image_size')
    caffe_bin = os.path.join(dir_flownet_meta, 'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,
                                 'model/flownet_official.caffemodel')

    text_1 = 'im_1.txt'
    text_2 = 'im_2.txt'
    deploy_file = 'deploy.prototxt'
    gpu = 0

    params_dict = {}
    params_dict['video_list_file'] = video_list_file
    params_dict['path_to_video_meta'] = path_to_video_meta
    params_dict['in_dir_meta'] = in_dir_meta
    params_dict['out_dir_meta'] = out_dir_meta
    params_dict['path_to_deploy'] = path_to_deploy
    params_dict['out_file_commands'] = out_file_commands
    params_dict['dir_flownet_meta'] = dir_flownet_meta
    params_dict['path_to_sizer'] = path_to_sizer
    params_dict['caffe_bin'] = caffe_bin
    params_dict['path_to_model'] = path_to_model
    params_dict['text_1'] = text_1
    params_dict['text_2'] = text_2
    params_dict['deploy_file'] = deploy_file
    params_dict['gpu'] = gpu

    params = createParams('writeFlownetCommands')
    params = params(**params_dict)
    # script_writeFlownetCommands(params);
    commands = util.readLinesFromFile(params.out_file_commands)
    commands = [c.replace('-gpu 1', '-gpu 0') for c in commands]
    util.writeFile(params.out_file_commands, commands)
    pickle.dump(params._asdict(),
                open(params.out_file_commands + '_meta_experiment.p', 'wb'))

    return
    video_list_file = '/disk2/video_data/video_list.txt'
    path_to_im_meta = '/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data'
    path_to_video_meta = '/disk2/video_data'
    commands_file_text = '/disk2/januaryExperiments/gettingFlows/resize_commands.txt'

    video_list = util.readLinesFromFile(video_list_file)
    print len(video_list)
    image_dirs = [
        video_curr.replace(path_to_video_meta, path_to_im_meta)[:-4]
        for video_curr in video_list
    ]
    print len(image_dirs), image_dirs[0]
    image_dirs = image_dirs[:1]

    commands = []
    command_conv = ['convert', '-resize 512x384']
    for image_dir in image_dirs:
        image_list = [
            os.path.join(image_dir, im) for im in os.listdir(image_dir)
            if im.endswith('.ppm')
        ]
        for image_curr in image_list:
            command_curr = [
                command_conv[0], image_curr, command_conv[1], image_curr
            ]
            command_curr = ' '.join(command_curr)
            commands.append(command_curr)

    print len(commands)
    print commands[0]
    util.writeFile(commands_file_text, commands)

    return
    video_list_file = '/disk2/video_data/video_list.txt'
    path_to_im_meta = '/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data'
    path_to_video_meta = '/disk2/video_data'
    path_to_txt_1 = '/disk2/januaryExperiments/gettingFlows/temp_im_1.txt'
    path_to_txt_2 = '/disk2/januaryExperiments/gettingFlows/temp_im_2.txt'

    video_list = util.readLinesFromFile(video_list_file)
    print len(video_list)
    image_dirs = [
        video_curr.replace(path_to_video_meta, path_to_im_meta)[:-4]
        for video_curr in video_list
    ]
    print len(image_dirs), image_dirs[0]

    list_1 = []
    list_2 = []
    for image_dir in image_dirs[:10]:
        list_1_curr, list_2_curr = getImageListForFlow(image_dir)
        list_1.extend(list_1_curr[:3])
        list_2.extend(list_2_curr[:3])

    assert len(list_1) == len(list_2)

    util.writeFile(path_to_txt_1, list_1)
    util.writeFile(path_to_txt_2, list_2)
def main():
    # list_params = ['out_file_html',
    #                     'out_dir_flo_im',
    #                     'flo_dir',
    #                     'im_file_1',
    #                     'im_file_2',
    #                     'height',
    #                     'width',
    #                     'rel_path'];
    # video_dir = '/disk2/marchExperiments/ucf-101- new/v_HeadMassage_g23_c07';
    # params_dict={};
    # params_dict['out_dir_flo_im'] = os.path.join(video_dir,'flo_im');
    # util.mkdir(params_dict['out_dir_flo_im']);
    # params_dict['out_file_html'] = os.path.join(video_dir,'visualize_flos.html');
    # params_dict['flo_dir'] = os.path.join(video_dir,'flo');
    # params_dict['im_file_1'] = os.path.join(video_dir,'flo','im_1.txt')
    # params_dict['im_file_2'] = os.path.join(video_dir,'flo','im_2.txt')
    # params_dict['height'] = 240;
    # params_dict['width'] = 320;
    # params_dict['rel_path'] = ['/disk2','../../../..'];

    # params=createParams('visualizeFlos');
    # params=params(**params_dict);
    
    # script_visualizeFlos(params);
    # pickle.dump(params._asdict(),open(params.out_file_html+'_meta_experiment.p','wb'));


    # return
    out_dir_old='/disk2/marchExperiments/ucf-101';
    out_dir='/disk2/marchExperiments/ucf-101-new';
    subset_size=100;
    # path_to_video_subset=os.path.join(out_dir_old,'video_files_subset_'+str(subset_size)+'.txt');
    path_to_video_subset=os.path.join(out_dir,'video_list_0.txt');
    videos=util.readLinesFromFile(path_to_video_subset);
    gpu=0;
    # random.shuffle(videos);
    # halfway=len(videos)/2;
    # videos_1=videos[:halfway];
    # videos_2=videos[halfway:];
    # print len(videos_1),len(videos_2),len(set(videos_1+videos_2)),len(videos)
    # util.writeFile(os.path.join(out_dir,'video_list_0.txt'),videos_1);
    # util.writeFile(os.path.join(out_dir,'video_list_1.txt'),videos_2);

    # return

    
    size_vid=[320,240];
    size_flo=[160,120];
    path_to_deploy='';
    out_deploy='deploy.prototxt';
    # gpu=0;

    
    args=[];
    for idx_curr,path_to_video in enumerate(videos):
    #     print path_to_video,
    # idx_curr=0;
    # path_to_video='/disk2/video_data/UCF-101/ApplyLipstick/v_ApplyLipstick_g06_c04.avi'
        args_curr=(path_to_video,out_dir,size_vid,size_flo,path_to_deploy,out_deploy,gpu)
        script_processVideo(args_curr);

        # args.append(args_curr)

    print len(args);
    # return
        # script_processVideo(path_to_video,out_dir,size_vid,size_flo,path_to_deploy,out_deploy,gpu)
        # break;
    # n_jobs=4;
    # t=time.time();
    # p=multiprocessing.Pool(n_jobs);
    # p.map(script_processVideo,args);
    # print time.time()-t
        # break;


    return
    out_dir='/disk2/marchExperiments/ucf-101';
    path_to_video_txt=os.path.join(out_dir,'video_files.txt');
    subset_size=100;
    
    videos=util.readLinesFromFile(path_to_video_txt);
    random.shuffle(videos);
    videos_subset=videos[:subset_size];
    util.writeFile(path_to_video_subset,videos_subset);




    return

    path_to_deploy='/disk2/januaryExperiments/gettingFlows/deploy_template.prototxt';
    dir_flownet_meta='/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet';
    path_to_sizer=os.path.join(dir_flownet_meta,'bin/get_image_size')
    caffe_bin = os.path.join(dir_flownet_meta,'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,'model/flownet_official.caffemodel')
    
    text_1='im_1.txt';
    text_2='im_2.txt';
    deploy_file='deploy.prototxt';
    gpu=0;

    im_dir='/disk2/marchExperiments/debug_data/image'
    out_deploy='deploy.prototxt'
    out_dir_curr='/disk2/marchExperiments/debug_data/flo/'
    list_1,list_2 = getImageListForFlow(im_dir)
    # print list_1
    util.writeFile(text_1,list_1);
    util.writeFile(text_2,list_2);

    dim_list = [int(dimstr) for dimstr in str(subprocess.check_output([path_to_sizer, list_1[0]])).split(',')]
    print dim_list
    dim_list=[160,120];
    replaceProto(path_to_deploy,out_deploy,dim_list,text_1,text_2,len(list_1),out_dir_curr)
        
    args = [caffe_bin, 'test', '-model', util.escapeString(out_deploy),
            '-weights', path_to_model,
            '-iterations', '1',
            '-gpu', str(gpu)]

    cmd = str.join(' ', args)

    print cmd;

    return
    # dir_meta='/disk2/flow_data';
    # dir_meta_old='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    # deploy_file='deploy.prototxt';
    # dir_mids=[os.path.join(dir_meta,dir_mid) for dir_mid in os.listdir(dir_meta) if os.path.isdir(os.path.join(dir_meta,dir_mid))]
    # dirs_left=[os.path.join(dir_mid,dir_curr) for dir_mid in dir_mids for dir_curr in os.listdir(dir_mid) if os.path.isdir(os.path.join(dir_mid,dir_curr))]

    # dirs_left=[os.path.join(dir_mid,dir_curr) for dir_mid in dirs_left for dir_curr in os.listdir(dir_mid) if os.path.isdir(os.path.join(dir_mid,dir_curr))]
    
    # print len(dirs_left);
    # print dirs_left[0];

    # for dir_curr in dirs_left:
    #     deploy_curr=os.path.join(dir_curr,deploy_file);
    #     print deploy_curr
    #     data=[];
    #     with open(deploy_curr,'r') as f:
    #         data = f.read()
        
    #     with open(deploy_curr+'_backup','w') as f:
    #         f.write(data);

    #     data = data.replace(dir_meta_old, dir_meta)
    #     with open(deploy_curr, "w") as f:
    #         f.write(data);

    # return
    # video_list_file='/disk2/video_data/video_list.txt'
    # path_to_video_meta='/disk2/video_data';

    # path_to_flo_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    # path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    
    # video_files=util.readLinesFromFile(video_list_file);
    # # image_dirs=[dir_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for dir_curr in video_files];
    # # flo_dirs=[dir_curr.replace(path_to_video_meta,path_to_flo_meta)[:-4] for dir_curr in video_files];
    # flo_dirs=pickle.load(open('/disk2/temp/dirs_done.p','rb'));
    # image_dirs=[dir_curr.replace(path_to_flo_meta,path_to_im_meta) for dir_curr in flo_dirs];
    # print len(image_dirs)
    # out_dir='/disk2/image_data_moved';

    # out_file='/disk2/image_data_moved/mv_commands_2.txt'
    # commands=[];
    # image_dirs_to_move=image_dirs[5000:7000];
    # for image_dir in image_dirs_to_move:
    #     image_dir=util.escapeString(image_dir);
    #     new_dir=image_dir.replace(path_to_im_meta,out_dir);
    #     command='mkdir -p '+new_dir+';';
    #     command=command+'mv '+image_dir+'/* '+new_dir;
    #     commands.append(command);
    # util.writeFile('/disk2/image_data_moved/dirs_moved_2.txt',image_dirs_to_move);
    # util.writeFile(out_file,commands);

    # return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_video_meta='/disk2/video_data';

    # path_to_flo_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    path_to_flo_meta='/disk2/flow_data';
    path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    
    video_files=util.readLinesFromFile(video_list_file);
    # image_dirs=[dir_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for dir_curr in video_files];
    # flo_dirs=[dir_curr.replace(path_to_video_meta,path_to_flo_meta)[:-4] for dir_curr in video_files];
    flo_dirs=pickle.load(open('/disk2/temp/dirs_done_disk2.p','rb'));
    image_dirs=[dir_curr.replace(path_to_flo_meta,path_to_im_meta) for dir_curr in flo_dirs];
    print len(image_dirs);
    finished=[];
    i=0;
    for image_dir,flo_dir in zip(image_dirs,flo_dirs):
        print i
        count_im_command='ls '+os.path.join(util.escapeString(image_dir),'*.ppm')+'| wc -l';
        count_flo_command='ls '+os.path.join(util.escapeString(flo_dir),'*.flo')+'| wc -l';
        
        # im_count=int(subprocess.check_output(count_im_command,shell=True));
        # flo_count=int(subprocess.check_output(count_flo_command,shell=True));
        im_count=len([file_curr for file_curr in os.listdir(image_dir) if file_curr.endswith('.ppm')]);
        flo_count=len([file_curr for file_curr in os.listdir(flo_dir) if file_curr.endswith('.flo')]);
        print i,flo_count,im_count
        if flo_count+1==im_count:
            finished.append(1);
        else:
            finished.append(0);
        
        i+=1;

    finished=np.array(finished);
    print 'done',sum(finished==1);
    print 'not done',sum(finished==0);

    pickle.dump([finished,image_dirs],open('/disk2/temp/to_rerun.p','wb'));
    
    return
    dir_flownet_meta='/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet';
    caffe_bin = os.path.join(dir_flownet_meta,'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,'model/flownet_official.caffemodel')

    video_list_file='/disk2/video_data/video_list.txt'
    path_to_video_meta='/disk2/video_data';

    in_dir_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    in_dir_meta='/disk2/flow_data';
    # if not os.path.exists(new_in_dir_meta):
    #     os.mkdir(new_in_dir_meta);

    deploy_name='deploy.prototxt';
    gpu=0;

    dirs=[dir_curr.replace(path_to_video_meta,in_dir_meta)[:-4] for dir_curr in util.readLinesFromFile(video_list_file)];
    dirs=[dir_curr for dir_curr in dirs if os.path.exists(dir_curr)]
    counts=[len(os.listdir(dir_curr)) for dir_curr in dirs if os.path.exists(dir_curr)];
    dirs_left=[]
    dirs_done=[];
    for idx_count,count in enumerate(counts):
        if count==4:
            dirs_left.append(dirs[idx_count])
            # dir_curr=dirs[idx_count]
            # deploy_curr=os.path.join(dir_curr,deploy_name);
            # im_file=os.path.join(dir_curr,'im_1.txt');
            # batch_size = sum(1 for line in open(im_file))
            
            # old_str='batch_size: '+str(int(ceil(batch_size/5)));
            # print old_str,
            
            # batch_size = int(ceil(batch_size/8));
            # new_str='batch_size: '+str(batch_size);
            # print new_str

            # data=[];
            # with open(deploy_curr,'r') as f:
            #     data = f.read()
            # # print data[:300];
            # assert old_str in data;
            # data = data.replace(old_str, new_str)
            # # print data[:300];
            # with open(deploy_curr, "w") as f:
            #     f.write(data);

            # out_dir_curr=dir_curr.replace(in_dir_meta,new_in_dir_meta);
            #mkdir of new location
            # mkdir_command='mkdir -p '+util.escapeString(out_dir_curr)
            # print mkdir_command
            # subprocess.call(mkdir_command, shell=True)

            #mv contents from old to new
            # mv_command='mv '+util.escapeString(dir_curr)+'/* '+util.escapeString(out_dir_curr);
            # print mv_command
            # subprocess.call(mv_command, shell=True)
            #append new to dirs_left
            # dirs_left.append(out_dir_curr);
            # raw_input();
        else:
            dirs_done.append(dirs[idx_count])



    print min(counts);
    counts=np.array(counts);
    print sum(counts==4);
    print len(dirs_left)
    
    mid_point=len(dirs_left)/2;
    
    print mid_point,len(dirs_left)-mid_point
    out_file_commands='/disk2/januaryExperiments/gettingFlows/flownet_commands_left_0.txt';
    gpu=0;
    # writeCommands_hacky(out_file_commands,dirs_left[:mid_point],caffe_bin,deploy_name,path_to_model,gpu)
    
    out_file_commands='/disk2/januaryExperiments/gettingFlows/flownet_commands_left_1.txt';
    gpu=1;
    # writeCommands_hacky(out_file_commands,dirs_left[mid_point:],caffe_bin,deploy_name,path_to_model,gpu)
    
    print len(dirs_done);
    pickle.dump(dirs_done,open('/disk2/temp/dirs_done_disk2.p','wb'));

    return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_video_meta='/disk2/video_data';

    in_dir_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    out_dir_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    path_to_deploy='/disk2/januaryExperiments/gettingFlows/deploy_template.prototxt';
    out_file_commands='/disk2/januaryExperiments/gettingFlows/flownet_commands.txt';

    dir_flownet_meta='/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet';
    path_to_sizer=os.path.join(dir_flownet_meta,'bin/get_image_size')
    caffe_bin = os.path.join(dir_flownet_meta,'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,'model/flownet_official.caffemodel')
    
    text_1='im_1.txt';
    text_2='im_2.txt';
    deploy_file='deploy.prototxt';
    gpu=0;

    params_dict={};
    params_dict['video_list_file'] = video_list_file;
    params_dict['path_to_video_meta'] = path_to_video_meta;
    params_dict['in_dir_meta'] = in_dir_meta;
    params_dict['out_dir_meta'] = out_dir_meta;
    params_dict['path_to_deploy'] = path_to_deploy;
    params_dict['out_file_commands'] = out_file_commands;
    params_dict['dir_flownet_meta'] = dir_flownet_meta;
    params_dict['path_to_sizer'] = path_to_sizer;
    params_dict['caffe_bin']  = caffe_bin;
    params_dict['path_to_model']  = path_to_model;
    params_dict['text_1'] = text_1;
    params_dict['text_2'] = text_2;
    params_dict['deploy_file'] = deploy_file;
    params_dict['gpu'] = gpu;
    
    params=createParams('writeFlownetCommands');
    params=params(**params_dict);
    # script_writeFlownetCommands(params);
    commands=util.readLinesFromFile(params.out_file_commands);
    commands=[c.replace('-gpu 1','-gpu 0') for c in commands];
    util.writeFile(params.out_file_commands,commands);
    pickle.dump(params._asdict(),open(params.out_file_commands+'_meta_experiment.p','wb'));




    return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    path_to_video_meta='/disk2/video_data';
    commands_file_text='/disk2/januaryExperiments/gettingFlows/resize_commands.txt';

    video_list=util.readLinesFromFile(video_list_file);
    print len(video_list);
    image_dirs=[video_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for video_curr in video_list];
    print len(image_dirs),image_dirs[0];
    image_dirs=image_dirs[:1];

    commands=[];
    command_conv=['convert','-resize 512x384'];
    for image_dir in image_dirs:
        image_list=[os.path.join(image_dir,im) for im in os.listdir(image_dir) if im.endswith('.ppm')];
        for image_curr in image_list:
            command_curr=[command_conv[0],image_curr,command_conv[1],image_curr];
            command_curr=' '.join(command_curr);
            commands.append(command_curr);

    print len(commands);
    print commands[0];
    util.writeFile(commands_file_text,commands);



    return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    path_to_video_meta='/disk2/video_data';
    path_to_txt_1='/disk2/januaryExperiments/gettingFlows/temp_im_1.txt';
    path_to_txt_2='/disk2/januaryExperiments/gettingFlows/temp_im_2.txt';

    video_list=util.readLinesFromFile(video_list_file);
    print len(video_list);
    image_dirs=[video_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for video_curr in video_list];
    print len(image_dirs),image_dirs[0];
    
    list_1=[];list_2=[];
    for image_dir in image_dirs[:10]:
        list_1_curr,list_2_curr=getImageListForFlow(image_dir);
        list_1.extend(list_1_curr[:3]);
        list_2.extend(list_2_curr[:3]);
    
    assert len(list_1)==len(list_2);

    util.writeFile(path_to_txt_1,list_1);
    util.writeFile(path_to_txt_2,list_2);
def script_processVideo((path_to_video,out_dir_meta,size_vid,size_flo,path_to_deploy,out_deploy,gpu)):
    video_name=path_to_video[path_to_video.rindex('/')+1:];
    video_name=video_name[:video_name.rindex('.')];
    out_dir=os.path.join(out_dir_meta,video_name)
    if os.path.exists(os.path.join(out_dir,'done')):
        print 'skipping';
        return;
    util.mkdir(out_dir);
    
    # temp_dir=os.path.join(out_dir,'temp')
    im_dir=os.path.join(out_dir,'im')
    flo_dir=os.path.join(out_dir,'flo')
    
    # util.mkdir(temp_dir);
    util.mkdir(im_dir);
    util.mkdir(flo_dir);
    out_video=os.path.join(out_dir,'temp_vid.avi');

    ffmpeg_command='ffmpeg -i '+util.escapeString(path_to_video)+' -vf scale='+str(size_vid[0])+':'+str(size_vid[1])+' -y '+util.escapeString(out_video);
    print ffmpeg_command;

    subprocess.call(ffmpeg_command, shell=True)

    ffmpeg_command_resize='ffmpeg -i '+util.escapeString(out_video)+' '+os.path.join(util.escapeString(im_dir),'image_%d.jpg');
    print ffmpeg_command_resize;
    
    subprocess.call(ffmpeg_command_resize, shell=True)

    list_1,list_2 = getImageListForFlow(im_dir,10)
    # print list_1
    text_1=os.path.join(flo_dir,'im_1.txt');
    text_2=os.path.join(flo_dir,'im_2.txt');
    util.writeFile(text_1,list_1);
    util.writeFile(text_2,list_2);

    out_deploy=os.path.join(flo_dir,out_deploy);
    path_to_deploy='/disk2/januaryExperiments/gettingFlows/deploy_template.prototxt';
    dir_flownet_meta='/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet';
    path_to_sizer=os.path.join(dir_flownet_meta,'bin/get_image_size')
    caffe_bin = os.path.join(dir_flownet_meta,'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,'model/flownet_official.caffemodel')

    if len(list_1)<500:
        batch_size = len(list_1);
        iterations = 1;
    else:
        # print len(list_1);
        batch_size = math.floor(len(list_1)/8);
        # print batch_size;
        iterations = 8;

    replaceProto(path_to_deploy,out_deploy,size_flo,text_1,text_2,batch_size,flo_dir)
        
    args = [caffe_bin, 'test', '-model', util.escapeString(out_deploy),
            '-weights', path_to_model,
            '-iterations', str(iterations),
            '-gpu', str(gpu)]

    cmd = str.join(' ', args)
    print cmd

    subprocess.call(cmd, shell=True)
    util.mkdir(os.path.join(out_dir,'done'));
    print 'done';
Esempio n. 12
0
def main():
    # dir_meta='/disk2/flow_data';
    # dir_meta_old='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    # deploy_file='deploy.prototxt';
    # dir_mids=[os.path.join(dir_meta,dir_mid) for dir_mid in os.listdir(dir_meta) if os.path.isdir(os.path.join(dir_meta,dir_mid))]
    # dirs_left=[os.path.join(dir_mid,dir_curr) for dir_mid in dir_mids for dir_curr in os.listdir(dir_mid) if os.path.isdir(os.path.join(dir_mid,dir_curr))]

    # dirs_left=[os.path.join(dir_mid,dir_curr) for dir_mid in dirs_left for dir_curr in os.listdir(dir_mid) if os.path.isdir(os.path.join(dir_mid,dir_curr))]
    
    # print len(dirs_left);
    # print dirs_left[0];

    # for dir_curr in dirs_left:
    #     deploy_curr=os.path.join(dir_curr,deploy_file);
    #     print deploy_curr
    #     data=[];
    #     with open(deploy_curr,'r') as f:
    #         data = f.read()
        
    #     with open(deploy_curr+'_backup','w') as f:
    #         f.write(data);

    #     data = data.replace(dir_meta_old, dir_meta)
    #     with open(deploy_curr, "w") as f:
    #         f.write(data);

    # return
    # video_list_file='/disk2/video_data/video_list.txt'
    # path_to_video_meta='/disk2/video_data';

    # path_to_flo_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    # path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    
    # video_files=util.readLinesFromFile(video_list_file);
    # # image_dirs=[dir_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for dir_curr in video_files];
    # # flo_dirs=[dir_curr.replace(path_to_video_meta,path_to_flo_meta)[:-4] for dir_curr in video_files];
    # flo_dirs=pickle.load(open('/disk2/temp/dirs_done.p','rb'));
    # image_dirs=[dir_curr.replace(path_to_flo_meta,path_to_im_meta) for dir_curr in flo_dirs];
    # print len(image_dirs)
    # out_dir='/disk2/image_data_moved';

    # out_file='/disk2/image_data_moved/mv_commands_2.txt'
    # commands=[];
    # image_dirs_to_move=image_dirs[5000:7000];
    # for image_dir in image_dirs_to_move:
    #     image_dir=util.escapeString(image_dir);
    #     new_dir=image_dir.replace(path_to_im_meta,out_dir);
    #     command='mkdir -p '+new_dir+';';
    #     command=command+'mv '+image_dir+'/* '+new_dir;
    #     commands.append(command);
    # util.writeFile('/disk2/image_data_moved/dirs_moved_2.txt',image_dirs_to_move);
    # util.writeFile(out_file,commands);

    # return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_video_meta='/disk2/video_data';

    # path_to_flo_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    path_to_flo_meta='/disk2/flow_data';
    path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    
    video_files=util.readLinesFromFile(video_list_file);
    # image_dirs=[dir_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for dir_curr in video_files];
    # flo_dirs=[dir_curr.replace(path_to_video_meta,path_to_flo_meta)[:-4] for dir_curr in video_files];
    flo_dirs=pickle.load(open('/disk2/temp/dirs_done_disk2.p','rb'));
    image_dirs=[dir_curr.replace(path_to_flo_meta,path_to_im_meta) for dir_curr in flo_dirs];
    print len(image_dirs);
    finished=[];
    i=0;
    for image_dir,flo_dir in zip(image_dirs,flo_dirs):
        print i
        count_im_command='ls '+os.path.join(util.escapeString(image_dir),'*.ppm')+'| wc -l';
        count_flo_command='ls '+os.path.join(util.escapeString(flo_dir),'*.flo')+'| wc -l';
        
        # im_count=int(subprocess.check_output(count_im_command,shell=True));
        # flo_count=int(subprocess.check_output(count_flo_command,shell=True));
        im_count=len([file_curr for file_curr in os.listdir(image_dir) if file_curr.endswith('.ppm')]);
        flo_count=len([file_curr for file_curr in os.listdir(flo_dir) if file_curr.endswith('.flo')]);
        print i,flo_count,im_count
        if flo_count+1==im_count:
            finished.append(1);
        else:
            finished.append(0);
        
        i+=1;

    finished=np.array(finished);
    print 'done',sum(finished==1);
    print 'not done',sum(finished==0);

    pickle.dump([finished,image_dirs],open('/disk2/temp/to_rerun.p','wb'));
    
    return
    dir_flownet_meta='/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet';
    caffe_bin = os.path.join(dir_flownet_meta,'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,'model/flownet_official.caffemodel')

    video_list_file='/disk2/video_data/video_list.txt'
    path_to_video_meta='/disk2/video_data';

    in_dir_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    in_dir_meta='/disk2/flow_data';
    # if not os.path.exists(new_in_dir_meta):
    #     os.mkdir(new_in_dir_meta);

    deploy_name='deploy.prototxt';
    gpu=0;

    dirs=[dir_curr.replace(path_to_video_meta,in_dir_meta)[:-4] for dir_curr in util.readLinesFromFile(video_list_file)];
    dirs=[dir_curr for dir_curr in dirs if os.path.exists(dir_curr)]
    counts=[len(os.listdir(dir_curr)) for dir_curr in dirs if os.path.exists(dir_curr)];
    dirs_left=[]
    dirs_done=[];
    for idx_count,count in enumerate(counts):
        if count==4:
            dirs_left.append(dirs[idx_count])
            # dir_curr=dirs[idx_count]
            # deploy_curr=os.path.join(dir_curr,deploy_name);
            # im_file=os.path.join(dir_curr,'im_1.txt');
            # batch_size = sum(1 for line in open(im_file))
            
            # old_str='batch_size: '+str(int(ceil(batch_size/5)));
            # print old_str,
            
            # batch_size = int(ceil(batch_size/8));
            # new_str='batch_size: '+str(batch_size);
            # print new_str

            # data=[];
            # with open(deploy_curr,'r') as f:
            #     data = f.read()
            # # print data[:300];
            # assert old_str in data;
            # data = data.replace(old_str, new_str)
            # # print data[:300];
            # with open(deploy_curr, "w") as f:
            #     f.write(data);

            # out_dir_curr=dir_curr.replace(in_dir_meta,new_in_dir_meta);
            #mkdir of new location
            # mkdir_command='mkdir -p '+util.escapeString(out_dir_curr)
            # print mkdir_command
            # subprocess.call(mkdir_command, shell=True)

            #mv contents from old to new
            # mv_command='mv '+util.escapeString(dir_curr)+'/* '+util.escapeString(out_dir_curr);
            # print mv_command
            # subprocess.call(mv_command, shell=True)
            #append new to dirs_left
            # dirs_left.append(out_dir_curr);
            # raw_input();
        else:
            dirs_done.append(dirs[idx_count])



    print min(counts);
    counts=np.array(counts);
    print sum(counts==4);
    print len(dirs_left)
    
    mid_point=len(dirs_left)/2;
    
    print mid_point,len(dirs_left)-mid_point
    out_file_commands='/disk2/januaryExperiments/gettingFlows/flownet_commands_left_0.txt';
    gpu=0;
    # writeCommands_hacky(out_file_commands,dirs_left[:mid_point],caffe_bin,deploy_name,path_to_model,gpu)
    
    out_file_commands='/disk2/januaryExperiments/gettingFlows/flownet_commands_left_1.txt';
    gpu=1;
    # writeCommands_hacky(out_file_commands,dirs_left[mid_point:],caffe_bin,deploy_name,path_to_model,gpu)
    
    print len(dirs_done);
    pickle.dump(dirs_done,open('/disk2/temp/dirs_done_disk2.p','wb'));

    return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_video_meta='/disk2/video_data';

    in_dir_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    out_dir_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/flow_data';
    path_to_deploy='/disk2/januaryExperiments/gettingFlows/deploy_template.prototxt';
    out_file_commands='/disk2/januaryExperiments/gettingFlows/flownet_commands.txt';

    dir_flownet_meta='/home/maheenrashid/Downloads/flownet/flownet-release/models/flownet';
    path_to_sizer=os.path.join(dir_flownet_meta,'bin/get_image_size')
    caffe_bin = os.path.join(dir_flownet_meta,'bin/caffe')
    path_to_model = os.path.join(dir_flownet_meta,'model/flownet_official.caffemodel')
    
    text_1='im_1.txt';
    text_2='im_2.txt';
    deploy_file='deploy.prototxt';
    gpu=0;

    params_dict={};
    params_dict['video_list_file'] = video_list_file;
    params_dict['path_to_video_meta'] = path_to_video_meta;
    params_dict['in_dir_meta'] = in_dir_meta;
    params_dict['out_dir_meta'] = out_dir_meta;
    params_dict['path_to_deploy'] = path_to_deploy;
    params_dict['out_file_commands'] = out_file_commands;
    params_dict['dir_flownet_meta'] = dir_flownet_meta;
    params_dict['path_to_sizer'] = path_to_sizer;
    params_dict['caffe_bin']  = caffe_bin;
    params_dict['path_to_model']  = path_to_model;
    params_dict['text_1'] = text_1;
    params_dict['text_2'] = text_2;
    params_dict['deploy_file'] = deploy_file;
    params_dict['gpu'] = gpu;
    
    params=createParams('writeFlownetCommands');
    params=params(**params_dict);
    # script_writeFlownetCommands(params);
    commands=util.readLinesFromFile(params.out_file_commands);
    commands=[c.replace('-gpu 1','-gpu 0') for c in commands];
    util.writeFile(params.out_file_commands,commands);
    pickle.dump(params._asdict(),open(params.out_file_commands+'_meta_experiment.p','wb'));




    return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    path_to_video_meta='/disk2/video_data';
    commands_file_text='/disk2/januaryExperiments/gettingFlows/resize_commands.txt';

    video_list=util.readLinesFromFile(video_list_file);
    print len(video_list);
    image_dirs=[video_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for video_curr in video_list];
    print len(image_dirs),image_dirs[0];
    image_dirs=image_dirs[:1];

    commands=[];
    command_conv=['convert','-resize 512x384'];
    for image_dir in image_dirs:
        image_list=[os.path.join(image_dir,im) for im in os.listdir(image_dir) if im.endswith('.ppm')];
        for image_curr in image_list:
            command_curr=[command_conv[0],image_curr,command_conv[1],image_curr];
            command_curr=' '.join(command_curr);
            commands.append(command_curr);

    print len(commands);
    print commands[0];
    util.writeFile(commands_file_text,commands);



    return
    video_list_file='/disk2/video_data/video_list.txt'
    path_to_im_meta='/media/maheenrashid/e5507fe3-2bff-4cbe-bc63-400de6deba92/maheen_data/image_data';
    path_to_video_meta='/disk2/video_data';
    path_to_txt_1='/disk2/januaryExperiments/gettingFlows/temp_im_1.txt';
    path_to_txt_2='/disk2/januaryExperiments/gettingFlows/temp_im_2.txt';

    video_list=util.readLinesFromFile(video_list_file);
    print len(video_list);
    image_dirs=[video_curr.replace(path_to_video_meta,path_to_im_meta)[:-4] for video_curr in video_list];
    print len(image_dirs),image_dirs[0];
    
    list_1=[];list_2=[];
    for image_dir in image_dirs[:10]:
        list_1_curr,list_2_curr=getImageListForFlow(image_dir);
        list_1.extend(list_1_curr[:3]);
        list_2.extend(list_2_curr[:3]);
    
    assert len(list_1)==len(list_2);

    util.writeFile(path_to_txt_1,list_1);
    util.writeFile(path_to_txt_2,list_2);