def running_plan(file, config_scripts, config_general): ########################################################## # Execution of the whole described plan in the config file ########################################################## execution = ConfigParser.ConfigParser() execution.read(file) logging.info(" ") logging.info(" ") logging.info(" ") logging.info(" ") logging.info("##########################################") logging.info("running plan from " + file) logging.info("##########################################") for execution_plan_section in execution.sections(): if execution_plan_section == "General": continue begin_time_section = timeit.default_timer() logging.info(" ") logging.info(" ") logging.info("-------------------------------------------") logging.info("running section : " + execution_plan_section) logging.info("-------------------------------------------") section = config_section_map(execution, execution_plan_section) logging.info("section : " + section['description']) dispatch(config_scripts, config_general, section) end_time_section = timeit.default_timer() logging.info('section ' + execution_plan_section + ' took ' + str(end_time_section - begin_time_section) + 's')
def main(argv): ############################### # Getting programme options ############################### help_str = 'no help provided' try: opts, args = getopt.getopt(argv, "ho:t:c:", ["config=", "template=", "output-folder="]) except getopt.GetoptError as err: print help_str print str(err) sys.exit(2) for opt, arg in opts: if opt == '-h': print help_str sys.exit() elif opt in ("-t", "--template"): url_list = arg elif opt in ("-o", "--output-folder"): res_folder = arg elif opt in ("-c", "--config"): config_file = arg ######################### # Chargement de la config ######################### config = ConfigParser.ConfigParser() config.read(config_file) config_general = config_section_map(config, 'General') ######################### # Configuration du logger ######################### log_dir = config_general['log_dir'] now = datetime.now() date_str = str(now.day) + '_' + str(now.hour) + '_' + str(now.minute) + "_" + str(now.second) + "_" + str(now.microsecond) logfile_name = os.path.basename(__file__).split('.')[0] + '-' + date_str + '.log' logging.basicConfig(filename=log_dir + '/' + logfile_name, level=logging.DEBUG) ########################## # Create output dir ########################## logging.info("results_dir = " + res_folder) if not os.path.exists(res_folder): logging.info('no output dir creating at ' + res_folder) if not subprocess.call(['mkdir', '-p', res_folder]) == 0: logging.warning('cannot create output dir, aborting') sys.exit(1) logging.info('output dir is ' + res_folder) gvalues = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] wvalues = [1, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 21, 22, 23, 24] with open(url_list, "r") as outfile: template = Template(outfile.read()) for g in gvalues: for w in wvalues: name = 'g-'+str(g)+'_w-'+str(w) with open(res_folder + "/" + name + ".ini", "w") as plan: config = template.substitute(g=g, w=w, name=name) plan.write(config)
def main(argv): # Creating config instance config = ConfigParser.ConfigParser() # Creating config file if not os.path.exists('project.ini'): subprocess.call(['touch', 'project.ini']) # Read the base config config.read('project.ini') # Open file config for write config_file = open('install.ini', 'w') # Add project base directory current_directory = os.path.dirname(os.path.realpath(__file__)) config.set('General', 'config_file', current_directory + '/install.ini') config.set('General', 'project_dir', current_directory) ###################################### # Crating config section ###################################### config.add_section('libC') config.add_section('libSvm') config.add_section('Scripts') config.add_section('trecEval') config.add_section('Predict') config_general = config_section_map(config, 'General') ####################################### # Create python related directory ####################################### # Creating log directory log_directory = current_directory + '/log' if not os.path.exists(log_directory): res = subprocess.call(['mkdir', '-p', log_directory]) if res != 0: print 'cannot create ' + log_directory sys.exit(1) config.set('General', 'log_dir', log_directory) # Creating download directory (to store photos) download_dir = current_directory + '/download' if not os.path.exists(download_dir): res = subprocess.call(['mkdir', '-p', download_dir]) if res != 0: print 'cannot create ' + download_dir sys.exit(1) config.set('General', 'download_dir', download_dir) scripts_dir = current_directory + '/scripts' if not os.path.exists(scripts_dir): res = subprocess.call(['mkdir', '-p', scripts_dir]) if res != 0: print 'cannot create ' + scripts_dir sys.exit(1) predict_files = current_directory + '/predict_files' config.set('General', 'scripts_dir', scripts_dir) config.set('General', 'predict_files', predict_files) config.set('Scripts', 'histogram', scripts_dir + '/histogram.py') config.set('Scripts', 'concept', scripts_dir + '/concept.py') config.set('Scripts', 'svm-train', scripts_dir + '/svm-train.py') config.set('Scripts', 'svm-predict', scripts_dir + '/svm-predict.py') config.set('Scripts', 'transform-trec_eval', scripts_dir + '/svmToTrecEval.py') config.set('Scripts', 'trec_eval', scripts_dir + '/trec_eval.py') config.set('Scripts', 'format_sift', scripts_dir + '/formatSift.py') config.set('Scripts', 'kmeans', scripts_dir + '/kmeans_clustering.R') config.set('Scripts', 'sift_kmeans', scripts_dir + '/siftKmeans.py') config.set('Scripts', 'mapping_kmeans', scripts_dir + '/generate_mapping.py') config.set('Scripts', 'sift_histogram', scripts_dir + '/sift_histograms.py') config.set('Scripts', 'late_fusion', scripts_dir + '/late_fusion.py') config.set('Scripts', '1nn', scripts_dir + '/1nn.R') config.set('Predict', 'best_results_sift', os.path.join(predict_files, 'best_results_sift.txt.sort')) config.set('Predict', 'best_results_color', os.path.join(predict_files, 'best_results_color.txt.sort')) config.set('Predict', 'best_results_fusion', os.path.join(predict_files, 'best_results_fusion.txt')) config.set('Predict', 'centers_folders', os.path.join(predict_files, 'centers')) config.set('Predict', 'color_folders', os.path.join(predict_files, 'color')) config.set('Predict', 'sift_folders', os.path.join(predict_files, 'sift')) config.set('Predict', 'concepts', os.path.join(predict_files, 'concepts.txt')) ####################################### # Installing C module ####################################### build_dir = current_directory + '/build' if os.path.exists(build_dir): print 'purging build dir' shutil.rmtree(build_dir) res = subprocess.call(['mkdir', '-p', build_dir]) if res != 0: print 'cannot create ' + download_dir sys.exit(1) # Building C Library res = subprocess.call(['cmake', '..'], cwd=build_dir) if res != 0: print 'problem for Cmake ' sys.exit(1) res = subprocess.call('make', cwd=build_dir) if res != 0: print 'Problem src not compiling' sys.exit(1) lib_histogram = build_dir + '/src/libHistogram.so' if os.path.exists(build_dir + '/src/libHistogram.so'): print 'libHistogram found at ' + lib_histogram config.set('libC', 'libHistogram', lib_histogram) ####################################### # Installing trec_eval ####################################### trec_eval_install_dir = current_directory + '/lib/trec_eval_latest' if os.path.exists(trec_eval_install_dir): subprocess.call(['rm', '-r', trec_eval_install_dir], cwd='lib') subprocess.call(['mkdir', trec_eval_install_dir], cwd='lib') tar_file = 'lib/trec_eval_latest.tar.gz' if not os.path.exists(tar_file): print "no trec eval provided " sys.exit(1) subprocess.call(['tar', '-zxf' 'trec_eval_latest.tar.gz', '--directory', 'trec_eval_latest', '--strip-components=2'], cwd='lib') subprocess.call(['make'], cwd='lib/trec_eval_latest') if not os.path.exists('lib/trec_eval_latest/trec_eval'): print "installation of trec eval failed" sys.exit(1) config.set('trecEval', 'trec_eval', current_directory + '/lib/trec_eval_latest/trec_eval') ####################################### # Installing SVM ####################################### current_version = config_general['libsvm_version'] svm_install_dir = current_directory + '/lib/libsvm-'+current_version if os.path.exists(svm_install_dir): subprocess.call(['rm', 'r', svm_install_dir], cwd='lib') tar_file = 'lib/libsvm-' + current_version + '.tar.gz' if not os.path.exists(tar_file): print "no svm provided " sys.exit(1) subprocess.call(['tar', '-zxvf' 'libsvm-' + current_version + '.tar.gz'], cwd='lib') config.set('libSvm', 'install_dir', svm_install_dir) if os.path.exists(svm_install_dir): subprocess.call(["make", "clean"], cwd=svm_install_dir) subprocess.call(["make"], cwd=svm_install_dir) svm_train = svm_install_dir + '/svm-train' svm_predict = svm_install_dir + '/svm-predict' if os.path.exists(svm_train) and os.path.exists(svm_predict): config.set('libSvm', 'svm_train', svm_train) config.set('libSvm', 'svm_predict', svm_predict) else: print 'problem svm exec not found' sys.exit(1) else: print 'Please check svm install, should be at ' + svm_install_dir sys.exit(1) ####################################### # Result folder ####################################### results_dir = current_directory + '/results' if not os.path.exists(results_dir): res = subprocess.call(['mkdir', '-p', results_dir]) if res != 0: print 'cannot create ' + results_dir sys.exit(1) config.set('General', 'results_dir', results_dir) ####################################### # Working dir ####################################### working_dir = current_directory + '/works' if not os.path.exists(working_dir): res = subprocess.call(['mkdir', '-p', working_dir]) if res != 0: print 'cannot create ' + working_dir sys.exit(1) config.set('General', 'working_dir', working_dir) config.write(config_file) config_file.close()
def main(argv): """ Entry point for Recherche multimedia. This scripts is used to make simple to run other scripts with pre-configured options """ ############################### # Getting program options ############################### parser = argparse.ArgumentParser(description='Main for recherche multimedia.') parser.add_argument('-f', action="store", dest='execution_file') parser.add_argument('-j', action="store", dest='job') parser.add_argument('-d', action="store", dest='directory') parser.add_argument('-i', action="store", dest='init') options = parser.parse_args() config_general = config_section_map(config, 'General') config_scripts = config_section_map(config, 'Scripts') ######################### # Configuration du logger ######################### log_dir = config_general['log_dir'] now = datetime.now() date_str = str(now.day) + '_' + str(now.hour) + '_' + str(now.minute) + "_" + str(now.second) + "_" + str(now.microsecond) logfile_name = os.path.basename(__file__).split('.')[0] + '-' + date_str + '.log' logging.basicConfig(filename=log_dir + '/' + logfile_name, level=logging.DEBUG) atexit.register(safe_quit) ############################### # Load execution config ############################### directory = options.directory job = options.job init_plan = options.init if init_plan is not None: logging.info("running initialization plan ") running_plan(init_plan, config_scripts, config_general) logging.info("End initialisation plan") if job is not None: execution = ConfigParser.ConfigParser() execution.read(options.execution_file) logging.info("running single job " + job) section = config_section_map(execution, job) dispatch(config_scripts, config_general, section) elif directory is not None: if not directory.startswith("/"): directory = os.path.join(config_general['project_dir'], directory) if options.execution_file is not None: logging.warning("Execution file provided, will be ignored in folder mode") logging.info("Executing plan from folder " + directory) plans = glob.glob(directory + "/*") for plan_file in plans: execution = ConfigParser.ConfigParser() execution.read(plan_file) running_plan(plan_file, config_scripts, config_general) else: execution = ConfigParser.ConfigParser() execution.read(options.execution_file) running_plan(options.execution_file, config_scripts, config_general)