def total_report(): import define import prepare import fselect import evaluate import improve # label = 'LocalizationNew_Tx' label = 'BQ_AQUARIS_E5_' # for i in range(7, 8): for i in ['20', '30', '40']: data_name = "./uploads/" + label + str(i) + ".csv" print(data_name) # data_name = "iris.csv" class_name = "class" definer = define.Define(data_path=data_name, header=None, response=class_name).pipeline() preparer = prepare.Prepare(definer).pipeline() selector = fselect.Select(definer).pipeline() evaluator = evaluate.Evaluate(definer, preparer, selector) improver = improve.Improve(evaluator).pipeline() improver.save_full_report('./market/' + label + str(i)) improver.save_score_report('./market/' + label + str(i))
def __init__(self, config, db_config): self.__threadNumOfDownload = config['thread_num_of_download'] self.__threadNumOfHandle = config['thread_num_of_handle'] self.__retryTimes = config['retry_times'] self.__tmpDataDir = config['tmp_data_dir'] self.__saveModule = config['save_module'] self.__id = config['id'] self.__fileName = config['start_date'] + '_' + config[ 'end_date'] + '_' + str(config['input_nodes']) + '_' + self.__id self.__config = config self.__dbConfig = db_config self.__uriLock = threading.Lock() self.__dataLock = threading.Lock() self.__fileLock = threading.Lock() self.__downloadThreadLock = threading.Lock() self.__failQueue = Queue.Queue() self.__failUriDict = {} self.__com = {} self.init() self.o_prepare = prepare.Prepare(self.__config, self.__dbConfig) self.__curlTime = 0 self.__curlCounts = 0 self.__handleTime = 0 self.__handleCount = 0
def report_improve(data_path, data_name, problem_type, optimizer, modelos): definer = define.Define(data_path=data_path,data_name=data_name,problem_type=problem_type).pipeline() preparer = prepare.Prepare(definer).pipeline() selector = fselect.Select(definer).pipeline() evaluator = evaluate.Evaluate(definer, preparer, selector) improver = improve.Improve(evaluator, optimizer, modelos).pipeline() plot = improver.plot_models() table = improver.report dict_report = {'plot': plot, 'table': table} #dict_report = {'table': table} return dict_report
def report_model(response, data_path, data_name, problem_type): definer = define.Define(data_path=data_path,data_name=data_name,problem_type=problem_type).pipeline() preparer = prepare.Prepare(definer).pipeline() # scaler selector = fselect.Select(definer).pipeline() # pca evaluator = evaluate.Evaluate(definer, preparer, selector).pipeline() plot = evaluator.plot_models() table = evaluator.report data_name = data_name.replace(".csv", "") plot_path = os.path.join(app.config['MARKET_DIR'], data_name, 'model') tools.path_exists(plot_path) plot_path_plot = os.path.join(plot_path, 'boxplot.html') evaluator.save_plot(plot_path_plot) plot_path_report = os.path.join(plot_path, 'report.csv') evaluator.save_report(plot_path_report) dict_report = {'plot': plot, 'table': table} return dict_report
def __init__(self, sample=False, model_name='VGG16', image_dir='data/images', caption_file='data/caption/Flickr8k.token.txt', clean_caption=False, load_pickle=False): self.prepare = prepare.Prepare(sample,model_name,image_dir,caption_file,clean_caption,load_pickle) training_dataset = self.prepare.load_identifiers('data/caption/Flickr_8k.trainImages.txt') testing_dataset = self.prepare.load_identifiers('data/caption/Flickr_8k.testImages.txt') validation_dataset = self.prepare.load_identifiers('data/caption/Flickr_8k.devImages.txt') train_feature = self.prepare.load_features(training_dataset) test_feature = self.prepare.load_features(testing_dataset) validation_feature = self.prepare.load_features(validation_dataset) train_description = self.prepare.load_description(training_dataset) test_description = self.prepare.load_description(testing_dataset) validation_description = self.prepare.load_description(validation_dataset) tokenizer, vocab_size, max_length = self.prepare.create_tokens(self.prepare.description) _, _, m1 = self.prepare.create_tokens(train_description) _, _, m2 = self.prepare.create_tokens(test_description) _, _, m3 = self.prepare.create_tokens(validation_description) max_length = max([max_length, m1, m2, m3]) detail = dict() detail['vocab_size'] = vocab_size detail['max_length'] = max_length pickle.dump(tokenizer, open('pkl/tokenizer.pkl', 'wb')) pickle.dump(detail, open('pkl/detail.pkl', 'wb')) X1train, X2train, ytrain = self.prepare.create_sequence(train_description, tokenizer, max_length, vocab_size, train_feature) X1test, X2test, ytest = self.prepare.create_sequence(test_description, tokenizer, max_length,vocab_size, test_feature) X1validation, X2validation, yvalidation = self.prepare.create_sequence(validation_description, tokenizer, max_length,vocab_size, validation_feature) self.prepare.define_caption_model(max_length, vocab_size) self.prepare.checkpoint_prepare() self.prepare.model_fit(X1train, X2train, ytrain,X1validation, X2validation, yvalidation)
search_dirs = [args.dir] if sdo: stats_list = {'sdo': search_dirs} else: stats_list = {'vendor': search_dirs} if args.run_integrity: stats_list = { 'vendor': [get_curr_dir(__file__) + '/../../vendor/cisco'] } LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare", yangcatalog_api_prefix) for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, args.result_html_dir, args.save_file_dir, private_credentials) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_modules(args.json_dir)
missing = [] my_files = find_missing_hello('./../../vendor/', '*.yang') for name in set(my_files): if '.incompatible' not in name and 'MIBS' not in name: missing.append(name) missing = ', '.join(missing).replace('./../..', '') stats_file.write( '<h3>Folders with yang files but missing hello message inside of vendor:</h3><p>' + missing + '</p>') stats_file.write('</body></html>') stats_file.close() start = time.time() index = 1 prepare = prepare.Prepare("prepare") update = True for filename in find_files('../../vendor/', '*capabilit*.xml'): try: file_modification = open( 'fileModificationDate/' + '-'.join(filename.split('/')[-4:]) + '.txt', 'rw') time_in_file = file_modification.readline() if time.strptime(time_in_file, '%a %b %d %H:%M:%S %Y') == time.ctime( os.path.getmtime(filename)): update = False file_modification.close() else: file_modification.seek(0) file_modification.write(time.ctime(os.path.getmtime(filename))) file_modification.truncate()
if sdo: stats_list = {'sdo': search_dirs} else: stats_list = {'vendor': search_dirs} if args.run_statistics: stats_list = { 'sdo': ['../../experimental', '../../standard'], 'vendor': ['../../vendor'] } statistics_in_catalog = statisticsInCatalog.StatisticsInCatalog() LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare") for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, statistics_in_catalog) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_sdo(args.json_dir) else: sdo = False prepare_vendor = prepare.Prepare("prepare")
sdo = args.sdo search_dirs = [args.dir] if sdo: stats_list = {'sdo': search_dirs} else: stats_list = {'vendor': search_dirs} if args.run_integrity: stats_list = {'vendor': [get_curr_dir(__file__) + '/../../vendor/cisco']} LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare", args.result_html_dir, args.api_port, args.api_ip, args.api_protocol) for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, args.result_html_dir, args.save_file_dir) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_modules(args.json_dir) else:
if sdo: stats_list = {'sdo': search_dirs} else: stats_list = {'vendor': search_dirs} if args.run_statistics: stats_list = { 'sdo': ['../../experimental', '../../standard'], 'vendor': ['../../vendor'] } LOGGER.info('Starting to iterate through files') for key in stats_list: search_dirs = stats_list[key] if key == 'sdo': sdo = True prepare_sdo = prepare.Prepare("prepare", args.result_html_dir) for search_dir in search_dirs: LOGGER.info('Found directory for sdo {}'.format(search_dir)) integrity = statistics.Statistics(search_dir) capability = cap.Capability(search_dir, index, prepare_sdo, integrity, args.api, sdo, args.json_dir, args.result_html_dir) LOGGER.info('Starting to parse files in sdo directory') capability.parse_and_dump_sdo() index += 1 prepare_sdo.dump_modules(args.json_dir) else: sdo = False
def setup(thing): thing.prepare = prepare.Prepare(thing.context) thing.prepare._pool = MagicMock() return thing