def create_path(): config_path = ConfigPath( 3, Config(name="config1", base_url="test1"), Config(name="config1", base_url="test1", auth={"field": "token"}), Config(name="test_factory", base_url="https://jsonplaceholder.typicode.com"), Config(name="config2", base_url="test2"), "foo")
def get_fully_trained_network(module_graph, data_augs, num_epochs=100, plot_best_graph=False): """takes a module graph, converts it into a module net NN and trains it for n epochs. used to fully train a saved module graph to find its max accuracy """ train, test = DataLoader.load_data(dataset=module_graph.dataset) sample, _ = DataLoader.sample_data(Config.get_device(), dataset=module_graph.dataset) module_graph_backup = copy.deepcopy(module_graph) model = create_nn(module_graph, sample, feature_multiplier=Config.feature_multiplier_for_fully_train) if plot_best_graph: module_graph.plot_tree_with_graphvis(title="after putting in model", file="after") print("fully training ~ num epochs:", num_epochs, "num augs:", len(data_augs), "feature multiplier:", Config.feature_multiplier_for_fully_train) Evaluator.print_epoch_every = 1 da_phenotypes = [dagenome.to_phenotype() for dagenome in data_augs] if len(data_augs) > 0: print("fully training using augs:", data_augs) acc = Evaluator.evaluate(model, num_epochs, Config.get_device(), train_loader=train, test_loader=test, print_accuracy=True, batch_size=Config.batch_size, augmentors=da_phenotypes, training_target=module_graph.fitness_values[0]) if Config.toss_bad_runs and acc == "toss": print("runn failed to take off. starting over") get_fully_trained_network(module_graph_backup, data_augs, num_epochs=num_epochs, plot_best_graph=plot_best_graph) print("model trained on", num_epochs, "epochs scored:", acc)
def test_duplicates_config_type(self): assert ApiConfig.configs.sort(key=lambda x: x.name) == [ Config(name="config1", base_url="test1"), Config(name="config2", base_url="test2"), Config(name="test_factory", base_url="https://jsonplaceholder.typicode.com"), ].sort(key=lambda x: x.name)
def testFullNameValidation(self): Config("Super Mario") with self.assertRaises(ValueError): Config(None) with self.assertRaises(ValueError): Config("") with self.assertRaises(ValueError): Config(" ")
def initialize_elements(self): self.cf = Config() self.lineEdit__local_certificado.setText(self.cf.get('cert_path')) if self.cf.get('cert_path'): self.lineEdit_senha_certificado.setEnabled(True) self.lineEdit_senha_certificado.setText( self.cf.get('cert_password')) self.lineEdit_input_xml.setText(self.cf.get('input_folder')) self.lineEdit_output_xml.setText(self.cf.get('output_folder')) self.enable_process_button()
def init(self): # 加载全局变量模块 # 加载配置模块 Config.init() self.conf = Config.get('system') # 创建缓存文件夹和日志文件夹 try: os.makedirs(self.conf['default_cache']) os.makedirs("./log/") except OSError: pass if not os.path.isdir(self.conf['default_cache']): Log.error( 'FATAL: Output path is not a directory or does not exist') sys.exit(1)
class GlobalSpace: CFG = Config() # mylock = threading.RLock() # data_buffer = [] def __init__(self): pass def append_data_buffer(self, dictV): global data_buffer global mylock mylock.acquire() data_buffer.append(dictV) mylock.release() def pop_data_buffer(self): global data_buffer global mylock mylock.acquire() pop_item = data_buffer.pop() mylock.release() return pop_item
def main(): parser = argparse.ArgumentParser() parser.add_argument( '-v', '--verbose', action='count', default=0, help='Indicates the level of messages to be displayed.') parser.add_argument('-k', '--keep', const=str, action='store_const', default=False, help='Keeps the temporary files for future runs.') parser.add_argument('--path', required=True, help='REQUIRED: Path with all images to work.') parser.add_argument( '--extension', required=True, choices=[ext.lower() for ext in ACCEPTED_IMAGE_FORMATS], help='REQUIRED: Images extension') arguments = parser.parse_args() logging.basicConfig(level=LOGGING_LEVELS[min( len(LOGGING_LEVELS) - 1, arguments.verbose)], format='[%(levelname)s] %(message)s') print(' - IMAGE SIZE CLASSIFIER - ') service = Service() service.execute(Config(arguments.path, arguments.extension, arguments.keep)) print(' - DONE - ') exit(0)
def __parse_service_config(self): """ Parse the YAML configuration file containing specific settings for the individual p2p loan platform """ with open(self.config_file, "r", encoding="utf-8") as ymlconfig: config = safe_load(ymlconfig) self.config = Config(config)
def __init__(self, assert_spec: AssertSpec, rundir, libdir, parallel=False, logstring="log>>>", condaenvname="py3.6_allennlp", threads=None, logger=None ): self.assert_spec = assert_spec self.parallel = parallel self.config = Config() self.patch = None self.logdir = None self.logstring = logstring self.logger = logger self.condaenvname = condaenvname self.rundir = rundir self.libdir = libdir self.threads = threads if self.parallel: exceptions = open("src/exceptionlist.txt").readlines() for l in exceptions: if assert_spec.test.filename.endswith(l.split(",")[1]) and assert_spec.test.classname == l.split(",")[ 2]: self.parallel = False print("Found in exception list")
class TestApiRequest: Config_obj = Config(name="test_name", base_url="https://jsonplaceholder.typicode.com", auth={"auth_field": "auth_token"}, headers={"header1": "val1", "header2": "val2"}) request_obj = ApiRequest() def test_set_config(self): self.request_obj.set_config(config=self.Config_obj) assert self.request_obj._headers == {"header1": "val1", "header2": "val2"} assert self.request_obj._auth == {"auth_field": "auth_token"} assert self.request_obj._base_url == "https://jsonplaceholder.typicode.com" def test_set_request(self): self.request_obj.set_request(end_url="todos/1") assert self.request_obj._end_url == "todos/1" def test_get_response(self): assert self.request_obj.get_response().json() == {'completed': False, 'id': 1, 'title': 'delectus aut autem', 'userId': 1}
def makeMimeText(self, helper): content = '<style>table,table tr th, table tr td { border:1px solid #0094ff; }table { width: 90%; min-height: 25px; line-height: 25px; text-align: center; border-collapse: collapse;}</style>' # 生成author of day content += self.html_header(2, 'Author of day') + '<table style="border:1px solid #F00;">' + \ '<tr><th>Author</th><th>Add</th><th>Delete</th><th>Commits</th></tr>' for author in helper.getAuthors( Config.get('system.git')['max_authors']): info = helper.author_of_day[author] content += '<tr><td>%s</td><td>%d</td><td>%d</td><td>%d</td></tr>' % ( author, info['lines_added'], info['lines_removed'], info['commit']) content += "</table>" # 生成author of month content += self.html_header(2, 'Author of Month') + '<table style="border:1px solid #F00;">' + \ '<tr><th>Month</th><th>Author</th><th>Commits (%)</th><th class="unsortable">Next top 5</th></tr>' for yymm in reversed(sorted(helper.author_of_month.keys())): authordict = helper.author_of_month[yymm] authors = Core.getkeyssortedbyvalues(authordict) authors.reverse() commits = helper.author_of_month[yymm][authors[0]] next = ', '.join(authors[1:5]) content += '<tr><td>%s</td><td>%s</td><td>%d (%.2f%% of %d)</td><td>%s</td></tr>' % ( yymm, authors[0], commits, (100.0 * commits) / helper.commits_by_month[yymm], helper.commits_by_month[yymm], next) content += "</table>" return content
def main(): try: config = Config(elitism=True, eliteCount=2, mutationProbability=0.1) masterImage = cv2.imread("images/facebook.jpg") masterImage = cv2.resize(masterImage, (10, 10)) cv2.imwrite('images/output/master.jpg', masterImage) population = Population(populationSize=50, masterImage=masterImage, config=config, populate=True) originalFitness = population.fitestCitizen.fitness generation = 0 Max_Generation = 100000000 while generation < Max_Generation: population = population.Evolve(generation) fitestCitizen = population.fitestCitizen fitnessPercentage = 100 * (originalFitness - fitestCitizen.fitness) / originalFitness print("Fitness = " + str(fitestCitizen.fitness) + " : FitnessPercentage = " + str(fitnessPercentage) + "%") if generation % 1000 == 0: cv2.imwrite( 'images/output/generation' + str(generation) + ".jpg", fitestCitizen.image) generation += 1 except IOError: pass
class TestData(unittest.TestCase): r = randint(0, 99999999) file = str(r) + '_test.csv' nrows = 999 ncols = 49 min_border = -1.0 max_border = 1.0 config = Config( file=file, nrows=nrows, ncols=ncols, ) @classmethod def setUpClass(cls) -> None: Data(TestData.config) @classmethod def tearDownClass(cls) -> None: os.remove(TestData.file) def test_if_file_is_created(self): self.assertTrue(os.path.isfile(TestData.file)) def test_if_numbers_are_valid(self): data = Data(TestData.config) mn = data.data_frame[data.data_frame.columns].min().min() mx = data.data_frame[data.data_frame.columns].max().min() self.assertTrue(mn > TestData.min_border and mx < TestData.max_border)
def get_accuracy_estimate_for_network(model, da_scheme=None, batch_size=Config.batch_size): """used in the evolutionary process to train a model for the number of epochs specified in Config settings""" if Config.dummy_run: acc = random.random() else: acc = Evaluator.evaluate(model, Config.number_of_epochs_per_evaluation, Config.get_device(), batch_size, augmentors=[da_scheme]) return acc
def __init__(self, populationSize, masterImage, config = Config(), populate = False): self.populationSize = populationSize self.masterImage = masterImage self.config = config self.rankedCitizens = [] if populate: self.Populate() self.UpdateRanking()
def test__from_file(): path_config = 'files/config/config.ini' abs_path = real_path_absolute c = Config(abs_path, path_config) assert c.path_in == '{}/in_file_name'.format(real_path_absolute) assert c.path_out == '{}/out_file_name'.format(real_path_absolute) assert c.processes_number == 16
def main(): args = arguments() config = Config(path_absolute, path_config, args.path_in, args.path_out, args.process_number) params = Params(config.path_in) counter = Counter(params.points, params.paths, config.processes_number) counter_result = counter.count() result = Result(config.path_out, counter_result) result.save()
def is_valid_config(configurator: Config) -> bool: """Checks if configurator is valid :param Config configurator: :return: """ if not configurator.path_exist(): logging.error("Work Folder '{}' does not exist!".format( configurator.get_work_path())) return False if configurator.get_work_count() == 0: logging.error("Work Folder '{}' doesn't have any {} image!".format( configurator.get_work_path(), configurator.get_work_extension().upper())) return False return True
def init(): Log('') Log('Starting...') Config.load() Log('Config loaded') Core.session = vk_api.VkApi(token=Config.token) Core.api = Core.session.get_api() Core.longpoll = LongPoll(Core.session, Config.group_id) Log('Authed') if Core.api.groups.getOnlineStatus( group_id=Config.group_id)['status'] == 'none': Core.api.groups.enableOnline(group_id=Config.group_id) db.load() Log('db loaded')
def check_db_exists(self): if self.db_path == '' or FileIO.file_exists(self.db_path) is False: LogFactory.MAIN_LOG.warning('DB does not exist! Initializing!') self.db_path = f"{Config.root_dir}/../{RandomGenerator.generate_random_string(10)}.zp" nConf: dict = Config.conf_file nConf['db'] = {} if 'db' not in nConf.keys() else nConf['db'] nConf['db']['path'] = self.db_path Config.update_config_file(nConf) self.db = {} self.db['masterpass'] = RandomGenerator.generate_random_string(10) LogFactory.MAIN_LOG.info( f"Your master password is {self.db['masterpass']}") self.write_db()
def test__with_args(): abs_path = '/app' path_in = 'input_file_name' path_out = 'out_file_name' processes_number = 8 c = Config(abs_path, None, path_in, path_out, processes_number) assert c.path_in == '/app/input_file_name' assert c.path_out == '/app/out_file_name' assert c.processes_number == 8
def __parse_service_config(self): """ Parse the YAML configuration file containing specific settings for the individual p2p loan platform :return: """ with open(self.config_file, 'r', encoding='utf-8') as ymlconfig: yaml = YAML(typ='safe') config = yaml.load(ymlconfig) self.config = Config(config)
class TestConfig: objectEqual1 = Config(name="equal", base_url="test1", auth={"field": "token"}) objectEqual2 = Config(name="equal", base_url="test2", headers={"header": "val"}) objectAttr = Config(name="attr", base_url="test_attr", auth={"field": "attr"}, headers={"head": "atr"}) def test_equality(self): assert self.objectEqual1 == self.objectEqual2 def test_inequality(self): assert self.objectEqual1 != self.objectAttr def test_eq_hash(self): assert self.objectEqual1.__hash__() == self.objectEqual2.__hash__() def test_in_eq_hash(self): assert self.objectEqual2.__hash__() != self.objectAttr.__hash__() def test_attr(self): assert self.objectAttr.name == "attr" assert self.objectAttr.base_url == "test_attr" assert self.objectAttr.auth == {"field": "attr"} assert self.objectAttr.headers == {"head": "atr"} def test_repr(self): assert self.objectAttr.__repr__( ) == "Config(name='attr', base_url='test_attr', auth={'field': 'attr'}, headers={'head': 'atr'})"
def __init__(self, config=Config()): self.desired_success_rate = 0.75 self.default_ease_rate = 2.50 self.easecap = 0.20 self.learning_steps = [15, 1440] self.grade_allocations = [ 0.01, 0.25, 0.50, 0.75, 0.99 ] #"No idea", "Unsure", "Half Right", "Almost Perfect", "Nailed It" self.config = config self.client = self.config.client() self.collection = self.client.get_collection_view( self.config.topics_collection_url())
def cdn_pareto_front(individuals): individuals.sort(key=lambda indv: indv.fitness_values[0], reverse=True) pf = [individuals[0] ] # pareto front populated with best individual in primary objective for indv in individuals[1:]: if Config.second_objective_comparator(indv.fitness_values[1], pf[-1].fitness_values[1]): pf.append(indv) return pf
def set_data(self, _data_frame, _dataset_name=None, _column_names=None): if _dataset_name is None: _dataset_name = 'demo' _data_frame = _data_frame.rename(columns=lambda n: str(n).replace(' ', '')) _data_frame = _data_frame.applymap(str) # to refresh the outputs self.all_posteriors[_dataset_name] = {} self.predicted_types = {} self.normal_types = {} self.missing_types = {} self.anomaly_types = {} self.p_z_columns = {} self.p_t_columns = {} # Check for dataframes without column names if _column_names is None: _column_names = _data_frame.columns # Creates a configuration object for the experiments if self.types is None: config = Config(_dataset_name=_dataset_name, _column_names=_column_names) else: config = Config(_dataset_name=_dataset_name, _column_names=_column_names, _types=self.types) # Ptype model for inference if self.model is None: self.model = PtypeModel(config, _data_frame=_data_frame) else: self.model.set_params(config, _data_frame=_data_frame) # Creates a df to report annotations for Wrattler self.missing_types = {} self.anomaly_types = {} self.p_z_columns = {} self.p_t_columns = {} self.print = False self.prediction_path = None self.predicted_types = {}
def init(self): config = Config(configFile) logger = PrintLogger() driver = ChromeDriver.getInstance() order = OrderPizza(config, logger, driver) order.acceptCookies() order.login() order.addProductsToCart() order.goToCheckout() order.selectDeliveryTime() order.selectPaymentType() order.send()
def __init__(self, configFile, limit, register): """ Initializes client with a given parameters :param configFile: :param limit: :param register: :type configFile: str :type limit: int :type register: bool """ self.userData = {'username': None, 'password': None} try: self.config = Config(configFile) self.request = Request() self.collection = None self.view = View(limit) self.registerMode = register except OSError: filename = configFile print(f"InitError: Configuration file {filename} can\'t be found!") exit(1)
def get_result(self, configurator: Config, selected_percent: int): """ Copy the final selected images to `selected` folder. :param Config configurator: :param int selected_percent: :return: """ result_dir = '{}{}/'.format( configurator.get_work_path(), FOLDER_NAME_OUTPUT_RESULT.format(selected_percent)) self.create_dir(result_dir) print('Selecting between {} images with >= {}%...'.format( configurator.get_work_count(), selected_percent)) for file in configurator.get_work_files(): if file.get_jpg_percent() >= selected_percent: copyfile(file.get_path(), result_dir + file.get_name()) else: break # My list is sorted! print("DONE - Results with {}% in '{}'!".format( selected_percent, result_dir))
from src.Config import Config config = Config() config.reload() temp = config.get_temp() print temp print config.get_day()