def create_path():
config_path = ConfigPath(
3, Config(name="config1", base_url="test1"),
Config(name="config1", base_url="test1", auth={"field": "token"}),
Config(name="test_factory",
base_url="https://jsonplaceholder.typicode.com"),
Config(name="config2", base_url="test2"), "foo")
def get_fully_trained_network(module_graph, data_augs, num_epochs=100, plot_best_graph=False):
"""takes a module graph, converts it into a module net NN and trains it for n epochs.
used to fully train a saved module graph to find its max accuracy
"""
train, test = DataLoader.load_data(dataset=module_graph.dataset)
sample, _ = DataLoader.sample_data(Config.get_device(), dataset=module_graph.dataset)
module_graph_backup = copy.deepcopy(module_graph)
model = create_nn(module_graph, sample, feature_multiplier=Config.feature_multiplier_for_fully_train)
if plot_best_graph:
module_graph.plot_tree_with_graphvis(title="after putting in model", file="after")
print("fully training ~ num epochs:", num_epochs, "num augs:", len(data_augs), "feature multiplier:",
Config.feature_multiplier_for_fully_train)
Evaluator.print_epoch_every = 1
da_phenotypes = [dagenome.to_phenotype() for dagenome in data_augs]
if len(data_augs) > 0:
print("fully training using augs:", data_augs)
acc = Evaluator.evaluate(model, num_epochs, Config.get_device(), train_loader=train, test_loader=test,
print_accuracy=True, batch_size=Config.batch_size, augmentors=da_phenotypes,
training_target=module_graph.fitness_values[0])
if Config.toss_bad_runs and acc == "toss":
print("runn failed to take off. starting over")
get_fully_trained_network(module_graph_backup, data_augs, num_epochs=num_epochs,
plot_best_graph=plot_best_graph)
print("model trained on", num_epochs, "epochs scored:", acc)
def test_duplicates_config_type(self):
assert ApiConfig.configs.sort(key=lambda x: x.name) == [
Config(name="config1", base_url="test1"),
Config(name="config2", base_url="test2"),
Config(name="test_factory",
base_url="https://jsonplaceholder.typicode.com"),
].sort(key=lambda x: x.name)
def testFullNameValidation(self):
Config("Super Mario")
with self.assertRaises(ValueError):
Config(None)
with self.assertRaises(ValueError):
Config("")
with self.assertRaises(ValueError):
Config(" ")
def initialize_elements(self):
self.cf = Config()
self.lineEdit__local_certificado.setText(self.cf.get('cert_path'))
if self.cf.get('cert_path'):
self.lineEdit_senha_certificado.setEnabled(True)
self.lineEdit_senha_certificado.setText(
self.cf.get('cert_password'))
self.lineEdit_input_xml.setText(self.cf.get('input_folder'))
self.lineEdit_output_xml.setText(self.cf.get('output_folder'))
self.enable_process_button()
def init(self):
# 加载全局变量模块
# 加载配置模块
Config.init()
self.conf = Config.get('system')
# 创建缓存文件夹和日志文件夹
try:
os.makedirs(self.conf['default_cache'])
os.makedirs("./log/")
except OSError:
pass
if not os.path.isdir(self.conf['default_cache']):
Log.error(
'FATAL: Output path is not a directory or does not exist')
sys.exit(1)
class GlobalSpace:
CFG = Config()
# mylock = threading.RLock()
# data_buffer = []
def __init__(self):
pass
def append_data_buffer(self, dictV):
global data_buffer
global mylock
mylock.acquire()
data_buffer.append(dictV)
mylock.release()
def pop_data_buffer(self):
global data_buffer
global mylock
mylock.acquire()
pop_item = data_buffer.pop()
mylock.release()
return pop_item
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'-v',
'--verbose',
action='count',
default=0,
help='Indicates the level of messages to be displayed.')
parser.add_argument('-k',
'--keep',
const=str,
action='store_const',
default=False,
help='Keeps the temporary files for future runs.')
parser.add_argument('--path',
required=True,
help='REQUIRED: Path with all images to work.')
parser.add_argument(
'--extension',
required=True,
choices=[ext.lower() for ext in ACCEPTED_IMAGE_FORMATS],
help='REQUIRED: Images extension')
arguments = parser.parse_args()
logging.basicConfig(level=LOGGING_LEVELS[min(
len(LOGGING_LEVELS) - 1, arguments.verbose)],
format='[%(levelname)s] %(message)s')
print(' - IMAGE SIZE CLASSIFIER - ')
service = Service()
service.execute(Config(arguments.path, arguments.extension,
arguments.keep))
print(' - DONE - ')
exit(0)
def __parse_service_config(self):
"""
Parse the YAML configuration file containing specific settings for the individual p2p loan platform
"""
with open(self.config_file, "r", encoding="utf-8") as ymlconfig:
config = safe_load(ymlconfig)
self.config = Config(config)
def __init__(self, assert_spec: AssertSpec, rundir,
libdir,
parallel=False,
logstring="log>>>",
condaenvname="py3.6_allennlp",
threads=None,
logger=None
):
self.assert_spec = assert_spec
self.parallel = parallel
self.config = Config()
self.patch = None
self.logdir = None
self.logstring = logstring
self.logger = logger
self.condaenvname = condaenvname
self.rundir = rundir
self.libdir = libdir
self.threads = threads
if self.parallel:
exceptions = open("src/exceptionlist.txt").readlines()
for l in exceptions:
if assert_spec.test.filename.endswith(l.split(",")[1]) and assert_spec.test.classname == l.split(",")[
2]:
self.parallel = False
print("Found in exception list")
class TestApiRequest:
Config_obj = Config(name="test_name",
base_url="https://jsonplaceholder.typicode.com",
auth={"auth_field": "auth_token"},
headers={"header1": "val1", "header2": "val2"})
request_obj = ApiRequest()
def test_set_config(self):
self.request_obj.set_config(config=self.Config_obj)
assert self.request_obj._headers == {"header1": "val1", "header2": "val2"}
assert self.request_obj._auth == {"auth_field": "auth_token"}
assert self.request_obj._base_url == "https://jsonplaceholder.typicode.com"
def test_set_request(self):
self.request_obj.set_request(end_url="todos/1")
assert self.request_obj._end_url == "todos/1"
def test_get_response(self):
assert self.request_obj.get_response().json() == {'completed': False, 'id': 1, 'title': 'delectus aut autem', 'userId': 1}
def makeMimeText(self, helper):
content = '<style>table,table tr th, table tr td { border:1px solid #0094ff; }table { width: 90%; min-height: 25px; line-height: 25px; text-align: center; border-collapse: collapse;}</style>'
# 生成author of day
content += self.html_header(2, 'Author of day') + '<table style="border:1px solid #F00;">' + \
'<tr><th>Author</th><th>Add</th><th>Delete</th><th>Commits</th></tr>'
for author in helper.getAuthors(
Config.get('system.git')['max_authors']):
info = helper.author_of_day[author]
content += '<tr><td>%s</td><td>%d</td><td>%d</td><td>%d</td></tr>' % (
author, info['lines_added'], info['lines_removed'],
info['commit'])
content += "</table>"
# 生成author of month
content += self.html_header(2, 'Author of Month') + '<table style="border:1px solid #F00;">' + \
'<tr><th>Month</th><th>Author</th><th>Commits (%)</th><th class="unsortable">Next top 5</th></tr>'
for yymm in reversed(sorted(helper.author_of_month.keys())):
authordict = helper.author_of_month[yymm]
authors = Core.getkeyssortedbyvalues(authordict)
authors.reverse()
commits = helper.author_of_month[yymm][authors[0]]
next = ', '.join(authors[1:5])
content += '<tr><td>%s</td><td>%s</td><td>%d (%.2f%% of %d)</td><td>%s</td></tr>' % (
yymm, authors[0], commits,
(100.0 * commits) / helper.commits_by_month[yymm],
helper.commits_by_month[yymm], next)
content += "</table>"
return content
def main():
try:
config = Config(elitism=True, eliteCount=2, mutationProbability=0.1)
masterImage = cv2.imread("images/facebook.jpg")
masterImage = cv2.resize(masterImage, (10, 10))
cv2.imwrite('images/output/master.jpg', masterImage)
population = Population(populationSize=50,
masterImage=masterImage,
config=config,
populate=True)
originalFitness = population.fitestCitizen.fitness
generation = 0
Max_Generation = 100000000
while generation < Max_Generation:
population = population.Evolve(generation)
fitestCitizen = population.fitestCitizen
fitnessPercentage = 100 * (originalFitness -
fitestCitizen.fitness) / originalFitness
print("Fitness = " + str(fitestCitizen.fitness) +
" : FitnessPercentage = " + str(fitnessPercentage) + "%")
if generation % 1000 == 0:
cv2.imwrite(
'images/output/generation' + str(generation) + ".jpg",
fitestCitizen.image)
generation += 1
except IOError:
pass
class TestData(unittest.TestCase):
r = randint(0, 99999999)
file = str(r) + '_test.csv'
nrows = 999
ncols = 49
min_border = -1.0
max_border = 1.0
config = Config(
file=file,
nrows=nrows,
ncols=ncols,
)
@classmethod
def setUpClass(cls) -> None:
Data(TestData.config)
@classmethod
def tearDownClass(cls) -> None:
os.remove(TestData.file)
def test_if_file_is_created(self):
self.assertTrue(os.path.isfile(TestData.file))
def test_if_numbers_are_valid(self):
data = Data(TestData.config)
mn = data.data_frame[data.data_frame.columns].min().min()
mx = data.data_frame[data.data_frame.columns].max().min()
self.assertTrue(mn > TestData.min_border and mx < TestData.max_border)
def get_accuracy_estimate_for_network(model, da_scheme=None, batch_size=Config.batch_size):
"""used in the evolutionary process to train a model for the number of epochs specified in Config settings"""
if Config.dummy_run:
acc = random.random()
else:
acc = Evaluator.evaluate(model, Config.number_of_epochs_per_evaluation, Config.get_device(), batch_size,
augmentors=[da_scheme])
return acc
def __init__(self, populationSize, masterImage, config = Config(), populate = False):
self.populationSize = populationSize
self.masterImage = masterImage
self.config = config
self.rankedCitizens = []
if populate:
self.Populate()
self.UpdateRanking()
def test__from_file():
path_config = 'files/config/config.ini'
abs_path = real_path_absolute
c = Config(abs_path, path_config)
assert c.path_in == '{}/in_file_name'.format(real_path_absolute)
assert c.path_out == '{}/out_file_name'.format(real_path_absolute)
assert c.processes_number == 16
def main():
args = arguments()
config = Config(path_absolute, path_config, args.path_in, args.path_out,
args.process_number)
params = Params(config.path_in)
counter = Counter(params.points, params.paths, config.processes_number)
counter_result = counter.count()
result = Result(config.path_out, counter_result)
result.save()
def is_valid_config(configurator: Config) -> bool:
"""Checks if configurator is valid
:param Config configurator:
:return:
"""
if not configurator.path_exist():
logging.error("Work Folder '{}' does not exist!".format(
configurator.get_work_path()))
return False
if configurator.get_work_count() == 0:
logging.error("Work Folder '{}' doesn't have any {} image!".format(
configurator.get_work_path(),
configurator.get_work_extension().upper()))
return False
return True
def init():
Log('')
Log('Starting...')
Config.load()
Log('Config loaded')
Core.session = vk_api.VkApi(token=Config.token)
Core.api = Core.session.get_api()
Core.longpoll = LongPoll(Core.session, Config.group_id)
Log('Authed')
if Core.api.groups.getOnlineStatus(
group_id=Config.group_id)['status'] == 'none':
Core.api.groups.enableOnline(group_id=Config.group_id)
db.load()
Log('db loaded')
def check_db_exists(self):
if self.db_path == '' or FileIO.file_exists(self.db_path) is False:
LogFactory.MAIN_LOG.warning('DB does not exist! Initializing!')
self.db_path = f"{Config.root_dir}/../{RandomGenerator.generate_random_string(10)}.zp"
nConf: dict = Config.conf_file
nConf['db'] = {} if 'db' not in nConf.keys() else nConf['db']
nConf['db']['path'] = self.db_path
Config.update_config_file(nConf)
self.db = {}
self.db['masterpass'] = RandomGenerator.generate_random_string(10)
LogFactory.MAIN_LOG.info(
f"Your master password is {self.db['masterpass']}")
self.write_db()
def test__with_args():
abs_path = '/app'
path_in = 'input_file_name'
path_out = 'out_file_name'
processes_number = 8
c = Config(abs_path, None, path_in, path_out, processes_number)
assert c.path_in == '/app/input_file_name'
assert c.path_out == '/app/out_file_name'
assert c.processes_number == 8
def __parse_service_config(self):
"""
Parse the YAML configuration file containing specific settings for the individual p2p loan platform
:return:
"""
with open(self.config_file, 'r', encoding='utf-8') as ymlconfig:
yaml = YAML(typ='safe')
config = yaml.load(ymlconfig)
self.config = Config(config)
class TestConfig:
objectEqual1 = Config(name="equal",
base_url="test1",
auth={"field": "token"})
objectEqual2 = Config(name="equal",
base_url="test2",
headers={"header": "val"})
objectAttr = Config(name="attr",
base_url="test_attr",
auth={"field": "attr"},
headers={"head": "atr"})
def test_equality(self):
assert self.objectEqual1 == self.objectEqual2
def test_inequality(self):
assert self.objectEqual1 != self.objectAttr
def test_eq_hash(self):
assert self.objectEqual1.__hash__() == self.objectEqual2.__hash__()
def test_in_eq_hash(self):
assert self.objectEqual2.__hash__() != self.objectAttr.__hash__()
def test_attr(self):
assert self.objectAttr.name == "attr"
assert self.objectAttr.base_url == "test_attr"
assert self.objectAttr.auth == {"field": "attr"}
assert self.objectAttr.headers == {"head": "atr"}
def test_repr(self):
assert self.objectAttr.__repr__(
) == "Config(name='attr', base_url='test_attr', auth={'field': 'attr'}, headers={'head': 'atr'})"
def __init__(self, config=Config()):
self.desired_success_rate = 0.75
self.default_ease_rate = 2.50
self.easecap = 0.20
self.learning_steps = [15, 1440]
self.grade_allocations = [
0.01, 0.25, 0.50, 0.75, 0.99
] #"No idea", "Unsure", "Half Right", "Almost Perfect", "Nailed It"
self.config = config
self.client = self.config.client()
self.collection = self.client.get_collection_view(
self.config.topics_collection_url())
def cdn_pareto_front(individuals):
individuals.sort(key=lambda indv: indv.fitness_values[0], reverse=True)
pf = [individuals[0]
] # pareto front populated with best individual in primary objective
for indv in individuals[1:]:
if Config.second_objective_comparator(indv.fitness_values[1],
pf[-1].fitness_values[1]):
pf.append(indv)
return pf
def set_data(self, _data_frame, _dataset_name=None, _column_names=None):
if _dataset_name is None:
_dataset_name = 'demo'
_data_frame = _data_frame.rename(columns=lambda n: str(n).replace(' ', ''))
_data_frame = _data_frame.applymap(str)
# to refresh the outputs
self.all_posteriors[_dataset_name] = {}
self.predicted_types = {}
self.normal_types = {}
self.missing_types = {}
self.anomaly_types = {}
self.p_z_columns = {}
self.p_t_columns = {}
# Check for dataframes without column names
if _column_names is None:
_column_names = _data_frame.columns
# Creates a configuration object for the experiments
if self.types is None:
config = Config(_dataset_name=_dataset_name, _column_names=_column_names)
else:
config = Config(_dataset_name=_dataset_name, _column_names=_column_names, _types=self.types)
# Ptype model for inference
if self.model is None:
self.model = PtypeModel(config, _data_frame=_data_frame)
else:
self.model.set_params(config, _data_frame=_data_frame)
# Creates a df to report annotations for Wrattler
self.missing_types = {}
self.anomaly_types = {}
self.p_z_columns = {}
self.p_t_columns = {}
self.print = False
self.prediction_path = None
self.predicted_types = {}
def init(self):
config = Config(configFile)
logger = PrintLogger()
driver = ChromeDriver.getInstance()
order = OrderPizza(config, logger, driver)
order.acceptCookies()
order.login()
order.addProductsToCart()
order.goToCheckout()
order.selectDeliveryTime()
order.selectPaymentType()
order.send()
def __init__(self, configFile, limit, register):
"""
Initializes client with a given parameters
:param configFile:
:param limit:
:param register:
:type configFile: str
:type limit: int
:type register: bool
"""
self.userData = {'username': None, 'password': None}
try:
self.config = Config(configFile)
self.request = Request()
self.collection = None
self.view = View(limit)
self.registerMode = register
except OSError:
filename = configFile
print(f"InitError: Configuration file {filename} can\'t be found!")
exit(1)
def get_result(self, configurator: Config, selected_percent: int):
""" Copy the final selected images to `selected` folder.
:param Config configurator:
:param int selected_percent:
:return:
"""
result_dir = '{}{}/'.format(
configurator.get_work_path(),
FOLDER_NAME_OUTPUT_RESULT.format(selected_percent))
self.create_dir(result_dir)
print('Selecting between {} images with >= {}%...'.format(
configurator.get_work_count(), selected_percent))
for file in configurator.get_work_files():
if file.get_jpg_percent() >= selected_percent:
copyfile(file.get_path(), result_dir + file.get_name())
else:
break # My list is sorted!
print("DONE - Results with {}% in '{}'!".format(
selected_percent, result_dir))
from src.Config import Config config = Config() config.reload() temp = config.get_temp() print temp print config.get_day()