def create_app(config='app/config/config.cfg', default_app_name='theonestore'):
"""创建app
@param config 配置文件路径
@param default_app_name 默认App名称
"""
app = Flask(default_app_name)
# config
if config:
osname = os.name.upper()
if osname == 'NT':
config = config.replace('/', '\\')
app.config.from_pyfile(config)
return app
def logger_handler(keyword, logpath="log", debug=0):
path = os.path.split(os.path.realpath(__file__))[0]
os.chdir(path)
os.chdir("./")
with open('./conf/logging.yaml', 'r') as f:
yaml_config = yaml.load(f)
config = json.dumps(yaml_config)
config = config.replace("XXXXX", keyword)
jconfig = json.loads(config)
rkeyword = u"app_%s" % keyword
jconfig["handlers"][rkeyword]["filename"] = os.path.join(
logpath, keyword + ".log")
if debug == 1:
jconfig["loggers"][keyword]["handlers"].append("console")
logging.config.dictConfig(jconfig)
return logging.getLogger(keyword)
def setup_logging(self):
'''
This function initialises the logging mechanism
'''
if 'seed' in self.options:
self.log_folder = 'log/' + self.options['job_name'] + '/' + str(
self.options['seed']) + '/'
else:
self.log_folder = 'log/' + self.options['job_name'] + '/'
try:
os.makedirs(self.log_folder)
except OSError:
pass
# Load config file
with open('logging.yaml', 'r') as f:
config = f.read()
# Prepend logging folder
config = config.replace('PATH/', self.log_folder)
# Setup logging
logging.config.dictConfig(yaml.load(config))
def hamprobe_conf():
probe_id = binascii.hexlify(os.urandom(16)).decode(
'ascii') # TODO rate limit registration?
probe_key = binascii.hexlify(os.urandom(16)).decode('ascii')
created = datetime.datetime.now()
session = get_session()
probe = db.Probe(id=probe_id,
key=probe_key,
created=created,
target_script="default",
target_policy="default")
session.add(probe)
session.commit()
with open('../hamprobe-probe/hamprobe.conf.sample', 'r') as f:
config = f.read()
config = config.replace("%PROBE_ID%", probe_id,
1).replace("%PROBE_KEY%", probe_key, 1)
return Response(
config,
mimetype="text/plain",
headers={"Content-disposition": "attachment; filename=hamprobe.conf"})
def bayesian_optimization(self):
'''
This function implements the main loop of Bayesian Optimization
'''
if 'seed' in self.options:
self.log_folder = 'log/' + self.options['job_name'] + '/' + \
str(self.options['seed']) + '/'
else:
self.log_folder = 'log/' + self.options['job_name'] + '/'
try:
os.makedirs(self.log_folder)
except OSError:
pass
# Load config file
with open('logging.yaml', 'r') as f:
config = f.read()
# Prepend logging folder
config = config.replace('PATH/', self.log_folder)
# Setup logging
logging.config.dictConfig(yaml.load(config))
# Copy the objective. This is essential when testing draws from GPs
objective = copy.copy(self.options['objective'])
X0 = self.random_sample(self.bounds, self.initial_size)
y0 = objective.f(X0)
# Set the data to the GP model
# Careful, we might normalize the function evaluations
# Provide only the first column of y0.
# The others columns contain auxiliary data.
self.X = X0
self.Y = self.normalize(y0[:, 0:1])
# X_all stores all the points where f was evaluated and
# y_all the respective function values
X_all = X0
y_all = y0
logger = logging.getLogger('evals')
logger.info('----------------------------')
logger.info('Bounds:\n' + str(self.bounds))
if hasattr(objective, 'fmin'):
logger.info('Minimum value:' + str(objective.fmin))
logger.info('----------------------------')
for i in range(len(X0)):
logger.info('X:' + str(X0[i, :]) + ' y: ' + str(y0[i, :]))
for i in range(self.iterations):
self.optimize_restarts(restarts=self.options['model_restarts'])
logging.getLogger('').info('#Iteration:' + str(i + 1))
# Remove non-printable characters (bold identifiers)
ansi_escape = re.compile(r'\x1b[^m]*m')
logging.getLogger('model').info(ansi_escape.sub('', str(self)))
if self.options['samples'] > 0:
# Draw samples of the hyperparameters from the posterior
samples_raw = self.sample(self.options['samples'])
self.samples = self.get_samples_df(samples_raw)
# Evaluate the black-box function at the suggested points
X_new = self.get_suggestion(self.batch_size)
y_new = objective.f(X_new)
for i in range(len(X_new)):
logging.getLogger('evals').info('X:' + str(X_new[i, :]) +
' y: ' + str(y_new[i, :]))
# Append the algorithm's choice X_new to X_all
# Add the function evaluations f(X_new) to y_all
X_all = np.concatenate((X_all, X_new))
y_all = np.concatenate((y_all, y_new))
# Update the GP model
# Careful, the model might normalize the function evaluations
# Provide only the first column of y_all.
# The others columns contain auxiliary data.
self.X = X_all
self.Y = self.normalize(y_all[:, 0:1])
return X_all, y_all
def __genRemoteConfig(self, jobid, jobnumer, component, category,
upstream_buildurl):
result = False
#Generate personalized config
category_low = category.lower()
if category_low == "buildly":
session_include = self.__config.getValue('Buildly_Include',
component)
session_exclude = self.__config.getValue('Buildly_Exclude',
component)
#category = "Smoke"
#self.__config.setValue('Build_Version',component, build)
if component == "bmn":
buildurl_list = self.__config.getValue('Build_Url',
component).split(",")
buildurl_list[0] = upstream_buildurl
self.__config.setValue(
'Build_Url', component,
buildurl_list[0] + "," + buildurl_list[1])
else:
self.__config.setValue('Build_Url', component,
upstream_buildurl)
tmp_urls = upstream_buildurl
urls = tmp_urls.split('/')
build = urls[len(urls) - 2]
self.__config.setValue('Build_Version', component, build)
self.__config.setValue('Job_Num', component, jobnumer)
else:
session_include = self.__config.getValue('Nightly_Include',
component)
session_exclude = self.__config.getValue('Nightly_Exclude',
component)
build = self.__config.getValue('Build_Version', component)
#upstream_buildurl = self.__config.getValue('Build_Url',component)
#category = "nightly"
#build_url=self.__config.getValue('Build_Url',component)+build+".exe"
suite_path = self.__config.getValue('Suite_Path', component)
log_ip = self.__config.getValue('RobotLogRepository', "ipaddress")
log_usr = self.__config.getValue('RobotLogRepository', "username")
log_pwd = self.__config.getValue('RobotLogRepository', "password")
log_path = self.__config.getValue('RobotLogRepository', "rootpath")
report_to = self.__config.getValue('Report_To', component)
target_server = self.__config.getValue('Target_Server',
component).split(",")
test_server = self.__config.getValue('Test_Server',
component).split(",")
#suite_path=self.__config.getValue('Suite_Path','path')+component
try:
#Read config from base configuration ini
fi = open(self.__base_configuration_ini, 'r')
configs = fi.readlines()
fi.close()
#After modification, generate a new tmp configuration ini
fo = open(self.__tmp_configuration_ini, 'w')
for config in configs:
config = config.replace('${target_ipaddress}$',
target_server[0])
config = config.replace('${target_user}$', target_server[1])
config = config.replace('${target_pwd}$', target_server[2])
config = config.replace('${session_include}$', session_include)
config = config.replace('${session_exclude}$', session_exclude)
config = config.replace('${session_component}$', component)
config = config.replace(
'${build_version}$',
self.__config.getValue('Build_Version', component))
config = config.replace(
'${build_url}$',
self.__config.getValue('Build_Url', component))
config = config.replace('${job_number}$', jobnumer)
config = config.replace('${jobid}$', jobid)
config = config.replace('${suite_path}$', suite_path)
config = config.replace('${category}$', category)
config = config.replace('${log_ip}$', log_ip)
config = config.replace('${log_user}$', log_usr)
config = config.replace('${log_pwd}$', log_pwd)
config = config.replace('${root_path}$', log_path)
config = config.replace('${log_component}$', component)
config = config.replace('${test_server}$', test_server[0])
config = config.replace('${report_to}$', report_to)
fo.write(config)
fo.close()
result = True
self.__logger.info("Temp configuration.ini had been generated.")
except IOError, e:
print e.strerror
self.__logger.error("IO exception: %s" % e.strerror)