def find_gate(self):
self._logger.log("finding the gate")
config = self.config['search']
MAX_TIME_SEC = config['max_time_sec']
MODE = config['mode']
stopwatch = Stopwatch()
stopwatch.start()
self._logger.log("started find gate loop")
while stopwatch.time() < config['max_time_sec']:
if MODE == "mvg_avg":
for i in range(config['number_of_samples']):
if self.is_this_gate():
return True
elif MODE == "simple":
bbox = False
bbox = self.darknet_client.predict()[0].normalize(480, 480)
if not bbox:
self._logger.log("gate not found")
return False
self._logger.log("gate found")
return True
#self.movements.rotate_angle(0, 0, config['rotation_angle'])
self._logger.log("gate not found")
return False
def center_on_flare(self):
"""
rotates in vertical axis so flare is in the middle of an image
TODO: obsługa dwóch flar
"""
config = self.config['centering']
flare_size = get_config(
"objects_size")["localization"]["flare"]["height"]
MAX_CENTER_ANGLE_DEG = config['max_center_angle_deg']
MAX_TIME_SEC = config['max_time_sec']
stopwatch = Stopwatch()
stopwatch.start()
while stopwatch <= MAX_TIME_SEC:
bbox = self.darknet_client.predict()[0].normalize(480, 480)
self.flare_position = location_calculator(bbox, flare_size,
"height")
angle = -m.degrees(
m.atan2(self.flare_position['x'],
self.flare_position['distance']))
if abs(angle) <= MAX_CENTER_ANGLE_DEG:
self._logger.log("centered on flare successfully")
return True
self._control.rotate_angle(0, 0, angle)
self._logger.log("couldn't center on flare")
return False
def find_gate(self):
self._logger.log("finding the gate")
config = self.config['search']
MAX_ANG_SPEED = config['max_ang_speed']
MOVING_AVERAGE_DISCOUNT = config['moving_avg_discount']
CONFIDENCE_THRESHOLD = config['confidence_threshold']
MAX_TIME_SEC = config['max_time_sec']
self._control.set_ang_velocity(0, 0, MAX_ANG_SPEED)
stopwatch = Stopwatch()
stopwatch.start()
self._logger.log("started to find gate loop")
result = self.darknet_client.predict()
if not result:
self._logger.log("Empty bounding box")
else:
self._logger.log("Num of bb: "+ str(len(result)))
self._logger.log(str(result[0]))
while stopwatch.time() < MAX_TIME_SEC:
if self.is_this_gate(img):
# TODO: zlokalizuj bramkę
gate = {"x", "y", "angle"}
# TODO: zlokalizuj przeszkodę
obstacle = "{"x", "y"}
# more info in create_path comment
# self.create_path(gate, obstacle) #W trajektorii musi byc uwzgledniona przeszkoda funkcja wyszukujaca przeszkode is_this_obstacle(bounding_box, img):
return True
def center_above_bucket(self):
'''
centering above bucket
'''
self.darknet_client.change_camera("bottom")
stopwatch = Stopwatch()
stopwatch.start()
bbox = self.darknet_client.predict()[0].normalize(480, 480)
while bbox is None and stopwatch.time() < self.MAX_TIME_SEC:
bbox = self.darknet_client.predict()[0].normalize(480, 480)
sleep(0.3)
if bbox is None:
self._logger.log("Could not locate bucket")
return 0
position_x = bbox.x
position_y = bbox.y
Kp = 0.001
i = 0
while position_x > self.POSITION_THRESHOLD and position_y > self.POSITION_THRESHOLD:
self._control.set_lin_velocity(front=position_y * Kp,
right=position_x * Kp)
bbox = self.darknet_client.predict()[0].normalize(480, 480)
if bbox is not None:
position_x = bbox.x
position_y = bbox.y
i += 1
if i == 1000:
self._logger.log("Could not center above bucket")
return 0
return 1
def run(self):
self._logger.log("Bucket grab task exector started")
self.darknet_client.load_model("bucket")
'''
TO DO: uncomment loading model when its done
'''
#self.darknet_client.load_model('buckets_task')
stopwatch = Stopwatch()
stopwatch.start()
## THIS LOOP SHOULD FIND AT LEAST FIRST BBOX WITH BUCKET
while not self.find_buckets():
self._logger.log("Finding buckets in progress")
if stopwatch.time() >= self.MAX_TIME_SEC:
self._logger.log("Finding buckets time expired")
return 0
#self.darknet_client.load_model("bucket")
i = 0
# THIS LOOP SHOULD FIND BUCKET WITH PINGER
if self.bucket == 'pinger':
while i < self.PINGER_LOOP_COUNTER:
if self.find_pinger_bucket():
self._logger.log("Found pinger bucket")
i = self.PINGER_LOOP_COUNTER
self.grab_marker()
self._logger.log("Marker grabbed")
return 1
i += 1
self._logger.log("Marker could not be grabbed")
return 0
elif self.bucket == 'blue':
k = 0
while k < self.BLUE_LOOP_COUNTER:
if self.find_blue_bucket():
self._logger.log("Found blue bucket")
k = self.BLUE_LOOP_COUNTER
self.grab_marker()
self._logger.log("Marker grabbed")
return 1
k += 1
self._logger.log("Marker could not be grabbed")
return 0
elif self.bucket == 'red':
l = 0
while l < self.ANY_BUCKET_COUNTER:
if self.find_random_bucket():
self._logger.log("Found random bucket")
l = self.ANY_BUCKET_COUNTER
self.grab_marker()
self._logger.log("Marker grabbed")
return 1
l += 1
self._logger.log("Grabbing marker failed")
return 0
def load_gene_expression_profile(gene_list_file_name, gene_expression_file_name, gene_filter_file_name=None, gene_list_path=None, gene_expression_path=None, gene_filter_path=None ,by_gene=False, list_mode = "FROM_DISK"):
stopwatch = Stopwatch()
stopwatch.start()
if list_mode == "ON_THE_FLY":
gene_list = gene_list_file_name
else:
gene_list = load_gene_list(gene_list_file_name=gene_list_file_name, gene_list_path=gene_list_path)
gene_list = [l.split(".")[0] for i, l in enumerate(gene_list)]
print stopwatch.stop("done loading gene list")
# random.shuffle(gene_list)
# gene_list = gene_list[:400]
if gene_filter_file_name:
stopwatch.start()
filter_gene_list = load_gene_list(gene_list_file_name=gene_filter_file_name, gene_list_path=gene_filter_path)
gene_list = [cur for cur in gene_list if cur in filter_gene_list]
print stopwatch.stop("done filter gene list")
if gene_expression_path == None:
gene_expression_path = os.path.join(constants.TCGA_DATA_DIR, gene_expression_file_name)
stopwatch.start()
f = open(gene_expression_path,'r')
expression_profiles_filtered = [l.strip().split() for i, l in enumerate(f) if i==0 or l[:l.strip().find('\t')].split(".")[0] in gene_list]
# or l.strip()[0:l.strip().find('\t')] in gene_list or l.strip()[0:l.strip().find('\t')].split(".")[0] in gene_list
f.close()
print stopwatch.stop("done filter gene expression")
if not by_gene:
stopwatch.start()
expression_profiles_filtered = np.flip(np.rot90(expression_profiles_filtered, k=1, axes=(1,0)),1)
print stopwatch.stop("done rotate gene expression")
return expression_profiles_filtered
def center_on_gate(self):
config = self.config['centering']
MAX_TIME_SEC = config['max_time_sec']
MAX_CENTER_DISTANCE = config['max_center_distance']
stopwatch = Stopwatch()
stopwatch.start()
while stopwatch.time() <= MAX_TIME_SEC:
bbox = self.darknet_client.predict()[0].normalize(480, 480)
if bbox.x <= MAX_CENTER_DISTANCE & bbox.y <= MAX_CENTER_DISTANCE:
self._logger.log("centered on gate successfully")
return True
center_rov(move=self._control,
Bbox=bbox,
depth_sensor=self.depth_sensor)
self._logger.log("couldn't center on gate")
return False
def run(self):
MAX_TIME_SEC = self.config['search']['max_time_sec'] #w configu trzeba zmienic bo na razie jest do samego gate
self._logger.log("Gate task executor started")
self._control.pid_turn_on()
stopwatch = Stopwatch()
stopwatch.start()
if stopwatch.time() >= MAX_TIME_SEC:
self._logger.log("TIME EXPIRED GATE NOT FOUND")
self._control.set_ang_velocity(0, 0, 0)
if not self.dive():
self._logger.log("GateTE: diving in progress")
if not self.find_gate():
self._logger.log("GateTE: finding the gate in progress")
if not self.go_trough_gate():
self._logger.log("GateTE: going through the gate")
def find_flare(self):
# TODO: obsługa wykrycia dwóch flar
self._logger.log("finding the flare")
config = self.config['search']
stopwatch = Stopwatch()
stopwatch.start()
self._logger.log("started find flare loop")
while stopwatch < config['max_time_sec']:
# sprawdza kilka razy, dla pewności
# (no i żeby confidence się zgadzało, bo bez tego to nawet jak raz wykryje, to nie przejdzie -
# - przy moving_avg_discount=0.9 musi wykryć 10 razy z rzędu
for i in range(config['number_of_samples']):
if self.is_this_flare():
return True
self._control.rotate_angle(0, 0, config['rotation_angle'])
self._logger.log("flare not found")
return False
def go_to_flare(self):
"""
moves distance to flare + a little more to knock it
:return: True - if managed to move distance in time
False - if didn't manage to move distance in time
"""
config = self.config['go']
MAX_TIME_SEC = config['max_time_sec']
stopwatch = Stopwatch()
stopwatch.start()
self._control.move_distance(
self.flare_position['distance'] +
self.config['go']['distance_to_add_m'], 0, 0)
if stopwatch <= MAX_TIME_SEC:
self._logger.log("go_to_flare - traveled whole distance")
return True
else:
self._logger.log("go_to_flare - didn't travel whole distance")
return False
def center_on_pinger(self):
"""
rotates in vertical axis so pinger signal is in front
"""
config = self.config['centering']
MAX_TIME_SEC = config['max_time_sec']
MAX_CENTER_ANGLE_DEG = config['max_center_angle_deg']
self._logger.log("centering on pinger")
stopwatch = Stopwatch()
stopwatch.start()
while stopwatch < MAX_TIME_SEC:
angle = self._hydrophones.get_angle()
if angle is None:
self._logger.log(
"no signal from hydrophones - locating pinger failed")
return False
if abs(angle) < MAX_CENTER_ANGLE_DEG:
self._logger.log("centered on pinger successfully")
return True
self._control.rotate_angle(0, 0, angle)
self._logger.log("couldn't ceneter on pinger")
return False
def gl_ProxGD_primal(x0: np.ndarray, A: np.ndarray, b: np.ndarray, mu_0, opts: dict):
default_opts = {
"maxit": 2500, # 最大迭代次数
"thres": 1e-3, # 判断小量是否被认为 0 的阈值
"step_type": "line_search", # 步长衰减的类型（见辅助函数）
"alpha0": 2e-3, # 步长的初始值
"ftol": 1e-6, # 停机准则，当目标函数历史最优值的变化小于该值时认为满足
"stable_len_threshold": 70,
"line_search_attenuation_coeffi": 0.9,
"maxit_line_search_iter": 5,
}
# The second dictionary's values overwrite those from the first.
opts = {**default_opts, **opts}
sparsity_func = lambda x: np.sum(np.abs(x) > 1e-6 * np.max(np.abs(x))) / x.size
def real_obj_func(x: np.ndarray):
fro_term = 0.5 * np.sum((A @ x - b) ** 2)
regular_term = np.sum(LA.norm(x, axis=1).reshape(-1, 1))
return fro_term + mu_0 * regular_term
out = {
"fvec": None, # 每一步迭代的 LASSO 问题目标函数值
"grad_hist": None, # 可微部分梯度范数的历史值
"f_hist": None, # 目标函数的历史值
"f_hist_best": None, # 目标函数每一步迭代对应的历史最优值
"tt": None, # 运行时间
"flag": None # 标记是否收敛
}
maxit, ftol, alpha0 = opts["maxit"], opts["ftol"], opts["alpha0"]
stable_len_threshold = opts["stable_len_threshold"]
thres = opts["thres"]
step_type = opts['step_type']
aten_coeffi = opts['line_search_attenuation_coeffi']
max_line_search_iter = opts['maxit_line_search_iter']
logger.debug("alpha0= {:10E}".format(alpha0))
f_hist, f_hist_best, sparsity_hist = [], [], []
f_best = np.inf
x = np.copy(x0)
stopwatch = Stopwatch()
stopwatch.start()
k = 0
for mu in [100 * mu_0, 10 * mu_0, mu_0]:
logger.debug("new mu= {:10E}".format(mu))
# min f(x) = g(x) + h(x)
# g(x) = 0.5 * |Ax-b|_F^2
# h(x) = mu * |x|_{1,2}
def g(x: np.ndarray):
return 0.5 * np.sum((A @ x - b) ** 2)
grad_g = None
def prox_th(x: np.ndarray, t):
""" Proximal operator of t * mu * h(x).
"""
t_mu = t * mu
row_norms = LA.norm(x, axis=1).reshape(-1, 1)
rv = x * np.clip(row_norms - t_mu, a_min=0, a_max=None) / ((row_norms < thres) + row_norms)
return rv
def Gt(x: np.ndarray, t):
return (x - prox_th(x - t * grad_g, t)) / t
inner_iter = 0
def set_step(step_type: str):
iter_hat = max(inner_iter, 1000) - 999
if step_type == 'fixed':
return alpha0
elif step_type == 'diminishing':
return alpha0 / np.sqrt(iter_hat)
elif step_type == 'diminishing2':
return alpha0 / iter_hat
elif step_type == 'line_search':
g_x = g(x)
def stop_condition(x, t):
gt_x = Gt(x, t)
return (g(x - t * gt_x)
<= g_x - t * np.sum(grad_g * gt_x) + 0.5 * t * np.sum(gt_x ** 2))
alpha = alpha0
for i in range(max_line_search_iter):
if stop_condition(x, alpha):
break
alpha *= aten_coeffi
return alpha
else:
logger.error("Unsupported type.")
stable_len = 0
while inner_iter < maxit:
# Record current objective value
f_now = real_obj_func(x)
f_hist.append(f_now)
f_best = min(f_best, f_now)
f_hist_best.append(f_best)
sparsity_hist.append(sparsity_func(x))
k += 1
inner_iter += 1
if (k > 1
and abs(f_hist[k - 1] - f_hist[k - 2]) / abs(f_hist[k - 2]) < ftol
and abs(sparsity_hist[k - 1] - sparsity_hist[k - 2]) / abs(sparsity_hist[k - 2]) < ftol):
stable_len += 1
else:
stable_len = 0
if stable_len > stable_len_threshold:
break
x[np.abs(x) < thres] = 0
grad_g = A.T @ (A @ x - b)
alpha_k = set_step(step_type)
# logger.debug("alpha_k: {}".format(alpha_k))
x = prox_th(x - alpha_k * grad_g, alpha_k)
if k % 100 == 0:
logger.debug(
'iter= {:5}, objective= {:10E}, sparsity= {:3f}'.format(k, f_now.item(), sparsity_func(x)))
elapsed_time = stopwatch.elapsed(time_format=Stopwatch.TimeFormat.kMicroSecond) / 1e6
out = {
"tt": elapsed_time,
"fval": real_obj_func(x),
"f_hist": f_hist,
"f_hist_best": f_hist_best
}
return x, k, out
def gl_FGD_primal(x0: np.ndarray, A: np.ndarray, b: np.ndarray, mu_0,
opts: dict):
default_opts = {
"maxit": 1500, # 最大迭代次数
"thres": 1e-3, # 判断小量是否被认为 0 的阈值
"step_type": "line_search", # 步长衰减的类型（见辅助函数）
"alpha0": 1e-3, # 步长的初始值
"ftol": 1e-6, # 停机准则，当目标函数历史最优值的变化小于该值时认为满足
"stable_len_threshold": 70,
"line_search_attenuation_coeffi": 0.98,
"maxit_line_search_iter": 5,
"delta": 1e-6 # 光滑化参数
}
# The second dictionary's values overwrite those from the first.
opts = {**default_opts, **opts}
sparsity_func = lambda x: np.sum(np.abs(x) > 1e-6 * np.max(np.abs(x))
) / x.size
def real_obj_func(x: np.ndarray):
fro_term = 0.5 * np.sum((A @ x - b)**2)
regular_term = np.sum(LA.norm(x, axis=1).reshape(-1, 1))
return fro_term + mu_0 * regular_term
out = {
"fvec": None, # 每一步迭代的 LASSO 问题目标函数值
"grad_hist": None, # 可微部分梯度范数的历史值
"f_hist": None, # 目标函数的历史值
"f_hist_best": None, # 目标函数每一步迭代对应的历史最优值
"tt": None, # 运行时间
"flag": None # 标记是否收敛
}
maxit, ftol, alpha0 = opts["maxit"], opts["ftol"], opts["alpha0"]
stable_len_threshold = opts["stable_len_threshold"]
thres = opts["thres"]
step_type = opts['step_type']
aten_coeffi = opts['line_search_attenuation_coeffi']
max_line_search_iter = opts['maxit_line_search_iter']
delta = opts["delta"]
logger.debug("alpha0= {:10E}".format(alpha0))
f_hist, f_hist_best, sparsity_hist = [], [], []
v_hist, t_hist = [], []
f_best = np.inf
x_k = np.copy(x0)
stopwatch = Stopwatch()
stopwatch.start()
k = 0
for mu in [100 * mu_0, 10 * mu_0, mu_0]:
logger.debug("new mu= {:10E}".format(mu))
# min f(x) = g(x) + h(x)
# g(x) = 0.5 * |Ax-b|_F^2 + mu * smoothed |x|_{1,2}
# h(x) = 0
def g_func(x: np.ndarray):
fro_term = 0.5 * np.sum((A @ x - b)**2)
regular_term = np.sum(
np.sqrt(np.sum(x**2, axis=1).reshape(-1, 1) + delta * delta) -
delta)
return fro_term + mu * regular_term
def grad_g_func(x: np.ndarray):
fro_term_grad = A.T @ (A @ x - b)
regular_term_grad = x / np.sqrt(
np.sum(x**2, axis=1).reshape(-1, 1) + delta * delta)
return fro_term_grad + mu * regular_term_grad
v_k = np.copy(x_k)
t_k = alpha0
def prox_th(x: np.ndarray, t):
""" Proximal operator of t * mu * h(x).
"""
return x
inner_iter = 0
def set_step(step_type: str):
iter_hat = max(inner_iter, 1000) - 999
if step_type == 'fixed':
return alpha0
elif step_type == 'diminishing':
return alpha0 / np.sqrt(iter_hat)
elif step_type == 'diminishing2':
return alpha0 / iter_hat
elif step_type == 'line_search':
t = t_k
g_y = g_func(y)
def stop_condition(t):
x = prox_th(y - t * grad_g_y, t)
g_x = g_func(x)
return g_x <= g_y + np.sum(grad_g_y * (x - y)) + np.sum(
(x - y)**2) / (2 * t)
for i in range(max_line_search_iter):
if stop_condition(t):
break
t *= aten_coeffi
return t
else:
logger.error("Unsupported type.")
stable_len = 0
while inner_iter < maxit:
# Record current objective value
f_now = real_obj_func(x_k)
f_hist.append(f_now)
f_best = min(f_best, f_now)
f_hist_best.append(f_best)
sparsity_hist.append(sparsity_func(x_k))
v_hist.append(v_k)
t_hist.append(t_k)
k += 1
inner_iter += 1
if (k > 1
and abs(f_hist[k - 1] - f_hist[k - 2]) / abs(f_hist[k - 2])
< ftol
and abs(sparsity_hist[k - 1] - sparsity_hist[k - 2]) /
abs(sparsity_hist[k - 2]) < ftol):
stable_len += 1
else:
stable_len = 0
if stable_len > stable_len_threshold:
break
x_k[np.abs(x_k) < thres] = 0
theta = 2 / (inner_iter + 1)
y = (1 - theta) * x_k + theta * v_k
grad_g_y = grad_g_func(y)
t = set_step(step_type)
x = prox_th(y - t * grad_g_y, t)
v = x_k + (x - x_k) / theta
x_k, v_k, t_k = x, v, t
if k % 100 == 0:
logger.debug(
'iter= {:5}, objective= {:10E}, sparsity= {:3f}'.format(
k, f_now.item(), sparsity_func(x)))
elapsed_time = stopwatch.elapsed(
time_format=Stopwatch.TimeFormat.kMicroSecond) / 1e6
out = {
"tt": elapsed_time,
"fval": real_obj_func(x),
"f_hist": f_hist,
"f_hist_best": f_hist_best
}
return x, k, out
def check_group_enrichment_goatools(tested_gene_file_name,
total_gene_file_name,
th=1):
if len(tested_gene_file_name) == 0 or len(total_gene_file_name) == 0:
return []
if type(total_gene_file_name) == str:
total_gene_list = load_gene_list(total_gene_file_name)
else:
total_gene_list = total_gene_file_name
if type(tested_gene_file_name) == str:
tested_gene_list = load_gene_list(tested_gene_file_name)
else:
tested_gene_list = tested_gene_file_name
if not os.path.exists(
os.path.join(constants.GO_DIR, constants.GO_FILE_NAME)):
download(constants.GO_OBO_URL, constants.GO_DIR)
obo_dag = GODag(os.path.join(constants.GO_DIR, constants.GO_FILE_NAME))
if not os.path.exists(
os.path.join(constants.GO_DIR,
constants.GO_ASSOCIATION_FILE_NAME)):
download(constants.GO_ASSOCIATION_GENE2GEO_URL, constants.GO_DIR)
with gzip.open(
os.path.join(
constants.GO_DIR,
os.path.basename(constants.GO_ASSOCIATION_GENE2GEO_URL)),
'rb') as f_in:
with open(
os.path.join(constants.GO_DIR,
constants.GO_ASSOCIATION_FILE_NAME),
'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
assoc = read_ncbi_gene2go(os.path.join(constants.GO_DIR,
constants.GO_ASSOCIATION_FILE_NAME),
no_top=True)
sw = Stopwatch()
sw.start()
g = GOEnrichmentStudy(
[int(cur) for cur in ensembl2entrez_convertor(total_gene_list)],
assoc,
obo_dag,
methods=[],
log=None) # "bonferroni", "fdr_bh"
g_res = g.run_study(
[int(cur) for cur in ensembl2entrez_convertor(tested_gene_list)])
print sw.stop("done GO analysis in ")
# GO_results = [(cur.NS, cur.GO, cur.goterm.name, cur.pop_count, cur.p_uncorrected, cur.p_fdr_bh) for cur in g_res if
# cur.p_fdr_bh <= 0.05]
GO_results = [(cur.NS, cur.GO, cur.goterm.name, cur.pop_count,
cur.p_uncorrected) for cur in g_res
if cur.p_uncorrected <= th]
hg_report = [{
HG_GO_ROOT: cur[0],
HG_GO_ID: cur[1],
HG_GO_NAME: cur[2],
HG_VALUE: cur[3],
HG_PVAL: cur[4],
HG_QVAL: 1
} for cur in GO_results] # , HG_QVAL : cur[5]
# hg_report.sort(key=lambda x: x[HG_QVAL])
hg_report.sort(key=lambda x: x[HG_PVAL])
if len(GO_results) > 0:
go_ns, go_terms, go_names, go_hg_value, uncorrectd_pvals = zip(
*GO_results) # , FDRs
else:
go_terms = []
uncorrectd_pvals = []
FDRs = []
go_names = []
go_ns = []
# output_rows = [("\r\n".join(e2g_convertor(tested_gene_list)), "\r\n".join(go_ns),
# "\r\n".join(go_terms), "\r\n".join(go_names), "\r\n".join(map(str, uncorrectd_pvals)),
# "\r\n".join(map(str, FDRs)))]
# print_to_excel(output_rows, str(tested_gene_file_name)[:10], str(total_gene_file_name)[:10])
return hg_report
def gl_Alm_dual(x0: np.ndarray, A: np.ndarray, b: np.ndarray, mu, opts: dict):
default_opts = {
"maxit": 100, # 最大迭代次数
"thres": 1e-3, # 判断小量是否被认为 0 的阈值
"tau": (1 + math.sqrt(5)) * 0.5,
"rho": 1e2,
"converge_len": 20,
}
# The second dictionary's values overwrite those from the first.
opts = {**default_opts, **opts}
def sparsity_func(x: np.ndarray):
return np.sum(np.abs(x) > 1e-6 * np.max(np.abs(x))) / x.size
def real_obj_func(x: np.ndarray):
fro_term = 0.5 * np.sum((A @ x - b)**2)
regular_term = np.sum(LA.norm(x, axis=1).reshape(-1, 1))
return fro_term + mu * regular_term
out = {
"fvec": None, # 每一步迭代的 LASSO 问题目标函数值
"f_hist": None, # 目标函数的历史值
"f_hist_best": None, # 目标函数每一步迭代对应的历史最优值
"tt": None, # 运行时间
}
def projection_functor(x: np.array):
row_norms = LA.norm(x, axis=1, ord=2).reshape(-1, 1)
return mu * x / np.clip(row_norms, a_min=mu, a_max=None)
maxit, thres = opts["maxit"], opts["thres"]
rho, tau = opts['rho'], opts['tau']
converge_len = opts['converge_len']
f_hist, f_hist_best, sparsity_hist = [], [], []
f_best = np.inf
x_k = np.copy(x0)
z_k = np.zeros_like(b)
u_k = np.zeros_like(x_k)
stopwatch = Stopwatch()
stopwatch.start()
L = LA.cholesky(np.identity(A.shape[0]) + rho * A @ A.T)
k = 0
length = 0
while k < maxit:
k += 1
u = solve_sub_problem(b, rho, A, x_k, L, mu)
z = LA.solve(L.T, LA.solve(L, A @ (x_k - rho * u) - b))
x = x_k - tau * rho * (u + A.T @ z)
r_k = u + A.T @ z # 原始可行性
s_k = A @ (u_k - u) # 对偶可行性
z_k, u_k, x_k = z, u, x
f_now = real_obj_func(x_k)
f_hist.append(f_now)
f_best = min(f_best, f_now)
f_hist_best.append(f_best)
sparsity_now = sparsity_func(x_k)
sparsity_hist.append(sparsity_now)
if k % 1 == 0:
logger.debug(
'iter= {:5}, objective= {:10E}, sparsity= {:3f}'.format(
k, f_now.item(), sparsity_now.item()))
r_k_norm = LA.norm(r_k, ord=2)
s_k_norm = LA.norm(s_k, ord=2)
if r_k_norm < thres and s_k_norm < thres:
length += 1
else:
length = 0
if length >= converge_len:
break
elapsed_time = stopwatch.elapsed(
time_format=Stopwatch.TimeFormat.kMicroSecond) / 1e6
out = {
"tt": elapsed_time,
"fval": real_obj_func(x_k),
"f_hist": f_hist,
"f_hist_best": f_hist_best
}
return x_k, k, out
def gl_Admm_primal(x0: np.ndarray, A: np.ndarray, b: np.ndarray, mu,
opts: dict):
default_opts = {
"maxit": 100, # 最大迭代次数
"thres": 1e-3, # 判断小量是否被认为 0 的阈值
"tau": (1 + math.sqrt(5)) * 0.5,
"rho": 1e-2,
"eta_0": 100,
"converge_len": 10,
"converge_thres": 1e-5,
"step_type": "fixed",
}
# The second dictionary's values overwrite those from the first.
opts = {**default_opts, **opts}
sparsity_func = lambda x: np.sum(np.abs(x) > 1e-6 * np.max(np.abs(x))
) / x.size
def real_obj_func(x: np.ndarray):
fro_term = 0.5 * np.sum((A @ x - b)**2)
regular_term = np.sum(LA.norm(x, axis=1).reshape(-1, 1))
return fro_term + mu * regular_term
out = {
"fvec": None, # 每一步迭代的 LASSO 问题目标函数值
"f_hist": None, # 目标函数的历史值
"f_hist_best": None, # 目标函数每一步迭代对应的历史最优值
"tt": None, # 运行时间
}
maxit, thres = opts["maxit"], opts["thres"]
rho, tau, eta_0 = opts['rho'], opts['tau'], opts['eta_0']
converge_len = opts['converge_len']
converge_thres = opts['converge_thres']
step_type = opts['step_type']
f_hist, f_hist_best, sparsity_hist = [], [], []
f_best = np.inf
def prox_tf(x: np.array, t):
t_mu = t * mu
row_norms = LA.norm(x, axis=1).reshape(-1, 1)
rv = x * np.clip(row_norms - t_mu, a_min=0, a_max=None) / (
(row_norms < thres) + row_norms)
return rv
x_k = np.copy(x0)
y_k = x_k
z_k = x_k
stopwatch = Stopwatch()
stopwatch.start()
k = 0
L = LA.cholesky(rho * np.identity(A.shape[1]) + A.T @ A)
AT_b = A.T @ b
length = 0
def set_step(step_type: str):
if step_type == 'fixed':
return eta_0
elif step_type == 'diminishing':
return eta_0 / np.sqrt(k)
elif step_type == 'diminishing2':
return eta_0 / k
while k < maxit:
k += 1
eta = set_step(step_type)
y = LA.solve(L.T, LA.solve(L, AT_b - z_k + rho * x_k))
# x = prox_tf(y + z_k / rho, 1/rho)
x = prox_tf(x_k - eta * rho * (x_k - y - z_k / rho), eta)
z = z_k - tau * rho * (x - y)
r_k = x - y # 原始可行性
s_k = y - y_k # 对偶可行性
x_k, y_k, z_k = x, y, z
f_now = real_obj_func(x_k)
f_hist.append(f_now)
f_best = min(f_best, f_now)
f_hist_best.append(f_best)
sparsity_hist.append(sparsity_func(x_k))
if k % 1 == 0:
logger.debug(
'iter= {:5}, objective= {:10E}, sparsity= {:3f}'.format(
k, f_now.item(), sparsity_func(x_k)))
r_k_norm = LA.norm(r_k, ord=2)
s_k_norm = LA.norm(s_k, ord=2)
if r_k_norm < thres and s_k_norm < thres:
length += 1
else:
length = 0
if length >= converge_len:
break
elapsed_time = stopwatch.elapsed(
time_format=Stopwatch.TimeFormat.kMicroSecond) / 1e6
out = {
"tt": elapsed_time,
"fval": real_obj_func(x_k),
"f_hist": f_hist,
"f_hist_best": f_hist_best
}
return x_k, k, out
def gl_SGD_primal(x0: np.ndarray, A: np.ndarray, b: np.ndarray, mu_0, opts: dict):
default_opts = {
"maxit": 2100, # 内循环最大迭代次数
"thres": 1e-3, # 判断小量是否被认为 0 的阈值
"step_type": "diminishing", # 步长衰减的类型（见辅助函数）
"alpha0": 1e-3, # 步长的初始值
"ftol": 1e-5, # 停机准则，当目标函数历史最优值的变化小于该值时认为满足
"stable_len_threshold": 100,
"continuous_subgradient_flag": False,
}
# The second dictionary's values overwrite those from the first.
opts = {**default_opts, **opts}
sparsity_func = lambda x: np.sum(np.abs(x) > 1e-6 * np.max(np.abs(x))) / x.size
out = {
"fvec": None, # 每一步迭代的 LASSO 问题目标函数值
"grad_hist": None, # 可微部分梯度范数的历史值
"f_hist": None, # 目标函数的历史值
"f_hist_best": None, # 目标函数每一步迭代对应的历史最优值
"tt": None, # 运行时间
"flag": None # 标记是否收敛
}
maxit, ftol, alpha0 = opts["maxit"], opts["ftol"], opts["alpha0"]
stable_len_threshold = opts["stable_len_threshold"]
thres = opts["thres"]
if opts["continuous_subgradient_flag"]:
L = np.max(LA.eigvals(A.T @ A))
alpha0 = 1. / L.real
logger.debug("alpha0= {:10E}".format(alpha0))
f_hist, f_hist_best = [], []
f_best = np.inf
x = np.copy(x0)
stopwatch = Stopwatch()
stopwatch.start()
k = 0
stable_len = 0
for mu in [100 * mu_0, 10 * mu_0, mu_0]:
logger.debug("new mu= {:10E}".format(mu))
def obj_func(x: np.ndarray):
fro_term = 0.5 * np.sum((A @ x - b) ** 2)
regular_term = np.sum(LA.norm(x, axis=1).reshape(-1, 1))
return fro_term + mu * regular_term
def subgrad(x: np.ndarray):
fro_term_grad = A.T @ (A @ x - b)
regular_term_norm = LA.norm(x, axis=1).reshape(-1, 1)
regular_term_grad = x / ((regular_term_norm < thres) + regular_term_norm)
grad = fro_term_grad + mu * regular_term_grad
return grad
inn_iter = 0
def set_step(step_type):
iter_hat = max(inn_iter, 1000) - 999
if step_type == 'fixed' or mu > mu_0:
return alpha0
elif step_type == 'diminishing':
return alpha0 / np.sqrt(iter_hat)
elif step_type == 'diminishing2':
return alpha0 / iter_hat
else:
logger.error("Unsupported type.")
while inn_iter < maxit:
# Record current objective value
f_now = obj_func(x)
f_hist.append(f_now)
f_best = min(f_best, f_now)
f_hist_best.append(f_best)
k += 1
inn_iter += 1
if k > 1 and abs(f_hist[k - 1] - f_hist[k - 2]) / abs(f_hist[k - 2]) < ftol:
stable_len += 1
else:
stable_len = 0
# if stable_len > stable_len_threshold:
# break
x[np.abs(x) < thres] = 0
sub_g = subgrad(x)
alpha = set_step(opts["step_type"])
x = x - alpha * sub_g
if k % 100 == 0:
logger.debug('iter= {:5}, objective= {:10E}, sparsity= {:3f}'.format(k, f_now.item(), sparsity_func(x)))
elapsed_time = stopwatch.elapsed(time_format=Stopwatch.TimeFormat.kMicroSecond) / 1e6
out = {
"tt": elapsed_time,
"fval": obj_func(x),
"f_hist": f_hist,
"f_hist_best": f_hist_best
}
return x, k, out
class Logger:
def __init__(self):
Context.logger = self
self._episodes = Context.config['exp.episodes']
self._work_path = os.path.join(Context.work_path, WORK_DIR)
self._sw = Stopwatch()
self._saved_time = 0.
self._train_history = []
self._eval_history = []
def __str__(self):
return "%s:\n\t%s\n\t%s\n\t%s" % (
self.__class__.__name__,
"saved_time: %s" % hms(self._saved_time),
"train_history: %d" % len(self._train_history),
"eval_history: %d" % len(self._eval_history),
)
def on_start(self):
self._sw.start()
def on_train_episode(self, e, n, r, q):
# episode, noise_rate, reward, q_max
t = self.time_spent
self._train_history.append([e, t, n, r, q])
self._log_training_episode(e, n, r, q)
self._update_training_title(self._episodes, e, n, r, q)
if (e + 1) % Context.config['report.write_every_episodes'] == 0:
self._write_train_report()
if (e + 1) % Context.config['report.summary_every_episodes'] == 0:
self._log_summary(e, self._episodes, self.time_spent)
if (e + 1) % Context.config['exp.save_every_episodes'] == 0:
self._save_state()
@staticmethod
def on_evaluiation_start():
Context.window_title['episode'] = "| Evaluating..."
def on_evaluiation_end(self, e, r):
self._log_evaluiation_end(r)
self._eval_history.append([e, self.time_spent, r]) # e,t,r
Context.window_title['episode'] = "| Trainig..."
@property
def episode(self):
return len(self._train_history)
@property
def time_left(self):
return int((self.time_spent * (1 - self.progress) /
self.progress) if self.progress != 0 else 0)
@property
def progress(self):
return self.episode / float(self._episodes)
@property
def time_spent(self):
return self._saved_time + self._sw.time_elapsed
@property
def _state_path(self):
return os.path.join(self._work_path, "state.pickle")
def _save_state(self):
with open(make_dir_if_not_exists(self._state_path), 'w') as f:
pickle.dump(
[self.time_spent, self._train_history, self._eval_history],
f,
protocol=pickle.HIGHEST_PROTOCOL)
def restore(self):
with open(self._state_path, 'r') as f:
[self._saved_time, self._train_history,
self._eval_history] = pickle.load(f)
def try_restore(self):
try:
self.restore()
except (ValueError, IOError):
return False
return True
def _write_train_report(self):
info = {
'ep': len(self._train_history),
'eps': self._episodes,
'spent': self.time_spent,
'left': self.time_left,
'progress': self.progress,
'train_history': self._train_history,
'eval_history': self._eval_history,
}
report = TrainReport(Context.config, info)
report.make()
report.save()
def _log_evaluiation_end(self, r):
pass
def _log_summary(self, ep, eps, spent):
line = '========================================================================='
self.log(line + ('\n' * 3))
self._log_summary_time(ep, eps, spent)
self.log(('\n' * 3) + line)
@staticmethod
def log(text, end='\n'):
print(text, end=end)
# noinspection PyStringFormat
def _log_summary_time(self, ep, eps, spent):
progress = ep / float(eps)
left = (spent * (1 - progress) / progress) if progress != 0 else 0
self.log("Time spent: %s | " % hms(spent), end='')
self.log("%.0f%% " % (progress * 100, ), end='')
self.log("+ %s = %s, " % (hms(left), hms(spent + left)), end='')
self.log("%.1f per sec" % (ep / float(spent), ))
@staticmethod
def _update_training_title(eps, e, n, r, q):
Context.window_title['episode'] = "| %d%% N=%.2f R=%+.0f Q=%+.0f" % (
e / float(eps) * 100, n, r, q)
def _log_training_episode(self, e, n, r, q):
self.log("Ep: %3d | NR: %.2f | Reward: %+7.0f | Qmax: %+8.1f" %
(e, n, r, q))