def processFirstQuarter(self, data, date, row): quarterProfit = row[ADJUST_NAME['QuarterProfit']] forecastQuarterProfit = np.nan try: forecastQuarterProfit = data.loc[ nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']] except KeyError as e: print(e) if quarterProfit <= 0: if util.isnan(forecastQuarterProfit): data.loc[date, self.key] = data.loc[ date:priorXQ(date, 3), ADJUST_NAME['QuarterProfit']].sum(skipna=False) else: data.loc[date, self.key] = data.loc[date:priorXQ(date, 2), ADJUST_NAME['QuarterProfit']].sum(skipna=False)+ \ forecastQuarterProfit else: if not util.isnan(forecastQuarterProfit): hpr = data.loc[priorQ(date), ADJUST_NAME['HalfYearProfitRatio']] data.loc[date, self.key] = (forecastQuarterProfit + quarterProfit) * (1 + hpr) else: total = data.loc[priorXQ(date, 1):priorXQ(date, 3), ADJUST_NAME['QuarterProfitRatio']].sum( skipna=False) data.loc[date, self.key] = (1 + total) * quarterProfit
def processFirstQuarter(self, data, date, row): quarterProfit = row[ADJUST_NAME['QuarterProfit']] forecastQuarterProfit = np.nan try: forecastQuarterProfit = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']] except KeyError as e: print(e) if quarterProfit <= 0: if util.isnan(forecastQuarterProfit): fq = priorXQ(date, 1) sq = priorXQ(date, 3) total = data.loc[date:sq, ADJUST_NAME['QuarterProfit']].sum(skipna=False) jbmgsy = data.loc[fq, KEY_NAME['jbmgsy']] data.loc[date, self.key] = total/jbmgsy - 1 else: # fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']] total = data.loc[date:priorXQ(date, 2), ADJUST_NAME['QuarterProfit']].sum(skipna=False) jbmgsy = data.loc[priorQ(date), KEY_NAME['jbmgsy']] data.loc[date, self.key] = (forecastQuarterProfit+total) / jbmgsy - 1 else: if not util.isnan(forecastQuarterProfit): hpr = data.loc[priorQ(date), ADJUST_NAME['HalfYearProfitRatio']] # fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']] jbmgsy = data.loc[priorQ(date), KEY_NAME['jbmgsy']] data.loc[date, self.key] = (forecastQuarterProfit + quarterProfit)*(1+hpr) / jbmgsy - 1 else: total = data.loc[priorXQ(date, 1):priorXQ(date, 3), ADJUST_NAME['QuarterProfitRatio']].sum(skipna=False) jbmgsy = data.loc[priorQ(date), KEY_NAME['jbmgsy']] data.loc[date, self.key] = (1 + total)*quarterProfit / jbmgsy - 1
def op(self, data): for date, row in data.iterrows(): try: valueMin = row[ADJUST_NAME['ValueMin']] valueMax = row[ADJUST_NAME['ValueMax']] if not util.isnan(valueMin): data.loc[date, self.keyMin] = (self.stock.lastPrice - valueMin) / valueMin if not util.isnan(valueMax): data.loc[date, self.keyMax] = ( valueMax - self.stock.lastPrice) / self.stock.lastPrice except TypeError as e: print(e) except KeyError as e: print(e)
def fix_chebi_id(id_value): """The Chebi ID should just be the number, we do not prepend the CHEBI:""" if not util.isnan(id_value): id_value = util.make_string(id_value) if id_value.startswith("CHEBI:"): id_value = id_value[6:] return id_value
def processSecondQuarter(self, data, date, row): fqp = np.nan try: fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']] except KeyError as e: print(e) profit = row[KEY_NAME['jbmgsy']] if not util.isnan(fqp): if profit > 0: data.loc[date, self.key] = (fqp + profit) * (1 + data.loc[ priorXQ(date, 2), ADJUST_NAME['ThreeQuarterProfitRatio']]) else: data.loc[date, self.key] = ( fqp + profit + data.loc[priorXQ(date, 2), ADJUST_NAME['QuarterProfit']]) else: if profit > 0: data.loc[date, self.key] = profit * (1 + data.loc[ priorXQ(date, 2), ADJUST_NAME['HalfYearProfitRatio']]) else: data.loc[date, self.key] = ( profit + data.loc[priorXQ(date, 2):priorXQ(date, 3), ADJUST_NAME['QuarterProfit']].sum(skipna=False))
def train(m, p=None): d = DataLoader(BSD3000(), o.batch_size, num_workers=o.num_workers) optimizer = torch.optim.Adam(m.parameters(), lr=o.lr) iter_num = len(d) num = 0 losss = [] stage = 1 if not p else p.stage + 1 for epoch in range(o.epoch): for i in tqdm(d): g, y, k, s = [x.to(o.device) for x in i] x = y optimizer.zero_grad() out = m(x) log("out", out) loss = npsnr(out, g) loss.backward() optimizer.step() losss.append(loss.detach().item()) assert not isnan(losss[-1]) print("stage", stage, "epoch", epoch + 1) log("loss", mean(losss[-5:])) num += 1 # if num > (o.epoch * iter_num - 4): if num % 50 == 1: show(torch.cat((y[0, 0], g[0, 0], out[0, 0]), 1), # save=f"save/{stage:02}{epoch:02}.png", ) plt.clf() plt.plot(range(len(losss)), losss) plt.xlabel("batch") plt.ylabel("loss") plt.title(f"{iter_num} iter x {o.epoch} epoch") plt.savefig(f"save/{stage:02}loss.png")
def op(self, data): for date, row in data.iterrows(): if util.isSameQuarter(date, util.FirstQuarter): data.loc[date, self.keyQ] = 1 else: firstQuarter = util.getFirstQuarter(date) try: firstData = data.loc[firstQuarter] firstQuarterProfit = firstData.loc[KN['QuarterProfit']] try: if firstQuarterProfit > 0: data.loc[date, self.keyQ] = data.loc[ date, KN['QuarterProfit']] / firstQuarterProfit else: data.loc[date, self.keyQ] = 1 if util.isSameQuarter(date, util.FourthQuarter): priorDate = priorQ(date) priorData = data.loc[priorDate] yearProfit = row[KEY_NAME['jbmgsy']] threeQuarterProfit = priorData.loc[ KEY_NAME['jbmgsy']] if not util.isnan(threeQuarterProfit): if threeQuarterProfit > 0: data.loc[date, self.keyT] = ( yearProfit - threeQuarterProfit ) / threeQuarterProfit else: data.loc[date, self.keyT] = float(1) / float(3) prior2Date = priorXQ(date, 2) prior2Data = data.loc[prior2Date] halfYearQuarterProfit = prior2Data.loc[ KEY_NAME['jbmgsy']] if not util.isnan(halfYearQuarterProfit): if halfYearQuarterProfit > 0: data.loc[date, self.keyH] = ( yearProfit - halfYearQuarterProfit ) / halfYearQuarterProfit else: data.loc[date, self.keyH] = 1 except TypeError as e: print(e) except KeyError as e: print(e)
def fix_inchi_parent(id_value): """The database requires the InChI= prepended to id""" if not util.isnan(id_value): id_value = util.make_string(id_value) if not id_value.startswith("InChI="): id_value = "InChI=" + id_value return id_value
def processThirdQuarter(self, data, date, row): fqp = np.nan try: fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']] except KeyError as e: print(e) profit = row[KEY_NAME['jbmgsy']] if not util.isnan(fqp): data.loc[date, self.key] = fqp+profit elif profit < 0: data.loc[date, self.key] = profit+data.loc[priorXQ(date, 3), ADJUST_NAME['QuarterProfit']] else: data.loc[date, self.key] = profit*(1+data.loc[priorXQ(date, 3), ADJUST_NAME['ThreeQuarterProfitRatio']])
def op(self, data): for date, row in data.iterrows(): try: fmg = row[ADJUST_NAME['ForecastMidGrowthRate']] if not util.isnan(fmg): if fmg > 0: data.loc[date, self.keyMin] = 80 * fmg data.loc[date, self.keyMax] = 150 * fmg else: data.loc[date, self.keyMin] = 1 data.loc[date, self.keyMax] = self.stock.gdp_speed * 100 except TypeError as e: print(e) except KeyError as e: print(e)
def get_metric(line): """ metric name: val""" from util import isnan i = line.rfind(':') if i == -1: return None, None else: try: val = line[i + 1:].strip() if val.endswith(" ms."): val = val[:-4] val = float(val) if isnan(val): return None, None return self._normalize(line[:i]), val except: return None, None
def get_metric(line): """ metric name: val""" from util import isnan i = line.rfind(':') if i == -1: return None, None else: try: val = line[i+1:].strip() if val.endswith(" ms."): val = val[:-4] val = float(val) if isnan(val): return None, None return self._normalize(line[:i]), val except: return None, None
def train(m, p=None): d = DataLoader(BSD3000(noise=False, edgetaper=False), o.batch_size, num_workers=o.num_workers) optimizer = torch.optim.Adam(m.parameters(), lr=o.lr) iter_num = len(d) num = 0 losss = [] stage = 1 if not p else p.stage + 1 for epoch in range(o.epoch): for i in tqdm(d): g, y, k, s = [x.to(o.device) for x in i] k = k.flip(1, 2) x = torch.tensor(y, requires_grad=True) if p: with torch.no_grad(): x = p([x, y, k, s]) optimizer.zero_grad() out = m([x, y, k, s]) log("out", out) out = center_crop(out, *g.shape[-2:]) loss = npsnr(out, g) loss.backward() optimizer.step() losss.append(loss.detach().item()) assert not isnan(losss[-1]) print("stage", stage, "epoch", epoch + 1) log("loss", mean(losss[-5:])) num += 1 # if num > (o.epoch * iter_num - 4): if num % 20 == 0: show( torch.cat((center_crop( y, *g.shape[-2:])[0, 0], g[0, 0], out[0, 0]), 1), save=f"save/{stage:02}{epoch:02}.png", ) plt.clf() plt.plot(range(len(losss)), losss) plt.xlabel("batch") plt.ylabel("loss") plt.title(f"{iter_num} iter x {o.epoch} epoch") plt.savefig(f"save/{stage:02}loss.png")
def train(m, p=None): d = DataLoader(BSD3000(), o.batch_size, num_workers=o.num_workers, shuffle=True) optimizer = torch.optim.Adam(m.parameters(), lr=o.lr) iter_num = len(d) num = 0 losss = [] mse = torch.nn.MSELoss() stage = 1 if not p else p.stage + 1 for epoch in range(o.epoch): for i in tqdm(d): g, y, k, s = [x.to(o.device) for x in i] k = k.flip(1, 2) x = y if p: with torch.no_grad(): x = p([x, y, k, s]) optimizer.zero_grad() out = m([x, y, k, s]) log("out", out) out = crop(out, k) loss = npsnr(out, g) loss.backward() optimizer.step() losss.append(loss.detach().item()) assert not isnan(losss[-1]) print("stage", stage, "epoch", epoch + 1) log("loss", mean(losss[-5:])) num += 1 if num > (o.epoch * iter_num - 4): # if num % 6 == 1: show(torch.cat((crop(y, k)[0, 0], g[0, 0], out[0, 0]), 1), # save=f"save/{stage:02}{epoch:02}.png", ) plt.clf() plt.plot([i + 1 for i in range(len(losss))], losss) plt.xlabel("batch") plt.ylabel("loss") plt.title(f"{iter_num} iter x {o.epoch} epoch") plt.savefig(f"save/{stage:02}loss.png")