def processFirstQuarter(self, data, date, row):
quarterProfit = row[ADJUST_NAME['QuarterProfit']]
forecastQuarterProfit = np.nan
try:
forecastQuarterProfit = data.loc[
nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']]
except KeyError as e:
print(e)
if quarterProfit <= 0:
if util.isnan(forecastQuarterProfit):
data.loc[date, self.key] = data.loc[
date:priorXQ(date, 3),
ADJUST_NAME['QuarterProfit']].sum(skipna=False)
else:
data.loc[date, self.key] = data.loc[date:priorXQ(date, 2), ADJUST_NAME['QuarterProfit']].sum(skipna=False)+ \
forecastQuarterProfit
else:
if not util.isnan(forecastQuarterProfit):
hpr = data.loc[priorQ(date),
ADJUST_NAME['HalfYearProfitRatio']]
data.loc[date, self.key] = (forecastQuarterProfit +
quarterProfit) * (1 + hpr)
else:
total = data.loc[priorXQ(date, 1):priorXQ(date, 3),
ADJUST_NAME['QuarterProfitRatio']].sum(
skipna=False)
data.loc[date, self.key] = (1 + total) * quarterProfit
def processFirstQuarter(self, data, date, row):
quarterProfit = row[ADJUST_NAME['QuarterProfit']]
forecastQuarterProfit = np.nan
try:
forecastQuarterProfit = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']]
except KeyError as e:
print(e)
if quarterProfit <= 0:
if util.isnan(forecastQuarterProfit):
fq = priorXQ(date, 1)
sq = priorXQ(date, 3)
total = data.loc[date:sq, ADJUST_NAME['QuarterProfit']].sum(skipna=False)
jbmgsy = data.loc[fq, KEY_NAME['jbmgsy']]
data.loc[date, self.key] = total/jbmgsy - 1
else:
# fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']]
total = data.loc[date:priorXQ(date, 2), ADJUST_NAME['QuarterProfit']].sum(skipna=False)
jbmgsy = data.loc[priorQ(date), KEY_NAME['jbmgsy']]
data.loc[date, self.key] = (forecastQuarterProfit+total) / jbmgsy - 1
else:
if not util.isnan(forecastQuarterProfit):
hpr = data.loc[priorQ(date), ADJUST_NAME['HalfYearProfitRatio']]
# fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']]
jbmgsy = data.loc[priorQ(date), KEY_NAME['jbmgsy']]
data.loc[date, self.key] = (forecastQuarterProfit + quarterProfit)*(1+hpr) / jbmgsy - 1
else:
total = data.loc[priorXQ(date, 1):priorXQ(date, 3), ADJUST_NAME['QuarterProfitRatio']].sum(skipna=False)
jbmgsy = data.loc[priorQ(date), KEY_NAME['jbmgsy']]
data.loc[date, self.key] = (1 + total)*quarterProfit / jbmgsy - 1
def op(self, data):
for date, row in data.iterrows():
try:
valueMin = row[ADJUST_NAME['ValueMin']]
valueMax = row[ADJUST_NAME['ValueMax']]
if not util.isnan(valueMin):
data.loc[date, self.keyMin] = (self.stock.lastPrice -
valueMin) / valueMin
if not util.isnan(valueMax):
data.loc[date, self.keyMax] = (
valueMax - self.stock.lastPrice) / self.stock.lastPrice
except TypeError as e:
print(e)
except KeyError as e:
print(e)
def fix_chebi_id(id_value):
"""The Chebi ID should just be the number, we do not prepend the CHEBI:"""
if not util.isnan(id_value):
id_value = util.make_string(id_value)
if id_value.startswith("CHEBI:"):
id_value = id_value[6:]
return id_value
def processSecondQuarter(self, data, date, row):
fqp = np.nan
try:
fqp = data.loc[nextXQ(date, 1),
ADJUST_NAME['ForecastQuarterProfit']]
except KeyError as e:
print(e)
profit = row[KEY_NAME['jbmgsy']]
if not util.isnan(fqp):
if profit > 0:
data.loc[date, self.key] = (fqp + profit) * (1 + data.loc[
priorXQ(date, 2), ADJUST_NAME['ThreeQuarterProfitRatio']])
else:
data.loc[date, self.key] = (
fqp + profit +
data.loc[priorXQ(date, 2), ADJUST_NAME['QuarterProfit']])
else:
if profit > 0:
data.loc[date, self.key] = profit * (1 + data.loc[
priorXQ(date, 2), ADJUST_NAME['HalfYearProfitRatio']])
else:
data.loc[date, self.key] = (
profit +
data.loc[priorXQ(date, 2):priorXQ(date, 3),
ADJUST_NAME['QuarterProfit']].sum(skipna=False))
def fix_chebi_id(id_value):
"""The Chebi ID should just be the number, we do not prepend the CHEBI:"""
if not util.isnan(id_value):
id_value = util.make_string(id_value)
if id_value.startswith("CHEBI:"):
id_value = id_value[6:]
return id_value
def train(m, p=None):
d = DataLoader(BSD3000(), o.batch_size, num_workers=o.num_workers)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
x = y
optimizer.zero_grad()
out = m(x)
log("out", out)
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
# if num > (o.epoch * iter_num - 4):
if num % 50 == 1:
show(torch.cat((y[0, 0], g[0, 0], out[0, 0]), 1),
# save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot(range(len(losss)), losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
def op(self, data):
for date, row in data.iterrows():
if util.isSameQuarter(date, util.FirstQuarter):
data.loc[date, self.keyQ] = 1
else:
firstQuarter = util.getFirstQuarter(date)
try:
firstData = data.loc[firstQuarter]
firstQuarterProfit = firstData.loc[KN['QuarterProfit']]
try:
if firstQuarterProfit > 0:
data.loc[date, self.keyQ] = data.loc[
date, KN['QuarterProfit']] / firstQuarterProfit
else:
data.loc[date, self.keyQ] = 1
if util.isSameQuarter(date, util.FourthQuarter):
priorDate = priorQ(date)
priorData = data.loc[priorDate]
yearProfit = row[KEY_NAME['jbmgsy']]
threeQuarterProfit = priorData.loc[
KEY_NAME['jbmgsy']]
if not util.isnan(threeQuarterProfit):
if threeQuarterProfit > 0:
data.loc[date, self.keyT] = (
yearProfit - threeQuarterProfit
) / threeQuarterProfit
else:
data.loc[date,
self.keyT] = float(1) / float(3)
prior2Date = priorXQ(date, 2)
prior2Data = data.loc[prior2Date]
halfYearQuarterProfit = prior2Data.loc[
KEY_NAME['jbmgsy']]
if not util.isnan(halfYearQuarterProfit):
if halfYearQuarterProfit > 0:
data.loc[date, self.keyH] = (
yearProfit - halfYearQuarterProfit
) / halfYearQuarterProfit
else:
data.loc[date, self.keyH] = 1
except TypeError as e:
print(e)
except KeyError as e:
print(e)
def fix_inchi_parent(id_value):
"""The database requires the InChI= prepended to id"""
if not util.isnan(id_value):
id_value = util.make_string(id_value)
if not id_value.startswith("InChI="):
id_value = "InChI=" + id_value
return id_value
def fix_inchi_parent(id_value):
"""The database requires the InChI= prepended to id"""
if not util.isnan(id_value):
id_value = util.make_string(id_value)
if not id_value.startswith("InChI="):
id_value = "InChI=" + id_value
return id_value
def processThirdQuarter(self, data, date, row):
fqp = np.nan
try:
fqp = data.loc[nextXQ(date, 1), ADJUST_NAME['ForecastQuarterProfit']]
except KeyError as e:
print(e)
profit = row[KEY_NAME['jbmgsy']]
if not util.isnan(fqp):
data.loc[date, self.key] = fqp+profit
elif profit < 0:
data.loc[date, self.key] = profit+data.loc[priorXQ(date, 3), ADJUST_NAME['QuarterProfit']]
else:
data.loc[date, self.key] = profit*(1+data.loc[priorXQ(date, 3), ADJUST_NAME['ThreeQuarterProfitRatio']])
def op(self, data):
for date, row in data.iterrows():
try:
fmg = row[ADJUST_NAME['ForecastMidGrowthRate']]
if not util.isnan(fmg):
if fmg > 0:
data.loc[date, self.keyMin] = 80 * fmg
data.loc[date, self.keyMax] = 150 * fmg
else:
data.loc[date, self.keyMin] = 1
data.loc[date,
self.keyMax] = self.stock.gdp_speed * 100
except TypeError as e:
print(e)
except KeyError as e:
print(e)
def get_metric(line):
""" metric name: val"""
from util import isnan
i = line.rfind(':')
if i == -1:
return None, None
else:
try:
val = line[i + 1:].strip()
if val.endswith(" ms."):
val = val[:-4]
val = float(val)
if isnan(val):
return None, None
return self._normalize(line[:i]), val
except:
return None, None
def get_metric(line):
""" metric name: val"""
from util import isnan
i = line.rfind(':')
if i == -1:
return None, None
else:
try:
val = line[i+1:].strip()
if val.endswith(" ms."):
val = val[:-4]
val = float(val)
if isnan(val):
return None, None
return self._normalize(line[:i]), val
except:
return None, None
def train(m, p=None):
d = DataLoader(BSD3000(noise=False, edgetaper=False),
o.batch_size,
num_workers=o.num_workers)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
k = k.flip(1, 2)
x = torch.tensor(y, requires_grad=True)
if p:
with torch.no_grad():
x = p([x, y, k, s])
optimizer.zero_grad()
out = m([x, y, k, s])
log("out", out)
out = center_crop(out, *g.shape[-2:])
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
# if num > (o.epoch * iter_num - 4):
if num % 20 == 0:
show(
torch.cat((center_crop(
y, *g.shape[-2:])[0, 0], g[0, 0], out[0, 0]), 1),
save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot(range(len(losss)), losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
def train(m, p=None):
d = DataLoader(BSD3000(),
o.batch_size,
num_workers=o.num_workers,
shuffle=True)
optimizer = torch.optim.Adam(m.parameters(), lr=o.lr)
iter_num = len(d)
num = 0
losss = []
mse = torch.nn.MSELoss()
stage = 1 if not p else p.stage + 1
for epoch in range(o.epoch):
for i in tqdm(d):
g, y, k, s = [x.to(o.device) for x in i]
k = k.flip(1, 2)
x = y
if p:
with torch.no_grad():
x = p([x, y, k, s])
optimizer.zero_grad()
out = m([x, y, k, s])
log("out", out)
out = crop(out, k)
loss = npsnr(out, g)
loss.backward()
optimizer.step()
losss.append(loss.detach().item())
assert not isnan(losss[-1])
print("stage", stage, "epoch", epoch + 1)
log("loss", mean(losss[-5:]))
num += 1
if num > (o.epoch * iter_num - 4):
# if num % 6 == 1:
show(torch.cat((crop(y, k)[0, 0], g[0, 0], out[0, 0]), 1),
# save=f"save/{stage:02}{epoch:02}.png",
)
plt.clf()
plt.plot([i + 1 for i in range(len(losss))], losss)
plt.xlabel("batch")
plt.ylabel("loss")
plt.title(f"{iter_num} iter x {o.epoch} epoch")
plt.savefig(f"save/{stage:02}loss.png")
