def processDocbook(doc) :
lib.normalize(doc)
for_text_nodes(handleTextReplacements, doc)
#convert @@foo to <link linkend="node.foo">foo</link>
def handleAtAt(match) :
text = match.group(1)
elem = doc.createElement('link')
elem.setAttribute('linkend', 'node.%s' % text)
elem.appendChild(doc.createTextNode(text))
return elem
for_text_nodes(
lambda x: replaceBy(x, r'@@([a-z\-]+)', handleAtAt),
doc)
#convert ##foo to <link linkend="def.foo">foo</link>
def handleHashHash(match) :
elem = doc.createElement('link')
elem.setAttribute('linkend', 'def.%s' % match.group(2))
elem.appendChild(doc.createTextNode(match.group(1)))
return elem
for_text_nodes(
lambda x: replaceBy(x, r'##(([a-z\-]+)(:?s))', handleHashHash),
doc)
for glossterm in doc.getElementsByTagName("glossterm") :
glossterm.setAttribute("xml:id", "def.%s" % lib.textValue(glossterm).replace(" ", "-"))
specTitle = lib.textValue(doc.getElementsByTagName("title")[0])
if specTitle == "Niviok 1.0 Specification" :
terminalTypenames = sorted( x["typename"] for x in nodeTypes if x["category"] == "terminal" )
listElem = lib.getElementsByAttribute(doc, "xml:id", "terminal-node-types-list")[0]
for typename in terminalTypenames :
memberElem = doc.createElement("member")
listElem.appendChild(memberElem)
memberElem.appendChild(doc.createTextNode(typename))
memberElem.setAttribute("linkend", "node.%s" % typename)
for elem in doc.getElementsByTagName("screen") :
processPreElem(elem)
for elem in doc.getElementsByTagName("programlistinge") :
processPreElem(elem)
for elem in doc.getElementsByTagName("member") :
role = elem.parentNode.getAttribute('role')
if role == 'layout' :
setupLayoutMember(elem)
elif role == 'family-members' :
setupFamilyMember(elem)
def preapre_learn_data(gps,
alt,
user,
window_fn,
window_params,
alpha=ALPHA,
const_norm=True,
use_data_perc=0.33):
altitude = gps['altitude']
speed = gps['speed']
user_data = user_events_to_data(user)
# TODO StandartScaler fit on training data
smoothed = altitude.ewm(alpha=alpha).mean()
if const_norm:
norm_v_speed = normalize(smoothed.diff(),
mean=ALT_SPEED_MEAN,
min=ALT_SPEED_MIN,
max=ALT_SPEED_MAX)
norm_speed = normalize(speed.ewm(alpha=alpha).mean(),
mean=SPEED_MEAN,
min=SPEED_MIN,
max=SPEED_MAX)
else:
norm_v_speed = normalize(smoothed.diff())
norm_speed = normalize(speed.ewm(alpha=alpha).mean())
X = []
y = []
i = 0
for ((ts, v_speed), (_, speed)) in zip(norm_v_speed.iteritems(),
norm_speed.iteritems()):
if not math.isnan(v_speed) and not math.isnan(
speed) and random() <= use_data_perc:
v_speed_feature = window_fn(i, norm_v_speed, **window_params)
speed_feature = window_fn(i, norm_speed, **window_params)
X.append(v_speed_feature + speed_feature)
u = get_intersecting_ts(ts, user_data)
y.append(0 if u['activity'] == "heli" else 1)
i += 1
return shuffle(X, y, random_state=42)
def normalize_keywords(query):
query = normalize(query)
query = lower(query)
for x in ('*', "'", '"', ':', '(', ')'):
query = query.replace(x, ' ')
kwds = []
for kwd in pat_divide_kwds.split(query):
if kwd:
kwds.append(kwd + '*')
return u' '.join(kwds)
def normalize_keywords(query):
query = normalize(query)
query = lower(query)
for x in ('*', "'", '"', ':', '(', ')'):
query = query.replace(x, ' ')
kwds = []
for kwd in pat_divide_kwds.split(query):
if kwd:
kwds.append(kwd + '*')
return u' '.join(kwds)
colors = ["#000000", "#222222", "#444444", "#666666", "#888888", "#aaaaaa"]
makers = ["o", "v", "^", "*", "s", "p"]
markerfacecolor = '#ffffff'
plt.figure(figsize=(10, 10))
instance_types = [
"m3.large", "m5.2xlarge", "m5.large", "m5.xlarge", "r3.xlarge",
"r5d.xlarge"
]
for it_index in range(len(instance_types)):
TARGET_TYPE = instance_types[it_index]
df = lib.load_data(DATA_PATH, TARGET_TYPE)
df, mean, std = lib.normalize(df)
(x_train, y_train), (x_test, y_test), columns = lib.train_test_split_lstm(
df["price"].values, df.index, PAST_HISTORY, TRAIN_RATIO)
# モデルを定義
model = create_model(x_train.shape[-2:])
# モデルを学習
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
validation_data=(x_test, y_test))
df_10days = lib.load_data(DATA_PATH_ACTUAL, TARGET_TYPE)
# モデルの検証
df_10days, mean, std = lib.normalize(df_10days)
test_data_unscaled = data[60000 + test_idx, :].astype(float)
test_labels = (labels[60000 + test_idx] == pos) * 2 - 1
# Preprocessing
train_data = sklearn.preprocessing.scale(train_data_unscaled,
axis=0,
with_std=False)
validation_data = sklearn.preprocessing.scale(validation_data_unscaled,
axis=0,
with_std=False)
test_data = sklearn.preprocessing.scale(test_data_unscaled,
axis=0,
with_std=False)
train_data = lib.normalize(train_data)
validation_data = lib.normalize(validation_data)
test_data = lib.normalize(test_data)
#
def sgd(samples, labels, c, lr, T):
w = np.zeros(samples.shape[1]).astype(np.double)
for i in range(T):
idx = np.random.randint(0, samples.shape[0])
current_lr = lr / (i + 1)
if labels[idx] * w.dot(samples[idx]) < 1: # yi*w*xi
w = (1 -
current_lr) * w + current_lr * c * labels[idx] * samples[idx]
else:
result = []
colors = ["red", "royalblue", "violet", "green", "cyan", "orange"]
instance_types = [
"m3.large", "m5.2xlarge", "m5.large", "m5.xlarge", "r3.xlarge",
"r5d.xlarge"
]
feature_importance = []
fig = plt.figure(figsize=(16, 9))
for i in range(len(instance_types)):
TARGET_TYPE = instance_types[i]
print("=" * 10, TARGET_TYPE, "=" * 10)
df = lib.load_data(DATA_PATH, TARGET_TYPE)
df, mean, std = lib.normalize(df)
(x_train, y_train), (x_test, y_test), columns = lib.train_test_split_lstm(
df["price"].values, df.index, PAST_HISTORY, TRAIN_RATIO)
# モデルを定義
model = create_model(x_train.shape[-2:])
# モデルを学習
hist = model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=1,
validation_data=(x_test, y_test))
y_pred = model.predict(x_test)
a = {}
from matplotlib.widgets import TextBox, Button
from pandas.plotting import register_matplotlib_converters
from lib import normalize, load_data, user_events_to_data, KW_GOING_HELI, KW_BACKWARD, widget_log
register_matplotlib_converters()
user_events_style = {"color": "red", "alpha": 0.33}
file_name = sys.argv[1]
(gps, alt, user2) = load_data(file_name)
user_marks_holder = [user2]
if __name__ == '__main__':
x = gps['altitude']
smoothed = x.ewm(alpha=0.1).mean()
norm_speed = normalize(gps['speed'].ewm(alpha=0.03).mean())
charts_ax = normalize(smoothed).plot(label="smoothed")
norm_speed.plot(label="speed", alpha=0.3)
plt.gca().xaxis.set_major_locator(mdates.HourLocator())
plt.gca().xaxis.set_major_formatter(
mdates.DateFormatter('%H:%M')) # %Y-%m-%d %H:%M
error_text = TextBox(plt.axes([0.04, 0.005, 0.8, 0.04]),
'Error',
initial="")
def draw_user_marks():
for span in draw_user_marks.spans:
span.remove()
draw_user_marks.spans = []
with open('../../file/model/struc_params.json', 'w') as f:
json.dump(model_struc_params, f)
# 保存损失函数记录
train_loss_list = [
float(p.detach().cpu().numpy()) for p in train_loss_record
]
with open('../../file/model/train_loss.json', 'w') as f:
json.dump(train_loss_list, f)
if __name__ == '__main__':
# %% 载入观测数据并进行列归一化
total_obs_data = pd.read_csv('../../data/runtime/total_obs_data.csv')
total_obs_data = normalize(total_obs_data)
# %% 构造样本
x0_tensor, x1_tensor, integrate_t_tensor, dt_tensor = build_samples(
total_obs_data)
# %% 搭建网络
input_size = x0_tensor.shape[1] + 1 # 输入为(x, t), 所以shape = dim(x) + dim(t)
output_size = x0_tensor.shape[1] # 输出为变量x的导数dx/dt, 所以shape = dim(x)
hidden_sizes = [
4 * input_size, 4 * input_size, 4 * output_size, 4 * output_size
]
pd_net = PartialDeriveNet(input_size, hidden_sizes, output_size)
# %% 指定优化器
register_matplotlib_converters()
from lib import normalize, load_data, ALPHA, ALT_SPEED_MEAN, ALT_SPEED_MIN, SPEED_MEAN, SPEED_MIN, ALT_SPEED_MAX, \
SPEED_MAX, user_events_to_data
user_events_style = {"color": "red", "alpha": 0.33}
(gps, alt, user) = load_data("part_1")
if __name__ == '__main__':
x = gps['altitude']
# x = alt["relativeAltitude"]
smoothed = x.ewm(alpha=ALPHA).mean()
norm_v_speed = normalize(smoothed.diff(),
mean=ALT_SPEED_MEAN,
min=ALT_SPEED_MIN,
max=ALT_SPEED_MAX)
norm_speed = normalize(gps['speed'].ewm(alpha=ALPHA).mean(),
mean=SPEED_MEAN,
min=SPEED_MIN,
max=SPEED_MAX)
normalize(smoothed).plot(label="smoothed")
# norm_v_speed.plot(label="vertical speed")
# norm_speed.plot(label="speed")
# mlp_model = pickle.load(open("./data/model.mlp", 'rb'))
# i = 0
# predictions = []
# for (a, b) in zip(norm_v_speed.iteritems(), norm_speed.iteritems()):
# if i % 50 == 0: