def processDocbook(doc) :
	lib.normalize(doc)
	
	for_text_nodes(handleTextReplacements, doc)
	
	#convert @@foo to <link linkend="node.foo">foo</link>
	def handleAtAt(match) :
		text = match.group(1)
		elem = doc.createElement('link')
		elem.setAttribute('linkend', 'node.%s' % text)
		elem.appendChild(doc.createTextNode(text))
		return elem
	for_text_nodes(
		lambda x: replaceBy(x, r'@@([a-z\-]+)', handleAtAt),
		doc)

	#convert ##foo to <link linkend="def.foo">foo</link>
	def handleHashHash(match) :
		elem = doc.createElement('link')
		elem.setAttribute('linkend', 'def.%s' % match.group(2))
		elem.appendChild(doc.createTextNode(match.group(1)))
		return elem
	for_text_nodes(
		lambda x: replaceBy(x, r'##(([a-z\-]+)(:?s))', handleHashHash),
		doc)

	for glossterm in doc.getElementsByTagName("glossterm") :
		glossterm.setAttribute("xml:id", "def.%s" % lib.textValue(glossterm).replace(" ", "-"))
	
	specTitle = lib.textValue(doc.getElementsByTagName("title")[0])
	if specTitle == "Niviok 1.0 Specification" :
		terminalTypenames = sorted( x["typename"] for x in nodeTypes if x["category"] == "terminal" )
		listElem = lib.getElementsByAttribute(doc, "xml:id", "terminal-node-types-list")[0]
		for typename in terminalTypenames :
			memberElem = doc.createElement("member")
			listElem.appendChild(memberElem)
			memberElem.appendChild(doc.createTextNode(typename))
			memberElem.setAttribute("linkend", "node.%s" % typename)

	for elem in doc.getElementsByTagName("screen") :
		processPreElem(elem)
	for elem in doc.getElementsByTagName("programlistinge") :
		processPreElem(elem)

	for elem in doc.getElementsByTagName("member") :
		role = elem.parentNode.getAttribute('role')
		if role == 'layout' :
			setupLayoutMember(elem)
		elif role == 'family-members' :
			setupFamilyMember(elem)
Esempio n. 2
0
def preapre_learn_data(gps,
                       alt,
                       user,
                       window_fn,
                       window_params,
                       alpha=ALPHA,
                       const_norm=True,
                       use_data_perc=0.33):
    altitude = gps['altitude']
    speed = gps['speed']
    user_data = user_events_to_data(user)

    # TODO StandartScaler fit on training data
    smoothed = altitude.ewm(alpha=alpha).mean()
    if const_norm:
        norm_v_speed = normalize(smoothed.diff(),
                                 mean=ALT_SPEED_MEAN,
                                 min=ALT_SPEED_MIN,
                                 max=ALT_SPEED_MAX)
        norm_speed = normalize(speed.ewm(alpha=alpha).mean(),
                               mean=SPEED_MEAN,
                               min=SPEED_MIN,
                               max=SPEED_MAX)
    else:
        norm_v_speed = normalize(smoothed.diff())
        norm_speed = normalize(speed.ewm(alpha=alpha).mean())

    X = []
    y = []
    i = 0
    for ((ts, v_speed), (_, speed)) in zip(norm_v_speed.iteritems(),
                                           norm_speed.iteritems()):
        if not math.isnan(v_speed) and not math.isnan(
                speed) and random() <= use_data_perc:
            v_speed_feature = window_fn(i, norm_v_speed, **window_params)
            speed_feature = window_fn(i, norm_speed, **window_params)
            X.append(v_speed_feature + speed_feature)
            u = get_intersecting_ts(ts, user_data)
            y.append(0 if u['activity'] == "heli" else 1)
        i += 1

    return shuffle(X, y, random_state=42)
Esempio n. 3
0
def normalize_keywords(query):
    query = normalize(query)
    query = lower(query)
    for x in ('*', "'", '"', ':', '(', ')'):
        query = query.replace(x, ' ')

    kwds = []
    for kwd in pat_divide_kwds.split(query):
        if kwd:
            kwds.append(kwd + '*')
    return u' '.join(kwds)
Esempio n. 4
0
def normalize_keywords(query):
    query = normalize(query)
    query = lower(query)
    for x in ('*', "'", '"', ':', '(', ')'):
        query = query.replace(x, ' ')

    kwds = []
    for kwd in pat_divide_kwds.split(query):
        if kwd:
            kwds.append(kwd + '*')
    return u' '.join(kwds)
Esempio n. 5
0
colors = ["#000000", "#222222", "#444444", "#666666", "#888888", "#aaaaaa"]
makers = ["o", "v", "^", "*", "s", "p"]
markerfacecolor = '#ffffff'
plt.figure(figsize=(10, 10))
instance_types = [
    "m3.large", "m5.2xlarge", "m5.large", "m5.xlarge", "r3.xlarge",
    "r5d.xlarge"
]

for it_index in range(len(instance_types)):
    TARGET_TYPE = instance_types[it_index]

    df = lib.load_data(DATA_PATH, TARGET_TYPE)

    df, mean, std = lib.normalize(df)
    (x_train, y_train), (x_test, y_test), columns = lib.train_test_split_lstm(
        df["price"].values, df.index, PAST_HISTORY, TRAIN_RATIO)
    # モデルを定義
    model = create_model(x_train.shape[-2:])
    # モデルを学習
    model.fit(x_train,
              y_train,
              batch_size=BATCH_SIZE,
              epochs=EPOCHS,
              verbose=1,
              validation_data=(x_test, y_test))

    df_10days = lib.load_data(DATA_PATH_ACTUAL, TARGET_TYPE)
    # モデルの検証
    df_10days, mean, std = lib.normalize(df_10days)
Esempio n. 6
0
test_data_unscaled = data[60000 + test_idx, :].astype(float)
test_labels = (labels[60000 + test_idx] == pos) * 2 - 1

# Preprocessing
train_data = sklearn.preprocessing.scale(train_data_unscaled,
                                         axis=0,
                                         with_std=False)
validation_data = sklearn.preprocessing.scale(validation_data_unscaled,
                                              axis=0,
                                              with_std=False)
test_data = sklearn.preprocessing.scale(test_data_unscaled,
                                        axis=0,
                                        with_std=False)

train_data = lib.normalize(train_data)
validation_data = lib.normalize(validation_data)
test_data = lib.normalize(test_data)

#


def sgd(samples, labels, c, lr, T):
    w = np.zeros(samples.shape[1]).astype(np.double)
    for i in range(T):
        idx = np.random.randint(0, samples.shape[0])
        current_lr = lr / (i + 1)
        if labels[idx] * w.dot(samples[idx]) < 1:  # yi*w*xi
            w = (1 -
                 current_lr) * w + current_lr * c * labels[idx] * samples[idx]
        else:
Esempio n. 7
0
result = []

colors = ["red", "royalblue", "violet", "green", "cyan", "orange"]
instance_types = [
    "m3.large", "m5.2xlarge", "m5.large", "m5.xlarge", "r3.xlarge",
    "r5d.xlarge"
]
feature_importance = []
fig = plt.figure(figsize=(16, 9))

for i in range(len(instance_types)):
    TARGET_TYPE = instance_types[i]
    print("=" * 10, TARGET_TYPE, "=" * 10)

    df = lib.load_data(DATA_PATH, TARGET_TYPE)
    df, mean, std = lib.normalize(df)

    (x_train, y_train), (x_test, y_test), columns = lib.train_test_split_lstm(
        df["price"].values, df.index, PAST_HISTORY, TRAIN_RATIO)
    # モデルを定義
    model = create_model(x_train.shape[-2:])
    # モデルを学習
    hist = model.fit(x_train,
                     y_train,
                     batch_size=BATCH_SIZE,
                     epochs=EPOCHS,
                     verbose=1,
                     validation_data=(x_test, y_test))
    y_pred = model.predict(x_test)

    a = {}
Esempio n. 8
0
from matplotlib.widgets import TextBox, Button
from pandas.plotting import register_matplotlib_converters

from lib import normalize, load_data, user_events_to_data, KW_GOING_HELI, KW_BACKWARD, widget_log

register_matplotlib_converters()

user_events_style = {"color": "red", "alpha": 0.33}
file_name = sys.argv[1]
(gps, alt, user2) = load_data(file_name)
user_marks_holder = [user2]

if __name__ == '__main__':
    x = gps['altitude']
    smoothed = x.ewm(alpha=0.1).mean()
    norm_speed = normalize(gps['speed'].ewm(alpha=0.03).mean())

    charts_ax = normalize(smoothed).plot(label="smoothed")
    norm_speed.plot(label="speed", alpha=0.3)

    plt.gca().xaxis.set_major_locator(mdates.HourLocator())
    plt.gca().xaxis.set_major_formatter(
        mdates.DateFormatter('%H:%M'))  # %Y-%m-%d %H:%M
    error_text = TextBox(plt.axes([0.04, 0.005, 0.8, 0.04]),
                         'Error',
                         initial="")

    def draw_user_marks():
        for span in draw_user_marks.spans:
            span.remove()
        draw_user_marks.spans = []
Esempio n. 9
0
    with open('../../file/model/struc_params.json', 'w') as f:
        json.dump(model_struc_params, f)

    # 保存损失函数记录
    train_loss_list = [
        float(p.detach().cpu().numpy()) for p in train_loss_record
    ]

    with open('../../file/model/train_loss.json', 'w') as f:
        json.dump(train_loss_list, f)


if __name__ == '__main__':
    # %% 载入观测数据并进行列归一化
    total_obs_data = pd.read_csv('../../data/runtime/total_obs_data.csv')
    total_obs_data = normalize(total_obs_data)

    # %% 构造样本
    x0_tensor, x1_tensor, integrate_t_tensor, dt_tensor = build_samples(
        total_obs_data)

    # %% 搭建网络
    input_size = x0_tensor.shape[1] + 1  # 输入为(x, t), 所以shape = dim(x) + dim(t)
    output_size = x0_tensor.shape[1]  # 输出为变量x的导数dx/dt, 所以shape = dim(x)
    hidden_sizes = [
        4 * input_size, 4 * input_size, 4 * output_size, 4 * output_size
    ]

    pd_net = PartialDeriveNet(input_size, hidden_sizes, output_size)

    # %% 指定优化器
Esempio n. 10
0
register_matplotlib_converters()

from lib import normalize, load_data, ALPHA, ALT_SPEED_MEAN, ALT_SPEED_MIN, SPEED_MEAN, SPEED_MIN, ALT_SPEED_MAX, \
    SPEED_MAX, user_events_to_data

user_events_style = {"color": "red", "alpha": 0.33}

(gps, alt, user) = load_data("part_1")

if __name__ == '__main__':
    x = gps['altitude']
    # x = alt["relativeAltitude"]
    smoothed = x.ewm(alpha=ALPHA).mean()
    norm_v_speed = normalize(smoothed.diff(),
                             mean=ALT_SPEED_MEAN,
                             min=ALT_SPEED_MIN,
                             max=ALT_SPEED_MAX)
    norm_speed = normalize(gps['speed'].ewm(alpha=ALPHA).mean(),
                           mean=SPEED_MEAN,
                           min=SPEED_MIN,
                           max=SPEED_MAX)

    normalize(smoothed).plot(label="smoothed")
    # norm_v_speed.plot(label="vertical speed")
    # norm_speed.plot(label="speed")

    # mlp_model = pickle.load(open("./data/model.mlp", 'rb'))
    # i = 0
    # predictions = []
    # for (a, b) in zip(norm_v_speed.iteritems(), norm_speed.iteritems()):
    #     if i % 50 == 0: