def gcn_model(size):
unit_num = 4
node_input = Input(shape=(get_config("node_len"), size), name='Input-Node')
edge_input = Input(shape=(get_config("node_len"), get_config("node_len")),
dtype='int32',
name='Input-Edge')
conv_layer = GraphConv(units=unit_num,
name='GraphConv')([node_input, edge_input])
x = Flatten()(conv_layer)
x = Dense(unit_num * get_config("node_len"), activation='relu')(x)
x = Dense(64, activation='relu')(x)
output_layer = Dense(2, activation='softmax')(x)
model = Model(inputs=[node_input, edge_input], outputs=output_layer)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def main():
# 读取配置文件
config = get_config()
for key in config["POOL"].keys():
config["POOL"][key] = int(config["POOL"][key])
config["MYSQL"]["port"] = int(config["MYSQL"]["port"])
Kit.print_azure("Read config success.")
# 运行启动
multiprocess_master(config)
def lstm_model(shape_0=get_config("seq_len"), shape_1=16):
model = Sequential()
model.add(LSTM(shape_1, input_shape=(shape_0, shape_1)))
model.add(Dense(32, activation='relu'))
model.add(Dense(16, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
return model
def main():
app_config = get_config()
mysql_config = app_config['MYSQL']
mysql_config["port"] = int(mysql_config["port"])
print(mysql_config)
conn = pymysql.connect(**mysql_config)
time_clock = 1601571610
username_set = set()
while time_clock < Kit.unix_time():
username_set = check_count(conn, time_clock, username_set)
time_clock += 3600
def __select_image(self):
file_dialog = QFileDialog()
MainWindow.__set_file_dialog_options(
file_dialog,
get_config().getboolean('UI', 'NO_NATIVE_DIALOG'))
path = file_dialog.getOpenFileName(
self, 'Open File', " ../../", 'Images (*.png *.svg *.jpg *.jpeg)',
'')[0]
if path:
self.controller.open_image(path)
self.ui.tabWidget.setCurrentIndex(0)
self.resize_image()
def advance_search_dataset(q, f, num, event_id):
_, db, _ = get_config()
collection = db.dataset_
tweetCriteria = got.manager.TweetCriteria().setQuerySearch(q).setTweetType(
f).setMaxTweets(num)
tweets = got.manager.TweetManager.getTweets(tweetCriteria)
for tweet in tweets:
if collection.find_one({'_id': tweet.id}) == None:
collection.insert_one({
'_id': tweet.id,
'tweet': tweet.__dict__,
'event_id': event_id,
'f': f,
'q': q
})
def train():
config = get_config()
#gpu_options = tf.GPUself.lossOptions(per_process_gpu_memory_fraction=config.gpu_fraction)
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=config.gpu_fraction)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
env = Environment(config)
agent = Agent(config, env, sess)
io = IO()
#Main Train Loop
for ep in range(config.epoch_size):
surob=np.zeros([config.screen_height,config.screen_width,config.screen_channals],dtype=np.float32)
reward ,fail , maxq = 0.0, 0.0, 0.0
io.log_in(config.log_path,config.log_pathb,1,ep,config.epoch_size,None,None)
for nu in range(config.num_eachepoch):
# 1. predict
action,cur_maxq = agent.get_action(surob,ep)
# 2. forward, getting current and next states/reward
surob,cur_reward,flag,surob_next = env.get_nextstate(action)
# 3.learning and update
agent.learn(surob,cur_reward,flag,surob_next,action,nu,ep)
if (flag == 1):
io.log_in(config.log_path,config.log_pathb,2,None,None,env.get_pos_info(),None)
if (flag == -1):
env.start_newgame()
io.log_in(config.log_path,config.log_pathb,4,None,None,env.get_pos_info(),None)
# str = '/home/moro/Desktop/pic/CurState1/%d.jpg'%nu
# cv2.imwrite(str,255*surob[:,:,0])
# print(cur_reward)
# # 3.learning and update
# agent.learn(surob,cur_reward,flag,surob_next,action,nu,ep)
reward += cur_reward
# fail += (flag == -1)
maxq = maxq + cur_maxq
reward = reward / config.num_eachepoch
maxq = maxq/config.num_eachepoch
str='epoch: %d/%d, sum_reward: %f, average_maxq: %f'%(ep+1, config.epoch_size,reward,maxq)
io.log_in(config.log_path,config.log_pathb,3,None,None,None,str)
agent.save_model(ep)
def main():
config = get_config("connect")
config.get('MYSQL')["port"] = int(config.get('MYSQL')["port"])
# 连接数据库
conn = pymysql.connect(**config['MYSQL'])
cursor = conn.cursor(pymysql.cursors.DictCursor)
sql = "SELECT `username`,`cookies` FROM `user` WHERE `online`='Yes'"
cursor.execute(sql)
user_list = cursor.fetchall()
print(len(user_list))
for user in user_list:
cookies = user["cookies"]
if cookies == "":
cookies = "{}"
base_info_update(conn, user["username"], cookies)
def cnn_model(shape_0=get_config("seq_len"), shape_1=16):
model = Sequential()
model.add(
Conv1D(32,
kernel_size=3,
activation='relu',
input_shape=(shape_0, shape_1)))
model.add(MaxPool1D(2))
model.add(Conv1D(16, kernel_size=5, activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dropout(0.5))
model.add(Dense(64, activation='relu'))
model.add(Dense(2, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def advance_search_test(type_id, q, f, num):
_, db, _ = get_config()
collection = [
db.Entertainment, db.Religion, db.Sport, db.Military, db.Politics,
db.Education, db.Technology, db.Economy, db.Agriculture
][type_id - 1]
tweetCriteria = got.manager.TweetCriteria().setQuerySearch(q).setTweetType(
f).setMaxTweets(num)
tweets = got.manager.TweetManager.getTweets(tweetCriteria)
for tweet in tweets:
if collection.find_one({'_id': tweet.id}) == None:
collection.insert_one({
'_id': tweet.id,
'tweet': tweet.__dict__,
'f': f,
'q': q
})
def advance_search_dataset(q, f, num, event_id):
_, db, _ = get_config()
collection = db.pos
tweetCriteria = got.manager.TweetCriteria().setQuerySearch(q).setTweetType(
f).setMaxTweets(num)
tweets = got.manager.TweetManager.getTweets(tweetCriteria)
for tweet in tweets:
if collection.find_one({'_id': tweet['id']}) == None:
collection.insert_one({
'_id':
tweet['id'],
'tweet':
tweet,
'event_id':
json.loads(event_id, object_hook=json_util.object_hook),
'f':
f,
'q':
q
})
db.close()
def feature_generate(output_path, feature_type, files, len_limit):
while True:
x = []
y = []
for file_name, label in files:
file_path = os.path.join(output_path, label, feature_type,
file_name)
if os.path.getsize(file_path) == 0:
continue
df = pd.read_csv(file_path, header=None)
feature_len = df.shape[0]
x_batch = []
if feature_type == "Edge":
for i in range(len_limit):
x_batch.append([0 for j in range(len_limit)])
for index, row in df.iterrows():
if int(row[0]) < len_limit and int(row[1]) < len_limit:
x_batch[int(row[0])][int(row[1])] = 1
else:
for index, row in df.iterrows():
if index < len_limit:
x_batch.append(list(row))
if feature_len < len_limit:
for i in range(feature_len, len_limit):
x_batch.append([0 for j in range(df.shape[1])])
x.append(x_batch)
y.append([1, 0] if label == "False" else [0, 1])
if len(x) == get_config("batch_size"):
# print(np.array(x).shape)
yield np.array(x), np.array(y)
x.clear()
y.clear()
while len(population) < config.pop_size:
child1, child2 = crossover(
parents[np.random.choice(len(parents)), :],
parents[np.random.choice(len(parents)), :])
if random.random() < config.mutate_chance:
child1 = mutate(child1)
if random.random() < config.mutate_chance:
child2 = mutate(child2)
population = np.append(population, [child1, child2], 0)
#print("Max fitness:")
fitnesses = calcGenerationsFitness(population, config, initialMessage)
maxFitnesses.append(fitnesses.max())
#print(fitnesses.max())
if i > config.convergence_number:
if np.average(
maxFitnesses[-1 * config.convergence_number:]
) == maxFitnesses[
-1] and maxFitnesses[-1] > config.convergence_threshold:
break
print("The Overall Best Key Found:")
print(convertIntArrayToString(population[fitnesses.argmax()]))
if __name__ == "__main__":
config, unparsed = get_config()
# If we have unparsed arguments, print usage and exit
if len(unparsed) > 0:
print_usage()
exit(1)
nltk.download('words')
main(config)
def setupUi(self, MainWindow):
MainWindow.setObjectName("MainWindow")
MainWindow.resize(933, 658)
self.centralwidget = QtWidgets.QWidget(MainWindow)
self.centralwidget.setObjectName("centralwidget")
self.gridLayout = QtWidgets.QGridLayout(self.centralwidget)
self.gridLayout.setObjectName("gridLayout")
self.tabWidget = QtWidgets.QTabWidget(self.centralwidget)
self.tabWidget.setObjectName("tabWidget")
self.tabOriginal = QtWidgets.QWidget()
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding,
QtWidgets.QSizePolicy.Expanding)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.tabOriginal.sizePolicy().hasHeightForWidth())
self.tabOriginal.setSizePolicy(sizePolicy)
self.tabOriginal.setLayoutDirection(QtCore.Qt.LeftToRight)
self.tabOriginal.setObjectName("tabOriginal")
self.verticalLayout = QtWidgets.QVBoxLayout(self.tabOriginal)
self.verticalLayout.setContentsMargins(0, 0, 0, 0)
self.verticalLayout.setObjectName("verticalLayout")
self.graphicsViewOriginal = QtWidgets.QGraphicsView(self.tabOriginal)
self.graphicsViewOriginal.setObjectName("graphicsViewOriginal")
self.verticalLayout.addWidget(self.graphicsViewOriginal)
self.tabWidget.addTab(self.tabOriginal, "")
self.tabReducedColors = QtWidgets.QWidget()
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding,
QtWidgets.QSizePolicy.Expanding)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.tabReducedColors.sizePolicy().hasHeightForWidth())
self.tabReducedColors.setSizePolicy(sizePolicy)
self.tabReducedColors.setObjectName("tabReducedColors")
self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.tabReducedColors)
self.verticalLayout_2.setContentsMargins(0, 0, 0, 0)
self.verticalLayout_2.setObjectName("verticalLayout_2")
self.graphicsViewReducedColors = QtWidgets.QGraphicsView(
self.tabReducedColors)
self.graphicsViewReducedColors.setObjectName(
"graphicsViewReducedColors")
self.verticalLayout_2.addWidget(self.graphicsViewReducedColors)
self.tabWidget.addTab(self.tabReducedColors, "")
self.tabTemplate = QtWidgets.QWidget()
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding,
QtWidgets.QSizePolicy.Expanding)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.tabTemplate.sizePolicy().hasHeightForWidth())
self.tabTemplate.setSizePolicy(sizePolicy)
self.tabTemplate.setObjectName("tabTemplate")
self.verticalLayout_3 = QtWidgets.QVBoxLayout(self.tabTemplate)
self.verticalLayout_3.setContentsMargins(0, 0, 0, 0)
self.verticalLayout_3.setObjectName("verticalLayout_3")
self.graphicsViewTemplate = QtWidgets.QGraphicsView(self.tabTemplate)
self.graphicsViewTemplate.setObjectName("graphicsViewTemplate")
self.verticalLayout_3.addWidget(self.graphicsViewTemplate)
self.tabWidget.addTab(self.tabTemplate, "")
self.gridLayout.addWidget(self.tabWidget, 0, 1, 1, 1)
self.horizontalLayout = QtWidgets.QHBoxLayout()
self.horizontalLayout.setObjectName("horizontalLayout")
self.verticalLayoutLeft = QtWidgets.QVBoxLayout()
self.verticalLayoutLeft.setSpacing(get_config().getint(
'UI', 'LEFT_COLUMN_SPACING'))
self.verticalLayoutLeft.setObjectName("verticalLayoutLeft")
self.labelOpenPhoto = QtWidgets.QLabel(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.labelOpenPhoto.sizePolicy().hasHeightForWidth())
self.labelOpenPhoto.setSizePolicy(sizePolicy)
self.labelOpenPhoto.setMinimumSize(QtCore.QSize(0, 22))
self.labelOpenPhoto.setAlignment(QtCore.Qt.AlignRight
| QtCore.Qt.AlignTrailing
| QtCore.Qt.AlignVCenter)
self.labelOpenPhoto.setObjectName("labelOpenPhoto")
self.verticalLayoutLeft.addWidget(self.labelOpenPhoto)
self.labelNumberOfColors = QtWidgets.QLabel(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.labelNumberOfColors.sizePolicy().hasHeightForWidth())
self.labelNumberOfColors.setSizePolicy(sizePolicy)
self.labelNumberOfColors.setMinimumSize(QtCore.QSize(0, 26))
self.labelNumberOfColors.setAlignment(QtCore.Qt.AlignRight
| QtCore.Qt.AlignTrailing
| QtCore.Qt.AlignVCenter)
self.labelNumberOfColors.setObjectName("labelNumberOfColors")
self.verticalLayoutLeft.addWidget(self.labelNumberOfColors)
self.labelPrintSize = QtWidgets.QLabel(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.labelPrintSize.sizePolicy().hasHeightForWidth())
self.labelPrintSize.setSizePolicy(sizePolicy)
self.labelPrintSize.setMinimumSize(QtCore.QSize(0, 26))
self.labelPrintSize.setAlignment(QtCore.Qt.AlignRight
| QtCore.Qt.AlignTrailing
| QtCore.Qt.AlignVCenter)
self.labelPrintSize.setObjectName("labelPrintSize")
self.verticalLayoutLeft.addWidget(self.labelPrintSize)
self.labelMinSurfaceSize = QtWidgets.QLabel(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.labelMinSurfaceSize.sizePolicy().hasHeightForWidth())
self.labelMinSurfaceSize.setSizePolicy(sizePolicy)
self.labelMinSurfaceSize.setMinimumSize(QtCore.QSize(0, 24))
self.labelMinSurfaceSize.setAlignment(QtCore.Qt.AlignRight
| QtCore.Qt.AlignTrailing
| QtCore.Qt.AlignVCenter)
self.labelMinSurfaceSize.setObjectName("labelMinSurfaceSize")
self.verticalLayoutLeft.addWidget(self.labelMinSurfaceSize)
self.label = QtWidgets.QLabel(self.centralwidget)
self.label.setObjectName("label")
self.verticalLayoutLeft.addWidget(self.label)
spacerItem = QtWidgets.QSpacerItem(20, 40,
QtWidgets.QSizePolicy.Minimum,
QtWidgets.QSizePolicy.Expanding)
self.verticalLayoutLeft.addItem(spacerItem)
self.pushButtonStart = QtWidgets.QPushButton(self.centralwidget)
self.pushButtonStart.setObjectName("pushButtonStart")
self.verticalLayoutLeft.addWidget(self.pushButtonStart)
self.horizontalLayout.addLayout(self.verticalLayoutLeft)
self.verticalLayoutRight = QtWidgets.QVBoxLayout()
self.verticalLayoutRight.setSpacing(8)
self.verticalLayoutRight.setObjectName("verticalLayoutRight")
self.toolButtonOpenPhoto = QtWidgets.QToolButton(self.centralwidget)
self.toolButtonOpenPhoto.setMinimumSize(QtCore.QSize(0, 22))
self.toolButtonOpenPhoto.setMaximumSize(QtCore.QSize(16777215, 22))
self.toolButtonOpenPhoto.setObjectName("toolButtonOpenPhoto")
self.verticalLayoutRight.addWidget(self.toolButtonOpenPhoto)
self.spinBoxNumberOfColors = QtWidgets.QSpinBox(self.centralwidget)
self.spinBoxNumberOfColors.setMinimumSize(QtCore.QSize(0, 26))
self.spinBoxNumberOfColors.setMaximumSize(QtCore.QSize(16777215, 26))
self.spinBoxNumberOfColors.setMinimum(1)
self.spinBoxNumberOfColors.setMaximum(100)
self.spinBoxNumberOfColors.setProperty("value", 21)
self.spinBoxNumberOfColors.setObjectName("spinBoxNumberOfColors")
self.verticalLayoutRight.addWidget(self.spinBoxNumberOfColors)
self.comboBoxPrintSize = QtWidgets.QComboBox(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.comboBoxPrintSize.sizePolicy().hasHeightForWidth())
self.comboBoxPrintSize.setSizePolicy(sizePolicy)
self.comboBoxPrintSize.setMinimumSize(QtCore.QSize(0, 26))
self.comboBoxPrintSize.setMaximumSize(QtCore.QSize(16777215, 26))
self.comboBoxPrintSize.setObjectName("comboBoxPrintSize")
self.comboBoxPrintSize.addItem("")
self.comboBoxPrintSize.addItem("")
self.comboBoxPrintSize.addItem("")
self.comboBoxPrintSize.addItem("")
self.comboBoxPrintSize.addItem("")
self.verticalLayoutRight.addWidget(self.comboBoxPrintSize)
self.spinBoxMinSurfaceSize = QtWidgets.QSpinBox(self.centralwidget)
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred,
QtWidgets.QSizePolicy.Fixed)
sizePolicy.setHorizontalStretch(0)
sizePolicy.setVerticalStretch(0)
sizePolicy.setHeightForWidth(
self.spinBoxMinSurfaceSize.sizePolicy().hasHeightForWidth())
self.spinBoxMinSurfaceSize.setSizePolicy(sizePolicy)
self.spinBoxMinSurfaceSize.setMinimumSize(QtCore.QSize(0, 24))
self.spinBoxMinSurfaceSize.setMaximumSize(QtCore.QSize(16777215, 24))
self.spinBoxMinSurfaceSize.setMinimum(1)
self.spinBoxMinSurfaceSize.setMaximum(100)
self.spinBoxMinSurfaceSize.setProperty("value", 21)
self.spinBoxMinSurfaceSize.setObjectName("spinBoxMinSurfaceSize")
self.verticalLayoutRight.addWidget(self.spinBoxMinSurfaceSize)
self.checkBox = QtWidgets.QCheckBox(self.centralwidget)
self.checkBox.setText("")
self.checkBox.setObjectName("checkBox")
self.verticalLayoutRight.addWidget(self.checkBox)
spacerItem1 = QtWidgets.QSpacerItem(20, 40,
QtWidgets.QSizePolicy.Minimum,
QtWidgets.QSizePolicy.Expanding)
self.verticalLayoutRight.addItem(spacerItem1)
self.pushButtonExport = QtWidgets.QPushButton(self.centralwidget)
self.pushButtonExport.setObjectName("pushButtonExport")
self.verticalLayoutRight.addWidget(self.pushButtonExport)
self.horizontalLayout.addLayout(self.verticalLayoutRight)
self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 1)
MainWindow.setCentralWidget(self.centralwidget)
self.statusbar = QtWidgets.QStatusBar(MainWindow)
self.statusbar.setObjectName("statusbar")
MainWindow.setStatusBar(self.statusbar)
self.retranslateUi(MainWindow)
self.tabWidget.setCurrentIndex(0)
QtCore.QMetaObject.connectSlotsByName(MainWindow)
import re
import json
import random
random.seed(1)
from Config import get_config
_,db,r = get_config()
stopwords_en = db.event_metadata.find_one({'name':'stopwords_en'})['data']
def get_stopwords():
return stopwords_en[random.randint(0,len(stopwords_en)-1)]
def get_task():
for item in db.dataset.find({},{'_id':1}):
if random.randint(0,1):
tweet_id = item['_id']
q = get_stopwords()
message = {'q':q,'f':'&f=tweets','num':10,'tweet_id':tweet_id}
print message
r.rpush('task:neg',json.dumps(message))
if __name__ == '__main__':
get_task()
import json
import pymysql
import logging
import requests
import sentry_sdk
from flask import Flask
from flask import jsonify
from flask import request
from flask_cors import CORS
from Config import get_config
from dbutils.pooled_db import PooledDB
from concurrent.futures import ThreadPoolExecutor
from sentry_sdk.integrations.flask import FlaskIntegration
# 获取配置
app_config = get_config()
base_path = os.path.split(os.path.abspath(__file__))[0]
# Sentry
sentry_sdk.init(dsn=app_config['SERVICE']['dsn'],
integrations=[FlaskIntegration()],
environment=app_config["RUN_ENV"])
# 初始化应用
app = Flask(__name__)
app.config.from_mapping(app_config)
# 服务日志
file_logger = logging.getLogger('file_log')
file_logger.setLevel(logging.INFO)
file_handler = logging.FileHandler(filename='{}/log/run.log'.format(base_path),
deepwalk_files = deepwalk_files[int(len(deepwalk_files) * 0.2):]
paragraph_files_test = paragraph_files[:int(
len(paragraph_files) * 0.2)]
paragraph_files = paragraph_files[int(len(paragraph_files) * 0.2):]
else:
vector_paths_test = vector_paths
deepwalk_files_test = deepwalk_files
paragraph_files_test = paragraph_files
model_list = [{
"name":
"cnn",
"model":
cnn_model(feature_size),
"generator":
vector_generate(sys.argv[1], vector_paths, get_config("seq_len")),
"steps":
int(len(vector_paths) / get_config("batch_size")),
"test_generator":
vector_generate(sys.argv[1], vector_paths_test,
get_config("seq_len")),
"test_steps":
int(len(vector_paths_test) / get_config("batch_size")),
}, {
"name":
"lstm",
"model":
lstm_model(feature_size),
"generator":
vector_generate(sys.argv[1], vector_paths, get_config("seq_len")),
"steps":
import os
import pandas as pd
import sys
sys.path.insert(1, './Config')
# Refrence: https://gist.github.com/wpm/52758adbf506fd84cff3cdc7fc109aad
from Config import get_config
configs = get_config()
DATASET_FOLDER = configs['dataset_dir'][1:-1]
SENTENCES_FILE = os.path.join(DATASET_FOLDER, 'datasetSentences.txt')
LABEL_FILE = os.path.join(DATASET_FOLDER, 'sentiment_labels.txt')
DICT_FILE = os.path.join(DATASET_FOLDER, 'dictionary.txt')
SPLIT_FILE = os.path.join(DATASET_FOLDER, 'datasetSplit.txt')
DATA_DIR = configs['data_dir'][1:-1]
def get_split_name(split_num):
name = {1: "train", 2: "test", 3: "dev"}
return name[split_num]
def export():
phrases_df = pd.read_csv(DICT_FILE, sep='|')
phrases_df.columns = ["phrase", "id"]
phrases_df = phrases_df.set_index("id")
score_df = pd.read_csv(LABEL_FILE, sep='|')
for f in os.listdir(model_path):
if len(f.split(".")) != 2 or f.split(".")[1] != "h5":
continue
if f not in model_choose:
continue
model_name = f.split(".")[0].split("_")[0]
feature_size = int(f.split(".")[0].split("_")[1])
data_path = os.path.join(sys.argv[2], f.split(".")[0].split("_")[1])
if model_name == "cnn" or model_name == "lstm":
model = load_model(os.path.join(model_path, f))
predict_files = get_feature_files(data_path)
predict_generator = vector_generate(data_path, predict_files,
get_config("seq_len"))
elif model_name == "deepwalk":
model = load_model(os.path.join(model_path, f),
custom_objects={"GraphConv": GraphConv})
predict_files = get_feature_files(data_path, "DeepWalk")
predict_generator = graph_generate(data_path, "DeepWalk",
predict_files,
get_config("node_len"))
elif model_name == "para2vec":
model = load_model(os.path.join(model_path, f),
custom_objects={"GraphConv": GraphConv})
predict_files = get_feature_files(data_path, "ParagraphVec")
predict_generator = graph_generate(data_path, "ParagraphVec",
predict_files,
get_config("node_len"))
else:
def test_config(self):
config = get_config()
stocks = list(config["STOCKS"].keys())
assert stocks[2] is not None
def train_main(data_dir,
model_path,
model_name,
epoch_num,
feature_size,
flag=True):
vector_paths = get_feature_files(data_dir)
random.shuffle(vector_paths)
deepwalk_files = get_feature_files(data_dir, "DeepWalk")
random.shuffle(deepwalk_files)
paragraph_files = get_feature_files(data_dir, "ParagraphVec")
random.shuffle(paragraph_files)
# print(vector_size, len(deepwalk_files))
if len(sys.argv) > 5:
vector_paths_test = vector_paths[:int(len(vector_paths) * 0.2)]
vector_paths = vector_paths[int(len(vector_paths) * 0.2):]
deepwalk_files_test = deepwalk_files[:int(len(deepwalk_files) * 0.2)]
deepwalk_files = deepwalk_files[int(len(deepwalk_files) * 0.2):]
paragraph_files_test = paragraph_files[:int(
len(paragraph_files) * 0.2)]
paragraph_files = paragraph_files[int(len(paragraph_files) * 0.2):]
else:
vector_paths_test = vector_paths
deepwalk_files_test = deepwalk_files
paragraph_files_test = paragraph_files
model_list = [{
"name":
"cnn",
"model":
cnn_model(feature_size),
"generator":
vector_generate(sys.argv[1], vector_paths, get_config("seq_len")),
"steps":
int(len(vector_paths) / get_config("batch_size")),
"test_generator":
vector_generate(sys.argv[1], vector_paths_test, get_config("seq_len")),
"test_steps":
int(len(vector_paths_test) / get_config("batch_size")),
}, {
"name":
"lstm",
"model":
lstm_model(feature_size),
"generator":
vector_generate(sys.argv[1], vector_paths, get_config("seq_len")),
"steps":
int(len(vector_paths) / get_config("batch_size")),
"test_generator":
vector_generate(sys.argv[1], vector_paths_test, get_config("seq_len")),
"test_steps":
int(len(vector_paths_test) / get_config("batch_size")),
}, {
"name":
"deepwalk",
"model":
gcn_model(feature_size),
"generator":
graph_generate(sys.argv[1], "DeepWalk", deepwalk_files,
get_config("node_len")),
"steps":
int(len(deepwalk_files) / get_config("batch_size")),
"test_generator":
graph_generate(sys.argv[1], "DeepWalk", deepwalk_files_test,
get_config("node_len")),
"test_steps":
int(len(deepwalk_files_test) / get_config("batch_size")),
}, {
"name":
"para2vec",
"model":
gcn_model(feature_size),
"generator":
graph_generate(sys.argv[1], "ParagraphVec", paragraph_files,
get_config("node_len")),
"steps":
int(len(paragraph_files) / get_config("batch_size")),
"test_generator":
graph_generate(sys.argv[1], "ParagraphVec", paragraph_files_test,
get_config("node_len")),
"test_steps":
int(len(paragraph_files_test) / get_config("batch_size")),
}]
for item in model_list:
if model_name != item["name"]:
continue
model = item["model"]
if flag:
g1, g2, g3 = itertools.tee(item["test_generator"], 3)
history = model.fit_generator(
item["generator"],
steps_per_epoch=item["steps"],
validation_data=g1,
validation_steps=item["test_steps"],
epochs=epoch_num,
verbose=0,
workers=0,
callbacks=[Metrics(g2, item["test_steps"])]).history
history["name"] = item["name"]
with open(
os.path.join(
model_path, "_".join(
[item["name"],
str(feature_size),
str(epoch_num)])), 'w') as f:
f.write(str(history).replace("'", "\""))
else:
model.fit_generator(item["generator"],
steps_per_epoch=item["steps"],
epochs=epoch_num,
verbose=(0 if flag else 1),
workers=0)
model.save(
os.path.join(
model_path,
"_".join([item["name"],
str(feature_size),
str(epoch_num)]) + ".h5"))
