def gcn_model(size): unit_num = 4 node_input = Input(shape=(get_config("node_len"), size), name='Input-Node') edge_input = Input(shape=(get_config("node_len"), get_config("node_len")), dtype='int32', name='Input-Edge') conv_layer = GraphConv(units=unit_num, name='GraphConv')([node_input, edge_input]) x = Flatten()(conv_layer) x = Dense(unit_num * get_config("node_len"), activation='relu')(x) x = Dense(64, activation='relu')(x) output_layer = Dense(2, activation='softmax')(x) model = Model(inputs=[node_input, edge_input], outputs=output_layer) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def main(): # 读取配置文件 config = get_config() for key in config["POOL"].keys(): config["POOL"][key] = int(config["POOL"][key]) config["MYSQL"]["port"] = int(config["MYSQL"]["port"]) Kit.print_azure("Read config success.") # 运行启动 multiprocess_master(config)
def lstm_model(shape_0=get_config("seq_len"), shape_1=16): model = Sequential() model.add(LSTM(shape_1, input_shape=(shape_0, shape_1))) model.add(Dense(32, activation='relu')) model.add(Dense(16, activation='relu')) model.add(Dense(2, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def main(): app_config = get_config() mysql_config = app_config['MYSQL'] mysql_config["port"] = int(mysql_config["port"]) print(mysql_config) conn = pymysql.connect(**mysql_config) time_clock = 1601571610 username_set = set() while time_clock < Kit.unix_time(): username_set = check_count(conn, time_clock, username_set) time_clock += 3600
def __select_image(self): file_dialog = QFileDialog() MainWindow.__set_file_dialog_options( file_dialog, get_config().getboolean('UI', 'NO_NATIVE_DIALOG')) path = file_dialog.getOpenFileName( self, 'Open File', " ../../", 'Images (*.png *.svg *.jpg *.jpeg)', '')[0] if path: self.controller.open_image(path) self.ui.tabWidget.setCurrentIndex(0) self.resize_image()
def advance_search_dataset(q, f, num, event_id): _, db, _ = get_config() collection = db.dataset_ tweetCriteria = got.manager.TweetCriteria().setQuerySearch(q).setTweetType( f).setMaxTweets(num) tweets = got.manager.TweetManager.getTweets(tweetCriteria) for tweet in tweets: if collection.find_one({'_id': tweet.id}) == None: collection.insert_one({ '_id': tweet.id, 'tweet': tweet.__dict__, 'event_id': event_id, 'f': f, 'q': q })
def train(): config = get_config() #gpu_options = tf.GPUself.lossOptions(per_process_gpu_memory_fraction=config.gpu_fraction) gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=config.gpu_fraction) with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess: env = Environment(config) agent = Agent(config, env, sess) io = IO() #Main Train Loop for ep in range(config.epoch_size): surob=np.zeros([config.screen_height,config.screen_width,config.screen_channals],dtype=np.float32) reward ,fail , maxq = 0.0, 0.0, 0.0 io.log_in(config.log_path,config.log_pathb,1,ep,config.epoch_size,None,None) for nu in range(config.num_eachepoch): # 1. predict action,cur_maxq = agent.get_action(surob,ep) # 2. forward, getting current and next states/reward surob,cur_reward,flag,surob_next = env.get_nextstate(action) # 3.learning and update agent.learn(surob,cur_reward,flag,surob_next,action,nu,ep) if (flag == 1): io.log_in(config.log_path,config.log_pathb,2,None,None,env.get_pos_info(),None) if (flag == -1): env.start_newgame() io.log_in(config.log_path,config.log_pathb,4,None,None,env.get_pos_info(),None) # str = '/home/moro/Desktop/pic/CurState1/%d.jpg'%nu # cv2.imwrite(str,255*surob[:,:,0]) # print(cur_reward) # # 3.learning and update # agent.learn(surob,cur_reward,flag,surob_next,action,nu,ep) reward += cur_reward # fail += (flag == -1) maxq = maxq + cur_maxq reward = reward / config.num_eachepoch maxq = maxq/config.num_eachepoch str='epoch: %d/%d, sum_reward: %f, average_maxq: %f'%(ep+1, config.epoch_size,reward,maxq) io.log_in(config.log_path,config.log_pathb,3,None,None,None,str) agent.save_model(ep)
def main(): config = get_config("connect") config.get('MYSQL')["port"] = int(config.get('MYSQL')["port"]) # 连接数据库 conn = pymysql.connect(**config['MYSQL']) cursor = conn.cursor(pymysql.cursors.DictCursor) sql = "SELECT `username`,`cookies` FROM `user` WHERE `online`='Yes'" cursor.execute(sql) user_list = cursor.fetchall() print(len(user_list)) for user in user_list: cookies = user["cookies"] if cookies == "": cookies = "{}" base_info_update(conn, user["username"], cookies)
def cnn_model(shape_0=get_config("seq_len"), shape_1=16): model = Sequential() model.add( Conv1D(32, kernel_size=3, activation='relu', input_shape=(shape_0, shape_1))) model.add(MaxPool1D(2)) model.add(Conv1D(16, kernel_size=5, activation='relu')) model.add(GlobalAveragePooling1D()) model.add(Dropout(0.5)) model.add(Dense(64, activation='relu')) model.add(Dense(2, activation='softmax')) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return model
def advance_search_test(type_id, q, f, num): _, db, _ = get_config() collection = [ db.Entertainment, db.Religion, db.Sport, db.Military, db.Politics, db.Education, db.Technology, db.Economy, db.Agriculture ][type_id - 1] tweetCriteria = got.manager.TweetCriteria().setQuerySearch(q).setTweetType( f).setMaxTweets(num) tweets = got.manager.TweetManager.getTweets(tweetCriteria) for tweet in tweets: if collection.find_one({'_id': tweet.id}) == None: collection.insert_one({ '_id': tweet.id, 'tweet': tweet.__dict__, 'f': f, 'q': q })
def advance_search_dataset(q, f, num, event_id): _, db, _ = get_config() collection = db.pos tweetCriteria = got.manager.TweetCriteria().setQuerySearch(q).setTweetType( f).setMaxTweets(num) tweets = got.manager.TweetManager.getTweets(tweetCriteria) for tweet in tweets: if collection.find_one({'_id': tweet['id']}) == None: collection.insert_one({ '_id': tweet['id'], 'tweet': tweet, 'event_id': json.loads(event_id, object_hook=json_util.object_hook), 'f': f, 'q': q }) db.close()
def feature_generate(output_path, feature_type, files, len_limit): while True: x = [] y = [] for file_name, label in files: file_path = os.path.join(output_path, label, feature_type, file_name) if os.path.getsize(file_path) == 0: continue df = pd.read_csv(file_path, header=None) feature_len = df.shape[0] x_batch = [] if feature_type == "Edge": for i in range(len_limit): x_batch.append([0 for j in range(len_limit)]) for index, row in df.iterrows(): if int(row[0]) < len_limit and int(row[1]) < len_limit: x_batch[int(row[0])][int(row[1])] = 1 else: for index, row in df.iterrows(): if index < len_limit: x_batch.append(list(row)) if feature_len < len_limit: for i in range(feature_len, len_limit): x_batch.append([0 for j in range(df.shape[1])]) x.append(x_batch) y.append([1, 0] if label == "False" else [0, 1]) if len(x) == get_config("batch_size"): # print(np.array(x).shape) yield np.array(x), np.array(y) x.clear() y.clear()
while len(population) < config.pop_size: child1, child2 = crossover( parents[np.random.choice(len(parents)), :], parents[np.random.choice(len(parents)), :]) if random.random() < config.mutate_chance: child1 = mutate(child1) if random.random() < config.mutate_chance: child2 = mutate(child2) population = np.append(population, [child1, child2], 0) #print("Max fitness:") fitnesses = calcGenerationsFitness(population, config, initialMessage) maxFitnesses.append(fitnesses.max()) #print(fitnesses.max()) if i > config.convergence_number: if np.average( maxFitnesses[-1 * config.convergence_number:] ) == maxFitnesses[ -1] and maxFitnesses[-1] > config.convergence_threshold: break print("The Overall Best Key Found:") print(convertIntArrayToString(population[fitnesses.argmax()])) if __name__ == "__main__": config, unparsed = get_config() # If we have unparsed arguments, print usage and exit if len(unparsed) > 0: print_usage() exit(1) nltk.download('words') main(config)
def setupUi(self, MainWindow): MainWindow.setObjectName("MainWindow") MainWindow.resize(933, 658) self.centralwidget = QtWidgets.QWidget(MainWindow) self.centralwidget.setObjectName("centralwidget") self.gridLayout = QtWidgets.QGridLayout(self.centralwidget) self.gridLayout.setObjectName("gridLayout") self.tabWidget = QtWidgets.QTabWidget(self.centralwidget) self.tabWidget.setObjectName("tabWidget") self.tabOriginal = QtWidgets.QWidget() sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.tabOriginal.sizePolicy().hasHeightForWidth()) self.tabOriginal.setSizePolicy(sizePolicy) self.tabOriginal.setLayoutDirection(QtCore.Qt.LeftToRight) self.tabOriginal.setObjectName("tabOriginal") self.verticalLayout = QtWidgets.QVBoxLayout(self.tabOriginal) self.verticalLayout.setContentsMargins(0, 0, 0, 0) self.verticalLayout.setObjectName("verticalLayout") self.graphicsViewOriginal = QtWidgets.QGraphicsView(self.tabOriginal) self.graphicsViewOriginal.setObjectName("graphicsViewOriginal") self.verticalLayout.addWidget(self.graphicsViewOriginal) self.tabWidget.addTab(self.tabOriginal, "") self.tabReducedColors = QtWidgets.QWidget() sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.tabReducedColors.sizePolicy().hasHeightForWidth()) self.tabReducedColors.setSizePolicy(sizePolicy) self.tabReducedColors.setObjectName("tabReducedColors") self.verticalLayout_2 = QtWidgets.QVBoxLayout(self.tabReducedColors) self.verticalLayout_2.setContentsMargins(0, 0, 0, 0) self.verticalLayout_2.setObjectName("verticalLayout_2") self.graphicsViewReducedColors = QtWidgets.QGraphicsView( self.tabReducedColors) self.graphicsViewReducedColors.setObjectName( "graphicsViewReducedColors") self.verticalLayout_2.addWidget(self.graphicsViewReducedColors) self.tabWidget.addTab(self.tabReducedColors, "") self.tabTemplate = QtWidgets.QWidget() sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Expanding) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.tabTemplate.sizePolicy().hasHeightForWidth()) self.tabTemplate.setSizePolicy(sizePolicy) self.tabTemplate.setObjectName("tabTemplate") self.verticalLayout_3 = QtWidgets.QVBoxLayout(self.tabTemplate) self.verticalLayout_3.setContentsMargins(0, 0, 0, 0) self.verticalLayout_3.setObjectName("verticalLayout_3") self.graphicsViewTemplate = QtWidgets.QGraphicsView(self.tabTemplate) self.graphicsViewTemplate.setObjectName("graphicsViewTemplate") self.verticalLayout_3.addWidget(self.graphicsViewTemplate) self.tabWidget.addTab(self.tabTemplate, "") self.gridLayout.addWidget(self.tabWidget, 0, 1, 1, 1) self.horizontalLayout = QtWidgets.QHBoxLayout() self.horizontalLayout.setObjectName("horizontalLayout") self.verticalLayoutLeft = QtWidgets.QVBoxLayout() self.verticalLayoutLeft.setSpacing(get_config().getint( 'UI', 'LEFT_COLUMN_SPACING')) self.verticalLayoutLeft.setObjectName("verticalLayoutLeft") self.labelOpenPhoto = QtWidgets.QLabel(self.centralwidget) sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Fixed) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.labelOpenPhoto.sizePolicy().hasHeightForWidth()) self.labelOpenPhoto.setSizePolicy(sizePolicy) self.labelOpenPhoto.setMinimumSize(QtCore.QSize(0, 22)) self.labelOpenPhoto.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.labelOpenPhoto.setObjectName("labelOpenPhoto") self.verticalLayoutLeft.addWidget(self.labelOpenPhoto) self.labelNumberOfColors = QtWidgets.QLabel(self.centralwidget) sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Fixed) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.labelNumberOfColors.sizePolicy().hasHeightForWidth()) self.labelNumberOfColors.setSizePolicy(sizePolicy) self.labelNumberOfColors.setMinimumSize(QtCore.QSize(0, 26)) self.labelNumberOfColors.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.labelNumberOfColors.setObjectName("labelNumberOfColors") self.verticalLayoutLeft.addWidget(self.labelNumberOfColors) self.labelPrintSize = QtWidgets.QLabel(self.centralwidget) sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Fixed) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.labelPrintSize.sizePolicy().hasHeightForWidth()) self.labelPrintSize.setSizePolicy(sizePolicy) self.labelPrintSize.setMinimumSize(QtCore.QSize(0, 26)) self.labelPrintSize.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.labelPrintSize.setObjectName("labelPrintSize") self.verticalLayoutLeft.addWidget(self.labelPrintSize) self.labelMinSurfaceSize = QtWidgets.QLabel(self.centralwidget) sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Fixed) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.labelMinSurfaceSize.sizePolicy().hasHeightForWidth()) self.labelMinSurfaceSize.setSizePolicy(sizePolicy) self.labelMinSurfaceSize.setMinimumSize(QtCore.QSize(0, 24)) self.labelMinSurfaceSize.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignTrailing | QtCore.Qt.AlignVCenter) self.labelMinSurfaceSize.setObjectName("labelMinSurfaceSize") self.verticalLayoutLeft.addWidget(self.labelMinSurfaceSize) self.label = QtWidgets.QLabel(self.centralwidget) self.label.setObjectName("label") self.verticalLayoutLeft.addWidget(self.label) spacerItem = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding) self.verticalLayoutLeft.addItem(spacerItem) self.pushButtonStart = QtWidgets.QPushButton(self.centralwidget) self.pushButtonStart.setObjectName("pushButtonStart") self.verticalLayoutLeft.addWidget(self.pushButtonStart) self.horizontalLayout.addLayout(self.verticalLayoutLeft) self.verticalLayoutRight = QtWidgets.QVBoxLayout() self.verticalLayoutRight.setSpacing(8) self.verticalLayoutRight.setObjectName("verticalLayoutRight") self.toolButtonOpenPhoto = QtWidgets.QToolButton(self.centralwidget) self.toolButtonOpenPhoto.setMinimumSize(QtCore.QSize(0, 22)) self.toolButtonOpenPhoto.setMaximumSize(QtCore.QSize(16777215, 22)) self.toolButtonOpenPhoto.setObjectName("toolButtonOpenPhoto") self.verticalLayoutRight.addWidget(self.toolButtonOpenPhoto) self.spinBoxNumberOfColors = QtWidgets.QSpinBox(self.centralwidget) self.spinBoxNumberOfColors.setMinimumSize(QtCore.QSize(0, 26)) self.spinBoxNumberOfColors.setMaximumSize(QtCore.QSize(16777215, 26)) self.spinBoxNumberOfColors.setMinimum(1) self.spinBoxNumberOfColors.setMaximum(100) self.spinBoxNumberOfColors.setProperty("value", 21) self.spinBoxNumberOfColors.setObjectName("spinBoxNumberOfColors") self.verticalLayoutRight.addWidget(self.spinBoxNumberOfColors) self.comboBoxPrintSize = QtWidgets.QComboBox(self.centralwidget) sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Fixed) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.comboBoxPrintSize.sizePolicy().hasHeightForWidth()) self.comboBoxPrintSize.setSizePolicy(sizePolicy) self.comboBoxPrintSize.setMinimumSize(QtCore.QSize(0, 26)) self.comboBoxPrintSize.setMaximumSize(QtCore.QSize(16777215, 26)) self.comboBoxPrintSize.setObjectName("comboBoxPrintSize") self.comboBoxPrintSize.addItem("") self.comboBoxPrintSize.addItem("") self.comboBoxPrintSize.addItem("") self.comboBoxPrintSize.addItem("") self.comboBoxPrintSize.addItem("") self.verticalLayoutRight.addWidget(self.comboBoxPrintSize) self.spinBoxMinSurfaceSize = QtWidgets.QSpinBox(self.centralwidget) sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Fixed) sizePolicy.setHorizontalStretch(0) sizePolicy.setVerticalStretch(0) sizePolicy.setHeightForWidth( self.spinBoxMinSurfaceSize.sizePolicy().hasHeightForWidth()) self.spinBoxMinSurfaceSize.setSizePolicy(sizePolicy) self.spinBoxMinSurfaceSize.setMinimumSize(QtCore.QSize(0, 24)) self.spinBoxMinSurfaceSize.setMaximumSize(QtCore.QSize(16777215, 24)) self.spinBoxMinSurfaceSize.setMinimum(1) self.spinBoxMinSurfaceSize.setMaximum(100) self.spinBoxMinSurfaceSize.setProperty("value", 21) self.spinBoxMinSurfaceSize.setObjectName("spinBoxMinSurfaceSize") self.verticalLayoutRight.addWidget(self.spinBoxMinSurfaceSize) self.checkBox = QtWidgets.QCheckBox(self.centralwidget) self.checkBox.setText("") self.checkBox.setObjectName("checkBox") self.verticalLayoutRight.addWidget(self.checkBox) spacerItem1 = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding) self.verticalLayoutRight.addItem(spacerItem1) self.pushButtonExport = QtWidgets.QPushButton(self.centralwidget) self.pushButtonExport.setObjectName("pushButtonExport") self.verticalLayoutRight.addWidget(self.pushButtonExport) self.horizontalLayout.addLayout(self.verticalLayoutRight) self.gridLayout.addLayout(self.horizontalLayout, 0, 0, 1, 1) MainWindow.setCentralWidget(self.centralwidget) self.statusbar = QtWidgets.QStatusBar(MainWindow) self.statusbar.setObjectName("statusbar") MainWindow.setStatusBar(self.statusbar) self.retranslateUi(MainWindow) self.tabWidget.setCurrentIndex(0) QtCore.QMetaObject.connectSlotsByName(MainWindow)
import re import json import random random.seed(1) from Config import get_config _,db,r = get_config() stopwords_en = db.event_metadata.find_one({'name':'stopwords_en'})['data'] def get_stopwords(): return stopwords_en[random.randint(0,len(stopwords_en)-1)] def get_task(): for item in db.dataset.find({},{'_id':1}): if random.randint(0,1): tweet_id = item['_id'] q = get_stopwords() message = {'q':q,'f':'&f=tweets','num':10,'tweet_id':tweet_id} print message r.rpush('task:neg',json.dumps(message)) if __name__ == '__main__': get_task()
import json import pymysql import logging import requests import sentry_sdk from flask import Flask from flask import jsonify from flask import request from flask_cors import CORS from Config import get_config from dbutils.pooled_db import PooledDB from concurrent.futures import ThreadPoolExecutor from sentry_sdk.integrations.flask import FlaskIntegration # 获取配置 app_config = get_config() base_path = os.path.split(os.path.abspath(__file__))[0] # Sentry sentry_sdk.init(dsn=app_config['SERVICE']['dsn'], integrations=[FlaskIntegration()], environment=app_config["RUN_ENV"]) # 初始化应用 app = Flask(__name__) app.config.from_mapping(app_config) # 服务日志 file_logger = logging.getLogger('file_log') file_logger.setLevel(logging.INFO) file_handler = logging.FileHandler(filename='{}/log/run.log'.format(base_path),
deepwalk_files = deepwalk_files[int(len(deepwalk_files) * 0.2):] paragraph_files_test = paragraph_files[:int( len(paragraph_files) * 0.2)] paragraph_files = paragraph_files[int(len(paragraph_files) * 0.2):] else: vector_paths_test = vector_paths deepwalk_files_test = deepwalk_files paragraph_files_test = paragraph_files model_list = [{ "name": "cnn", "model": cnn_model(feature_size), "generator": vector_generate(sys.argv[1], vector_paths, get_config("seq_len")), "steps": int(len(vector_paths) / get_config("batch_size")), "test_generator": vector_generate(sys.argv[1], vector_paths_test, get_config("seq_len")), "test_steps": int(len(vector_paths_test) / get_config("batch_size")), }, { "name": "lstm", "model": lstm_model(feature_size), "generator": vector_generate(sys.argv[1], vector_paths, get_config("seq_len")), "steps":
import os import pandas as pd import sys sys.path.insert(1, './Config') # Refrence: https://gist.github.com/wpm/52758adbf506fd84cff3cdc7fc109aad from Config import get_config configs = get_config() DATASET_FOLDER = configs['dataset_dir'][1:-1] SENTENCES_FILE = os.path.join(DATASET_FOLDER, 'datasetSentences.txt') LABEL_FILE = os.path.join(DATASET_FOLDER, 'sentiment_labels.txt') DICT_FILE = os.path.join(DATASET_FOLDER, 'dictionary.txt') SPLIT_FILE = os.path.join(DATASET_FOLDER, 'datasetSplit.txt') DATA_DIR = configs['data_dir'][1:-1] def get_split_name(split_num): name = {1: "train", 2: "test", 3: "dev"} return name[split_num] def export(): phrases_df = pd.read_csv(DICT_FILE, sep='|') phrases_df.columns = ["phrase", "id"] phrases_df = phrases_df.set_index("id") score_df = pd.read_csv(LABEL_FILE, sep='|')
for f in os.listdir(model_path): if len(f.split(".")) != 2 or f.split(".")[1] != "h5": continue if f not in model_choose: continue model_name = f.split(".")[0].split("_")[0] feature_size = int(f.split(".")[0].split("_")[1]) data_path = os.path.join(sys.argv[2], f.split(".")[0].split("_")[1]) if model_name == "cnn" or model_name == "lstm": model = load_model(os.path.join(model_path, f)) predict_files = get_feature_files(data_path) predict_generator = vector_generate(data_path, predict_files, get_config("seq_len")) elif model_name == "deepwalk": model = load_model(os.path.join(model_path, f), custom_objects={"GraphConv": GraphConv}) predict_files = get_feature_files(data_path, "DeepWalk") predict_generator = graph_generate(data_path, "DeepWalk", predict_files, get_config("node_len")) elif model_name == "para2vec": model = load_model(os.path.join(model_path, f), custom_objects={"GraphConv": GraphConv}) predict_files = get_feature_files(data_path, "ParagraphVec") predict_generator = graph_generate(data_path, "ParagraphVec", predict_files, get_config("node_len")) else:
def test_config(self): config = get_config() stocks = list(config["STOCKS"].keys()) assert stocks[2] is not None
def train_main(data_dir, model_path, model_name, epoch_num, feature_size, flag=True): vector_paths = get_feature_files(data_dir) random.shuffle(vector_paths) deepwalk_files = get_feature_files(data_dir, "DeepWalk") random.shuffle(deepwalk_files) paragraph_files = get_feature_files(data_dir, "ParagraphVec") random.shuffle(paragraph_files) # print(vector_size, len(deepwalk_files)) if len(sys.argv) > 5: vector_paths_test = vector_paths[:int(len(vector_paths) * 0.2)] vector_paths = vector_paths[int(len(vector_paths) * 0.2):] deepwalk_files_test = deepwalk_files[:int(len(deepwalk_files) * 0.2)] deepwalk_files = deepwalk_files[int(len(deepwalk_files) * 0.2):] paragraph_files_test = paragraph_files[:int( len(paragraph_files) * 0.2)] paragraph_files = paragraph_files[int(len(paragraph_files) * 0.2):] else: vector_paths_test = vector_paths deepwalk_files_test = deepwalk_files paragraph_files_test = paragraph_files model_list = [{ "name": "cnn", "model": cnn_model(feature_size), "generator": vector_generate(sys.argv[1], vector_paths, get_config("seq_len")), "steps": int(len(vector_paths) / get_config("batch_size")), "test_generator": vector_generate(sys.argv[1], vector_paths_test, get_config("seq_len")), "test_steps": int(len(vector_paths_test) / get_config("batch_size")), }, { "name": "lstm", "model": lstm_model(feature_size), "generator": vector_generate(sys.argv[1], vector_paths, get_config("seq_len")), "steps": int(len(vector_paths) / get_config("batch_size")), "test_generator": vector_generate(sys.argv[1], vector_paths_test, get_config("seq_len")), "test_steps": int(len(vector_paths_test) / get_config("batch_size")), }, { "name": "deepwalk", "model": gcn_model(feature_size), "generator": graph_generate(sys.argv[1], "DeepWalk", deepwalk_files, get_config("node_len")), "steps": int(len(deepwalk_files) / get_config("batch_size")), "test_generator": graph_generate(sys.argv[1], "DeepWalk", deepwalk_files_test, get_config("node_len")), "test_steps": int(len(deepwalk_files_test) / get_config("batch_size")), }, { "name": "para2vec", "model": gcn_model(feature_size), "generator": graph_generate(sys.argv[1], "ParagraphVec", paragraph_files, get_config("node_len")), "steps": int(len(paragraph_files) / get_config("batch_size")), "test_generator": graph_generate(sys.argv[1], "ParagraphVec", paragraph_files_test, get_config("node_len")), "test_steps": int(len(paragraph_files_test) / get_config("batch_size")), }] for item in model_list: if model_name != item["name"]: continue model = item["model"] if flag: g1, g2, g3 = itertools.tee(item["test_generator"], 3) history = model.fit_generator( item["generator"], steps_per_epoch=item["steps"], validation_data=g1, validation_steps=item["test_steps"], epochs=epoch_num, verbose=0, workers=0, callbacks=[Metrics(g2, item["test_steps"])]).history history["name"] = item["name"] with open( os.path.join( model_path, "_".join( [item["name"], str(feature_size), str(epoch_num)])), 'w') as f: f.write(str(history).replace("'", "\"")) else: model.fit_generator(item["generator"], steps_per_epoch=item["steps"], epochs=epoch_num, verbose=(0 if flag else 1), workers=0) model.save( os.path.join( model_path, "_".join([item["name"], str(feature_size), str(epoch_num)]) + ".h5"))