def lstm_test(path, content): data = StringIO(content) ### Read data from S3. cr = csv.reader(data) num_lines = sum(1 for line in cr) num_instances = num_lines - 1 ### The first line shouldn't be considered. train_size = int(math.ceil(num_lines / 2.0)) test_size = int(num_instances - train_size) ### Create an instance of lstm class params = {} params["raw_rows"] = content ### Update the lstm params["train_size"] = train_size params["test_size"] = test_size params["class_type"] = "Sentiment" run_lstm = lstm(params=params) run_lstm.build_model() run_lstm.train_model() run_lstm.test_model()
''' ### Create the Json file name as well as the path of the json file, and store the updated information into our Json file new_json_name = ff new_json_name = new_json_name.strip('.csv') ### Delete ".csv" new_json_name += ".json" new_json_path = os.path.join(lstm_params_dir, new_json_name) ''' jsonFile = open(json_path, "w+") jsonFile.write(json.dumps(tmp_data)) jsonFile.close() ### temp = Load_LSTM_Params(lstm_params_dir, param_file) temp = lstm(params_dir=lstm_params_dir, param_file=param_file) print temp.model_options ##temp.preprocess() ##temp.update_options() PD_list[str(index)] = temp index += 1 for item in PD_list: print item.model_options pass ''' PD = Load_LSTM_Params(lstm_params_dir, param_file)
''' import os, sys, inspect this_dir = os.path.realpath( os.path.abspath( os.path.split( inspect.getfile( inspect.currentframe() ))[0])) lstm_dir = os.path.realpath( os.path.abspath( os.path.join( this_dir, "../models/"))) lstm_params_dir = os.path.realpath( os.path.abspath( os.path.join( lstm_dir, "lstm/params/"))) lstm_data_dir = os.path.realpath( os.path.abspath( os.path.join( lstm_dir, "lstm/data/"))) lstm_code_dir = os.path.realpath( os.path.abspath( os.path.join( lstm_dir, "lstm/scode/"))) if lstm_dir not in sys.path: sys.path.insert(0, lstm_dir) sys.path.insert(0, lstm_code_dir) from load_params import Load_LSTM_Params from lstm_class import LSTM as lstm # param_file = 'orig_params.json' param_file = 'ruofan_params.json' data_file = '' PD = Load_LSTM_Params(lstm_params_dir, param_file) PD.preprocess() PD.update_options() print PD.model_options # Here I can pickle the PD object for use later. Good if the data is HUGE LSTM = lstm(PD) LSTM.build_model() LSTM.train_model().test_model() # IN.gen_sent_tvt(0,5,100,100)