def Upload_house():
if request.method == 'POST':
## step 1. load the file
f = request.files['file']
## step 2. save the file
f.save(secure_filename(f.filename))
## step 3. connect the service from ssh
### SSHConnection is the Class in ssh.py,its parameters contains(host=str, port=int, username=str,pwd=str)
ssh_2 = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
dic_s['password'])
ssh_2.connect()
ssh_2.upload(
str(os.getcwd()) + '/' + str(f.filename),
dic_s['remote work directory path'] + '/lgbm/train1.csv')
## ssh_2.upload(LOCAL FILE PATH,REMOTE PATH)
##step 4. run data_v.py to get data of HousePrice
global mydata ## mydata is the visualization data of HousePrice data
mydata = eval(ssh_2.cmd('cd lgbm;python data_v.py'))
return render_template('Housedata_description.html', mydata=mydata)
else:
return render_template('Upload_house.html')
def setup_ssh(self, host, username, password, port, destfile):
"""
Open SSH connection
"""
print(host, username, password, port)
self.ssh_connection = SSHConnection(host, username, password, port)
print("Log SSH:", self.ssh_connection)
self.destfile = destfile
self.ssh_open = True
def upload_s():
if request.method == 'POST':
f = request.files['file']
f.save(secure_filename(f.filename))
print(dic_s['password'])
ssh_1=SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
ssh_1.connect()
ssh_1.upload(str(os.getcwd()) + '\\' + str(f.filename),dic_s['remote work directory path'] + '/FlaskIndexPrediction/data/data.csv') # INPUT YOUR PATH
return render_template('index_s.html')
else:
return render_template('upload_s.html')
def connect(self):
if 'host' in self.config['target']:
host = self.config['target']['host']
else:
host = raw_input('Target host: ')
if 'username' in self.config['target']:
username = self.config['target']['username']
else:
username = raw_input('Target username: '******'password' in self.config['target']:
password = self.config['target']['password']
else:
password = getpass()
self.target = SSHConnection(host, username, password)
def upload_h():
if request.method == 'POST':
f = request.files['file']
f.save(secure_filename(f.filename))
print(f.filename)
global ssh
ssh = SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
ssh.connect()
ssh.upload(str(os.getcwd()) + '\\' + str(f.filename),dic_s['remote work directory path'] + '/lgbm/train1.csv')
global mydata
mydata = eval(ssh.cmd('cd py2;source bin/activate;cd ..;cd lgbm;python data_v.py'))
return render_template('index_h.html',mydata=mydata)
else:
return render_template('upload_h.html')
def upload_p():
def get(name):
return request.values.get(name)
if request.method == 'POST':
dic={}
dic['e'] = int(get('e'))
dic['lb'] = int (get('lb'))
dic['lr'] = float (get('lr'))
dic['tp'] = float (get('tp'))
ssh_s=SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
ssh_s.connect( )
s='cd py2;source bin/activate;cd ..;cd FlaskIndexPrediction;python main.py -e {e} -lb {lb} -lr {lr} -tp{tp}'
ssh_s.cmd(s.format(e=dic['e'],lb=dic['lb'],lr=dic['lr'],tp=dic['tp']))
ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTest.csv', str(os.getcwd()) + '\\' + 'outputTest.csv')
ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTrain.csv',str(os.getcwd()) + '\\' + 'outputTrain.csv')
ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTrainTest.csv',str(os.getcwd()) + '\\' + 'outputTrainTest.csv')
outputTest = pd.read_csv('outputTest.csv')
outputTrain = pd.read_csv('outputTrain.csv')
outputTrainTest = pd.read_csv('outputTrainTest.csv')
Test_time = outputTest['Unnamed: 0']
Test_origin = outputTest['origin']
Test_predict = outputTest['predict']
Train_time = outputTrain['Unnamed: 0']
Train_origin = outputTrain['origin']
Train_predict = outputTrain['predict']
TrainTest_time = outputTrainTest['Unnamed: 0']
TrainTest_origin = outputTrainTest['origin']
TrainTest_predict = outputTrainTest['predict']
stock_data={'Test_time':Test_time,'Test_origin':Test_origin,
'Test_predict':Test_predict,'Train_time':Train_time,
'Train_origin':Train_origin,'Train_predict':Train_predict,
'TrainTest_time':TrainTest_time,'TrainTest_origin':TrainTest_origin,
'TrainTest_predict':TrainTest_predict}
return render_template('ZhangYD.html',stock_data = stock_data)
else:
return render_template('index_p.html')
def connection(self, user_name=None, password=None, private_key=None):
"""
connection target host by ssh protocal
:param user_name:
:param password:
:param private_key:
:return:
"""
hosts = set()
map(lambda _host: hosts.add(parse_host_port(_host[0])[0]),
self.config["ps_hosts"])
map(lambda _host: hosts.add(parse_host_port(_host[0])[0]),
self.config["worker_hosts"])
_ssh_clients = {}
for host in hosts:
logger.debug("ssh connect to host[%s]" % host)
_ssh_clients[host] = SSHConnection(host,
username=user_name,
password=password,
private_key=private_key)
return _ssh_clients
def Upload_stock():
if request.method == 'POST':
## step 1. load the file
f = request.files['file']
## step 2. save the file
f.save(secure_filename(f.filename))
## step 3. connect the service from ssh
### SSHConnection is the Class in ssh.py,its parameters contains(host=str, port=int, username=str,pwd=str)
ssh_1 = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
dic_s['password'])
ssh_1.connect()
ssh_1.upload(
str(os.getcwd()) + '/' + str(f.filename),
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/data/data.csv') # INPUT YOUR PATH
## ssh_1.upload(LOCAL FILE PATH,REMOTE PATH)
return render_template('Choose_LSTM.html')
else:
return render_template('Upload_stock.html')
def __init__(
self,
_id: str,
address: str,
username: str,
ssh_password: str,
jobs_db,
skip_gpus: Sequence[int] = (),
gpu_runner_on: bool = False,
app=None,
):
self._id = _id
self.address = address
self.username = username
self.jobs_db = jobs_db
self.skip_gpus = skip_gpus
self.gpu_runner_on = gpu_runner_on
self.app = app
self.new_processes = []
self._client = SSHConnection(self.address,
self.username,
ssh_password,
auto_add_host=True)
self._client_lock = Lock()
def Stockdata_parameter():
def get(
name
): ## get values from the HTML part ,'name' is the 'id' of HTML <input>
return request.values.get(name)
if request.method == 'POST':
## step 1. get the parameters of stock data from HTML
stock_parameter = {
} ## stock_parameter is a dic to store teh parameters user input in HTML
stock_parameter['e'] = int(get('e'))
stock_parameter['lb'] = int(get('lb'))
stock_parameter['lr'] = float(get('lr'))
stock_parameter['tp'] = float(get('tp'))
## step 2. running the LSTM module in service
ssh_s = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
dic_s['password'])
ssh_s.connect()
s = 'cd FlaskIndexPrediction;python main.py -e {e} -lb {lb} -lr {lr} -tp{tp}'
ssh_s.cmd(
s.format(e=stock_parameter['e'],
lb=stock_parameter['lb'],
lr=stock_parameter['lr'],
tp=stock_parameter['tp']))
## step 3.download the main.py output file to local
ssh_s.download(
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/output/outputTest.csv',
str(os.getcwd()) + '/module/YD/' + 'outputTest.csv')
ssh_s.download(
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/output/outputTrain.csv',
str(os.getcwd()) + '/module/YD/' + 'outputTrain.csv')
ssh_s.download(
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/output/outputTrainTest.csv',
str(os.getcwd()) + '/module/YD/' + 'outputTrainTest.csv')
## step 4. read csv from local
outputTest = pd.read_csv('module/YD/outputTest.csv')
outputTrain = pd.read_csv('module/YD/outputTrain.csv')
outputTrainTest = pd.read_csv('module/YD/outputTrainTest.csv')
Test_time = outputTest['Unnamed: 0']
Test_origin = outputTest['origin']
Test_predict = outputTest['predict']
Train_time = outputTrain['Unnamed: 0']
Train_origin = outputTrain['origin']
Train_predict = outputTrain['predict']
TrainTest_time = outputTrainTest['Unnamed: 0']
TrainTest_origin = outputTrainTest['origin']
TrainTest_predict = outputTrainTest['predict']
stock_data = {
'Test_time': Test_time,
'Test_origin': Test_origin,
'Test_predict': Test_predict,
'Train_time': Train_time,
'Train_origin': Train_origin,
'Train_predict': Train_predict,
'TrainTest_time': TrainTest_time,
'TrainTest_origin': TrainTest_origin,
'TrainTest_predict': TrainTest_predict
}
return render_template('Stock_final_output.html',
stock_data=stock_data)
else:
return render_template('Stockdata_parameter.html')
def Housedata_parameter():
def get(
name
): ## get values from the HTML part ,'name' is the 'id' of HTML <input>
return request.values.get(name)
if request.method == 'POST':
##step 1. get housedata parameters from HTML
LotArea = get("LotArea")
Neighborhood = get("Neighborhood")
YearBuilt = get("YearBuilt")
GrLivArea = get("GrLivArea")
Street = get("Street")
Utilities = get("Utilities")
LotConfig = get("LotConfig")
HouseStyle = get("HouseStyle")
RoofStyle = get("RoofStyle")
SaleType = get('SaleType')
SaleCondition = get('SaleCondition')
lgbm_data = [
LotArea, Street, Utilities, LotConfig, Neighborhood, HouseStyle,
YearBuilt, RoofStyle, GrLivArea, SaleType, SaleCondition
]
lgbm_data_1 = ",".join(lgbm_data)
##step 2. Data Processing.
def getSortedValues(
row
): ## make data in a certain order('row' is the key for the data need to be sorted)
sortedValues = []
keys = row.keys()
keys.sort()
for key in keys:
sortedValues.append(row[key])
return sortedValues
rows = [
{
'Column1': 1,
'Column2': LotArea,
'Column3': Street,
'Column4': Utilities,
'Column5': LotConfig,
'Column6': Neighborhood,
'Column7': HouseStyle,
'Column8': YearBuilt,
'Column9': RoofStyle,
'Column10': GrLivArea,
'Column11': SaleType,
'Column12': SaleCondition
},
]
names = {
'Column1': 'Id',
'Column2': 'LotArea',
'Column3': 'Street',
'Column4': 'Utilities',
'Column5': 'LotConfig',
'Column6': 'Neighborhood',
'Column7': 'HouseStyle',
'Column8': 'YearBuilt',
'Column9': 'RoofStyle',
'Column10': 'GrLivArea',
'Column11': 'SaleType',
'Column12': 'SaleCondition'
}
## step 3. write the sorted data in a csv file('test.csv')
fileobj = open("module/YX/test.csv", 'wb')
fileobj.write('\xEF\xBB\xBF')
writer = csv.writer(fileobj)
sortedValues = getSortedValues(names)
writer.writerow(sortedValues)
for row in rows:
sortedValues = getSortedValues(row)
print(sortedValues)
writer.writerow(sortedValues)
fileobj.close()
## step 4. upload test.csv
ssh_3 = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
dic_s['password'])
ssh_3.connect()
ssh_3.upload(
str(os.getcwd()) + '/module/YX/' + 'test.csv',
'/root/lgbm/test.csv')
## step 5. running LGBM moudle to process the data
show = ssh_3.cmd('cd lgbm;python lgbm.py')
show_l = show.split('\n') ## the output of lgbm.py in terminal
# print "showl_l"
# print show_l
pred_y = eval(show_l[-2]) ## the price output in list[]
# print "pred_y"
# print pred_y
pred_y_show = pred_y[0] ## the price output in float
# print "pred_y_show"
# print pred_y_show
quantitative = re.findall(
"\d+", show_l[3])[0] ## number of quantitative features
qualitative = re.findall(
"\d+", show_l[3])[1] ## number of qualitative features
train_x1 = show_l[6] ## the shape of train_X(1018,11)
# print "train_x1"
# print train_x1
train_y1 = show_l[7] ## the shape of train_y(1018,)
# print "train_y1"
# print train_y1
test_x1 = show_l[8] ## the shape of test_x(1,11)
# print "text_x1"
# print test_x1
return render_template('House_final_output.html',
pred_y=round(pred_y_show, 2),
train_x1=train_x1,
train_y1=train_y1,
test_x1=test_x1,
quantitative=quantitative,
qualitative=qualitative)
else:
return render_template('Housedata_parameter.html', mydata=mydata)
