def upload_p():
def get(name):
return request.values.get(name)
if request.method == 'POST':
dic={}
dic['e'] = int(get('e'))
dic['lb'] = int (get('lb'))
dic['lr'] = float (get('lr'))
dic['tp'] = float (get('tp'))
ssh_s=SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
ssh_s.connect( )
s='cd py2;source bin/activate;cd ..;cd FlaskIndexPrediction;python main.py -e {e} -lb {lb} -lr {lr} -tp{tp}'
ssh_s.cmd(s.format(e=dic['e'],lb=dic['lb'],lr=dic['lr'],tp=dic['tp']))
ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTest.csv', str(os.getcwd()) + '\\' + 'outputTest.csv')
ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTrain.csv',str(os.getcwd()) + '\\' + 'outputTrain.csv')
ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTrainTest.csv',str(os.getcwd()) + '\\' + 'outputTrainTest.csv')
outputTest = pd.read_csv('outputTest.csv')
outputTrain = pd.read_csv('outputTrain.csv')
outputTrainTest = pd.read_csv('outputTrainTest.csv')
Test_time = outputTest['Unnamed: 0']
Test_origin = outputTest['origin']
Test_predict = outputTest['predict']
Train_time = outputTrain['Unnamed: 0']
Train_origin = outputTrain['origin']
Train_predict = outputTrain['predict']
TrainTest_time = outputTrainTest['Unnamed: 0']
TrainTest_origin = outputTrainTest['origin']
TrainTest_predict = outputTrainTest['predict']
stock_data={'Test_time':Test_time,'Test_origin':Test_origin,
'Test_predict':Test_predict,'Train_time':Train_time,
'Train_origin':Train_origin,'Train_predict':Train_predict,
'TrainTest_time':TrainTest_time,'TrainTest_origin':TrainTest_origin,
'TrainTest_predict':TrainTest_predict}
return render_template('ZhangYD.html',stock_data = stock_data)
else:
return render_template('index_p.html')
def Stockdata_parameter():
def get(
name
): ## get values from the HTML part ,'name' is the 'id' of HTML <input>
return request.values.get(name)
if request.method == 'POST':
## step 1. get the parameters of stock data from HTML
stock_parameter = {
} ## stock_parameter is a dic to store teh parameters user input in HTML
stock_parameter['e'] = int(get('e'))
stock_parameter['lb'] = int(get('lb'))
stock_parameter['lr'] = float(get('lr'))
stock_parameter['tp'] = float(get('tp'))
## step 2. running the LSTM module in service
ssh_s = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
dic_s['password'])
ssh_s.connect()
s = 'cd FlaskIndexPrediction;python main.py -e {e} -lb {lb} -lr {lr} -tp{tp}'
ssh_s.cmd(
s.format(e=stock_parameter['e'],
lb=stock_parameter['lb'],
lr=stock_parameter['lr'],
tp=stock_parameter['tp']))
## step 3.download the main.py output file to local
ssh_s.download(
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/output/outputTest.csv',
str(os.getcwd()) + '/module/YD/' + 'outputTest.csv')
ssh_s.download(
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/output/outputTrain.csv',
str(os.getcwd()) + '/module/YD/' + 'outputTrain.csv')
ssh_s.download(
dic_s['remote work directory path'] +
'/FlaskIndexPrediction/output/outputTrainTest.csv',
str(os.getcwd()) + '/module/YD/' + 'outputTrainTest.csv')
## step 4. read csv from local
outputTest = pd.read_csv('module/YD/outputTest.csv')
outputTrain = pd.read_csv('module/YD/outputTrain.csv')
outputTrainTest = pd.read_csv('module/YD/outputTrainTest.csv')
Test_time = outputTest['Unnamed: 0']
Test_origin = outputTest['origin']
Test_predict = outputTest['predict']
Train_time = outputTrain['Unnamed: 0']
Train_origin = outputTrain['origin']
Train_predict = outputTrain['predict']
TrainTest_time = outputTrainTest['Unnamed: 0']
TrainTest_origin = outputTrainTest['origin']
TrainTest_predict = outputTrainTest['predict']
stock_data = {
'Test_time': Test_time,
'Test_origin': Test_origin,
'Test_predict': Test_predict,
'Train_time': Train_time,
'Train_origin': Train_origin,
'Train_predict': Train_predict,
'TrainTest_time': TrainTest_time,
'TrainTest_origin': TrainTest_origin,
'TrainTest_predict': TrainTest_predict
}
return render_template('Stock_final_output.html',
stock_data=stock_data)
else:
return render_template('Stockdata_parameter.html')
class PostgresManager(object):
"""docstring for PostgresManager"""
def __init__(self, node):
super(PostgresManager, self).__init__()
self.node = node
if not node['local'] == 'true':
self.connection = SSHConnection(
node['hostname'],
node['username'],
node['password'])
def run(self, cmd):
if self.node['local'] == 'true':
return local.shell(cmd)
else:
return self.connection.execute(cmd)
def start(self):
self.run('sudo /etc/init.d/postgresql start')
def stop(self):
self.run('sudo /etc/init.d/postgresql stop')
def restart(self):
self.run('sudo /etc/init.d/postgresql restart')
def reload(self):
self.run('sudo /etc/init.d/postgresql reload')
def write_file(self, path, data):
if self.node['local'] == 'true':
local.write_file(path, data)
else:
self.connection.upload_data(data, path)
def read_file(self, path):
if self.node['local'] == 'true':
return local.read_file(path)
else:
return self.connection.download(path)
def init(self, master):
environ = os.environ.copy()
environ['PATH'] = '/usr/lib/pgclust:' + environ['PATH']
self.stop()
self.run('sudo pg_dropcluster %(pgversion)s %(cluster)s' % self.node)
self.run('sudo pg_createcluster %(pgversion)s %(cluster)s' % self.node)
self.run('sudo chown -R %(pguser)s:%(pguser)s /var/lib/postgresql/%(pgversion)s/%(cluster)s' % self.node)
self.write_file('/tmp/pgpostgresql.conf', template.PG_CONFIG_TEMPLATE % self.node)
self.write_file('/tmp/pgpg_hba.conf', template.PG_HBA_CONFIG_TEMPLATE % self.node)
self.write_file('/tmp/pgrepmgr.conf', template.REPMGR_CONFIG_TEMPLATE % self.node)
self.run('sudo cp /tmp/pgpostgresql.conf /etc/postgresql/%(pgversion)s/%(cluster)s/postgresql.conf' % self.node)
self.run('sudo cp /tmp/pgpg_hba.conf /etc/postgresql/%(pgversion)s/%(cluster)s/pg_hba.conf' % self.node)
self.run('sudo cp /tmp/pgrepmgr.conf /etc/postgresql/%(pgversion)s/%(cluster)s/repmgr.conf' % self.node)
self.run('sudo chmod 644 /etc/postgresql/%(pgversion)s/%(cluster)s/*' % self.node)
self.run('sudo rm /tmp/pgpostgresql.conf /tmp/pgpg_hba.conf /tmp/pgrepmgr.conf')
self.run('sudo chown %(pguser)s:%(pguser)s -R /etc/postgresql' % self.node)
self.write_file('/tmp/sshid_rsa' % self.node, local.read_file(self.node['privkey']))
self.write_file('/tmp/sshid_rsa.pub' % self.node, local.read_file(self.node['pubkey']))
self.write_file('/tmp/sshconfig', template.SSH_CONFIG)
self.run('sudo -u %(pguser)s mkdir -p ~%(pguser)s/.ssh' % self.node)
self.run('sudo cp /tmp/sshid_rsa ~%(pguser)s/.ssh/id_rsa' % self.node)
self.run('sudo cp /tmp/sshid_rsa.pub ~%(pguser)s/.ssh/id_rsa.pub' % self.node)
self.run('sudo cp /tmp/sshconfig ~%(pguser)s/.ssh/config' % self.node)
self.run('sudo rm /tmp/sshconfig /tmp/sshid_rsa /tmp/sshid_rsa.pub')
self.run('sudo chown %(pguser)s:%(pguser)s ~%(pguser)s/.ssh/config' % self.node)
self.run('sudo chown %(pguser)s:%(pguser)s ~%(pguser)s/.ssh/id_rsa' % self.node)
self.run('sudo chown %(pguser)s:%(pguser)s ~%(pguser)s/.ssh/id_rsa.pub' % self.node)
self.run('sudo chmod 644 ~%(pguser)s/.ssh/config' % self.node)
self.run('sudo chmod 600 ~%(pguser)s/.ssh/id_rsa' % self.node)
self.run('sudo chmod 644 ~%(pguser)s/.ssh/id_rsa.pub' % self.node)
if self.node['type'] == 'master':
self.start()
self.run('sudo -u %(pguser)s createuser --login --superuser repmgr' % self.node)
self.run('sudo -u %(pguser)s createdb repmgr' % self.node)
self.run('sudo -u %(pguser)s repmgr --verbose -f /etc/postgresql/%(pgversion)s/%(cluster)s/repmgr.conf master register' % self.node)
else:
self.run('sudo -u %(pguser)s rm -rf /var/lib/postgresql/%(pgversion)s/%(cluster)s' % self.node)
self.run('sudo -u %(pguser)s mkdir /var/lib/postgresql/%(pgversion)s/%(cluster)s' % self.node)
self.run('sudo -u %(pguser)s chmod 700 /var/lib/postgresql/%(pgversion)s/%(cluster)s' % self.node)
self.run(('sudo -u %(pguser)s PATH="' + environ['PATH'] + '" repmgr --verbose --force -D /var/lib/postgresql/%(pgversion)s/%(cluster)s -d repmgr -p 5432 -U repmgr -R %(pguser)s standby clone ') % self.node + master['hostname'])
self.start()
self.run('sudo -u %(pguser)s repmgr -f /etc/postgresql/%(pgversion)s/%(cluster)s/repmgr.conf --verbose standby register' % self.node)
def update_nodes(self, nodes):
records = []
for node in nodes:
node['hostname'] = socket.gethostbyname(node['hostname']) + '/32'
records.append(template.REPLICATION_NODE_TEMPLATE % node)
pg_hba = '\n\n'.join(records) + template.PG_HBA_CONFIG_TEMPLATE % self.node
self.write_file('/tmp/pgpg_hba.conf' % self.node, pg_hba)
self.run('sudo cp /tmp/pgpg_hba.conf /etc/postgresql/%(pgversion)s/%(cluster)s/pg_hba.conf' % self.node)
self.run('sudo rm /tmp/pgpg_hba.conf')
def update_keys(self, nodes):
self.run('sudo -u %(pguser)s mkdir -p ~%(pguser)s/.ssh' % self.node)
self.run('sudo chmod 700 ~%(pguser)s/.ssh' % self.node)
keys = []
for node in nodes:
key = local.read_file(node['pubkey'])
keys.append(key)
self.write_file('/tmp/sshauthorized_keys' % self.node, '\n'.join(keys))
self.run('sudo cp /tmp/sshauthorized_keys ~%(pguser)s/.ssh/authorized_keys' % self.node)
self.write_file('/tmp/sshid_rsa' % self.node, local.read_file(self.node['privkey']))
self.write_file('/tmp/sshid_rsa.pub' % self.node, local.read_file(self.node['pubkey']))
self.run('sudo cp /tmp/sshid_rsa ~%(pguser)s/.ssh/id_rsa' % self.node)
self.run('sudo cp /tmp/sshid_rsa.pub ~%(pguser)s/.ssh/id_rsa.pub' % self.node)
self.run('sudo rm /tmp/sshauthorized_keys /tmp/sshid_rsa /tmp/sshid_rsa.pub')
self.run('sudo chown %(pguser)s:%(pguser)s ~%(pguser)s/.ssh/authorized_keys' % self.node)
self.run('sudo chown %(pguser)s:%(pguser)s ~%(pguser)s/.ssh/id_rsa' % self.node)
self.run('sudo chown %(pguser)s:%(pguser)s ~%(pguser)s/.ssh/id_rsa.pub' % self.node)
self.run('sudo chmod 600 ~%(pguser)s/.ssh/authorized_keys' % self.node)
self.run('sudo chmod 600 ~%(pguser)s/.ssh/id_rsa' % self.node)
self.run('sudo chmod 644 ~%(pguser)s/.ssh/id_rsa.pub' % self.node)
def update(self, nodes):
self.update_nodes(nodes.values())
self.update_keys(nodes.values())
self.reload()
