Пример #1
0
def Upload_house():
    if request.method == 'POST':
        ## step 1. load the file
        f = request.files['file']

        ## step 2. save the file
        f.save(secure_filename(f.filename))

        ## step 3. connect the service from ssh
        ### SSHConnection is the Class in ssh.py,its parameters contains(host=str, port=int, username=str,pwd=str)
        ssh_2 = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
                              dic_s['password'])
        ssh_2.connect()
        ssh_2.upload(
            str(os.getcwd()) + '/' + str(f.filename),
            dic_s['remote work directory path'] + '/lgbm/train1.csv')
        ## ssh_2.upload(LOCAL FILE PATH,REMOTE PATH)

        ##step 4. run data_v.py to get data of HousePrice
        global mydata  ## mydata is the visualization data of HousePrice data
        mydata = eval(ssh_2.cmd('cd lgbm;python data_v.py'))

        return render_template('Housedata_description.html', mydata=mydata)
    else:
        return render_template('Upload_house.html')
Пример #2
0
 def setup_ssh(self, host, username, password, port, destfile):
     """
     Open SSH connection
     """
     print(host, username, password, port)
     self.ssh_connection = SSHConnection(host, username, password, port)
     print("Log SSH:", self.ssh_connection)
     self.destfile = destfile
     self.ssh_open = True
Пример #3
0
def upload_s():
	if request.method == 'POST':

		f = request.files['file']
		f.save(secure_filename(f.filename))
		print(dic_s['password'])
		ssh_1=SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
		ssh_1.connect()
		ssh_1.upload(str(os.getcwd()) + '\\' + str(f.filename),dic_s['remote work directory path'] + '/FlaskIndexPrediction/data/data.csv')  # INPUT YOUR PATH
		return render_template('index_s.html')

	else:
		return render_template('upload_s.html')
Пример #4
0
    def connect(self):
        if 'host' in self.config['target']:
            host = self.config['target']['host']
        else:
            host = raw_input('Target host: ')

        if 'username' in self.config['target']:
            username = self.config['target']['username']
        else:
            username = raw_input('Target username: '******'password' in self.config['target']:
            password = self.config['target']['password']
        else:
            password = getpass()

        self.target = SSHConnection(host, username, password)
Пример #5
0
def upload_h():
	if request.method == 'POST':

		f = request.files['file']
		f.save(secure_filename(f.filename))
		print(f.filename)
		global ssh
		ssh = SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
		ssh.connect()
		ssh.upload(str(os.getcwd()) + '\\' + str(f.filename),dic_s['remote work directory path'] + '/lgbm/train1.csv')
		
		
		global mydata 
		mydata = eval(ssh.cmd('cd py2;source bin/activate;cd ..;cd lgbm;python data_v.py'))
		
		return render_template('index_h.html',mydata=mydata)
	else:
		return render_template('upload_h.html')
Пример #6
0
def upload_p():
	def get(name):
		return request.values.get(name)
	if request.method == 'POST':
		dic={}
		dic['e'] = int(get('e'))
		dic['lb'] = int (get('lb'))
		dic['lr'] = float (get('lr'))
		dic['tp'] = float (get('tp'))
		ssh_s=SSHConnection(dic_s['host'],dic_s['port'],dic_s['username'],dic_s['password'])
		ssh_s.connect(  )
		s='cd py2;source bin/activate;cd ..;cd FlaskIndexPrediction;python main.py -e {e} -lb {lb} -lr {lr} -tp{tp}'
		ssh_s.cmd(s.format(e=dic['e'],lb=dic['lb'],lr=dic['lr'],tp=dic['tp']))

		
		ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTest.csv', str(os.getcwd())  + '\\' +  'outputTest.csv')
		ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTrain.csv',str(os.getcwd())  + '\\' +  'outputTrain.csv')
		ssh_s.download(dic_s['remote work directory path'] + '/FlaskIndexPrediction/output/outputTrainTest.csv',str(os.getcwd())  + '\\' +  'outputTrainTest.csv')


		outputTest = pd.read_csv('outputTest.csv')
		outputTrain = pd.read_csv('outputTrain.csv')
		outputTrainTest = pd.read_csv('outputTrainTest.csv')
		Test_time = outputTest['Unnamed: 0']
		Test_origin = outputTest['origin']
		Test_predict = outputTest['predict']
		Train_time = outputTrain['Unnamed: 0']
		Train_origin = outputTrain['origin']
		Train_predict = outputTrain['predict']
		TrainTest_time = outputTrainTest['Unnamed: 0']
		TrainTest_origin = outputTrainTest['origin']
		TrainTest_predict = outputTrainTest['predict']
		
		stock_data={'Test_time':Test_time,'Test_origin':Test_origin,
		'Test_predict':Test_predict,'Train_time':Train_time,
		'Train_origin':Train_origin,'Train_predict':Train_predict,
		'TrainTest_time':TrainTest_time,'TrainTest_origin':TrainTest_origin,
		'TrainTest_predict':TrainTest_predict}
		return render_template('ZhangYD.html',stock_data = stock_data)
	else:
		return render_template('index_p.html')
Пример #7
0
 def connection(self, user_name=None, password=None, private_key=None):
     """
         connection target host by ssh protocal
     :param user_name:
     :param password:
     :param private_key:
     :return:
     """
     hosts = set()
     map(lambda _host: hosts.add(parse_host_port(_host[0])[0]),
         self.config["ps_hosts"])
     map(lambda _host: hosts.add(parse_host_port(_host[0])[0]),
         self.config["worker_hosts"])
     _ssh_clients = {}
     for host in hosts:
         logger.debug("ssh connect to host[%s]" % host)
         _ssh_clients[host] = SSHConnection(host,
                                            username=user_name,
                                            password=password,
                                            private_key=private_key)
     return _ssh_clients
Пример #8
0
def Upload_stock():
    if request.method == 'POST':
        ## step 1. load the file
        f = request.files['file']

        ## step 2. save the file
        f.save(secure_filename(f.filename))

        ## step 3. connect the service from ssh
        ### SSHConnection is the Class in ssh.py,its parameters contains(host=str, port=int, username=str,pwd=str)
        ssh_1 = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
                              dic_s['password'])
        ssh_1.connect()
        ssh_1.upload(
            str(os.getcwd()) + '/' + str(f.filename),
            dic_s['remote work directory path'] +
            '/FlaskIndexPrediction/data/data.csv')  # INPUT YOUR PATH
        ## ssh_1.upload(LOCAL FILE PATH,REMOTE PATH)
        return render_template('Choose_LSTM.html')

    else:
        return render_template('Upload_stock.html')
Пример #9
0
 def __init__(
     self,
     _id: str,
     address: str,
     username: str,
     ssh_password: str,
     jobs_db,
     skip_gpus: Sequence[int] = (),
     gpu_runner_on: bool = False,
     app=None,
 ):
     self._id = _id
     self.address = address
     self.username = username
     self.jobs_db = jobs_db
     self.skip_gpus = skip_gpus
     self.gpu_runner_on = gpu_runner_on
     self.app = app
     self.new_processes = []
     self._client = SSHConnection(self.address,
                                  self.username,
                                  ssh_password,
                                  auto_add_host=True)
     self._client_lock = Lock()
Пример #10
0
def Stockdata_parameter():
    def get(
        name
    ):  ##  get values from the HTML part ,'name' is the 'id' of HTML <input>
        return request.values.get(name)

    if request.method == 'POST':
        ## step 1. get the parameters of stock data from HTML
        stock_parameter = {
        }  ##  stock_parameter is a dic to store teh parameters user input in HTML
        stock_parameter['e'] = int(get('e'))
        stock_parameter['lb'] = int(get('lb'))
        stock_parameter['lr'] = float(get('lr'))
        stock_parameter['tp'] = float(get('tp'))

        ## step 2. running the LSTM module in service
        ssh_s = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
                              dic_s['password'])
        ssh_s.connect()
        s = 'cd FlaskIndexPrediction;python main.py -e {e} -lb {lb} -lr {lr} -tp{tp}'
        ssh_s.cmd(
            s.format(e=stock_parameter['e'],
                     lb=stock_parameter['lb'],
                     lr=stock_parameter['lr'],
                     tp=stock_parameter['tp']))

        ## step 3.download the main.py output file to local
        ssh_s.download(
            dic_s['remote work directory path'] +
            '/FlaskIndexPrediction/output/outputTest.csv',
            str(os.getcwd()) + '/module/YD/' + 'outputTest.csv')
        ssh_s.download(
            dic_s['remote work directory path'] +
            '/FlaskIndexPrediction/output/outputTrain.csv',
            str(os.getcwd()) + '/module/YD/' + 'outputTrain.csv')
        ssh_s.download(
            dic_s['remote work directory path'] +
            '/FlaskIndexPrediction/output/outputTrainTest.csv',
            str(os.getcwd()) + '/module/YD/' + 'outputTrainTest.csv')

        ## step 4. read csv from local
        outputTest = pd.read_csv('module/YD/outputTest.csv')
        outputTrain = pd.read_csv('module/YD/outputTrain.csv')
        outputTrainTest = pd.read_csv('module/YD/outputTrainTest.csv')
        Test_time = outputTest['Unnamed: 0']
        Test_origin = outputTest['origin']
        Test_predict = outputTest['predict']
        Train_time = outputTrain['Unnamed: 0']
        Train_origin = outputTrain['origin']
        Train_predict = outputTrain['predict']
        TrainTest_time = outputTrainTest['Unnamed: 0']
        TrainTest_origin = outputTrainTest['origin']
        TrainTest_predict = outputTrainTest['predict']

        stock_data = {
            'Test_time': Test_time,
            'Test_origin': Test_origin,
            'Test_predict': Test_predict,
            'Train_time': Train_time,
            'Train_origin': Train_origin,
            'Train_predict': Train_predict,
            'TrainTest_time': TrainTest_time,
            'TrainTest_origin': TrainTest_origin,
            'TrainTest_predict': TrainTest_predict
        }
        return render_template('Stock_final_output.html',
                               stock_data=stock_data)
    else:
        return render_template('Stockdata_parameter.html')
Пример #11
0
def Housedata_parameter():
    def get(
        name
    ):  ##  get values from the HTML part ,'name' is the 'id' of HTML <input>
        return request.values.get(name)

    if request.method == 'POST':
        ##step 1. get housedata parameters from HTML
        LotArea = get("LotArea")
        Neighborhood = get("Neighborhood")
        YearBuilt = get("YearBuilt")
        GrLivArea = get("GrLivArea")
        Street = get("Street")
        Utilities = get("Utilities")
        LotConfig = get("LotConfig")
        HouseStyle = get("HouseStyle")
        RoofStyle = get("RoofStyle")
        SaleType = get('SaleType')
        SaleCondition = get('SaleCondition')
        lgbm_data = [
            LotArea, Street, Utilities, LotConfig, Neighborhood, HouseStyle,
            YearBuilt, RoofStyle, GrLivArea, SaleType, SaleCondition
        ]
        lgbm_data_1 = ",".join(lgbm_data)

        ##step 2. Data Processing.
        def getSortedValues(
            row
        ):  ## make data in a certain order('row' is the key for the data need to be sorted)
            sortedValues = []
            keys = row.keys()
            keys.sort()
            for key in keys:
                sortedValues.append(row[key])
            return sortedValues

        rows = [
            {
                'Column1': 1,
                'Column2': LotArea,
                'Column3': Street,
                'Column4': Utilities,
                'Column5': LotConfig,
                'Column6': Neighborhood,
                'Column7': HouseStyle,
                'Column8': YearBuilt,
                'Column9': RoofStyle,
                'Column10': GrLivArea,
                'Column11': SaleType,
                'Column12': SaleCondition
            },
        ]

        names = {
            'Column1': 'Id',
            'Column2': 'LotArea',
            'Column3': 'Street',
            'Column4': 'Utilities',
            'Column5': 'LotConfig',
            'Column6': 'Neighborhood',
            'Column7': 'HouseStyle',
            'Column8': 'YearBuilt',
            'Column9': 'RoofStyle',
            'Column10': 'GrLivArea',
            'Column11': 'SaleType',
            'Column12': 'SaleCondition'
        }

        ## step 3. write the sorted data in a csv file('test.csv')
        fileobj = open("module/YX/test.csv", 'wb')
        fileobj.write('\xEF\xBB\xBF')
        writer = csv.writer(fileobj)
        sortedValues = getSortedValues(names)
        writer.writerow(sortedValues)
        for row in rows:
            sortedValues = getSortedValues(row)
            print(sortedValues)
            writer.writerow(sortedValues)
        fileobj.close()

        ## step 4. upload test.csv
        ssh_3 = SSHConnection(dic_s['host'], dic_s['port'], dic_s['username'],
                              dic_s['password'])
        ssh_3.connect()
        ssh_3.upload(
            str(os.getcwd()) + '/module/YX/' + 'test.csv',
            '/root/lgbm/test.csv')

        ## step 5. running LGBM moudle to process the data
        show = ssh_3.cmd('cd lgbm;python lgbm.py')
        show_l = show.split('\n')  ## the output of lgbm.py in terminal
        # print "showl_l"
        # print show_l

        pred_y = eval(show_l[-2])  ##  the price output in list[]
        # print "pred_y"
        # print pred_y
        pred_y_show = pred_y[0]  ##  the price output in float
        # print "pred_y_show"
        # print pred_y_show
        quantitative = re.findall(
            "\d+", show_l[3])[0]  ## number of quantitative features
        qualitative = re.findall(
            "\d+", show_l[3])[1]  ## number of qualitative features
        train_x1 = show_l[6]  ## the shape of train_X(1018,11)
        # print "train_x1"
        # print  train_x1
        train_y1 = show_l[7]  ## the shape of train_y(1018,)
        # print "train_y1"
        # print  train_y1
        test_x1 = show_l[8]  ## the shape of test_x(1,11)
        # print "text_x1"
        # print  test_x1
        return render_template('House_final_output.html',
                               pred_y=round(pred_y_show, 2),
                               train_x1=train_x1,
                               train_y1=train_y1,
                               test_x1=test_x1,
                               quantitative=quantitative,
                               qualitative=qualitative)

    else:
        return render_template('Housedata_parameter.html', mydata=mydata)