def create_spider():
s = Spider()
Config.init()
s.config_from_object(Config)
Parse.init_spider(s)
return s
def getConnection(self):
if (self.conn == None):
config = Config()
return pymssql.connect(config.dbServer, config.SIIS_User,
config.SIIS_Pwd, config.SIIS_db)
else:
return self.conn
def create_app():
"""
create flask web manager
from environ
:return: Flask
"""
# environ bootstrap
Config.bootstrap()
# register app logger
logger_register(os.path.dirname(__file__), Config.LOGGER)
# make flask manager
app = Flask(__name__)
app.config.from_object(Config)
database.init_app(app) # init database use SQLAlchmy
redis_store.init_app(app) # init redis store database use flask-redis
jwt_manager.init_app(app) # init json web token manager
CSRFProtect(app) # init CSRF protect
return load_blueprints(load_blinker(app))
def GetData(self, path):
config = Config()
conn = pymssql.connect(host=config.dbServer,
user=config.dbUser,
password=config.dbPwd,
database=config.dbName)
cur = conn.cursor()
workbook = xlrd.open_workbook(path)
sheet = workbook.sheet_by_index(0)
for rows in range(1, sheet.nrows):
arr = []
for cols in range(3):
arr.append(self.finaldata(sheet.cell_value(rows, cols)))
cur.execute("INSERT INTO dbo.city VALUES (%s, %s, %s)", tuple(arr))
conn.commit()
return 'success'
def GetData(self, path):
config = Config()
conn = pymssql.connect(host=config.dbServer,
user=config.dbUser,
password=config.dbPwd,
database=config.dbName)
cur = conn.cursor()
data = xlrd.open_workbook(path)
table = data.sheets()[0]
num_cols = table.ncols
for row_index in range(1, table.nrows):
for col_index in range(0, num_cols):
#idCity = table.cell(row_index, 0).value
Name = table.cell(row_index, 1).value
latitude = table.cell(row_index, 2).value
Longitude = table.cell(row_index, 3).value
print Name, latitude, Longitude
SalestrSQL = """INSERT INTO dbo.city (Name,latitude,Longitude) VALUES(%s,%s,%s)"""
Values = (Name, latitude, Longitude)
cur.execute(SalestrSQL, Values)
conn.commit()
from etlpipeline import SinglePredictPipelineConstructor
################
import os
def create_app(conf):
app = Flask(__name__)
app.config.from_object(conf)
#db.init_app(app)
#db.create_all()
return app
conf = Config()
app = create_app(conf)
session = boto3.session.Session(profile_name='lost')
s3 = session.resource('s3')
bucket = 'lost-pagesource-staging'
@app.route('/', methods=['POST'])
def detect():
domain_name = request.json.get('domain')
response = s3.Object(bucket, domain_name)
source = response.get()['Body'].read()
predictor = SinglePredictPipelineConstructor()
data = predictor.predict(domain_name, source)
import numpy as np
import torch
from setting import Config
import model
from get_feature import Feature
config=Config()
feature=Feature(config)
class Parser(object):
def __init__(self,sentence):
self.sentence=sentence
self.stack=["<ROOT>"]
self.buffer=list(sentence)
self.dep=[]
def parse_step(self,transition):
if transition=="S" and len(self.buffer)>0:
word=self.buffer.pop(0)
self.stack.append(word)
if transition=="L":
head=self.stack[-1]
dependent=self.stack.pop(-2)
self.dep.append((head,dependent))
if transition=='R':
head=self.stack[-2]
dependent=self.stack.pop()
self.dep.append((head,dependent))
def parse(self,transitions):
""" # -*- encoding: utf-8 -*- @Time : 2021/12/23 12:45:18 @Author : yylime @Contact : [email protected] """ from numpy.testing._private.utils import clear_and_catch_warnings from utils import Slider from setting import Config import argparse parser = argparse.ArgumentParser() parser.add_argument( "--name", default="yidun", type=str, help="find useful name in setting.py", ) args = parser.parse_args() if __name__ == "__main__": cfg = Config(args.name) slider = Slider(cfg) slider.run()
