def allCategory(url):
s = requests.session()
s.headers.update(headers)
db = dbHandle()
r = s.get(url)
if r.status_code == 200:
r.encoding = 'utf-8'
text = r.text
obj = etree.HTML(text)
#职位分类
jobCategory = r'//*[@id="sidebar"]/div//div'
#职位细分-分类
detailCategory = r'./div[2]//dl/dd//a'
#职位名
jobTitle = r'./text()'
#职位ur
jobUrl = r'./@href'
allCategorys = obj.xpath(jobCategory)
for catetory in allCategorys:
if len(catetory.xpath(detailCategory)) != 0:
for job in catetory.xpath(detailCategory):
# print(job.xpath(jobTitle)[0])
# print(job.xpath(jobUrl)[0])
sql = """insert into jobCategories(category_name,category_url) values('{0}','{1}')""".\
format(job.xpath(jobTitle)[0],job.xpath(jobUrl)[0])
db.insert_db(sql)
def __init__(self):
# 初始化session,数据链接
self.db = dbHandle()
self.s = requests.session()
self.s.headers.update(headers)
self.post_url = """https://www.lagou.com/jobs/positionAjax.json?city=%E6%B7%B1%E5%9C%B3&needAddtionalResult=false&isSchoolJob=0"""
def ChangeRepl(self, _content):
try:
groupname, region, type = _content[0], _content[1], _content[-1]
if type == 'dow': #宕机任务，需重新选择节点并监听同步
for i in range(0, 3):
host, port = self.__get_master_for_region(
region, groupname)
with closing(dbHandle(host, port)) as dbhandle:
mysqlstate = dbhandle.RetryConn() # 检测是否能正常连接
time.sleep(1)
if mysqlstate:
zkHander().CreateWatch(
host=host.replace('.', '-'),
addition=True,
region=region,
region_for_groupname=groupname) # 重新创建master检测
else:
return self.__change_new_master(region=region,
groupname=groupname)
elif type == 'up': #只进行监听，用于手动添加了同步任务
self.__up_watch_master(region=region, groupname=groupname)
return True
except:
Logging(msg='addition task failed!', level='error')
return False
def dbInit():
handle = dbHandle('survey.db')
handle.clearDB()
courses = getCSVList('courses.csv')
classIDs = []
for c in courses:
cls = formatClass(c)
try:
classIDs.append(handle.addClass(cls))
except:
pass
users = getCSVList('passwords.csv')
for u in users:
usr = formatUser(u)
try:
handle.addUser(usr)
except:
pass
enrollments = getCSVList('enrolments.csv')
for e in enrollments:
classID = getClassID(e[1:], courses, classIDs)
try:
handle.addEnrollment(e[0], classID)
except:
pass
del handle
def __init__(self, dfData, dfInfo, aucTime):
self.df = dfData
self.date = datetime.datetime.strptime(self.df["date"][0], "%Y%m%d")
self.dfInfo = dfInfo
self.db = dbHandle()
self.AucTime = aucTime
self.initCleanRegulation()
def salary():
db = dbHandle()
#工资区间划分
#0-5,5-10,10-15,15-25,25-40,40+
less_5 = db.query_db(less_5_sql)
less_10 = db.query_db(less_10_sql)
less_15 = db.query_db(less_15_sql)
less_25 = db.query_db(less_25_sql)
less_40 = db.query_db(less_40_sql)
more_40 = db.query_db(more_40_sql)
print("less 5:{0}".format(less_5))
print("less 10:{0}".format(less_10))
print("less 15:{0}".format(less_15))
print("less 25:{0}".format(less_25))
print("less 40:{0}".format(less_40))
print("more 40:{0}".format(more_40))
# new_list = []
# for data in datas:
# newdata = data[0].split('-')
# if len(newdata)>1:
# total = int(newdata[0].strip('K').strip('k'))+int(newdata[1].strip('K').strip('k'))
# avg = total/2
# new = (avg,data[1],data[2])
# new_sql = "update position_detail_bak set salary={0} where salary='{1}';".format(avg,data[0])
# print(new_sql)
# #print(new)
# new_list.append(new)
#
# print(new_list)
# key_lis = ['1-3年','3-5年','不限','5-10年','1年以下','10年以上','应届毕业生']
return render_template('salary.html')
def __init__(self, dfInfo, date, aucTime):
self.timeFilePath = os.getcwd() + '/' + 'timeSeriesFile/'
self.barDict = {}
self.splitDict = {}
self.dfInfo = dfInfo
self.db = dbHandle()
self.timePoint = date
self.cycle = [1, 5, 15, 30, 60]
self.AucTime = aucTime
self.initStart()
def getContent(table_name):
db = dbHandle()
jieba.del_word("电影")
jieba.del_word("导演")
jieba.del_word("没有")
jieba.del_word("影片")
jieba.del_word("看到")
query_sql = "select comment_content from {0}".format(table_name)
texts = db.query_db(query_sql)
text = ''
for t in texts:
text += t[0]
#result = jieba.analyse.textrank(text, topK=1000, withWeight=True)
result = jieba.cut(text, cut_all=True)
wl_space_split = " ".join(result)
return wl_space_split
def insert_data(self, data):
#db = dbHandle()
data = data
db = dbHandle()
print(data)
texts = self.get_job_detail(data)
text = '\n'.join(texts)
text = text.replace('"', "'")
# 如果职位id对应的描述为空则插入数据
job_detail = db.query_db(job_detail_sql.format(data))
print(job_detail)
if job_detail[0][0] == None:
update_sql = """update position_detail set positionText="{0}" where positionId='{1}'""".format(
text, data)
print("正在执行更新语句:{0}".format(update_sql))
db.update_db(update_sql.format(text, data))
financeStage = result['financeStage']
industryField = result['industryField']
companySize = result['companySize']
companyLabelList = result['companyLabelList']
companyFullName = result['companyFullName']
#拼装公司sql数据
company_sql = """insert into company_detail(companyId,city,district,financeStage,industryField,companySize,companyLabelList,companyFullName) \
values('{0}','{1}','{2}','{3}','{4}','{5}',"{6}",'{7}')""".format(
companyId, city, district, financeStage, industryField,
companySize, companyLabelList, companyFullName)
print("company_sql:{0}".format(company_sql))
#公司数据入库
db = dbHandle()
company_datas = db.query_db(
query_companyid_sql.format(companyId))
#如果公司id不存在则入库
if len(company_datas) == 0:
db.insert_db(company_sql)
#职位数据
position_id = result['positionId']
position_name = result['positionName']
position_workYear = result['workYear']
position_education = result['education']
position_companyId = result['companyId']
position_salary = result['salary']
position_Advantage = result['positionAdvantage']
from dbHandle import dbHandle
handle = dbHandle("survey.db")
def authenticator(username, password):
myUser = handle.getUser(username)
if (password == myUser.getPass()):
return myUser
else:
return -1
def education():
db = dbHandle()
datas = db.query_db(education_sql)
print(datas)
return render_template('workyear.html', datas=datas)
def year():
db = dbHandle()
datas = db.query_db(work_year_sql)
print(datas)
return render_template('education.html', datas=datas)
def index():
db = dbHandle()
datas = db.query_db(category_count_sql)
print(datas)
return render_template('data.html', datas=datas[:50])
def __init__(self):
self.s = requests.session()
self.db = dbHandle()