def submit(club_id, gala_id, swimmer_id):
"""
Retrieve and store heat entry details
Send confirmation
:param club_id:
:param gala_id:
:param swimmer_id:
:return: template
"""
if swimmer_id != session['swimmer']:
return 'Error, invalid swimmer id detected'
selected = request.form.getlist('selected')
heats = []
for heat in selected:
v = ast.literal_eval(heat)
v["id"] = str(v["id"])
heats.append(v)
with Mongo(db_config) as db:
gala = db.gala.find_one({'title': gala_id, 'club.id': club_id})
swimmer = db.swimmers.find_one({
'swimmer_id': swimmer_id,
'club_id': club_id
})
entry = {
'gala_id': gala_id,
'swimmer_id': swimmer_id,
'club_id': club_id,
'heats': heats,
}
with Mongo(db_config) as db:
db.entries.update_one(
{
'gala_id': gala_id,
'swimmer_id': swimmer_id,
'club_id': club_id
}, {'$set': entry}, True)
template = render_template("submitted.html",
heats=heats,
swimmer=swimmer,
gala=gala)
if swimmer["email"]:
link = constants.FORM_ADDRESS + link_generator.generate_payload(
gala_id, club_id, swimmer['swimmer_id'])
mailer.send_mail(swimmer["email"],
'Gala invitation.',
body=render_template("confirmation_mail.txt",
heats=heats,
swimmer=swimmer,
gala=gala,
link=link),
html=render_template("confirmation_mail.html",
heats=heats,
swimmer=swimmer,
gala=gala,
link=link))
return template
def get_baiduqianxi():
m = Mongo('192.168.0.235', 27017, 'baiduqianxi', 'baiduqianxi')
coll = m.get_collection_object()
browser = webdriver.ChromeOptions()
browser.add_argument('--headless')
browser = webdriver.Chrome(chrome_options=browser,
executable_path='chromedriver.exe')
browser.set_window_size(1600, 900)
for city in CITY_LIST:
browser.get('http://qianxi.baidu.com/')
try:
time.sleep(3)
elem = browser.find_element_by_id('input_cityName')
elem.send_keys(city)
time.sleep(1)
browser.find_element_by_class_name('input-group-addon').click()
time.sleep(5)
in_data = browser.find_element_by_class_name(
'div_list_container')
in_info = in_data.text
print(in_info)
in_list = in_info.split('\n')
in_dict = {}
for i in in_list:
list_ = i.split(' ')
city_name = list_[1].replace(city, '')
number = list_[2].replace('％', '')
in_dict[city_name] = number
browser.find_element_by_xpath(
'//a[@class="btn btn-default"]').click()
out_data = browser.find_element_by_class_name(
'div_list_container')
out_info = out_data.text
print(out_info)
out_list = out_info.split('\n')
out_dict = {}
for i in out_list:
list_ = i.split(' ')
city_name = list_[1].replace(city, '')
number = list_[2].replace('％', '')
out_dict[city_name] = number
coll.insert_one({
'city':
city,
'in':
in_dict,
'out':
out_dict,
'insert_time':
datetime.datetime.now(),
'date':
int(str(datetime.date.today()).replace('-', ''))
})
except Exception as e:
print(e)
print(city)
browser.close()
def _load_data_from_files(mdb_file, zip_file):
"""
Build the gala/club/swimmer object
:param mdb_file:
:param zip_file:
:return: gala_details.html
"""
try:
o = Objectmaker(os.path.join(constants.UPLOAD_PATH, mdb_file),
os.path.join(constants.UPLOAD_PATH, zip_file))
except Exception as e:
return render_template(
'gala_details.html',
data='Could not read mdb or zip file. <br/>Error: ' + str(e))
data = o.get_data()
swimmers = data.pop('swimmers')
with Mongo(db_config) as db:
db.gala.update_one(
{
"title": data["title"],
"club.id": data["club"]["id"]
}, {'$set': data}, True)
for swimmer in swimmers.values():
db.swimmers.update_one(
{
"swimmer_id": swimmer["swimmer_id"],
"club_id": data["club"]["id"]
}, {'$set': swimmer}, True)
os.remove(os.path.join(constants.UPLOAD_PATH, mdb_file))
os.remove(os.path.join(constants.UPLOAD_PATH, zip_file))
for heat in data["heats"]:
heat["id"] = int(heat["id"])
return render_template("gala_details.html", data=data)
class Baiduqianxi:
connect = Mongo(setting['baiduqianxi']['mongo']['host']).connect
coll = connect[setting['baiduqianxi']['mongo']['db']][
setting['baiduqianxi']['mongo']['collection']]
now_time = datetime.datetime.now()
today_int = int(now_time.strftime('%Y%m%d'))
q = Queue(maxsize=0)
def put_in_queue(self):
for city_name in city_id_dict:
city_id = city_id_dict[city_name]
self.q.put((city_name, city_id))
def start_consume(self):
count = 0
self.put_in_queue()
while not self.q.empty():
city_info = self.q.get()
city_name = city_info[0]
city_id = city_info[1]
try:
type_ = 'move_in'
city = city_name
time_str = str(self.today_int)
url = 'http://huiyan.baidu.com/migration/api/cityrank?dt=city&id=' + city_id + '&type=' + type_ + '&date=' + time_str
res_in = requests.get(url)
time.sleep(10)
in_info = re.search(r'\[(.*?)\]', res_in.text).group()
in_list = eval(in_info)
in_all_list = []
for i in in_list:
in_all_list.append(i)
type_ = 'move_out'
url = 'http://huiyan.baidu.com/migration/api/cityrank?dt=city&id=' + city_id + '&type=' + type_ + '&date=' + time_str
res_out = requests.get(url)
time.sleep(10)
out_info = re.search(r'\[(.*?)\]', res_out.text).group()
out_list = eval(out_info)
out_all_list = []
for i in out_list:
out_all_list.append(i)
count += 1
log.info('count={},城市={}'.format(count, city_name))
data = {
'city': city_name,
'date': int(time_str),
'insert_time': datetime.datetime.now(),
'in': in_all_list,
'out': out_all_list,
}
if not in_all_list and not in_all_list:
log.info('迁入和迁出为空,{}'.format(city))
else:
log.info('插入一条数据,{}'.format(data))
self.coll.insert_one(data)
self.q.task_done()
except Exception as e:
log.error('错误,e={}'.format(e))
self.q.put(city_name)
def __init__(self, bot):
self.bot = bot
self.db = Mongo.init_db(Mongo())
self.server_db = None
self.leaderboards = {'exp': {'exp', 'experience', 'xp'},
'thanks.thanks_received': {'thankee', 'thanks received'},
'thanks.thanks_given': {'thanker', 'thanks given'}}
def send_emails(club_id, gala_id):
"""
Split swimmers into those who have emails or those who don't
Send to to first list
Allow print forms for second list
:param club_id:
:param gala_id:
:return: sentemail.html
"""
with Mongo(db_config) as db:
swimmers = db.swimmers.find({"club_id": club_id})
got_email = []
no_email = []
for swimmer in swimmers:
link = constants.FORM_ADDRESS + link_generator.generate_payload(
gala_id, club_id, swimmer['swimmer_id'])
if not swimmer["inactive"]:
if swimmer["email"] is None or swimmer["email"] == "":
no_email.append({"swimmer": swimmer, "link": link})
else:
got_email.append({"swimmer": swimmer, "link": link})
mailer.send_mail(
swimmer["email"], 'Gala invitation.',
render_template("link_mail.txt",
first_name=swimmer["first"],
link=link,
club_id=club_id),
render_template("link_mail.html",
first_name=swimmer["first"],
link=link,
club_id=club_id))
return render_template("sentemail.html",
got_email=got_email,
no_email=no_email,
gala_id=gala_id)
def __init__(self):
cmd2.Cmd.__init__(self,
completekey='tab',
persistent_history_file=get_option('core', 'hist_file', config),
persistent_history_length=int(get_option('core', 'hist_size', config)))
self.allow_cli_args = False
self.default_to_shell = False
self.intro = 'Welcome to the Omnibus shell! Type "session" to get started or "help" to view all commands.'
self.allow_redirection = True
self.prompt = 'omnibus >> '
self.redirector = '>'
self.quit_on_sigint = False
del cmd2.Cmd.do_alias
del cmd2.Cmd.do_edit
del cmd2.Cmd.do_eof
del cmd2.Cmd.do_shell
del cmd2.Cmd.do_eos
del cmd2.Cmd.do_load
del cmd2.Cmd.do_py
del cmd2.Cmd.do_pyscript
del cmd2.Cmd.do_shortcuts
del cmd2.Cmd.do_unalias
del cmd2.Cmd.do__relative_load
self.db = Mongo(config)
self.dispatch = Dispatch(self.db)
self.session = None
if DEBUG:
self.do_set('debug true')
def __init__(self, name, persistent=True):
self.name = name
self.sessions = []
self.session_id = ""
self.epoch_id = 0
self.persistent = persistent
self.database = Mongo()
self.log = Logger("story-{}".format(name))
def __init__(self,article_id=None,text=None,comment_id=None,user_name=None,user_id=None,create_time=None,good_count=None):
self.comment_text = text
self._id = comment_id
self.user_name = user_name
self.user_id = user_id
self.create_time = create_time
self.good_count = good_count
self.article_id = article_id
self.coll = Mongo(setting['mongo']['host'], setting['mongo']['port'], setting['mongo']['db_name'],
setting['mongo']['comments']).get_collection_object()
def __init__(self, bot):
self.bot = bot
self.db = Mongo.init_db(Mongo())
self.server_db = None
self.sys_aliases = {
'ps': {'ps', 'psn', 'ps4', 'ps5', 'playstation'},
'xb': {'xb', 'xb1', 'xbox', 'microsoft'},
'steam': {'steam', 'steam64', 'valve'},
'ubi': {'uplay', 'ubi', 'ubisoft'},
'xiv': {'ffxiv', 'xiv', 'ff'}
}
def login():
print(request)
if request.method == 'POST':
username = request.form['inputUserName']
password = generate_password_hash(request.form['inputPassword'])
with Mongo(db_config) as db:
data = db.admin.find_one({'username': username})
if check_password_hash(password, data['password']):
session['username'] = username
session["clubid"] = data["clubid"]
return redirect(url_for('menu'))
return redirect(url_for('main'))
def galas():
"""
Retrieve and display current galas
:return: galas.html
"""
with Mongo({}) as db:
galas = list(
db.gala.find({
"club.id": session["clubid"],
'deleted': False
}))
return render_template("galas.html", galas=galas)
def __init__(self):
self.s = requests.session()
self.g = GetPhone('小资家')
self.phone = self.g.phone
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
}
self.code = ''
self.m = Mongo('114.80.150.196',
27777,
user_name='goojia',
password='******')
self.coll = self.m.connect['friends']['xiaozijia_user']
def delete_gala(gala):
"""
Flag gala as deleted
:param gala:
:return: redirect to menu
"""
club = session['clubid']
with Mongo(db_config) as db:
db.gala.update_one({
"title": gala,
"club.id": club
}, {'$set': {
'deleted': True
}})
return redirect(url_for('menu'))
def __init__(self):
self.password = '******' + str(random.randint(0, 5)) + '567yhn'
self.s = requests.session()
self.g = GetPhone('小资家')
self.phone = self.g.phone
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134',
'Referer': 'http://www.xiaozijia.cn/'
}
self.code = ''
self.m = Mongo('114.80.150.196',
27777,
user_name='goojia',
password='******')
self.coll = self.m.connect['friends']['xiaozijia_user']
self.result = ''
def __init__(self):
self.password = '******'
self.s = requests.session()
self.g = GetPhone('小资家')
self.phone = self.g.phone
self.headers = {
'Connection': 'keep-alive',
'Host': 'www.xiaozijia.cn:8002',
'User-Agent':
'xiao zi jiaiOS/1.2.1 (iPhone; iOS 11.4.1; Scale/2.00)',
}
self.code = ''
self.m = Mongo('114.80.150.196',
27777,
user_name='goojia',
password='******')
self.coll = self.m.connect['friends']['xiaozijia_user']
self.result = ''
def add_swimmers_with_entries(self):
"""
Add E1 records
:return: Lines
"""
with Mongo() as db:
swimmers_data = db.swimmers.find({'club_id': self.club_id})
entries_data = db.entries.find({
'gala_id': self.gala_id,
'club_id': self.club_id
})
for swimmer in swimmers_data:
swimmer_id = swimmer['swimmer_id']
self.swimmers[swimmer_id] = swimmer
self.swimmers[swimmer_id]['entry'] = None
for entry in entries_data:
swimmer_id = entry['swimmer_id']
if swimmer_id in self.swimmers:
self.swimmers[swimmer_id]['entry'] = entry
else:
raise Exception(
'Entry for an unknown swimmer has been detected.')
for swimmer in self.swimmers.values():
if swimmer['entry'] and swimmer['entry']['heats']:
swimmer_entry = SwimmerEntry(gender=swimmer['sex'],
event_swimmer_id=swimmer['id'],
surname=swimmer['last'],
first_name=swimmer['first'],
uss_number=swimmer['swimmer_id'],
date_of_birth=strftime(
'%m%d%Y',
swimmer['dob'].timetuple()))
self.output_lines.append(swimmer_entry.as_line())
for heat in swimmer['entry']['heats']:
event_entry = EventEntry(gender=swimmer['sex'],
event_swimmer_id=swimmer['id'],
abbreviated_name=swimmer['last'],
gender_category=heat['gender'],
distance=heat['distance'],
event_number=heat['id'],
stroke=heat['style'],
age_lower=heat['minage'],
age_upper=heat['maxage'])
self.output_lines.append(event_entry.as_line())
def form(payload):
"""
This is where a swimmer enters preferences
:param payload:
:return: form.html
"""
decrypted_payload = link_generator.load_payload(payload)
gala = decrypted_payload['gala_id']
swimmer_id = decrypted_payload['swimmer_id']
club_id = decrypted_payload['club_id']
session['swimmer'] = swimmer_id
with Mongo(db_config) as db:
gala = db.gala.find_one({"title": gala, "club.id": club_id})
swimmer = db.swimmers.find_one({
'swimmer_id': swimmer_id,
"club_id": club_id
})
gala = filter_heats(gala, swimmer)
return render_template("form.html", swimmer=swimmer, gala=gala)
def registration():
"""
Manual registration
:return: form.html|error
"""
if request.method == "GET":
return render_template("manual_entry_form.html")
swimmer_id = request.form['swimmer_id']
gala_id = request.form['gala_id']
club_id = session["clubid"]
invalid = {"value": False, "message": ""}
with Mongo() as db:
gala = db.gala.find_one({
'title': gala_id,
'club.id': club_id,
'deleted': False
})
swimmer = db.swimmers.find_one({
'swimmer_id': swimmer_id,
'club_id': club_id
})
if gala is None:
invalid["value"] = True
invalid["message"] += "Gala ID Not Found"
if swimmer is None:
invalid["value"] = True
if not invalid["message"] == "":
invalid["message"] += ", "
invalid["message"] += "Swimmer ID Not Found"
if invalid["value"]:
return render_template("manual_entry_form.html",
invalid=invalid,
swimmer_id=swimmer_id,
gala_id=gala_id)
else:
gala = filter_heats(gala, swimmer)
return render_template("form.html", swimmer=swimmer, gala=gala)
def add_headers(self):
"""
Add the records A,B,C
:return: Lines
"""
with Mongo() as db:
gala = db.gala.find_one({
'title': self.gala_id,
'club.id': self.club_id
})
self.output_lines.append(FileInformation(strftime("%m%d%Y")).as_line())
self.output_lines.append(
MeetInformation(
title=self.gala_id,
facility=gala['location'],
start_date=gala['date'].strftime("%m%d%Y"),
end_date=gala['end_date'].strftime("%m%d%Y"),
age_up_date=gala['age_up_date'].strftime("%m%d%Y")).as_line())
self.output_lines.append(
ClubInformation(self.club_id, gala['club']['name']).as_line())
def external_form(gala_id):
"""
Manual form entry for swimmers with no email
:param gala_id:
:return: print_forms.html
"""
with Mongo(db_config) as db:
gala = db.gala.find_one({
'title': gala_id,
"club.id": session['clubid']
})
swimmers = db.swimmers.find({
"email": None,
"club_id": session['clubid'],
"inactive": False
})
data = []
for swimmer in swimmers:
d = {"swimmer": swimmer, "gala": filter_heats(gala, swimmer)}
data.append(d)
return render_template("print_forms.html", data=data)
def get_entries(gala_id):
"""
Retrieve and display entries per gala/club id
:param gala_id:
:return: gala_entries.html|error
"""
with Mongo(db_config) as db:
entries = list(
db.entries.find({
"gala_id": gala_id,
"club_id": session["clubid"]
}))
gala = db.gala.find_one({
'title': gala_id,
"club.id": session["clubid"],
'deleted': False
})
swimmers = list(db.swimmers.find({"club_id": session["clubid"]}))
if gala:
def get_swimmer(swimmers, id):
for swimmer in swimmers:
if swimmer["swimmer_id"] == id:
return swimmer
return None
heats = []
for heat in gala["heats"]:
heat_with_swimmers = heat.copy()
heat_with_swimmers["swimmers"] = []
for entry in entries:
if heat in entry["heats"]:
swimmer = get_swimmer(swimmers, entry["swimmer_id"])
heat_with_swimmers["swimmers"].append(swimmer)
heat_with_swimmers["id"] = int(heat_with_swimmers["id"])
heats.append(heat_with_swimmers)
return render_template("gala_entries.html", gala=gala, heats=heats)
return "Invalid Gala Id"
import requests
from auction import Auction
from sql_mysql import inquire, TypeAuction
from lib.log import LogHandler
from lib.mongo import Mongo
from lxml import etree
import datetime
import yaml
import re
setting = yaml.load(open('config.yaml'))
client = Mongo(host=setting['mongo']['host'], port=setting['mongo']['port']).connect
coll = client[setting['mongo']['db']][setting['mongo']['collection']]
source = 'ali'
log = LogHandler(__name__)
s = requests.session()
class Ali:
def __init__(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 \
(KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36'
}
self.type_list = inquire(TypeAuction, source)
def start_crawler(self):
url = 'https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.13.pwypBc&city=&province=&\
auction_start_seg=-1'
try:
"""
青岛市拍卖中心
已经有id去重不请求详情页
"""
import requests
from lxml import etree
from auction import Auction
import datetime
import re
import yaml
from lib.mongo import Mongo
from lib.log import LogHandler
setting = yaml.load(open('config.yaml'))
client = Mongo(setting['mongo']['host'],
setting['mongo']['port'],
user_name=setting['mongo']['user_name'],
password=setting['mongo']['password']).connect
coll = client[setting['mongo']['db']][setting['mongo']['collection']]
source = '青岛市拍卖中心'
auction_type = '其他'
log = LogHandler(__name__)
class Qingdaopaimai(object):
def __init__(self):
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36'
}
self.start_url = 'http://auction.qdauction.com/items'
import requests
from lib.mongo import Mongo
from lxml import etree
import re
import datetime
import time
import random
from multiprocessing import Process
from proxy_connection import Proxy_contact
m = Mongo('192.168.0.235', 27017)
zhuge = m.connect['comm_price']
coll = zhuge['zhugefang']
class Zhugefang():
def __init__(self):
self.start_url = 'http://zhugefang.com'
self.headers = {
'User-Agent':
'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
# 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.146 Safari/537.36',
}
self.proxies = [
{
"http": "http://192.168.0.96:4234"
},
{
"http": "http://192.168.0.93:4234"
},
import json
import requests
import random
from fanggugu.login_fgg import Login
from lib.rabbitmq import Rabbit
from lib.mongo import Mongo
import datetime
import yaml
setting = yaml.load(open('config.yaml'))
# 连接 MongoDB
m = Mongo(setting['comm_price']['host'], setting['comm_price']['port'])
fgg = m.connect[setting['comm_price']['db']]
coll = fgg[setting['comm_price']['fgg_coll']]
coll_login = fgg[setting['fgg']['user_info']]
# 连接rabbit
r = Rabbit('192.168.0.235', 5673)
channel = r.get_channel()
IPS = ["192.168.0.90:4234",
"192.168.0.93:4234",
"192.168.0.94:4234",
"192.168.0.96:4234",
"192.168.0.98:4234",
"192.168.0.99:4234",
"192.168.0.100:4234",
"192.168.0.101:4234",
"192.168.0.102:4234",
"""
根据小区名称 和 城市找到 小区地址
"""
import requests
from lib.proxy_iterator import Proxies
from lib.mongo import Mongo
import pika
import json
from multiprocessing import Process
m = Mongo(host='114.80.150.196',
port=27777,
user_name='goojia',
password='******')
collection = m.connect['amap']['amap_road_clean']
p = Proxies()
proxies = p.get_one()
city_code = {
'绍兴': '330600',
'烟台': '370600',
'湖州': '330500',
'枣庄': '370400',
'丽水': '331100',
'金华': '330700',
'衢州': '330800',
'临沂': '371300',
'舟山': '330900',
'莱芜': '371200',
'威海': '371000',
import requests
import re
import yaml
from lib.mongo import Mongo
import time
setting = yaml.load(open('config.yaml'))
connect = Mongo(setting['old_county']['mongo']['host']).connect
coll = connect[setting['old_county']['mongo']['db']][setting['old_county']
['mongo']['collection']]
class Country:
def __init__(self):
self.url = 'https://cun.isoyu.com/laoxiangcun/PufXOt1NNtLrTj.html'
self.headers = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36',
'upgrade-insecure-requests':
'1',
'referer':
'https://cun.isoyu.com/laoxiangcun/QkNPPtLnGOxURNRJPuDNGEDqRj1EPuSGW0SnVREoH0pfIQSoVGNiHt.html',
'cookie':
'Hm_lvt_b564ae7a10584a875141e8357b5b20d1=1525313743,1525315892; Hm_lvt_b4b83be64d9a09729ed6dee742b57543=1525313743,1525315892; Hm_lpvt_b4b83be64d9a09729ed6dee742b57543=1525324259; Hm_lpvt_b564ae7a10584a875141e8357b5b20d1=1525324259',
}
def start(self):
time.sleep(0.1)
response = requests.get(self.url, headers=self.headers)
html = response.text
def __init__(self, bot):
self.bot = bot
self.db = Mongo.init_db(Mongo())
self.server_db = None
from lib.mongo import Mongo
setting = yaml.load(open('config.yaml'))
log = LogHandler('dianping_detail')
# rabbit
r = Rabbit(setting['dianping']['rabbit']['host'],
setting['dianping']['rabbit']['port'])
connection = r.connection
channel = connection.channel()
all_url_queue = setting['dianping']['rabbit']['queue']['all_url_queue']
channel.queue_declare(queue=all_url_queue)
# mongo
m = Mongo(setting['dianping']['mongo']['host'],
setting['dianping']['mongo']['port'])
coll = m.connect[setting['dianping']['mongo']['db']][setting['dianping']
['mongo']['save_coll']]
coll_html = m.connect[setting['dianping']['mongo']['db']]['html']
coll_update = m.connect['crawler_dianping']['shop_info_all_type']
kind_list = {
'ch10': '美食',
'ch30': '休闲娱乐',
'ch50': '丽人',
'ch35': '周边游',
'ch45': '运动健身',
'ch20': '购物',
'ch75': '学习培训',
'ch80': '生活服务',
'ch85': '医疗健康',