async def post(self):
id = self.get_argument('article_id', None)
raw_text = self.get_argument("text")
text = self.get_escape_argument('text')
title = self.get_escape_argument('title')
html = markdown.markdown(raw_text,
extensions=[
'markdown.extensions.toc',
'markdown.extensions.fenced_code'
])
html = aiomysql.escape_string(html)
print(html)
if id != 'None':
sql = "select count(*) as count from article where id='{}'".format(
id)
result = await self.query(sql)
if result[0]['count'] > 0:
sql = "update article set title='{}',text='{}',html='{}',updated='{}' where id={}".format(
title, text, html, self.get_format_time(), id)
await self.execute(sql)
else:
sql = "insert into article(`title`,`text`,`html`,`published`,`updated`,`is_published`) values('{}','{}','{}','{}','{}','{}')".format(
title, text, html, self.get_format_time(),
self.get_format_time(), '1')
print(sql)
await self.execute(sql)
self.redirect('/')
def safestr(val):
cls = type(val)
if cls is str:
val = aiomysql.escape_string(val)
elif cls is dict:
val = aiomysql.escape_dict(val)
else:
val = str(val)
return val
async def get_metainfo(self, infohash, addr):
fail = False
async with self.fetch_metainfo_semaphore:
filename = '{}{}{}.torrent'.format(cfg.get('torrent', 'save_path'), os.sep, infohash.lower())
if len(glob.glob(filename)) == 0:
try:
metainfo = await asyncio.wait_for(
get_metadata(
infohash, addr[0], addr[1], loop=self.loop
),
timeout=self.interval * 20)
except:
fail = True
if fail or (isinstance(metainfo, bool) and metainfo is False):
async with self.database_semaphore:
async with self.connection_pool.acquire() as connect:
cursor = await connect.cursor()
await cursor.execute(base_sql.remove_from_announce_queue.format(info_hash=infohash))
await connect.commit()
await cursor.close()
return
if metainfo is not None:
# hash error
if infohash != get_meta_hash(metainfo):
return
name = get_filename(metainfo)
size = get_file_size(metainfo)
logging.info(
'Hash: {}. Name: {}. Size: {}'.format(
infohash, name, size
)
)
file_content = bencoder.bencode({b'info': metainfo})
async with aiofiles.open(filename, mode='wb') as f:
await f.write(file_content)
async with self.database_semaphore:
async with self.connection_pool.acquire() as connect:
cursor = await connect.cursor()
try:
await cursor.execute(base_sql.insert_into_torrent.format(
name=aiomysql.escape_string(name),
info_hash=infohash,
size=size))
except pymysql.err.IntegrityError:
await cursor.rollback()
finally:
await cursor.execute(base_sql.remove_from_announce_queue.format(info_hash=infohash))
await connect.commit()
await cursor.close()
async def handler(self, info):
# 这里建议写成连接池，每次都连接很没效率
conn = await aiomysql.connect(
db='dht',
user='******',
password='******',
host='127.0.0.1',
charset='utf8',
unix_socket='/var/run/mysqld/mysqld.sock')
cur = await conn.cursor()
info_hash = info['info_hash'].lower()
if info['q_type'] == 'get_peers':
sql = 'Select 1 From `get_peers_%s` Where `info_hash`="%s";' % (
info_hash[:1], info_hash)
await cur.execute(sql)
ret = await cur.fetchone()
if ret is None or len(ret) < 1:
sql = 'Insert Into `get_peers_%s` (`info_hash`) Values ("%s");' % (
info_hash[:1], info_hash)
await cur.execute(sql)
await conn.commit()
if info['q_type'] == 'announce_peer':
sql = 'Select 1 From `announce_peer_%s` Where `info_hash`="%s" And `ip`="%s" And `port`="%s";' % (
info_hash[:1], info_hash, info['ip'], info['port'])
await cur.execute(sql)
ret = await cur.fetchone()
if ret is None or len(ret) < 1:
sql = 'Insert Into `announce_peer_%s` (`info_hash`,`ip`,`port`,`name`) Values ("%s","%s","%s","%s");' % (
info_hash[:1], info_hash, info['ip'], info['port'],
aiomysql.escape_string(info['name']))
await cur.execute(sql)
await conn.commit()
logging.info(info_hash)
await cur.close()
conn.close()
def escape_string(self, name):
name = aiomysql.escape_string(name)
return '' if name is None else name
async def get(self, loop, url):
pool = await aiomysql.create_pool(host='47.97.166.98',
port=3306,
user='******',
password='******',
db='yzz',
loop=loop,
autocommit=True,
charset='utf8')
async with aiohttp.ClientSession() as self.session:
content = await self.par(url) # 跳到par()
html = etree.HTML(content)
goods_name = html.xpath(
self.rules['goods_name'])[0].strip().replace('/', '').replace(
'*', '_') # 商品名
goods_price = html.xpath(
self.rules['goods_price'])[0].strip() # 商品价格
# 提取规格，几乎完全tmd不用改，2020年3月18日15:41:23
info = re.findall(r'var meta = ([\s\S]*?);', content)[0]
sku_ = re.findall(r'"public_title":"(.*?)"', info)
s_id_ = re.findall(r'"sku":"(.*?)"', info)
s_price_ = re.findall(r'"price":(\d+)', info)
specs_ = dict()
all_specs = []
for sku, s_id, s_price in zip(sku_, s_id_, s_price_):
# print(sku, s_id, s_price)
specs = dict()
specs["sku"] = sku.replace('\\', '')
specs["s_id"] = s_id
specs["s_price"] = int(s_price) / 100
all_specs.append(specs)
specs_["specs"] = all_specs
# 商品属性
goods_norms = html.xpath(self.rules['goods_norms'])
goods_value = html.xpath(self.rules['goods_value'])
norms = dict() # 这个是json， ok
for n, v in zip(goods_norms, goods_value):
norms[n.strip()] = v.strip()
# 图片链接
links = dict()
link_list = html.xpath(self.rules['img_links'])
links["links"] = link_list
data = dict()
data["tittle"] = goods_name
data["price"] = goods_price
data["domain"] = self.domain
data["link"] = url
data["sku"] = json.dumps(specs_)
data["specs"] = json.dumps(norms)
data["img_link"] = json.dumps(links)
# print(data)
# 整一个异步连接池，使用异步插入
async with pool.acquire() as conn:
async with conn.cursor() as cur:
insert = "INSERT IGNORE INTO shopify(tittle,price,domain,link,sku,specs,img_link) VALUES " \
"(\'%s\','%s','%s','%s',\'%s\',\'%s\',\'%s\')" % \
(aiomysql.escape_string(data["tittle"]), data["price"], data["domain"], data['link'],
aiomysql.escape_string(data["sku"]), aiomysql.escape_string(data["specs"]),
aiomysql.escape_string(data["img_link"]))
# print(insert)
await cur.execute(insert)
def escape_string(name):
return '' if not name else aiomysql.escape_string(name)
async def escape_string(self, strs):
return '' if not strs else aiomysql.escape_string(strs)
def get_escape_string(self, q):
return aiomysql.escape_string(q) if q else ""
def get_escape_argument(self, q):
value = self.get_argument(q, None)
return aiomysql.escape_string(value) if value else value