def process_item(self, item, spider): if item['title'] and item['video_link']: sql = "INSERT INTO `pornhub_videos`(`title`, `thumb`, `duration`, `video_link`, `video_link_480p`) " \ "VALUES(%s, %s, %s, %s, %s)" #sql = DB.connect().conn.cursor().mogrify(sql, (item['province'], item['city'], item['district'], item['path'], item['amount'], item['page_num'])) DB.connect().execute( sql, (item['title'], item['thumb'], item['duration'], item['video_link'], item['video_link_480p'])) return item else: raise DropItem('item由于不完整被丢弃')
class App: def __init__(self, config): self.config = config self.loop = asyncio.get_event_loop() self.ircHandle = IRCHandle(app=self) self.rconHandle = RconHandle(app=self) self.auth = Modlist(self) self.db = DB(app=self) self.iopool = ThreadPoolExecutor() self.dnsresolver = aiodns.DNSResolver(loop=self.loop) self.renx = Renx(self) async def runIrc(self): while True: await self.ircHandle.sock.run() async def runRcon(self): lastPupdate = time.time() while True: await self.rconHandle.sock.run() if time.time() - lastPupdate > 5: lastPupdate = time.time() asyncio.ensure_future(self.renx.periodic_playerList_update(), loop=self.loop) async def run(self): while True: await self.rconHandle.sock.run() async def getHost(self, ip): try: ip = ip_address(ip).reverse_pointer return (await self.dnsresolver.query(ip, 'PTR'))[0] except aiodns.error.DNSError as e: return '-' def start(self): self.loop.run_until_complete(self.ircHandle.sock.connect()) self.loop.run_until_complete(self.rconHandle.sock.connect()) self.loop.run_until_complete(self.db.connect()) asyncio.ensure_future(self.runIrc(), loop=self.loop) asyncio.ensure_future(self.runRcon(), loop=self.loop) try: self.loop.run_forever() except KeyboardInterrupt: exit()
import os import re from lib.db import DB root_path = os.path.realpath(os.path.dirname(__file__)) file_path = os.path.join(root_path, 'data', 'cnarea20160320-2.sql') # f = open(file_path, 'rb') # line.decode() f = open(file_path, 'r', encoding='utf8') for line in f: if line: column = line.replace('\n', '') column = re.sub('([^,"\d]+),([^,"]+)', '\\1|\\2', column) column = re.sub('([^,"\d]+),([^,"]+)', '\\1|\\2', column) column = re.sub('([^,"\d]+),([^,"]+)', '\\1|\\2', column) column = column.replace('"', '').split(',') if column: sql = 'INSERT INTO `china_area`(`id`, `parent_id`, `level`, `area_code`, `zip_code`, `city_code`, `name`, `short_name`, `merger_name`, `pinyin`, `longitude`, `latitude`) ' \ 'VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)' DB.connect().execute(sql, tuple(column)) else: print('ignored:' + line) f.close()