Example #1
0
 def process_item(self, item, spider):
     if item['title'] and item['video_link']:
         sql = "INSERT INTO `pornhub_videos`(`title`, `thumb`, `duration`, `video_link`, `video_link_480p`) " \
               "VALUES(%s, %s, %s, %s, %s)"
         #sql = DB.connect().conn.cursor().mogrify(sql, (item['province'], item['city'], item['district'], item['path'], item['amount'], item['page_num']))
         DB.connect().execute(
             sql, (item['title'], item['thumb'], item['duration'],
                   item['video_link'], item['video_link_480p']))
         return item
     else:
         raise DropItem('item由于不完整被丢弃')
Example #2
0
class App:
	def __init__(self, config):
		self.config = config
		self.loop = asyncio.get_event_loop()
		self.ircHandle = IRCHandle(app=self)
		self.rconHandle = RconHandle(app=self)
		self.auth = Modlist(self)
		self.db = DB(app=self)
		self.iopool = ThreadPoolExecutor()
		self.dnsresolver = aiodns.DNSResolver(loop=self.loop)
		self.renx = Renx(self)

	async def runIrc(self):
		while True:
			await self.ircHandle.sock.run()

	async def runRcon(self):
		lastPupdate = time.time()
		while True:
			await self.rconHandle.sock.run()
			if time.time() - lastPupdate > 5:
				lastPupdate = time.time()
				asyncio.ensure_future(self.renx.periodic_playerList_update(), loop=self.loop)

	async def run(self):
		while True:
			await self.rconHandle.sock.run()

	async def getHost(self, ip):
		try:
			ip = ip_address(ip).reverse_pointer
			return (await self.dnsresolver.query(ip, 'PTR'))[0]
		except aiodns.error.DNSError as e:
			return '-'


	def start(self):
		self.loop.run_until_complete(self.ircHandle.sock.connect())
		self.loop.run_until_complete(self.rconHandle.sock.connect())
		self.loop.run_until_complete(self.db.connect())
		asyncio.ensure_future(self.runIrc(), loop=self.loop)
		asyncio.ensure_future(self.runRcon(), loop=self.loop)
		try:
			self.loop.run_forever()
		except KeyboardInterrupt:
			exit()
Example #3
0
import os
import re

from lib.db import DB

root_path = os.path.realpath(os.path.dirname(__file__))

file_path = os.path.join(root_path, 'data', 'cnarea20160320-2.sql')

# f = open(file_path, 'rb')
# line.decode()
f = open(file_path, 'r', encoding='utf8')
for line in f:
    if line:
        column = line.replace('\n', '')
        column = re.sub('([^,"\d]+),([^,"]+)', '\\1|\\2', column)
        column = re.sub('([^,"\d]+),([^,"]+)', '\\1|\\2', column)
        column = re.sub('([^,"\d]+),([^,"]+)', '\\1|\\2', column)
        column = column.replace('"', '').split(',')
        if column:
            sql = 'INSERT INTO `china_area`(`id`, `parent_id`, `level`, `area_code`, `zip_code`, `city_code`, `name`, `short_name`, `merger_name`, `pinyin`, `longitude`, `latitude`) ' \
                  'VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)'

            DB.connect().execute(sql, tuple(column))
        else:
            print('ignored:' + line)
f.close()