Python Queue 예제들

예제 #1

파일: get_tenant_food.py 프로젝트: czf1009/crawler_meituan

import config
import os
import time
from url_queue import Queue
from downloader import Downloader
import copy
import logging
import asyncio
import pandas as pd
from sqlalchemy import create_engine
import pymysql
pymysql.install_as_MySQLdb()
import get_tenant_info
from pymongo import MongoClient

queue_food = Queue()
downloader = Downloader(queue_food)

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('get_tenant_food')

info_url = 'http://i.waimai.meituan.com/wxi/ajax/v8/poi/food'
post_data = {}
headers = {
    'user-agent':
    'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
    'referer': 'http://i.waimai.meituan.com/home?lat=22.544102&lng=113.947104',
}
cookies = {}

dir_name = 'json_food'

예제 #2

import hashlib
import json
from connect_db import connect_mysql
import config
import os
import threading
import time
from url_queue import Queue
from downloader import Downloader
import copy
import logging
import asyncio
from geohash import encode
import re

queue = Queue()
downloader = Downloader(queue)

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('get_tenant_info')

info_url = 'http://i.waimai.meituan.com/wxi/ajax/v6/poi/info'
post_data = {}
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
    'referer': 'http://i.waimai.meituan.com/home?lat=22.544102&lng=113.947104',
}
cookies = {}

dir_name = 'json_info'

예제 #3

import hashlib
import json
from connect_db import connect_mysql
import config
import os
import threading
import time
from url_queue import Queue
from downloader import Downloader
import copy
import logging
import asyncio
from geohash import encode
import coord_transform

queue_info = Queue()
downloader = Downloader(queue_info)

logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger('get_tenant_info')

info_url = 'http://i.waimai.meituan.com/wxi/ajax/v6/poi/info'
post_data = {}
headers = {
    'user-agent':
    'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
    'referer': 'http://i.waimai.meituan.com/home?lat=22.544102&lng=113.947104',
}
cookies = {}

dir_name = 'json_info'