reload(sys)
sys.setdefaultencoding("utf-8")
sys.path.append(os.path.join(os.path.split(os.path.realpath(__file__))[0], '../../../util'))
sys.path.append(os.path.join(os.path.split(os.path.realpath(__file__))[0], '../../support'))
import loghelper
import config
import util
import proxy_pool
#logger
loghelper.init_logger("itunes_crawler", stream=True)
logger = loghelper.get_logger("itunes_crawler")
#mongo
(mongodb_host, mongodb_port) = config.get_mongodb_config()
mongo = MongoClient(mongodb_host, mongodb_port)
itunes_collection = mongo.crawler_v2.market_itunes
itunes_collection.create_index([("appId", pymongo.DESCENDING)], unique=True)
cates1 = [
{"name":"商务", "url":"https://itunes.apple.com/cn/genre/ios-shang-wu/id6000?mt=8"},
]
cates = [
{"name":"商务", "url":"https://itunes.apple.com/cn/genre/ios-shang-wu/id6000?mt=8"},
{"name":"商品指南", "url":"https://itunes.apple.com/cn/genre/ios-shang-pin-zhi-nan/id6022?mt=8"},
{"name":"教育", "url":"https://itunes.apple.com/cn/genre/ios-jiao-yu/id6017?mt=8"},
{"name":"娱乐", "url":"https://itunes.apple.com/cn/genre/ios-yu-le/id6016?mt=8"},
{"name":"财务", "url":"https://itunes.apple.com/cn/genre/ios-cai-wu/id6015?mt=8"},
from frappymongocontent import ContentStore
if __name__ == "__main__":
logger = Logger("MAIN", log_folder=get_log_folder())
logger.info("==========================================")
logger.info(" Starting application")
logger.info("==========================================")
# load config from .env file
config = load_config()
# ---- database and stores ----
# fetch mongo config
from config import get_mongodb_config
mongo_url, mongo_db = get_mongodb_config()
# initialise stores
stores = {
"user":
UserStore(mongo_url=mongo_url,
mongo_db=mongo_db,
collection_name="users"),
"user_tokens":
UserTokenStore(mongo_url=mongo_url,
mongo_db=mongo_db,
collection_name="userTokens"),
"data":
DataStore(mongo_url=mongo_url,
mongo_db=mongo_db,
collection_name="dataSets"),
