reload(sys) sys.setdefaultencoding("utf-8") sys.path.append(os.path.join(os.path.split(os.path.realpath(__file__))[0], '../../../util')) sys.path.append(os.path.join(os.path.split(os.path.realpath(__file__))[0], '../../support')) import loghelper import config import util import proxy_pool #logger loghelper.init_logger("itunes_crawler", stream=True) logger = loghelper.get_logger("itunes_crawler") #mongo (mongodb_host, mongodb_port) = config.get_mongodb_config() mongo = MongoClient(mongodb_host, mongodb_port) itunes_collection = mongo.crawler_v2.market_itunes itunes_collection.create_index([("appId", pymongo.DESCENDING)], unique=True) cates1 = [ {"name":"商务", "url":"https://itunes.apple.com/cn/genre/ios-shang-wu/id6000?mt=8"}, ] cates = [ {"name":"商务", "url":"https://itunes.apple.com/cn/genre/ios-shang-wu/id6000?mt=8"}, {"name":"商品指南", "url":"https://itunes.apple.com/cn/genre/ios-shang-pin-zhi-nan/id6022?mt=8"}, {"name":"教育", "url":"https://itunes.apple.com/cn/genre/ios-jiao-yu/id6017?mt=8"}, {"name":"娱乐", "url":"https://itunes.apple.com/cn/genre/ios-yu-le/id6016?mt=8"}, {"name":"财务", "url":"https://itunes.apple.com/cn/genre/ios-cai-wu/id6015?mt=8"},
from frappymongocontent import ContentStore if __name__ == "__main__": logger = Logger("MAIN", log_folder=get_log_folder()) logger.info("==========================================") logger.info(" Starting application") logger.info("==========================================") # load config from .env file config = load_config() # ---- database and stores ---- # fetch mongo config from config import get_mongodb_config mongo_url, mongo_db = get_mongodb_config() # initialise stores stores = { "user": UserStore(mongo_url=mongo_url, mongo_db=mongo_db, collection_name="users"), "user_tokens": UserTokenStore(mongo_url=mongo_url, mongo_db=mongo_db, collection_name="userTokens"), "data": DataStore(mongo_url=mongo_url, mongo_db=mongo_db, collection_name="dataSets"),