def init_start_urls(self): self.redis.delete(self.start_urls_redis_key) self.redis.delete(self.items_redis_key) buffer = [] buffer_size = 1024 with op.DBManger() as m: pipeline = [ { "$match": { "_status": 3 } }, ] data_set = collections.DataSet( m.read_from(db_collect=("jingdong", self.last_retry_collect), out_field=("_seed", "_status"), pipeline=pipeline)) should_exit = True for i, (seed, status) in enumerate(data_set.distinct()): should_exit = False seed = Seed(value=seed, type=3) buffer.append(str(seed)) if len(buffer) % buffer_size == 0: self.redis.sadd(self.start_urls_redis_key, *buffer) buffer = [] if buffer: self.redis.sadd(self.start_urls_redis_key, *buffer) if should_exit: import sys sys.exit(0)
def __init__(self, seeds_file, **kwargs): super(GetBrands, self).__init__(**kwargs) self.ua = UserAgent() with open(seeds_file) as infile: data_set = collections.DataSet(infile) for i, seed in enumerate(data_set.map(lambda line: line.strip('\n').split("\t")[0].replace('-', ',')) .shuffle(1024)): self.seeds_queue.put(Seed(seed, kwargs["retries"])) self.pattern = re.compile(r'<li id="brand-(\d+)[\s\S]*?品牌::([\s\S]*?)\'\)"')
def __init__(self, seeds_file, dateindex, **kwargs): super(GetComment, self).__init__(**kwargs) self.ua = UserAgent() with open(seeds_file) as infile: data_set = collections.DataSet(infile) for i, seed in enumerate( data_set.map(lambda line: line.strip('\n').split("\t")[0]). shuffle(2048)): self.seeds_queue.put(Seed(seed, kwargs["retries"])) self.allcnt_pattern = re.compile(r'"commentCount":(\d+),') self.dateindex = dateindex
def __init__(self, seeds_file, **kwargs): super(GetBrands1, self).__init__(**kwargs) self.proxies = list( map(lambda x: ("http://u{}:[email protected]:3128".format(x)), range(28))) self.ua = UserAgent() with open(seeds_file) as infile: data_set = collections.DataSet(infile) for i, seed in enumerate( data_set.map(lambda line: line.strip('\n').split("\t")[0]. replace('-', ',')).shuffle(1024)): self.seeds_queue.put(Seed(seed, kwargs["retries"])) self.pattern = re.compile(r'"id":.*?"name":".*?"')
def __init__(self, seeds_file, dateindex, **kwargs): super(GetComment1, self).__init__(**kwargs) self.proxies = list( map(lambda x: ("http://u{}:[email protected]:3128".format(x)), range(28))) self.ua = UserAgent() with open(seeds_file) as infile: data_set = collections.DataSet(infile) for i, seed in enumerate( data_set.map(lambda line: line.strip('\n').split("\t")[0]). shuffle(2048)): self.seeds_queue.put(Seed(seed, kwargs["retries"])) self.allcnt_pattern = re.compile(r'"CommentCount": "(\d+)"') self.dateindex = dateindex
def __init__(self, **kwargs): super(GetProductId, self).__init__(**kwargs) self.retries = 3 self.proxies = HttpProxy.getHttpProxy() self.ua = UserAgent() with op.DBManger() as m: last_brand_collect = m.get_lasted_collection( "jingdong", filter={"name": { "$regex": r"^brand20\d\d\d\d\d\d$" }}) pipeline = [{ "$match": { "cate_id": { "$ne": None } } }, { "$match": { "brand_id": { "$ne": None } } }, { "$match": { "name": { "$ne": None } } }, { "$match": { "_status": 0 } }] data_set = collections.DataSet( m.read_from(db_collect=("jingdong", last_brand_collect), out_field=("cate_id", "brand_id", "name"), pipeline=pipeline)) for i, seed in enumerate(data_set.distinct()): self.seeds_queue.put( Seed(value=seed, retries=self.retries, type=0)) self.first_pettern = re.compile(r"search000014_log:{wids:'([,\d]*?)',") self.skuids_pettern = re.compile(r'{.*?"skuId":(\d+).*?}') self.totalpage_perttern = re.compile( r'<div id="J_topPage"[\s\S]*?<b>\d+</b><em>/</em><i>(\d+)</i>')
#!/usr/bin/env python # -*- coding: utf-8 -*- from multiprocess.tools import timeUtil, collections from mongo import op print(timeUtil.getdate(0, format="%Y-%m%d")) print(timeUtil.current_time()) dt = collections.DataSet([1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 2, 3, 4, 5, 6]) for i in dt.shuffle( buffer_size=3).map(lambda x: x * 2).map(lambda x: x + 1).distinct(): print(i) with op.DBManger() as m: last_brand_collect = m.get_lasted_collection( "jingdong", filter={"name": { "$regex": r"^brand20\d\d\d\d\d\d$" }}) pipeline = [{ "$match": { "cate_id": { "$ne": None } } }, { "$match": { "brand_id": { "$ne": None } } }, { "$match": {