def __init__(self): self.ssdb_client = pyssdb.Client( get_project_settings().get('SSDB_HOST'), 8888) self.key = base64.b64decode(self.key) self.get_pdd_login_info() if not self.cookie: return False self.pdd_class = pddSign.pddSign()
def __init__(self, hash_num = 0, process_nums = 1): self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888) self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数 #创建连接池 pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'),port=6379,db=10,password=get_project_settings().get('PROXY_REDIS_AUTH'),decode_responses=True) #创建链接对象 self.redis_client=redis.Redis(connection_pool=pool)
def __init__(): c = pyssdb.Client('127.0.0.1', 8992) pool = redis.ConnectionPool( host='127.0.0.1', port=8990, db=2, password='******') r = redis.Redis(connection_pool=pool)
def __init__(self): self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888) self.key = base64.b64decode(self.key) pool = redis.ConnectionPool(host=get_project_settings().get('TOKEN_REDIS_HOST'), port='6379', db=10, password=get_project_settings().get("PROXY_REDIS_AUTH")) self.redis_client = redis.Redis(connection_pool=pool) self.get_pdd_login_info() self.pdd_class = pddSign.pddSign()
def __init__(self, hash_num=0, process_nums=1): self.today = datetime.date.today() self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数 self.list_name = self.mall_id_list + str(self.today) self.pageSize = 1000 ##每次抓取的产品数 最大只返回500 self.ssdb = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888)
def __init__(self, hash_num=0, process_nums=1): self.today = datetime.date.today() self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数 self.list_name = self.mall_id_list + str(self.today) self.pageSize = 1000 ##每次抓取的产品数 最大只返回500 self.ssdb = pyssdb.Client('172.16.0.5', 8888) self.pdd_class = pddSign.pddSign()
def __init__(self): self.ssdb = pyssdb.Client('172.16.0.5', 8888) self.ssdb.hclear('pdd_keywords_hash') self.ssdb.hclear('pdd_keywords_extend_hash') self.ssdb.hclear('pdd_mall_category_hash') self.get_pdd_login_info() if not self.cookie: return False
def __init__(self, hash_num=0, process_nums=1, save_logs_flag = False): self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888) self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数 #创建连接池 pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'),port=6379,db=10,password='******',decode_responses=True) #创建链接对象 self.redis_client=redis.Redis(connection_pool=pool) self.save_logs_flag = save_logs_flag self.anti_token_key = get_project_settings().get('PDD_ANTI_TOKEN_KEY')
def __init__(self, hash_num=0, process_nums=1): self.hash_num = int(hash_num) self.process_nums = int(process_nums) today = datetime.date.today() self.retry_list_name = self.retry_list_name + ':' + str(today) self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888) # 创建连接池 pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'), port=6379, db=10, password=get_project_settings().get('PROXY_REDIS_AUTH'), decode_responses=True) # 创建链接对象 self.redis_client = redis.Redis(connection_pool=pool)
def __init__(self, hash_num = 0, process_nums = 1): self.today = datetime.date.today() self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888) self.goods_list = self.goods_list + str(self.today) #创建连接池 pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'),port=6379,db=10,password='******',decode_responses=True) #创建链接对象 self.redis_client=redis.Redis(connection_pool=pool) self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数
def start_requests(self): ssdb_class = pyssdb.Client('172.16.0.5', 8888) hash_names = ['pdd_scroll_activity_v2'] for hash_name in hash_names: subject_data = ssdb_class.hgetall(hash_name) if subject_data: for subject_info in subject_data: subject_info = subject_info.decode('utf-8') # self.save_log(json.dumps({'subject_begin_info': subject_info})) try: subject_info = eval(subject_info) except Exception: subject_info = None if type(subject_info) == dict and subject_info["api_type"] in [51, 71, 61, 62, 63, 64, 31, 72]: self.save_log(json.dumps({'subject_get_info': subject_info})) subject_id = subject_info['subject_id'] url = "" meta = {"subject_info": subject_info} logging.debug(json.dumps({"meta": meta, "subject_id": subject_id, "api_type": subject_info["api_type"]})) if subject_info["api_type"] == 51: # 爱逛街 url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list?resource_type=7&page_size=400&tab_id=' + str( subject_id) elif subject_info["api_type"] == 61: # 首页轮播 url = 'https://mobile.yangkeduo.com/proxy/api/api/carnival/goods_list/list?group=cat_promotion' elif subject_info["api_type"] == 62: # 首页轮播 url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list?subject_id=' + str( subject_id) elif subject_info["api_type"] == 63: # 首页轮播 url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list?subject_id=' + str( subject_id) elif subject_info["api_type"] == 64: # 首页轮播 subject_id_str = subject_info["subject_str"] url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list_by_goods_ids?goods_ids=' + subject_id_str elif subject_info["api_type"] == 31: # 品牌馆 url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/15/resource_goods?tab_id=' + str( subject_id) elif subject_info["api_type"] == 71: # 首页商品信息 subject_id 为71 url = 'https://mobile.yangkeduo.com/proxy/api/api/alexa/v1/goods?' elif subject_info["api_type"] == 72: # 首页子活动信息 url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/brand_goods?size=400&page=2&resource_type=15&brand_id=' + str( subject_id) for page in range(1, 5): meta["page"] = page yield scrapy.Request(url=url + "&page=" + str(page) + "&page_size=500", headers=self.make_headers(), callback=self.parse, dont_filter=True, meta=meta)
def __init__(self): self.ssdb_client = pyssdb.Client( get_project_settings().get('SSDB_HOST'), 8888) pool = redis.ConnectionPool( host=get_project_settings().get('PROXY_REDIS_HOST'), port=6379, db=10, password='******', decode_responses=True) # 创建链接对象 self.redis_client = redis.Redis(connection_pool=pool)
def get_ssdb(): ssdb1 = pyssdb.Client('127.0.0.1', 8866) try: l = 'safe_act|10.18.0.1|change_mobile_send_daily' s = ssdb1.get(l) print(s) except Exception as e: print(e) finally: ssdb1.disconnect()
def ssdbConnection(self): try: config = self.ssdb_config.getConfig() result = pyssdb.Client(config['ip'], int(config['port'])) return result except TimeoutError as e: print(e) return False except TypeError as t: print(t) return False
def __init__(self, hash_num=0, process_nums=1, p_time=0): self.hash_num = int(hash_num) self.process_nums = int(process_nums) self.ssdb = pyssdb.Client('172.16.0.5', 8888) self.p_time = int(time.time()) if p_time == 0: self.p_time = int(time.time()) else: self.p_time = p_time self.redis = redis.Redis(host='172.16.0.27', port='6379', db=10, password='******')
def __init__(self, host, port): """ init :param host: ssdb host ip :param port: ssdb host port :return """ try: self.__conn = pyssdb.Client(host, int(port)) except Exception as e: self.__conn = None raise e
def create_client(self, try_re_connect=True): try: client = pyssdb.Client(self.host, self.port) #client.connect() self.clients.append(client) return client except socket.error as e: message = getattr(e, 'message', '') or getattr(e, 'strerror', '') if 'refused' in message: if try_re_connect: self.re_connect() return None except: return None
def pyssdb_test(): conn = pyssdb.Client(HOST, PORT) start_time = time() map(lambda x: conn.hset('cssdbpy',str(i), 1), [i for i in xrange(10000,90000)]) print 'hset', FIELDS/(time()-start_time) start_time = time() conn.hscan('cssdbpy','','', -1) print 'hscan', FIELDS/(time()-start_time) start_time = time() map(lambda x: conn.hget('cssdbpy',str(i)), [i for i in xrange(10000,90000)]) print 'hget', FIELDS/(time()-start_time) start_time = time() map(lambda x: conn.hdel('cssdbpy',str(i)), [i for i in xrange(10000,90000)]) print 'hdel', FIELDS/(time()-start_time)
def __init__(self, hash_num=0, process_nums=1): self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数 self.ssdb = pyssdb.Client('172.16.0.5', 8888) self.ssdb_client = pyssdb.Client( get_project_settings().get('SSDB_HOST'), 8888) self.key = base64.b64decode(self.key) pool = redis.ConnectionPool( host=get_project_settings().get('TOKEN_REDIS_HOST'), port='6379', db=10, password=get_project_settings().get("PROXY_REDIS_AUTH")) self.redis_client = redis.Redis(connection_pool=pool) # 创建连接池 pool_ip = redis.ConnectionPool( host=get_project_settings().get('PROXY_REDIS_HOST'), port=6379, db=10, password='******', decode_responses=True) # 创建链接对象 self.redis_client_ip = redis.Redis(connection_pool=pool_ip) self.get_pdd_login_info() self.pdd_class = pddSign.pddSign()
def __init__(self, hash_num=0, process_nums=1, p_time=0): self.hash_num = int(hash_num) self.process_nums = int(process_nums) self.ssdb_client = pyssdb.Client( get_project_settings().get('SSDB_HOST'), 8888) self.p_time = int(time.time()) if p_time == 0: self.p_time = int(time.time()) else: self.p_time = p_time self.redis = redis.Redis(host='172.16.0.27', port='6379', db=10, password='******')
def ssdbConnection(self): try: ip_addr = self.ssdb_config.getConfig() if self.master: #区别主从服务器 ip_addr = ip_addr['master'] else: ip_addr_dict = ip_addr['slave'] ip_addr = self.weightList(ip_addr_dict) ip_list = ip_addr.split(":") ssdb_resources = pyssdb.Client(ip_list[0], int(ip_list[1])) return ssdb_resources except TimeoutError as e: print(e) except TypeError as t: print(t)
def start_requests(self): '''先从ssdb获取首页下的分类信息''' ssdb_class = pyssdb.Client('172.16.0.5', 8888) hash_names = ['pdd_scroll_activity_v2'] for hash_name in hash_names: subject_data = ssdb_class.hgetall(hash_name) if subject_data: for subject_info in subject_data: subject_info = subject_info.decode('utf-8') self.save_log(json.dumps({'subject_begin_info': subject_info})) try: subject_info = eval(subject_info) except Exception: subject_info = None if type(subject_info) == dict and subject_info["api_type"] in [11, 14, 21, 31, 41]: self.save_log(json.dumps({'subject_get_info': subject_info})) subject_id = subject_info['subject_id'] url = "" meta = {"subject_info": subject_info} logging.debug(json.dumps({"meta": meta, "subject_id": subject_id, "api_type": subject_info["api_type"]})) if subject_info["api_type"] == 11 and subject_info["name"] == "ongoing": # 正在秒杀app url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v6/list/ongoing?' elif subject_info["api_type"] == 11 and subject_info["name"] == "future": # 即将秒杀app url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v4/list/future?' elif subject_info["api_type"] == 11 and subject_info["name"] == "more": # 明日预告秒杀活动app url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v4/list/more?' elif subject_info["api_type"] == 11 and subject_info["name"] == "brand_top": # 品牌秒杀app url = "https://mobile.yangkeduo.com/proxy/api/api/spike/v4/list/brand_top?time_type=101&list_id=WTKoKBDscx&page=1&size=20&pdduid=0" # 限制size elif subject_info["api_type"] == 14: # 品牌秒杀子活动 app url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v4/brand/tab_goods?spike_brand_id=' + str( subject_id) elif subject_info["api_type"] == 21: # 断码清仓 app url = 'https://mobile.yangkeduo.com/proxy/api/api/milk/clearance_sale/goods_list?tab_id=' + str( subject_id) elif subject_info["api_type"] == 31: # 品牌馆 app url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/15/resource_goods?tab_id=' + str( subject_id) elif subject_info["api_type"] == 41: # 9块9特卖 app接口需要登录 url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/2/resource_goods?tab_id=' + str( subject_id) for page in range(1, 500): meta["page"] = page yield scrapy.Request(url=url + "&page=" + str(page) + "&page_size=500", headers=self.make_headers(), callback=self.parse, dont_filter=True, meta=meta)
def start_requests(self): '''先从ssdb获取首页下的分类信息''' ssdb_class = pyssdb.Client('172.16.0.5', 8888) hash_names = ['pdd_goods_activity_1'] for hash_name in hash_names: subject_data = ssdb_class.hgetall(hash_name) if subject_data: for subject_info in subject_data: subject_info = json.loads(subject_info.decode('utf-8')) if type(subject_info) == dict: page = 1 headers = self.make_headers() meta = { 'subject_info': subject_info, 'page': page, 'goods_list': [] } if subject_info['type'] == 1: if subject_info['subject_id'] == 0: ## 首页跳过 continue ##获取活动基本信息里的产品列表 url = 'http://apiv3.yangkeduo.com/subject/' + str( subject_info['subject_id']) yield scrapy.Request( url, meta=meta, callback=self.parse_subject_info, headers=headers, dont_filter=True, errback=self.errback_httpbin) '''拉取活动下产品列表api''' page_size = 100 if subject_info[ 'type'] == 1 and subject_info[ 'subject_id'] == 0 else 500 meta['page_size'] = page_size url = self.get_activity_url(subject_info, page, page_size) #print(url) yield scrapy.Request(url, meta=meta, callback=self.parse, headers=headers, dont_filter=True, errback=self.errback_httpbin)
def __init__(self): self.ssdb_client = pyssdb.Client( get_project_settings().get('SSDB_HOST'), 8888) self.today = datetime.date.today() # 创建连接池 pool = redis.ConnectionPool( host=get_project_settings().get('PROXY_REDIS_HOST'), port=6379, db=10, password=get_project_settings().get('PROXY_REDIS_AUTH'), decode_responses=True) # 创建链接对象 self.redis_client = redis.Redis(connection_pool=pool) self.logUtil = LogUtil() self.platform = "pdd" self.appSecret = "C3v2jTXJtFWDbRqL1VxPoEOA0NHnf9zQGuiB" self.get_token_url = 'http://172.16.0.27:88/api/v1/proxy/getToken' self.report_token_url = 'http://172.16.0.27:88/api/v1/proxy/tokenReport'
def register(): error = None if request.method == 'POST': gdata = request.get_json(force=True) payload = eval(gdata.get('payload', {})) uname = payload.get('username') data = str(payload) import pyssdb c = pyssdb.Client() ret = c.hset('skillExchange', uname, data) if ret: return str(ret) else: error = 'Invalid username/password' return render_template('hello.html', error=error)
def __init__(self, hash_num=0, process_nums=1): self.today = datetime.date.today() self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数 self.list_name = self.mall_id_list + str(self.today) self.pageSize = 1000 ##每次抓取的产品数 最大只返回500 self.ssdb_client = pyssdb.Client( get_project_settings().get('SSDB_HOST'), 8888) #创建连接池 pool = redis.ConnectionPool( host=get_project_settings().get('PROXY_REDIS_HOST'), port=6379, db=10, password='******', decode_responses=True) #创建链接对象 self.redis_client = redis.Redis(connection_pool=pool) self.pdd_class = pddSign.pddSign()
def __init__(self, host, port, timeout=0, prefix='', **kw): Base.__init__(timeout) # Currently most ssdb clients have not been updated and have a lot of quirks # There are no clients for Python 3.4+ available that support all ssdb functions properly import warnings warnings.warn('SSDB cache is deprecated, avoid using', DeprecationWarning) self.prefix = prefix kw['host'] = host kw['port'] = port if 'socket_timeout' not in kw: kw['socket_timeout'] = _DEFAULT_SOCKET_TIMEOUT try: # This module don't support PY3.4+ import ssdb # Patch ssdb import datetime ssdb.connection.iteritems = six.iteritems def expire(self, name, ttl): if isinstance(ttl, datetime.timedelta): ttl = ttl.seconds + ttl.days * 24 * 3600 return self.execute_command('expire', name, ttl) ssdb.SSDB.expire = expire if 'socket_connect_timeout' not in kw: kw['socket_connect_timeout'] = _DEFAULT_SOCKET_TIMEOUT if 'socket_keepalive' not in kw: kw['socket_keepalive'] = 1 if 'socket_keepalive_options' not in kw: kw['socket_keepalive_options'] = _TCP_KEEP_ALIVE_OPTIONS connection_pool = _SsdbConnectionPool(ssdb.Connection(**kw)) self._client = ssdb.SSDB(connection_pool=connection_pool) except ImportError: try: # This module don't support socket keepalive yet, and do not support some features import pyssdb self._client = pyssdb.Client(**kw) except ImportError: raise RuntimeError('no ssdb module found')
def speed_test_ssdb(): db = pyssdb.Client() start_time = time.time() for i in range(200): #db[bytes(i)] = b'test' db.set(bytes([i]), b'testfgsdgsdfgdsfgdsfg' + bytes(i)) end_time = time.time() print('ssdb write took leveldb {} seconds'.format(end_time - start_time)) #leveldb read speed start_time = time.time() for i in range(200): #db[bytes(i)] = b'test' test = db.get(bytes([i])) end_time = time.time() print('ssdb read took leveldb {} seconds'.format(end_time - start_time))
def clear_ssdb(*mobile): ssdb = pyssdb.Client('127.0.0.1', 8866) try: l = [ 'safe_act|10.18.0.1|change_mobile_send_daily', 'safe_act|10.18.0.1|change_mobile_send_daily|warn', 'safe_act|10.18.0.1|login-fail', 'safe_act|10.18.0.1|login_ip', 'safe_act|10.18.0.1|login_ok', 'safe_act|10.18.0.1|register_mobile_send_daily', 'safe_act|10.18.0.1|register_mobile_send_daily|warn' ] for a in mobile: m_login = '******' + str(a) + '|login' m_login_fail = 'safe_act|' + str(a) + '|login-fail' m_login_name = 'safe_act|' + str(a) + '|login_name' m_login_ok = 'safe_act|' + str(a) + '|login_ok' l.append(m_login) l.append(m_login_fail) l.append(m_login_name) l.append(m_login_ok) for i in l: ssdb.set(i, 0) finally: ssdb.disconnect()
def __init__(self, hash_num=0, process_nums=1): self.ssdb = pyssdb.Client('172.16.0.5', 8888) self.hash_num = int(hash_num) ##当前脚本号 self.process_nums = int(process_nums) ##脚本总数