def __init__(self):
self.ssdb_client = pyssdb.Client(
get_project_settings().get('SSDB_HOST'), 8888)
self.key = base64.b64decode(self.key)
self.get_pdd_login_info()
if not self.cookie:
return False
self.pdd_class = pddSign.pddSign()
def __init__(self, hash_num = 0, process_nums = 1):
self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888)
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
#创建连接池
pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'),port=6379,db=10,password=get_project_settings().get('PROXY_REDIS_AUTH'),decode_responses=True)
#创建链接对象
self.redis_client=redis.Redis(connection_pool=pool)
def __init__():
c = pyssdb.Client('127.0.0.1', 8992)
pool = redis.ConnectionPool(
host='127.0.0.1',
port=8990,
db=2,
password='******')
r = redis.Redis(connection_pool=pool)
def __init__(self):
self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888)
self.key = base64.b64decode(self.key)
pool = redis.ConnectionPool(host=get_project_settings().get('TOKEN_REDIS_HOST'), port='6379', db=10,
password=get_project_settings().get("PROXY_REDIS_AUTH"))
self.redis_client = redis.Redis(connection_pool=pool)
self.get_pdd_login_info()
self.pdd_class = pddSign.pddSign()
def __init__(self, hash_num=0, process_nums=1):
self.today = datetime.date.today()
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
self.list_name = self.mall_id_list + str(self.today)
self.pageSize = 1000 ##每次抓取的产品数 最大只返回500
self.ssdb = pyssdb.Client(get_project_settings().get('SSDB_HOST'),
8888)
def __init__(self, hash_num=0, process_nums=1):
self.today = datetime.date.today()
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
self.list_name = self.mall_id_list + str(self.today)
self.pageSize = 1000 ##每次抓取的产品数 最大只返回500
self.ssdb = pyssdb.Client('172.16.0.5', 8888)
self.pdd_class = pddSign.pddSign()
def __init__(self):
self.ssdb = pyssdb.Client('172.16.0.5', 8888)
self.ssdb.hclear('pdd_keywords_hash')
self.ssdb.hclear('pdd_keywords_extend_hash')
self.ssdb.hclear('pdd_mall_category_hash')
self.get_pdd_login_info()
if not self.cookie:
return False
def __init__(self, hash_num=0, process_nums=1, save_logs_flag = False):
self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888)
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
#创建连接池
pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'),port=6379,db=10,password='******',decode_responses=True)
#创建链接对象
self.redis_client=redis.Redis(connection_pool=pool)
self.save_logs_flag = save_logs_flag
self.anti_token_key = get_project_settings().get('PDD_ANTI_TOKEN_KEY')
def __init__(self, hash_num=0, process_nums=1):
self.hash_num = int(hash_num)
self.process_nums = int(process_nums)
today = datetime.date.today()
self.retry_list_name = self.retry_list_name + ':' + str(today)
self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888)
# 创建连接池
pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'), port=6379, db=10, password=get_project_settings().get('PROXY_REDIS_AUTH'), decode_responses=True)
# 创建链接对象
self.redis_client = redis.Redis(connection_pool=pool)
def __init__(self, hash_num = 0, process_nums = 1):
self.today = datetime.date.today()
self.ssdb_client = pyssdb.Client(get_project_settings().get('SSDB_HOST'), 8888)
self.goods_list = self.goods_list + str(self.today)
#创建连接池
pool = redis.ConnectionPool(host=get_project_settings().get('PROXY_REDIS_HOST'),port=6379,db=10,password='******',decode_responses=True)
#创建链接对象
self.redis_client=redis.Redis(connection_pool=pool)
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
def start_requests(self):
ssdb_class = pyssdb.Client('172.16.0.5', 8888)
hash_names = ['pdd_scroll_activity_v2']
for hash_name in hash_names:
subject_data = ssdb_class.hgetall(hash_name)
if subject_data:
for subject_info in subject_data:
subject_info = subject_info.decode('utf-8')
# self.save_log(json.dumps({'subject_begin_info': subject_info}))
try:
subject_info = eval(subject_info)
except Exception:
subject_info = None
if type(subject_info) == dict and subject_info["api_type"] in [51, 71, 61, 62, 63, 64, 31, 72]:
self.save_log(json.dumps({'subject_get_info': subject_info}))
subject_id = subject_info['subject_id']
url = ""
meta = {"subject_info": subject_info}
logging.debug(json.dumps({"meta": meta, "subject_id": subject_id, "api_type": subject_info["api_type"]}))
if subject_info["api_type"] == 51: # 爱逛街
url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list?resource_type=7&page_size=400&tab_id=' + str(
subject_id)
elif subject_info["api_type"] == 61: # 首页轮播
url = 'https://mobile.yangkeduo.com/proxy/api/api/carnival/goods_list/list?group=cat_promotion'
elif subject_info["api_type"] == 62: # 首页轮播
url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list?subject_id=' + str(
subject_id)
elif subject_info["api_type"] == 63: # 首页轮播
url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list?subject_id=' + str(
subject_id)
elif subject_info["api_type"] == 64: # 首页轮播
subject_id_str = subject_info["subject_str"]
url = 'https://mobile.yangkeduo.com/proxy/api/api/george/resource_goods/query_list_by_goods_ids?goods_ids=' + subject_id_str
elif subject_info["api_type"] == 31: # 品牌馆
url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/15/resource_goods?tab_id=' + str(
subject_id)
elif subject_info["api_type"] == 71: # 首页商品信息 subject_id 为71
url = 'https://mobile.yangkeduo.com/proxy/api/api/alexa/v1/goods?'
elif subject_info["api_type"] == 72: # 首页子活动信息
url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/brand_goods?size=400&page=2&resource_type=15&brand_id=' + str(
subject_id)
for page in range(1, 5):
meta["page"] = page
yield scrapy.Request(url=url + "&page=" + str(page) + "&page_size=500",
headers=self.make_headers(), callback=self.parse,
dont_filter=True, meta=meta)
def __init__(self):
self.ssdb_client = pyssdb.Client(
get_project_settings().get('SSDB_HOST'), 8888)
pool = redis.ConnectionPool(
host=get_project_settings().get('PROXY_REDIS_HOST'),
port=6379,
db=10,
password='******',
decode_responses=True)
# 创建链接对象
self.redis_client = redis.Redis(connection_pool=pool)
def get_ssdb():
ssdb1 = pyssdb.Client('127.0.0.1', 8866)
try:
l = 'safe_act|10.18.0.1|change_mobile_send_daily'
s = ssdb1.get(l)
print(s)
except Exception as e:
print(e)
finally:
ssdb1.disconnect()
def ssdbConnection(self):
try:
config = self.ssdb_config.getConfig()
result = pyssdb.Client(config['ip'], int(config['port']))
return result
except TimeoutError as e:
print(e)
return False
except TypeError as t:
print(t)
return False
def __init__(self, hash_num=0, process_nums=1, p_time=0):
self.hash_num = int(hash_num)
self.process_nums = int(process_nums)
self.ssdb = pyssdb.Client('172.16.0.5', 8888)
self.p_time = int(time.time())
if p_time == 0:
self.p_time = int(time.time())
else:
self.p_time = p_time
self.redis = redis.Redis(host='172.16.0.27', port='6379', db=10, password='******')
def __init__(self, host, port):
"""
init
:param host: ssdb host ip
:param port: ssdb host port
:return
"""
try:
self.__conn = pyssdb.Client(host, int(port))
except Exception as e:
self.__conn = None
raise e
def create_client(self, try_re_connect=True):
try:
client = pyssdb.Client(self.host, self.port)
#client.connect()
self.clients.append(client)
return client
except socket.error as e:
message = getattr(e, 'message', '') or getattr(e, 'strerror', '')
if 'refused' in message:
if try_re_connect:
self.re_connect()
return None
except:
return None
def pyssdb_test():
conn = pyssdb.Client(HOST, PORT)
start_time = time()
map(lambda x: conn.hset('cssdbpy',str(i), 1), [i for i in xrange(10000,90000)])
print 'hset', FIELDS/(time()-start_time)
start_time = time()
conn.hscan('cssdbpy','','', -1)
print 'hscan', FIELDS/(time()-start_time)
start_time = time()
map(lambda x: conn.hget('cssdbpy',str(i)), [i for i in xrange(10000,90000)])
print 'hget', FIELDS/(time()-start_time)
start_time = time()
map(lambda x: conn.hdel('cssdbpy',str(i)), [i for i in xrange(10000,90000)])
print 'hdel', FIELDS/(time()-start_time)
def __init__(self, hash_num=0, process_nums=1):
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
self.ssdb = pyssdb.Client('172.16.0.5', 8888)
self.ssdb_client = pyssdb.Client(
get_project_settings().get('SSDB_HOST'), 8888)
self.key = base64.b64decode(self.key)
pool = redis.ConnectionPool(
host=get_project_settings().get('TOKEN_REDIS_HOST'),
port='6379',
db=10,
password=get_project_settings().get("PROXY_REDIS_AUTH"))
self.redis_client = redis.Redis(connection_pool=pool)
# 创建连接池
pool_ip = redis.ConnectionPool(
host=get_project_settings().get('PROXY_REDIS_HOST'),
port=6379,
db=10,
password='******',
decode_responses=True)
# 创建链接对象
self.redis_client_ip = redis.Redis(connection_pool=pool_ip)
self.get_pdd_login_info()
self.pdd_class = pddSign.pddSign()
def __init__(self, hash_num=0, process_nums=1, p_time=0):
self.hash_num = int(hash_num)
self.process_nums = int(process_nums)
self.ssdb_client = pyssdb.Client(
get_project_settings().get('SSDB_HOST'), 8888)
self.p_time = int(time.time())
if p_time == 0:
self.p_time = int(time.time())
else:
self.p_time = p_time
self.redis = redis.Redis(host='172.16.0.27',
port='6379',
db=10,
password='******')
def ssdbConnection(self):
try:
ip_addr = self.ssdb_config.getConfig()
if self.master: #区别主从服务器
ip_addr = ip_addr['master']
else:
ip_addr_dict = ip_addr['slave']
ip_addr = self.weightList(ip_addr_dict)
ip_list = ip_addr.split(":")
ssdb_resources = pyssdb.Client(ip_list[0], int(ip_list[1]))
return ssdb_resources
except TimeoutError as e:
print(e)
except TypeError as t:
print(t)
def start_requests(self):
'''先从ssdb获取首页下的分类信息'''
ssdb_class = pyssdb.Client('172.16.0.5', 8888)
hash_names = ['pdd_scroll_activity_v2']
for hash_name in hash_names:
subject_data = ssdb_class.hgetall(hash_name)
if subject_data:
for subject_info in subject_data:
subject_info = subject_info.decode('utf-8')
self.save_log(json.dumps({'subject_begin_info': subject_info}))
try:
subject_info = eval(subject_info)
except Exception:
subject_info = None
if type(subject_info) == dict and subject_info["api_type"] in [11, 14, 21, 31, 41]:
self.save_log(json.dumps({'subject_get_info': subject_info}))
subject_id = subject_info['subject_id']
url = ""
meta = {"subject_info": subject_info}
logging.debug(json.dumps({"meta": meta, "subject_id": subject_id, "api_type": subject_info["api_type"]}))
if subject_info["api_type"] == 11 and subject_info["name"] == "ongoing": # 正在秒杀app
url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v6/list/ongoing?'
elif subject_info["api_type"] == 11 and subject_info["name"] == "future": # 即将秒杀app
url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v4/list/future?'
elif subject_info["api_type"] == 11 and subject_info["name"] == "more": # 明日预告秒杀活动app
url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v4/list/more?'
elif subject_info["api_type"] == 11 and subject_info["name"] == "brand_top": # 品牌秒杀app
url = "https://mobile.yangkeduo.com/proxy/api/api/spike/v4/list/brand_top?time_type=101&list_id=WTKoKBDscx&page=1&size=20&pdduid=0" # 限制size
elif subject_info["api_type"] == 14: # 品牌秒杀子活动 app
url = 'https://mobile.yangkeduo.com/proxy/api/api/spike/v4/brand/tab_goods?spike_brand_id=' + str(
subject_id)
elif subject_info["api_type"] == 21: # 断码清仓 app
url = 'https://mobile.yangkeduo.com/proxy/api/api/milk/clearance_sale/goods_list?tab_id=' + str(
subject_id)
elif subject_info["api_type"] == 31: # 品牌馆 app
url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/15/resource_goods?tab_id=' + str(
subject_id)
elif subject_info["api_type"] == 41: # 9块9特卖 app接口需要登录
url = 'https://mobile.yangkeduo.com/proxy/api/api/gentian/2/resource_goods?tab_id=' + str(
subject_id)
for page in range(1, 500):
meta["page"] = page
yield scrapy.Request(url=url + "&page=" + str(page) + "&page_size=500",
headers=self.make_headers(), callback=self.parse,
dont_filter=True, meta=meta)
def start_requests(self):
'''先从ssdb获取首页下的分类信息'''
ssdb_class = pyssdb.Client('172.16.0.5', 8888)
hash_names = ['pdd_goods_activity_1']
for hash_name in hash_names:
subject_data = ssdb_class.hgetall(hash_name)
if subject_data:
for subject_info in subject_data:
subject_info = json.loads(subject_info.decode('utf-8'))
if type(subject_info) == dict:
page = 1
headers = self.make_headers()
meta = {
'subject_info': subject_info,
'page': page,
'goods_list': []
}
if subject_info['type'] == 1:
if subject_info['subject_id'] == 0: ## 首页跳过
continue
##获取活动基本信息里的产品列表
url = 'http://apiv3.yangkeduo.com/subject/' + str(
subject_info['subject_id'])
yield scrapy.Request(
url,
meta=meta,
callback=self.parse_subject_info,
headers=headers,
dont_filter=True,
errback=self.errback_httpbin)
'''拉取活动下产品列表api'''
page_size = 100 if subject_info[
'type'] == 1 and subject_info[
'subject_id'] == 0 else 500
meta['page_size'] = page_size
url = self.get_activity_url(subject_info, page,
page_size)
#print(url)
yield scrapy.Request(url,
meta=meta,
callback=self.parse,
headers=headers,
dont_filter=True,
errback=self.errback_httpbin)
def __init__(self):
self.ssdb_client = pyssdb.Client(
get_project_settings().get('SSDB_HOST'), 8888)
self.today = datetime.date.today()
# 创建连接池
pool = redis.ConnectionPool(
host=get_project_settings().get('PROXY_REDIS_HOST'),
port=6379,
db=10,
password=get_project_settings().get('PROXY_REDIS_AUTH'),
decode_responses=True)
# 创建链接对象
self.redis_client = redis.Redis(connection_pool=pool)
self.logUtil = LogUtil()
self.platform = "pdd"
self.appSecret = "C3v2jTXJtFWDbRqL1VxPoEOA0NHnf9zQGuiB"
self.get_token_url = 'http://172.16.0.27:88/api/v1/proxy/getToken'
self.report_token_url = 'http://172.16.0.27:88/api/v1/proxy/tokenReport'
def register():
error = None
if request.method == 'POST':
gdata = request.get_json(force=True)
payload = eval(gdata.get('payload', {}))
uname = payload.get('username')
data = str(payload)
import pyssdb
c = pyssdb.Client()
ret = c.hset('skillExchange', uname, data)
if ret:
return str(ret)
else:
error = 'Invalid username/password'
return render_template('hello.html', error=error)
def __init__(self, hash_num=0, process_nums=1):
self.today = datetime.date.today()
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
self.list_name = self.mall_id_list + str(self.today)
self.pageSize = 1000 ##每次抓取的产品数 最大只返回500
self.ssdb_client = pyssdb.Client(
get_project_settings().get('SSDB_HOST'), 8888)
#创建连接池
pool = redis.ConnectionPool(
host=get_project_settings().get('PROXY_REDIS_HOST'),
port=6379,
db=10,
password='******',
decode_responses=True)
#创建链接对象
self.redis_client = redis.Redis(connection_pool=pool)
self.pdd_class = pddSign.pddSign()
def __init__(self, host, port, timeout=0, prefix='', **kw):
Base.__init__(timeout)
# Currently most ssdb clients have not been updated and have a lot of quirks
# There are no clients for Python 3.4+ available that support all ssdb functions properly
import warnings
warnings.warn('SSDB cache is deprecated, avoid using',
DeprecationWarning)
self.prefix = prefix
kw['host'] = host
kw['port'] = port
if 'socket_timeout' not in kw:
kw['socket_timeout'] = _DEFAULT_SOCKET_TIMEOUT
try:
# This module don't support PY3.4+
import ssdb
# Patch ssdb
import datetime
ssdb.connection.iteritems = six.iteritems
def expire(self, name, ttl):
if isinstance(ttl, datetime.timedelta):
ttl = ttl.seconds + ttl.days * 24 * 3600
return self.execute_command('expire', name, ttl)
ssdb.SSDB.expire = expire
if 'socket_connect_timeout' not in kw:
kw['socket_connect_timeout'] = _DEFAULT_SOCKET_TIMEOUT
if 'socket_keepalive' not in kw:
kw['socket_keepalive'] = 1
if 'socket_keepalive_options' not in kw:
kw['socket_keepalive_options'] = _TCP_KEEP_ALIVE_OPTIONS
connection_pool = _SsdbConnectionPool(ssdb.Connection(**kw))
self._client = ssdb.SSDB(connection_pool=connection_pool)
except ImportError:
try:
# This module don't support socket keepalive yet, and do not support some features
import pyssdb
self._client = pyssdb.Client(**kw)
except ImportError:
raise RuntimeError('no ssdb module found')
def speed_test_ssdb():
db = pyssdb.Client()
start_time = time.time()
for i in range(200):
#db[bytes(i)] = b'test'
db.set(bytes([i]), b'testfgsdgsdfgdsfgdsfg' + bytes(i))
end_time = time.time()
print('ssdb write took leveldb {} seconds'.format(end_time - start_time))
#leveldb read speed
start_time = time.time()
for i in range(200):
#db[bytes(i)] = b'test'
test = db.get(bytes([i]))
end_time = time.time()
print('ssdb read took leveldb {} seconds'.format(end_time - start_time))
def clear_ssdb(*mobile):
ssdb = pyssdb.Client('127.0.0.1', 8866)
try:
l = [
'safe_act|10.18.0.1|change_mobile_send_daily',
'safe_act|10.18.0.1|change_mobile_send_daily|warn',
'safe_act|10.18.0.1|login-fail', 'safe_act|10.18.0.1|login_ip',
'safe_act|10.18.0.1|login_ok',
'safe_act|10.18.0.1|register_mobile_send_daily',
'safe_act|10.18.0.1|register_mobile_send_daily|warn'
]
for a in mobile:
m_login = '******' + str(a) + '|login'
m_login_fail = 'safe_act|' + str(a) + '|login-fail'
m_login_name = 'safe_act|' + str(a) + '|login_name'
m_login_ok = 'safe_act|' + str(a) + '|login_ok'
l.append(m_login)
l.append(m_login_fail)
l.append(m_login_name)
l.append(m_login_ok)
for i in l:
ssdb.set(i, 0)
finally:
ssdb.disconnect()
def __init__(self, hash_num=0, process_nums=1):
self.ssdb = pyssdb.Client('172.16.0.5', 8888)
self.hash_num = int(hash_num) ##当前脚本号
self.process_nums = int(process_nums) ##脚本总数
