def iter_file_user_trade_list(user=0):
if not user:
from syncer import sync
sync_get_user = U.get_or_set('sync_get_user',
sync(taobao_trade.get_user))
user = sync_get_user()
print('sync_get_user :'******'C:\test\taobao_trade'):
if user not in f:
continue
b = F.read_byte(f)
d = T.json_loads(b)
assert d['user'] == user
for n, t in enumerate(d['ts']):
e = T.xpath(t, "//a[contains(., '查看物流')]")
if len(e) != 1:
t = T.html2text(t)
t = T.replace_all_space(t)[:99]
u = py.No(t)
else:
e = e[0]
u = convert_wuliu_url(e.attrib['href'])
row = [d['user'], d['n'], n, u]
v.append(row)
if not v:
return py.No('can not found user: %s trade_list' % user)
U.set(user, v)
return F.dill_dump(obj=v, file=f'{user}-{len(v)}-trade_list')
async def get_current_front_page():
pv = [(pa, await pa.evaluate("document.visibilityState"))
for pa in await taobao_trade.get_all_pages()]
r = []
for pa, v in pv:
if pa.url.startswith('devtools://'):
continue
if v == 'visible':
r.append(pa)
if not r:
return py.No('no visible page?')
if py.len(r) > 1:
return py.No('multi visible page?', pv, r)
return r[0]
def eng_audio(response, word):
f = 'google_translate_tts/%s.dill' % word
F.mkdir(U.gst + F.dir(f))
q = F.dill_load(f)
if not q:
q = F.dill_load('google_translate_tts/%s.dill' %
U.StrRepr(word, size=15))
if not q:
if N.check_port(21080):
proxy = 'socks5://127.0.0.1:21080'
else:
proxy = None
try:
u = 'https://translate.googleapis.com/translate_tts?client=gtx&ie=UTF-8&tl=en&tk=775040.775040&q=' + T.url_encode(
word)
q = N.HTTP.request(u, proxy=proxy)
b = q.content
F.dill_dump(file=f, obj=q)
except Exception as e:
return py.No(e)
if response:
return N.copy_request_to_flask_response(q, response)
else:
return q
async def save_item_html_and_sku(url, close=False, timeout=30 * 1000):
tb = taobao_trade
f = T.filename_legalized(url)[:250]
if not f.endswith('.html'): f += '.html'
if F.exist(U.gst + f[:-5] + '.dill'):
return py.No('exist', f)
# return py.pdb()()
pa = await tb.get_or_new_page(url, timeout=timeout)
nspeed = 1024 * 99
while nspeed > 1024 * 80:
nt, nm = U.ftime(), U.get_net_io_bytes()
await A.sleep(1)
nspeed = F.IntSize((U.get_net_io_bytes() - nm) / (U.ftime() - nt))
h = await pa.evaluate("document.documentElement.outerHTML")
f = F.write(f, h)
try:
sku = await pa.evaluate(
''' [window.Hub,KISSY.Env.mods["item-detail/promotion/index"].exports.get("promoData") ] '''
)
if not sku or not sku[1]:
sku = await pa.evaluate(''' window.Hub ''')
sku.append(await pa.evaluate(
''' KISSY.Env.mods["item-detail/promotion/index"].exports '''
))
except Exception as e:
sku = e
fd = F.dp(obj=sku, file=f[:-5])
if close: await pa.close()
return pa, f, fd
async def set_viewport(pages=py.No('auto all pages'), **ka):
''' #1347 看不到，1322 有留白'''
if 'width' not in ka:
ka['width'] = 1366 - 26
if 'height' not in ka:
ka['height'] = 768 - 98
a = {'width': ka['width'], 'height': ka['height']}
if not pages: pages = await taobao_trade.browser.pages()
if not py.islist(pages):
if not isinstance(pages, pyppeteer.page.Page):
return py.No(U.v.isinstance(pages, pyppeteer.page.Page))
pages = [pages]
for page in pages:
await page.setViewport(a)
return len(pages), ka
def clone(url, path=None):
client, target = dulwich.client.get_transport_and_path(
url) #c , 'user/repo'
if not path:
if target.endswith('.git'):
target = target[:-4]
path = target
path = F.mkdir(path)
ls = F.ls(path)
if len(ls) > 1: #skip .git
return py.No('Not empty dir:' + path, ls)
if len(ls) == 1:
r = dulwich.repo.Repo(path)
else:
r = dulwich.repo.Repo.init(path)
print(U.stime(), 'fetching url...')
remote_refs = client.fetch(url, r)
r[b"HEAD"] = remote_refs[b"HEAD"]
print(U.stime(), 'build_index_from_tree ...')
index.build_index_from_tree(r.path, r.index_path(), r.object_store,
r[b'HEAD'].tree)
r.close()
return path, r
def get_app_id(rsignin):
if not rsignin:return rsignin
if py.getattr(rsignin,'app_id',''):
return rsignin.app_id
# 'cookie': rsignin.cookies,
headers_apps = {
'authority': 'us-w1-console-api.leancloud.app',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="90", "Microsoft Edge";v="90"',
'accept': 'application/json',
'x-xsrf-token': rsignin.cookies['XSRF-TOKEN'],
'accept-language': 'zh',
'sec-ch-ua-mobile': '?0',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.62',
'origin': 'https://console.leancloud.app',
'sec-fetch-site': 'same-site',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://console.leancloud.app/',
}
ra = requests.get('https://us-w1-console-api.leancloud.app/1.1/clients/self/apps', headers=headers_apps,cookies=rsignin.cookies)
try:
return ra.json()[0]['app_id']
except Exception as e:
return py.No(ra,rsignin,msg=ra.text,)
def taobao_login(user, pw=py.No('auto load'),
user_input_xy=(1050, 383 + 9)): #incorrect tips + 9
if not pw:
pw = U.get_or_dill_load_and_set(user + '.pw')
if not pw: return pw
return U.simulate_system_actions([('for', '淘宝网 - 淘！我喜欢'), user_input_xy,
'ctrl+a', 'backspace', user, 'tab', pw,
'Enter'])
def read_file_by_url(url):
f = 'C:/test/qgb.tests.taobao_trade/' + T.fileName(url)
s = F.read(f, encoding='utf-8')
if not s:
return py.No(url + ' Not Found!', s)
r = U.StrRepr(s,
repr=f'<U.StrRepr {F.ssize(len(s))} {f} >') # |{s[:44]}...
r.f = r.file = f
return r
def write(q):
if py.isbytes(q):
b = q
else:
b = q.get_data()
if not b: return py.No(b, U.dir(q))
f = 'taobao_trade/' + T.fileName(b[:55].decode('utf-8'))
md5s = U.get('md5s')
if not md5s:
md5s = U.set('md5s', file_md5s())
m = U.md5(b)
if m in md5s:
U.msgbox(f, m + ' duplicated in md5s')
return py.No('duplicated md5 ' + m, m, f)
md5s.add(m)
return F.write(f, b, mkdir=1)
def sign_in(email,pw,dill_file=py.No('auto use email'),set_app_id=False):
# global rsignin
if '@' not in email and not dill_file:
raise py.ArgumentError('email',email)
if not dill_file:
dill_file=T.filename(email)
dill_file=F.auto_path(dill_file,ext='.dill')
if F.exist(dill_file):
r=F.dill_load(dill_file)
if check_login(r):
return r,dill_file
else:
print('can not load',F.ll(dill_file),'skip')
headers = {
'authority': 'us-w1-console-api.leancloud.app',
'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="90", "Microsoft Edge";v="90"',
'accept': 'application/json',
'accept-language': 'zh',
'sec-ch-ua-mobile': '?0',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safa',
'content-type': 'application/json; charset=utf-8',
'origin': 'https://console.leancloud.app',
'sec-fetch-site': 'same-site',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://console.leancloud.app/',
# 'cookie': '_ga=GA1.2.1912224529.1620372988; _gid=GA1.2.855550602.1623739540',
}
#<Response [400]> {"code":107,"error":"Malformed json object. A json dictionary is expected."}
# data = r'''{'email':%r,'password':%r}'''%(email,pw) # 400
data='{"email":"%s","password":"******"}'%(email,pw)
print(U.v.requests.post('https://us-w1-console-api.leancloud.app/1.1/signin', headers=headers, data=data))
rsignin = requests.post('https://us-w1-console-api.leancloud.app/1.1/signin', headers=headers, data=data)
print(rsignin,rsignin.text[:99])
if 'XSRF-TOKEN' not in rsignin.cookies or not rsignin.cookies['XSRF-TOKEN']:
return py.No(rsignin,rsignin.text,)
if set_app_id:
rsignin.app_id=get_app_id(rsignin)
f=F.dill_dump(obj=rsignin,file=dill_file)
return rsignin,f
async def _get_page_by_url(browser, url):
for i in range(8):
try:
all_pages = await get_pages_onload()
break
except Exception as e:
print('_get_page_by_url ', U.stime(), e)
print()
continue
for n, detected, page, u in all_pages:
if u.startswith(url):
return page
else:
for n, detected, page, u in all_pages:
if url in u: # 优先 startswith，因为是两次循环匹配
return page
else:
return py.No(
'can not found page.url.startswith or in page.url:' + url)
return py.No('can not found page url.startswith:' + url)
def flask_get_all_upload_files(
upload_dir=py.No('U.gst/upload_dir', no_raise=1),
save_size=py.No('8 MB', no_raise=1),
request=None,
):
'''save_size <=0 : save_all
'''
from shutil import copyfileobj
# U,T
if not request: from flask import request
if not upload_dir:
upload_dir = U.get_or_set('rpc.server.upload_dir',
U.gst + 'upload_dir/')
else:
upload_dir = U.set('rpc.server.upload_dir', upload_dir)
F.mkdir(upload_dir)
if not save_size:
save_size = U.get('rpc.server.upload.save_size', 1024 * 1024 * 8)
save_size = F.IntSize(py.int(save_size))
U.set('rpc.server.upload.save_size', save_size)
d = {}
for k, f in request.files.items(multi=True): # 默认multi=False，始终只能获取一个，坑！
b = f.stream.read(save_size)
if py.len(b) < save_size:
if py.len(b) > 99:
sr = "<{0} >'{1}...{2}'".format(F.readable_size(b),
repr(b[:50])[2:-1],
repr(b[-50:])[2:-1])
d[f] = U.object_custom_repr(b, repr=sr)
else:
d[f] = b
else:
fn = upload_dir + f.filename
with py.open(fn, 'wb') as fp:
fp.write(b)
copyfileobj(f.stream, fp)
d[f] = fn
return d
def get_recognized(a):
st = ''
if not py.isdict(a[5]): return py.No('a[5] is not dict', a)
for i in a[5]:
d = a[5][i]
if d['success']:
if not st:
st = '<br>' + '=' * 110
st += '<br><img src="{}" ><br>'.format(i)
for line in d['result']:
st += line['content']
return st
def commit(repo, commit_msg):
if not commit_msg:
commit_msg = 'dulwich_' + U.stime()
if not py.isbytes(commit_msg):
commit_msg = commit_msg.encode('utf-8')
index = repo.open_index()
new_tree = index.commit(repo.object_store)
if new_tree != repo[repo.head()].tree:
bhash = repo.do_commit(commit_msg, tree=new_tree)
return bhash, commit_msg
else:
return b'', py.No("Empty commit!")
async def get_page(page=None,
url=py.No(URL_TRADE_LIST),
wait=None,
browser=None):
global JS_ONLOAD
# py.pdb()()
if py.istr(page) and not url: # and ('://' in page)
url, page = page, None
if not browser: browser = await get_browser()
if not page and not url:
return (await browser.pages())[-1]
if not url: url = URL_TRADE_LIST
if not wait and not page:
page = await _get_page_by_url(browser, url)
if wait and not page:
start = U.timestamp()
while not page:
page = await _get_page_by_url(browser, url)
await A.sleep(py.max(0.1, wait / 10))
if U.timestamp() - start > wait:
return py.No('%s sec timeout!,can not found page.url:%s' %
(wait, url))
return U.set('page', page)
async def backup_cookie(user='', page=py.No('auto last -1')):
if not user:
user = await get_taobao_user()
if not page: page = await get_page()
ck = await get_all_cookies(page)
print('get_all_cookies len:', py.len(ck))
if not ck: return ck
f = F.dill_dump(obj=ck, file=f'{user}-{len(ck)}-cookies')
await page.deleteCookie(*ck)
cks = await get_all_cookies(page)
print('deleteCookie', 'done !', 'now len == %s' % py.len(cks))
return f
def request(url,
method='GET',
headers=gheaders,
proxies=AUTO_GET_PROXY,
verify=False,
no_raise=False,
print_req=False,
**ka):
'''
'''
import requests
U, T, N, F = py.importUTNF()
proxies, ka = auto_proxy(proxies, ka)
print_req = U.get_duplicated_kargs(ka,
'show',
'print',
'p',
'print_req',
default=print_req)
if (py.istr(url) and url.upper() in HTTP_METHODS):
# ka.pop('method','')#D.pop(k[,d]) -> v,
ka['method'] = url
url = ka['url'] # test url exists
elif method:
ka['method'] = method
if headers:
ka['headers'] = headers
for k in py.list(ka):
v = ka[k]
if py.istr(k) and py.istr(v):
if k[0].isupper():
ka.pop(k)
headers[k] = v
ka['verify'] = verify
if url and 'url' not in ka:
ka['url'] = url
ka['url'] = N.auto_url(ka['url'])
if print_req: print(U.v.requests.request(**ka))
if no_raise:
try:
return requests.request(**ka)
except Exception as e:
return py.No(e, ka)
return requests.request(**ka)
def textarea(
response,
name='t',
upload_dir=py.No('U.gst/upload_dir', no_raise=1),
):
"""
<input type="text" name="t">
"""
U.r(N.HTML)
if not U.all_in(name, T.aZ + '_'):
raise py.ArgumentError(name, 'must be alphabet_')
r = T.html_template(
globals=globals(),
locals=locals(),
s=ghtml_txt,
)
# r=format(r,name=name)
response.headers['Content-Type'] = 'text/html;charset=utf-8'
return response.set_data(r)
def get_price_list_from_setMdskip(d):
'''https://mdskip.taobao.com/core/initItemDetail.htm?isUseInventoryCenter=false&cartEnable=true&service3C=false&isApparel=false&isSecKill=false&tmallBuySupport=true&isAreaSell=false&tryBeforeBuy=false&offlineShop=false&itemId=597606333219&showShopProm=true&isPurchaseMallPage=false&itemGmtModified=1586583685000&isRegionLevel=false&household=false&sellerPreview=false&queryMemberRight=true&addressLevel=2&isForbidBuyItem=false&callback=setMdskip×tamp=1586591444224&isg=dBNwXtilqQnbNHidBOfgSZlsvAbToIOb4sPP3hrRQICPO41H77k5WZXXqRTMCnGVH6YMR35fHFhYBeYBqMIKnxvOa6Fy_7kSndC..&isg2=BLq611-JsKSWiz8nzofghCAaC-Dcaz5FTP5lBsSyas0Jt1rxrPkTVYCOB0NrJ7bd
'''
r = []
d = d['defaultModel']
d = d['itemPriceResultDO']
p = d['priceInfo']
for skuid, v in p.items():
old = v['price']
old = U.FloatRepr(old, size=9)
pl = v.get('promotionList', [])
if len(pl) == 1:
new = pl[0]['price']
new = U.FloatRepr(new, size=9)
else:
new = py.No('promotionList.len != 1', skuid, pl)
r.append([skuid, new, old]) # 1 , 2 , 3
return r
def getTopWords(text, n=11, keyword_length_priority=True):
# from qgb import U, es
U = py.importU()
ws = {}
for i in analyze(text)['tokens']:
if i['type'] != 'CN_WORD':
continue
w = i['token']
if w in ws:
ws[w] += 1
else:
ws[w] = 1
if not ws: return py.No('no CN_WORD result', text, n)
ws = U.getDict(ws, len(ws))
if keyword_length_priority:
#按次数 ，关键词长度 从大到小 倒序
ws.sort(key=lambda i: float('%s.%03i' % (i[1], len(i[0]))),
reverse=True)
else:
ws.sort(key=lambda i: i[1], reverse=True)
return ws[:n]
async def get_browser(
browserURL='http://127.0.0.1:9222',
browserWSEndpoint='',
executablePath=r'C:\Users\qgb\AppData\Local\CentBrowser\Application\chrome.exe'
):
''' await tb.pyppeteer.launcher.get_ws_endpoint('http://127.0.0.1:9222')
# if not chrome.exe or : BrowserError: Browser closed unexpectedly:
'''
global browser
browser = U.get(uk)
if browser:
try:
await browser.version()
# pages = await browser.pages()#这样不能检测浏览器是否关闭
return browser
except:
browser = U.set(uk, py.No(U.stime()))
try:
if browserWSEndpoint:
browser = await pyppeteer.connect(
browserWSEndpoint=browserWSEndpoint)
else:
with pyppeteer.launcher.urlopen(browserURL) as f:
data = T.json_loads(f.read().decode())
browser = await pyppeteer.connect(browserURL=browserURL)
except Exception as e: #
U.log(e)
# if is_chrome_process_exists(): # 会自动开一个 全新环境
# ret
browser = await pyppeteer.launcher.launch({
'executablePath': executablePath,
'headless': False,
'slowMo': 30
})
return U.set(uk, browser)
async def try_evaluate(page, code):
try:
return await page.evaluate(code)
except Exception as e:
return py.No(e)
async def del_cookies(ck, page=py.No('auto last -1')):
page = await get_or_new_page(page)
if py.islist(ck):
await page.deleteCookie(*ck)
if py.isdict(ck):
await page.deleteCookie(ck)
async def set_cookies(ck, page=py.No('auto last -1')):
if not page: page = await get_page()
if py.islist(ck):
await page.setCookie(*ck)
if py.isdict(ck):
await page.setCookie(ck)
def check_login(resp): if not resp or not py.getattr(resp,'cookies',0): return py.No(resp) if 'XSRF-TOKEN' not in resp.cookies: return py.No(resp.cookies) return get_apps(resp)
def up(url,
path=None,
commit_msg=None,
username=None,
password=None,
branch='master',
retry=3,
**ka):
r'''
porcelain.push(repo.path,"https://[email protected]/...",'master',username='******',password=_) ## 也会GitProtocolError: unexpected http resp 401 for
porcelain.push(repo.path,"https://*****:*****@' not in url:
domain = T.netloc(url)
if not username:
username = U.get_or_input(domain + '_git.username')
ka['username'] = username
if not password:
password = U.get_or_input(domain + '_git.password')
ka['password'] = password
if not 'refspecs' in ka:
ka['refspecs'] = branch
if not path and '/qpsu' in url.lower():
path = U.get_qpsu_dir()
if not path:
_client, target = dulwich.client.get_transport_and_path(url)
if target.endswith('.git'):
target = target[:-4]
path = target
path = F.auto_path(path)
ls = [i[len(path):] for i in F.ls(path, d=0, f=1, r=1)]
if len(ls) < 1:
return py.No('Not contains .git dir:' + path, ls)
repo = dulwich.repo.Repo(path)
U.pln(path, 'adding ...', U.stime())
repo.stage(['git.py'], )
dulwich.porcelain.add(repo.path) #TODO pwd 必须在当前repo目录下 ，否则
# 1-> 212 treepath = os.path.relpath(path, repopath)
# 213 if treepath.startswith(b'..'):
# 214 raise ValueError('Path not in repo')
# ipdb> !(path, repopath)
# (b'C:\\test\\www\\png\\0010005960.png', b'C:/QGB/babun/cygwin/bin/qgb/')
U.pln('commit ...', U.stime())
bhash, commit_msg = commit(repo, commit_msg)
r = bhash, commit_msg, ka
repo.close()
U.pln('push ...', U.stime())
try:
#repo.path == 'C:/QGB/babun/cygwin/bin/qgb/'
if 'git@' in url:
rp = push_with_key(repo_path=repo.path,
remote=url,
retry=retry,
**ka)
if not rp: return rp
else:
dulwich.porcelain.push(repo.path, url, **ka)
except Exception as e:
U.print_tb()
U.v.dulwich.porcelain.push(repo.path, url, **ka)
r = py.No(e, r)
return r
def push_with_key(repo_path,
remote="ssh://[email protected]:22/QGB/QPSU.git",
private_key='',
refspecs='master',
errstream=getattr(sys.stderr, 'buffer', None),
private_key_password=None,
retry=3):
r''' #important# 三引号string中不能出现 \util 这种字符（常见于路径）
# 会导致 SyntaxError: (unicode error) 'unicodeescape' codec can't decode bytes in position 1-2: truncated \uXXXX escape 错误
# 最好 引号前加r 强制用 raw-string
push ... 2021-04-15__06.26.44__.288
File "C:/QGB/babun/cygwin/bin\qgb\git.py", line 106, in up
push_with_key(repo.path,url,**ka)
File "C:/QGB/babun/cygwin/bin\qgb\git.py", line 205, in push_with_key
progress=errstream.write)
File "C:\QGB\Anaconda3\lib\site-packages\dulwich\client.py", line 789, in send_pack
proto, unused_can_read, stderr = self._connect(b'receive-pack', path)
File "C:\QGB\Anaconda3\lib\site-packages\dulwich\client.py", line 1410, in _connect
**kwargs)
File "C:\QGB\Anaconda3\lib\site-packages\dulwich\contrib\paramiko_vendor.py", line 103, in run_command
client.connect(**connection_kwargs)
File "C:\QGB\Anaconda3\lib\site-packages\paramiko\client.py", line 349, in connect
retry_on_signal(lambda: sock.connect(addr))
File "C:\QGB\Anaconda3\lib\site-packages\paramiko\util.py", line 283, in retry_on_signal
return function()
File "C:\QGB\Anaconda3\lib\site-packages\paramiko\client.py", line 349, in <lambda>
retry_on_signal(lambda: sock.connect(addr))
Out[26]: 'C:/QGB/babun/cygwin/bin/qgb/'
'''
from dulwich.contrib.paramiko_vendor import ParamikoSSHVendor
import dulwich.porcelain
from dulwich.protocol import (
Protocol,
ZERO_SHA,
)
from dulwich.objectspec import (
parse_commit,
parse_object,
parse_ref,
parse_reftuples,
parse_tree,
)
from dulwich.errors import (
GitProtocolError,
NotGitRepository,
SendPackError,
UpdateRefsError,
)
DEFAULT_ENCODING = 'utf-8'
selected_refs = []
if not private_key:
private_key = U.get_or_set('id_rsa', F.get_home() + '.ssh/id_rsa')
private_key = U.set('id_rsa', private_key)
with open_private_key(
private_key) as fpkey, dulwich.porcelain.open_repo_closing(
repo_path) as r:
def update_refs(refs):
selected_refs.extend(parse_reftuples(r.refs, refs, refspecs))
new_refs = {}
# TODO: Handle selected_refs == {None: None}
for (lh, rh, force) in selected_refs:
if lh is None:
new_refs[rh] = ZERO_SHA
else:
new_refs[rh] = r.refs[lh]
return new_refs
import paramiko
pkey = paramiko.RSAKey.from_private_key(fpkey,
password=private_key_password)
if 'git@' not in repo_path:
# raise py.NotImplementedError("'git@' not in repo_path")
repo_path
client, path = dulwich.client.get_transport_and_path(
remote, vendor=ParamikoSSHVendor(pkey=pkey))
err_encoding = getattr(errstream, 'encoding', None) or DEFAULT_ENCODING
remote_location_bytes = client.get_url(path).encode(err_encoding)
#b'ssh://[email protected]:22/QGB/QPSU.git'
try:
client.send_pack(
path,
update_refs,
generate_pack_data=r.object_store.generate_pack_data,
progress=errstream.write)
# print(remote_location_bytes)
errstream.write(b"Push to " + remote_location_bytes +
b" successful.\n")
errstream.flush() # 与 stdout 混合打印时 ，顺序可能不同
except (UpdateRefsError, SendPackError) as e:
errstream.write(b"Push to " + remote_location_bytes +
b" failed -> " + e.message.encode(err_encoding) +
b"\n")
return py.No(e)
finally:
errstream.flush()
return client
async def get_taobao_sku(page=None,
return_item_id=False,
debug=False,
padding_name=30,
**ka):
'''return 【 [0.10 , '2', 'M4*5*7', False, '3802079080101']。。】
#dict_keys(['config', 'mods', 'loadCSS', 'add', 'fire', 'listeners', 'fired'])
# ['config', 'get', 'set']
#dict_keys(['sku', 'desc', 'async_dc', 'support', 'async_sys', 'video'])
'''
return_item_id = U.get_duplicated_kargs(ka,
'id',
'item_id',
default=return_item_id)
if not page:
page = await taobao_trade.get_current_front_page()
if not page or 'item.taobao.com/item.htm' not in page.url:
page = await taobao_trade.get_page('item.taobao.com/item.htm')
sku_raw = await page.evaluate(
''' [window.Hub,KISSY.Env.mods["item-detail/promotion/index"].exports.get("promoData") ] '''
) # [{..},None]
d, none = sku_raw
dc = d['config']['config']
# sku,desc=U.get_dict_multi_values_return_list(dc,'sku', 'desc')
sku = dc['sku']
valItemInfo = sku['valItemInfo']
skuMap, propertyMemoMap = U.get_dict_multi_values_return_list(
valItemInfo, 'skuMap', 'propertyMemoMap')
if not propertyMemoMap:
propertyMemoMap = await page.evaluate('''
function get_sku_id_text(){
var lis=document.querySelectorAll('ul.J_TSaleProp>li')
//var r=[]
var d=new Map()
for(var i of lis){
var id=i.getAttribute('data-value')
var text=i.querySelector('a').textContent.trim()
d[id]=text
//r.push([i.getAttribute('data-value'),i.querySelector('a').textContent.trim()])
}
return d
} ''') #颜色分类
r = []
html = ''
try:
for i, d in skuMap.items():
id = T.sub_last(i, ';', ';')
if padding_name:
str_name = U.StrRepr(propertyMemoMap.get(id, ''),
size=padding_name)
else:
str_name = propertyMemoMap.get(id, '')
if not str_name:
if not html:
html = await page.evaluate(
"document.documentElement.outerHTML")
str_name = T.sub(
html, i[1:-1] +
'" class="tb-txt">\n <a href="javascript:;">\n <span>',
'</span>')
if not str_name and debug:
return html, d, dc, sku, skuMap, propertyMemoMap
sp = d['price']
row = [
U.FloatRepr(py.float(sp), repr=sp, size=6), d['stock'],
str_name, d['oversold'], d['skuId']
]
r.append(row)
except Exception as e:
return py.No(e, sku_raw, skuMap, page)
if return_item_id:
return T.url_get_arg(page.url, 'id'), r
return r
async def add_cart_by_gui(id, sku=py.No('click all')):
'''
await post('https://okfw.net/b=q.get_data();s=T.json_loads(b);r=len(s)',$0.innerHTML)
'''
tb = taobao_trade
id = str(id)
r = []
p = await tb.get_page(url=id)
if not p:
if '://' not in id:
id = 'https://item.taobao.com/item.htm?id=%s' % id
p = await tb.new_page(url=id)
t = await p.title()
r.append([
U.stime(),
p,
[T.sub(p.url, 'id=', ''), t],
])
await p.bringToFront()
if 'detail.tmall.com/item.htm' in p.url:
await p.evaluate(
'''r=xpath('//a[contains(@id,"J_LinkBasket" )]');r.click();''')
raise NotImplementedError()
await page_scroll_bottom(p)
top_add_cart_btn = await p.querySelector('a#J_TabShopCart')
await top_add_cart_btn.click() # 页面初始加载必须先点击才会出现 popsku
popsku = await p.querySelector('div.tb-popsku')
async def check_popsku():
style = await popsku.getProperty('style')
display = await style.getProperty('display')
value = await display.jsonValue()
if 'none' in value:
await top_add_cart_btn.click()
await A.sleep(0.01)
add = await p.querySelector('a.J_PopSKUAddCart')
cancel = await p.querySelector('a.J_PopSKUCancer')
try:
await cancel.click()
except:
pass
props = await p.querySelectorAll('.J_PopSKUProps>.J_Prop ')
if len(props) > 1: #TODO
U.beep()
await A.sleep(3)
U.beep()
await A.sleep(6)
return r, len(props)
ses = await p.querySelectorAll(
'.J_PopSKUProps > dl.J_Prop > dd > ul > li > a > span')
for se in ses:
await check_popsku()
t = await (await se.getProperty('textContent')).jsonValue()
for s in sku:
if t in s:
break
else:
if sku: # 指定了sku，但是没有找到（not break）
continue
await se.click()
await A.sleep(0.1)
await add.click()
await A.sleep(1)
r.append(t)
return r
