def get(self, shareFile, index=0):
with open(shareFile, 'r') as fp:
content = fp.read()
try:
obj = json.loads(content.decode('utf-8', 'ignore'))
except ValueError as e:
raise Exception('{} is not valid config file.'.format(shareFile))
if index >= 0 and index < obj['num']: # Get one
data = self.create(obj['list'][index])
return JsonResult.succeed(data)
# Get all
dataList = list()
for special in obj.pop('list'):
data = self.create(special)
if data is not None:
dataList.append(data)
randomSleep(1, 2)
obj['num'] = len(dataList)
obj['list'] = dataList
return JsonResult.succeed(obj)
def getImageByAnchor(anchor):
options = Options()
options.add_argument('--headless')
tmpDriver = webdriver.Firefox(options=options)
# tmpDriver = webdriver.Firefox()
tmpDriver.get(anchor)
WebDriverWait(tmpDriver, 10).until(
EC.element_to_be_clickable(
(By.XPATH, '//button[contains(text(), "Accept")]'))).click()
el = tmpDriver.find_elements_by_tag_name('img')
image = [i.get_attribute('src') for i in el]
print(image[1])
u.randomSleep(3, 5)
tmpDriver.close()
return image[1]
def isLoginned(uuid):
for count in range(10):
status = int(itchat.check_login(uuid))
if status is 200:
return True
if status is 201:
print 'Wait for confirm in mobile #', count
randomSleep(1, 2)
continue
print 'Error status:', status
return False
return False
def explore(self, content):
if content is None:
return JsonResult.error(-101, 'Empty inputted content')
r = Network.post(self.url, data={'content': content})
if r is None:
print 'No result for', content
return JsonResult.error(-102, 'No result')
try:
obj = json.loads(r.content.decode('utf-8', 'ignore'))
except ValueError as e:
print 'Error (', e, ') of json: "', r.content, '"'
return JsonResult.error(-103, 'Parse result error')
num = obj['num']
if num is 0:
print 'Error content: "', r.content, '"'
return JsonResult.error(-104, 'No result')
print 'Found', num, 'SKUs with "', content, '"'
# Get all
dataList = list()
for special in obj.pop('list'):
data = self.create(special)
if data is not None:
dataList.append(data)
randomSleep(1, 2)
obj['num'] = len(dataList)
obj['list'] = dataList
return JsonResult.succeed(obj)
def plogin(self):
def isValidAuthCode(code):
if code is None or len(code) != 4:
return False
for c in code:
if c.isdigit():
continue
if c.isalpha():
continue
# Wrong word
return False
return True
if self.pCookies is not None:
return True
# https://github.com/mozilla/geckodriver/releases
# browser = webdriver.Firefox()
# https://chromedriver.storage.googleapis.com/index.html
browser = webdriver.Chrome()
try:
# Plogin
browser.get(self.ploginUrl)
# Login by username and password
# Username and password
randomSleep(1, 2)
inputElement(browser.find_element_by_id('username'), self.pin)
randomSleep(1, 2)
inputElement(browser.find_element_by_id('password'), self.password)
# Submit
buttonElement = browser.find_element_by_id('loginBtn')
# Code
codeElement = browser.find_element_by_id('code')
imageElement = browser.find_element_by_id('imgCode')
times = 0
if codeElement.is_displayed():
while codeElement.is_displayed() and times < 50:
times += 1
# Image to text
path = OutputPath.getAuthPath(self.pin)
ImageKit.saveCapture(browser, imageElement, path)
code = ImageKit.getText(path)
codeElement.send_keys(code)
if not isValidAuthCode(code):
# Refresh auth code
randomSleep(0.5, 1)
imageElement.click()
# Wait for updating auth code
randomSleep(1, 2)
codeElement.clear()
continue
# Submit
randomSleep(1, 2)
buttonElement.click()
error = self.getBrowserError(browser)
if error is None:
print 'Succeed after', times, 'tries.'
break
if u'验证码' not in error:
raise Exception('Unable to login for "{}": {}'.format(
self.pin, error))
randomSleep(1, 2)
codeElement.clear()
randomSleep(1, 2)
else:
raise Exception('Unable to login for "{}"'.format(
self.pin))
else:
# Submit
randomSleep(1, 2)
buttonElement.click()
wait = WebDriverWait(browser, 3)
error = self.getBrowserError(browser)
if error is not None:
raise Exception('Unable to login for "{}": {}'.format(
self.pin, error))
print 'Loginned for', self.pin
# Redirect to wqs
qwsUrl = getProperty(self.configFile, 'cps-qwd-wqs-url')
browser.get(qwsUrl)
time.sleep(10)
# Save as type of cookie for requests
self.pCookies = dict()
for cookie in browser.get_cookies():
k = cookie['name']
v = cookie['value']
self.pCookies[k] = v
except Exception as e:
print e
finally:
browser.quit()
def login(self,
screenshotPath=None,
inputPath=None,
outputPath=None,
config='templates/login.json',
retries=10):
result = JsonResult.error()
if config is None:
config = 'templates/login.json'
with open(config) as fp:
content = fp.read()
try:
configObj = json.loads(content.decode('utf-8', 'ignore'))
except ValueError as e:
raise Exception('{} is not valid config file for user {}.'.format(
config, self.userId))
obj = configObj.pop('config')
driverType = obj.pop('driver')
loginUrl = obj.pop('login-url')
verificationUrl = obj.pop('verification-url')
display = None
if screenshotPath is not None:
display = Display(visible=0, size=(800, 600))
display.start()
if 'firefox' == driverType:
# https://github.com/mozilla/geckodriver/releases
driver = webdriver.Firefox()
else: # Chrome
# https://chromedriver.storage.googleapis.com/index.html
driver = webdriver.Chrome()
try:
driver.get(loginUrl)
driver.set_script_timeout(10)
self.saveScreenshot(driver, screenshotPath)
# Username and password
randomSleep(1, 2)
inputElement(driver.find_element_by_id('username'), self.username)
randomSleep(1, 2)
inputElement(driver.find_element_by_id('password'), self.password)
self.saveScreenshot(driver, screenshotPath)
# Submit
buttonElement = driver.find_element_by_id('loginBtn')
randomSleep(1, 2)
buttonElement.click()
authCodeInputter = Inputter('AuthCode', inputPath, outputPath)
verificationInputter = Inputter('Verification', inputPath,
outputPath)
for i in range(retries):
if loginUrl != driver.current_url:
break
time.sleep(1)
self.saveScreenshot(driver, screenshotPath)
if self.inputAuthCode(driver, authCodeInputter):
continue
# Need code
if self.isPhoneCodeNeeded(driver):
continue
# Verification
if self.verify(driver, verificationInputter):
continue
error = self.getLoginError(driver)
if error is not None:
if u'账号或密码不正确' in error:
return JsonResult.error(message=error)
if u'验证码' not in error:
return JsonResult.error(message=error)
else:
raise Exception('Unable to login for user {} in {}.'.format(
self.userId, loginUrl))
codeInputter = Inputter('Code', inputPath, outputPath, 30)
for i in range(retries):
if verificationUrl != driver.current_url:
break
time.sleep(1)
self.saveScreenshot(driver, screenshotPath)
self.inputPhoneCode(driver, codeInputter)
else:
raise Exception('Unable to login for user {} in {}.'.format(
self.userId, verificationUrl))
time.sleep(3)
self.updateDb(driver)
result = JsonResult.succeed()
except KeyboardInterrupt:
pass
except Exception as e:
print 'Error occurs at', datetime.now().strftime(
'%Y-%m-%d %H:%M:%S')
traceback.print_exc(file=sys.stdout)
result = JsonResult.error(message=e.message)
finally:
driver.quit()
if display is not None:
display.stop()
time.sleep(1)
return result
def _get(self, url, para):
res = self._subGet(url, para)
while (res.get('ret') != 0):
mytools.randomSleep()
res = self._subGet(url, para)
return res
def plogin(self, retries=0, force=False):
def isValidAuthCode(code):
if code is None or len(code) != 4:
return False
for c in code:
if c.isdigit():
continue
if c.isalpha():
continue
# Wrong word
return False
return True
if self.pCookies is not None:
return True
if force:
self.entryCookies = None
self.keyCookies = None
if retries is 0:
pass
elif retries is 1:
self.keyCookies = None
elif retries is 2:
self.entryCookies = None
else:
return False
cookies = None
if self.entryCookies is not None:
try:
cookies = json.loads(self.entryCookies.decode('utf-8', 'ignore'))
except ValueError as e:
pass
if cookies is None:
self.dbUpdated = True
display = Display(visible=0, size=(800, 600))
display.start()
if 'firefox' == self.ploginType:
# https://github.com/mozilla/geckodriver/releases
browser = webdriver.Firefox()
else: # Chrome
# https://chromedriver.storage.googleapis.com/index.html
browser = webdriver.Chrome()
try:
# Plogin
browser.get(self.ploginUrl)
# Login by username and password
# Username and password
randomSleep(1, 2)
inputElement(browser.find_element_by_id('username'), self.username)
randomSleep(1, 2)
inputElement(browser.find_element_by_id('password'), self.password)
# Submit
buttonElement = browser.find_element_by_id('loginBtn')
# Code
codeElement = browser.find_element_by_id('code')
imageElement = browser.find_element_by_id('imgCode')
times = 0
if codeElement.is_displayed():
print 'Auth code is needed ...'
while codeElement.is_displayed() and times < 50:
times += 1
# Image to text
path = OutputPath.getAuthPath(self.username)
ImageKit.saveCapture(browser, imageElement, path)
code = ImageKit.getText(path)
codeElement.send_keys(code)
if not isValidAuthCode(code):
# Refresh auth code
randomSleep(0.5, 1)
imageElement.click()
# Wait for updating auth code
randomSleep(1, 2)
codeElement.clear()
continue
# Submit
randomSleep(1, 2)
buttonElement.click()
error = self.getBrowserError(browser)
if error is None:
print 'Succeed after', times, 'tries.'
break
if u'验证码' not in error:
raise Exception('Unable to login for "{}": {}'.format(self.userId, error))
randomSleep(1, 2)
codeElement.clear()
randomSleep(1, 2)
else:
raise Exception('Unable to login for "{}"'.format(self.userId))
else:
# Submit
randomSleep(1, 2)
buttonElement.click()
wait = WebDriverWait(browser, 3)
error = self.getBrowserError(browser)
if error is not None:
raise Exception('Unable to login for "{}": {}'.format(self.userId, error))
print 'Loginned for user', self.userId, 'with type', self.loginType
# Redirect to wqs
time.sleep(1)
# Save as type of cookie for requests
cookies = dict()
for cookie in browser.get_cookies():
k = cookie['name']
v = cookie['value']
cookies[k] = v
self.entryCookies = reprDict(cookies)
except Exception as e:
print 'Unable to get entry cookie with an error:\n', e
return False
finally:
browser.quit()
if display is not None:
display.stop()
# Update pCookies
self.pCookies = cookies
cookies = None
if self.keyCookies is not None:
try:
cookies = json.loads(self.keyCookies.decode('utf-8', 'ignore'))
except ValueError as e:
pass
if cookies is None:
self.dbUpdated = True
try:
# Headers
headers = {'User-Agent': self.userAgent}
params = {'rurl': self.wqsUrl}
r = requests.get(self.wqUrl, params=params, cookies=self.pCookies, headers=headers, allow_redirects=False)
except Exception as e:
print 'Unable to get key cookie with an error:\n', e
return False
cookies = dict()
for cookie in r.cookies:
cookies[cookie.name] = cookie.value
self.keyCookies = reprDict(cookies)
# Update pCookies
self.pCookies.update(cookies)
return True
def dataPlan(self, g_tk, own_id):
res_list = []
cf = self.colloctConf
url = cf['campaign_data']['url']
data = {
"biz_filter": {
"status": 999
},
"rpt_filter": {
"time_dimension": 3,
"time_range": {
"start_date": "20201009",
"end_date": "20201009"
},
"time_line": 1
},
"order_by": {},
"page": 1,
"page_size": 20,
"data": ["list"]
}
self.req.headers = {
"accept": "application/json, text/javascript, */*; q=0.01",
"accept-encoding": "gzip, deflate, br",
"accept-language": "zh-CN,zh;q=0.9",
"content-length": "211",
"content-type": "application/json",
"origin": "https://ad.qq.com",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.135 Safari/537.36 OPR/70.0.3728.154",
"x-requested-with": "XMLHttpRequest",
}
parm = {
"g_tk": g_tk,
"owner": own_id,
"advertiser_id": own_id,
# "trace_id": "294e3e58-14ab-ef91-dd0d-0d03b660cf13",
# "g_trans_id": "d7629a32-00a0-8d08-3f1f-b9c4a648b19a",
# "unicode": "1",
# "post_format": "json"
}
for key in parm:
url += str(key) + "=" + str(parm[key]) + "&"
for dt in mytools.dateList():
day = dt.strftime('%Y%m%d')
day_str = str(dt)
data['page'] = 1 # 重新计页
data['rpt_filter']['time_range']['start_date'] = day
data['rpt_filter']['time_range']['end_date'] = day
data_json_str = json.dumps(data)
res = self._post(url, data_json_str)
res_list += self.dataDeal(res, day_str)
while res['data']['conf']['page'] < res['data']['conf']['total_page']:
mytools.randomSleep()
data['page'] = data['page'] + 1 # 翻页
data_json_str = json.dumps(data)
res = self._post(url, data_json_str)
res_list += self.dataDeal(res, day_str)
return res_list
def _post(self, url, para):
res = self._subPost(url, para)
while (res.get('code') != 0) and (res.get('errno') != 0):
mytools.randomSleep()
res = self._subPost(url, para)
return res