def shopcar_trigger(context, dag_run_obj):
ip = context['task_instance'].xcom_pull(
task_ids='task_get_ip_%s' % (context["params"]["cur_number"]))
task = context['task_instance'].xcom_pull(
task_ids='task_get_task_%s' % (context["params"]["cur_number"]))
if task == 'shopcar':
logger.info('run shopcar')
dag_run_obj.payload = {'ip': ip}
return dag_run_obj
def set_address(self, user_info):
self.driver.get(
'https://{}/a/addresses/add?ref_=ya_add_address_yaab'.format(
self.host))
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressFullName').send_keys(
user_info['username'])
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressLine1').send_keys(
user_info['address'])
if self.host.endswith('co.jp'):
webdriver.support.select.Select(
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressStateOrRegion-dropdown-nativeId'
)).select_by_value(user_info['state'])
zip_ = user_info['zip'].split('-')
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressPostalCodeOne').send_keys(
zip_[0])
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressPostalCodeTwo').send_keys(
zip_[1])
else:
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressCity').send_keys(
user_info['city'])
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressStateOrRegion').send_keys(
user_info['state'])
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressPostalCode').send_keys(
user_info['zip'])
self.driver.find_element_by_id(
'address-ui-widgets-enterAddressPhoneNumber').send_keys(
user_info['phone'])
self.driver.find_element_by_xpath(
'//*[@id="a-autoid-0"]/span/input').click()
self.driver.implicitly_wait(5)
s_url = self.driver.current_url
# ???? what's this
if s_url.count('yaab-enterAddressSucceed', 0, len(s_url)) == 0:
logger.error('the address error ')
else:
logger.info('now set address success!')
def login(self, email, password):
logger.info("current_url:" + self.driver.current_url)
if not self.driver.current_url.startswith(
'https://{host}/ap/signin'.format(host=self.host)):
logger.info("go_home...host:" + self.host)
self.go_home()
try:
self.click(
self.until(lambda d: self.driver.find_element_by_id(
'nav-link-yourAccount')))
except:
self.click(
self.until(lambda d: self.driver.find_element_by_id(
'nav-link-accountList')))
self.until(lambda d: self.driver.find_element_by_id('signInSubmit'))
self.driver.find_element_by_id('ap_email').send_keys(email)
self.driver.find_element_by_id('ap_password').send_keys(password)
self.driver.find_element_by_id('signInSubmit').click()
def op_get_task(*args, **kwargs):
# 从任务队列中获取任务
# task = get_task()
logger.info('start the get task')
try:
#task = get_task()
task = random.choice(['wishlist', 'register', 'shopcar'])
return task
except Exception as e:
message = e.message
logger.error('=======get the error message=====')
logger.error(e)
if 'error' in message:
ip = message.split(':')[-1]
manage.return_ip(ip, 'default')
# make sure always return the ip as require
raise ValueError(
'finally return the ip and do stop the rest task ')
def op_shopcar(*args, **kwargs):
ip = kwargs['dag_run'].conf['ip']
taskid = kwargs['dag_run'].conf['taskid']
host = kwargs['dag_run'].conf['host']
# time.sleep(300)
logger.info('taskid ---- {}'.format(taskid))
logger.info('host ---- {}'.format(host))
logger.info('shopcar run IP {}'.format(ip))
manage.return_ip(ip, 'default')
def register(self, username, email, password):
logger.info('start to get the main page')
self.go_home()
logger.info('finish render the home page and do next task')
e = self.until(
lambda d: self.driver.find_element_by_id('nav-flyout-ya-newCust'))
a = e.find_element_by_tag_name('a')
self.go(a.get_attribute('href'))
button = self.until(
lambda d: self.driver.find_element_by_id('continue'))
self.driver.find_element_by_id('ap_customer_name').send_keys(username)
if self.host.endswith('co.jp'):
self.driver.find_element_by_id(
'ap_customer_name_pronunciation').send_keys(username)
if self.host.endswith('.in'):
self.driver.find_element_by_id('ap_use_email').click()
self.driver.find_element_by_id('ap_email').send_keys(email)
self.driver.find_element_by_id('ap_password').send_keys(password)
if not self.host.endswith('.in'):
self.driver.find_element_by_id('ap_password_check').send_keys(
password)
# find the continue button
logger.info('find the continue button and do click')
button.click()
#time.sleep(10) # 缓迟五秒判断结果
self.driver.implicitly_wait(10)
if 'https://{host}/ap/register'.format(
host=self.host) in self.driver.current_url: # 发生错误
logger.error('quite the driver after the register')
self.driver.quit()
logger.error('register have error')
return False
logger.info('finish register the user: {} - {} - {}'.format(
username, email, password))
return True
def op_scheduler(*args, **kwargs):
tag = 'default'
while True:
#获取有用的IP,如果没有就等待
logger.info('start the get ip')
url = urljoin(IP_PROXY_SERVER, '/api/ip/get')
data = {
'direct': 'left',
'tag': tag,
}
ip = requests.post(url=url, data=data).json()
if not ip:
time.sleep(5)
continue
# {"result":"192.168.1.3:8888","status":200}
result = ip.get('result', '')
if not result:
time.sleep(5)
continue
# 从任务队列中获取任务
# task = get_task()
logger.info('start the get task')
url = urljoin('http://192.168.1.251:9001', '/api/task/get')
#task_json = requests.post(url=url, data=data).json()
r = requests.get(url=url)
dirc = json.loads(r.text)
task_result = eval(dirc["result"])
task = task_result["type"]
task_id = task_result["tid"]
task_host = task_result["host"]
#task = get_task()
#task = random.choice(['wishlist', 'register', 'shopcar'])
if task == 'register':
status, output = commands.getstatusoutput(
"airflow trigger_dag -c '{\"ip\":\"" + result +
"\",\"taskid\":\"" + task_id + "\",\"host\":\"" + task_host +
"\"}' dag_register")
if task == 'wishlist':
status, output = commands.getstatusoutput(
"airflow trigger_dag -c '{\"ip\":\"" + result +
"\",\"taskid\":\"" + task_id + "\",\"host\":\"" + task_host +
"\"}' dag_wishlist")
elif task == 'shoppingcart':
status, output = commands.getstatusoutput(
"airflow trigger_dag -c '{\"ip\":\"" + result +
"\",\"taskid\":\"" + task_id + "\",\"host\":\"" + task_host +
"\"}' dag_shopcar")
logger.info(output)
time.sleep(5)
def op_get_ip(*args, **kwargs):
logger.info('start the get ip')
try:
#ip = refresh_4g_and_ready_ip()
#ip = '192.168.1.1'
tag = 'default'
ip = manage.get_ip(tag)
logger.info('get the ip:{} from proxy server'.format(ip))
#host = ip.split(':')[0]
return ip
except Exception as e:
message = e.message
logger.error('=======get the error message=====')
logger.error(e)
if 'error' in message:
ip = message.split(':')[-1]
manage.return_ip(ip, 'default')
# make sure always return the ip as require
raise ValueError(
'finally return the ip and do stop the rest task ')
if 'not invalidate ip address' in message:
raise ValueError(' do stop the rest task ')
def op_return_ip(*args, **kwargs):
ip = kwargs['task_instance'].xcom_pull(
task_ids='task_get_ip_%s' % (kwargs["params"]["cur_number"]))
manage.return_ip(ip, 'default')
logger.info('return ip------{}'.format(ip))
def op_pass_ip(*args, **kwargs):
ip = kwargs['task_instance'].xcom_pull(
task_ids='task_get_ip_%s' % (kwargs["params"]["cur_number"]))
logger.info('op_pass_ip get the ip and do pass the ip {}'.format(ip))