def set_logon_id(driver, logon_id):
"""
向登录框填入登录账户，并返回当前页面的session
:param driver:
:param logon_id:
:return:
"""
# 用户名输入框
ele_user_id = None
try:
# ele_user_id = driver.find_element_by_id(LOGON_ID_INPUT)
# ele_user_id.clear()
# driver.find_element_by_id("storeUserid").send_keys(user)
driver.execute_script('document.getElementById("' + _get_input_id() +
'").value = "' + logon_id + '"')
except Exception as ex:
LOGGER.exception("填充登录名%s. Exception:%s", logon_id, ex)
# [{u'value': u'0000daj0cacDkAIEeAaGHCyf_bM:-1', u'name': u'JSESSIONID', u'httpOnly': True, u'secure': False}]
cookies = driver.get_cookies()
session_id = ''
for cookie in cookies:
if cookie.get('name') == 'JSESSIONID':
session_id = cookie.get('value')
return session_id
def get_ie_driver_bak2():
"""
https://docs.seleniumhq.org/docs/04_webdriver_advanced.jsp#using-a-proxy
:return:
"""
PROXY = "47.96.0.157:4444"
# Create a copy of desired capabilities object.
desired_capabilities = webdriver.DesiredCapabilities.INTERNETEXPLORER.copy(
)
# Change the proxy properties of that copy.
desired_capabilities['proxy'] = {
"httpProxy": PROXY,
"ftpProxy": PROXY,
"sslProxy": PROXY,
"noProxy": None,
"proxyType": "MANUAL",
"class": "org.openqa.selenium.Proxy",
"autodetect": False
}
LOGGER.info("try to use proxy: %s", PROXY)
# you have to use remote, otherwise you'll have to code it yourself in python to
# dynamically changing the system proxy preferences
driver = webdriver.Remote("http://localhost:4444/wd/hub",
desired_capabilities)
return driver
def _format_merchant_data(error, summary, orders, tip=None):
"""
格式商户数据
:param error:
:param summary:
:param orders:
:return: `dict`
"""
res = {
'status': False,
'summary': [],
'orders': [],
'error': None,
'tip': tip
}
try:
summary_data = dict()
orders_data = list()
if error:
res['status'] = False
res['error'] = error
else:
res['status'] = True
if isinstance(summary, TradeSummary):
summary_data = summary.to_dict()
if isinstance(orders, list):
for order in orders:
if isinstance(order, TradeDetail):
orders_data.append(order.to_dict())
res['summary'].append(summary_data)
res['orders'] = orders_data
except Exception as ex:
LOGGER.exception("下载商户订单数据异常%s", ex)
res['error'] = "下载商户订单数据发生异常:%s" % ex.message
return res
def wait_element_loaded(driver, delay, element_id):
try:
elem_founded = WebDriverWait(driver, delay).until(
EC.presence_of_element_located((By.ID, element_id)))
return True
except TimeoutException:
LOGGER.warn("等待页面加载元素%s在%s秒内超时", element_id, delay)
return False
def check_status_merchant(merchant_info):
if not isinstance(merchant_info, MerchantInfo):
return False
res, error = check_session_valid(merchant_info.session_id)
if res:
LOGGER.debug(u"商户(%s)的session有效性=%s", merchant_info.alias, res)
merchant_info.status = res
return res
def format_date(timestamp, format_str):
if isinstance(timestamp, unicode):
return timestamp
if isinstance(timestamp, str):
return unicode(timestamp)
try:
date_obj = datetime.datetime.fromtimestamp(timestamp / 1e3)
return datetime.datetime.strftime(date_obj, unicode(format_str))
except Exception as ex:
# ignore
LOGGER.error(u"日期转换出错，输入timestamp=%s, Exception:%s", timestamp, ex)
return unicode(timestamp)
def get_all_data_from_cache(order_download_date):
"""
从缓存中获取所有商户信息数据
:return:
"""
try:
cache_file_full_path = get_cache_full_path(order_download_date)
with open(cache_file_full_path, 'rb') as cache_file:
res = pickle.load(cache_file)
LOGGER.info("加载商户缓存数据成功")
return res
except Exception as ex:
LOGGER.error("加载商户缓存数据异常，Exception=%s", ex)
def _download_order_by_logon_id_helper(logon_id, order_download_date):
"""
根据登陆账号下载数据
:param logon_id:
:param order_download_date:
:return: error, `TradeSummary`, `TradeDetail`
"""
error = None
if not logon_id:
error = "登陆账号为空，请检查"
return error, None, None
merchant = get_merchant(logon_id)
if not merchant:
error = "根据%s未查找到合适的商户数据，请检查" % logon_id
return error, None, None
# 检查session有效性
session_validation = check_status_merchant(merchant)
if not session_validation:
error = "有部分商户未登录无法下载（已自动忽略），请检查后重新登录该部分商户"
return error, None, None
# 开始下载数据
summary = None
orders = None
try:
error, summary, orders = query_order_detail(merchant.session_id,
order_download_date)
except NoDataException as ex:
# 商户无数据
return "商户[%s]%s无数据，请重新选择日期下载" % (merchant.alias,
order_download_date), None, None
except Exception as ex:
LOGGER.exception("下载商户订单数据异常")
error = "下载商户订单数据发生异常:%s" % ex.message
return error, None, None
try:
if isinstance(summary, TradeSummary):
# 更新缓存里的商户数据
merchant.store_name = summary.store_name
merchant.shop_id = summary.shop_id
except Exception as ex:
LOGGER.exception("更新缓存商户%s数据发送异常，不影响业务，忽略。 Exception:%s", logon_id, ex)
# 替换掉商户名称，加上alias名称，解决同名商户不同mcc问题
replace_merchant_name(logon_id, summary, orders)
return error, summary, orders
def refresh_merchant_config(from_front=False):
"""
刷新商户状态和配置等
:return:
"""
LOGGER.info(u"获取商户配置数据")
cfgs = get_merchant_config()
if can_refresh(from_front):
check_status(cfgs)
res = list()
count = 0
for d in cfgs:
res.append(d.to_dict())
if d.status:
count += 1
LOGGER.info(u"在线商户:%s", count)
record_offline_time(count)
return sort_merchant(res)
def read_merchant_cfg():
"""
读取商户json配置，返回dict
:return: `dict`
"""
cfg_file = get_merchant_cfg_file()
if not os.path.exists(cfg_file):
raise Exception(u"商户配置json文件不存在，请检查,cfg_file=%s" % cfg_file)
res = None
with open(cfg_file, 'r') as data:
try:
content = data.read()
if content.startswith(codecs.BOM_UTF8):
content = content.decode('utf-8-sig')
res = json.loads(content)
except Exception as ex:
LOGGER.exception(u"商户配置json文件格式不合法，请检查%s", ex)
raise Exception(u"商户配置json文件格式不合法，请检查")
return res
def get_ie_driver():
"""
https://docs.seleniumhq.org/docs/04_webdriver_advanced.jsp#using-a-proxy
https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities#proxy-json-object
使用方式：先用正常页面不带代理的，启动多个窗口登录。登录成功后，再切换回代理。
TODO: 需要设置刷新的时候也走代理，本地模式。 服务模式不需要。
:return:
"""
proxy = get_proxy()
driver = None
if proxy:
prox = Proxy()
prox.proxy_type = ProxyType.MANUAL
prox.http_proxy = proxy
prox.socks_proxy = proxy
prox.ssl_proxy = proxy
prox.ftp_proxy = proxy
LOGGER.info("try to use proxy: %s", proxy)
caps = webdriver.DesiredCapabilities.INTERNETEXPLORER.copy()
# caps['proxy'] = {
# "httpProxy": proxy,
# "ftpProxy": proxy,
# "sslProxy": proxy,
# "socks_proxy": proxy,
# "noProxy": None,
# "proxyType": "MANUAL",
# "autodetect": False
# }
prox.add_to_capabilities(caps)
# caps["proxy"] = {"proxyType": "manual", "httpProxy": proxy}
opt = options.Options()
opt.use_per_process_proxy = True
driver = webdriver.Ie(capabilities=caps, options=opt)
else:
driver = webdriver.Ie()
return driver
def save_all_data_to_cache(order_download_date, res):
"""
保存所有信息到缓存
:return:
"""
if not is_valid_data(res):
LOGGER.warn("跳过刷新商户数据缓存，无效的缓存。tip=%s", res.get('tip'))
return None
try:
cache_file_full_path = get_cache_full_path(order_download_date)
with open(cache_file_full_path, 'wb') as cache_file:
pickle.dump(res, cache_file)
LOGGER.info("刷新商户数据缓存成功,tip=%s", res.get('tip'))
except Exception as ex:
LOGGER.error("刷新商户数据缓存异常，%s", ex)
def download_all_orders(order_download_date=None, enable_cache=False):
"""
下载所有的商户，过滤session无效的
:param order_download_date:
:return:
res = {'status': False, 'summary': summary, 'orders': orders, 'errors': errors, 'warnings': warnings, 'tip': ''}
"""
if not order_download_date:
order_download_date = get_now_date_str()
if enable_cache:
return get_all_data_from_cache(order_download_date)
jobs = list()
jobs_dict = dict()
for key in MERCHANTS_DATA:
merchant = MERCHANTS_DATA[key]
if isinstance(merchant, MerchantInfo):
logon_id = merchant.logon_id
greenlet = gevent.spawn(_download_order_by_logon_id_helper,
logon_id, order_download_date)
jobs.append(greenlet)
jobs_dict[logon_id] = greenlet
gevent.joinall(jobs)
# 获取每个任务的结果输出
warnings = list()
errors = list()
# 用于前台返回
summary = list()
orders = list()
# 用于生成excel
summary_dict = dict()
orders_dict = dict()
res = {
'status': True,
'summary': summary,
'orders': orders,
'errors': errors,
'warnings': warnings,
'tip': ''
}
# 按照配置文件的顺序组装最终返回数据
logon_ids = get_merchants_logon_ids()
valid_logon_ids = list()
for logon_id in logon_ids:
job = jobs_dict.get(logon_id)
merchant = get_merchant(logon_id)
if isinstance(job, gevent.greenlet.Greenlet):
job_error, job_summary, job_orders = job.get()
else:
LOGGER.warn("%s对应的返回数据为None，跳过,请刷新页面，加载配置数据", logon_id)
continue
if is_no_data(job_error):
warnings.append("商户[%s]%s无数据，跳过" %
(merchant.alias, order_download_date))
elif job_error:
errors.append(job_error)
# 有一个状态失败，则都失败
res['status'] = False
else:
summary.append(job_summary.to_dict())
for job_order in job_orders:
orders.append(job_order.to_dict())
summary_dict[logon_id] = job_summary
orders_dict[logon_id] = job_orders
valid_logon_ids.append(logon_id)
try:
file_path = batch_save_order_data_to_excel(order_download_date,
valid_logon_ids,
summary_dict, orders_dict)
res['tip'] = "成功处理了%s个商户,%s条订单，文件路径：%s" % (len(valid_logon_ids),
len(orders), file_path)
except Exception as ex:
LOGGER.exception("批量保存邮件异常：%s", ex)
errors.append("保存excel异常%s" % ex.message)
save_all_data_to_cache(order_download_date, res)
return res