def visit_url_action(self):
self.check_login()
self.driver.get(self.siteurl + "daily_report/url_action")
if "URL" in self.driver.page_source:
msg = "daily url action work success"
else:
msg = "daily url action work failed"
utils.record_log(msg)
def run_jobs(self):
self.visit_24hours_url_view()
self.save_and_uload_snapshot()
utils.record_log('upload yama 24 hours success')
self.visit_url_action()
self.save_and_uload_snapshot()
utils.record_log('upload yama 24 hours success')
def visit_24hours_url_view(self):
self.check_login()
self.driver.get(self.siteurl + "daily_report/hour_url_action")
if "用户量".decode("utf-8") in self.driver.page_source:
msg = "24hours url work success"
else:
msg = "24hours url work failed"
utils.record_log(msg)
def search_php(self):
page_info = self.baidu_search("python")
if "No results found" in page_info:
msg = "search python success"
else:
msg = "search php success"
utils.record_log(msg)
def cnblog_search_nosql(self):
page_info = self.cnblog_search('nosql')
if 'nosql' in page_info:
msg = "cnblog search work succes"
else:
msg = "cnblog search work failed"
utils.record_log(msg)
def save_snapshot(self, save_name="", width=1000, height=1000):
#self.driver.set_window_size(width, height)
if not save_name:
save_name = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
save_path = "Shot/" + save_name + ".png"
utils.record_log(save_path + " save success")
self.driver.save_screenshot(save_path)
return save_name + ".png"
def cnblog_search_golang(self):
self.select_driver_adapter()
page_info = self.cnblog_search('golang')
if 'golang' in page_info:
msg = "cnblog search work succes"
else:
msg = "cnblog search work failed"
utils.record_log(msg)
def save_and_uload_snapshot(self, save_name="", width=1000, height=1000):
auto_save_file_name = self.save_snapshot(save_name, width, height)
if not save_name:
save_name = auto_save_file_name
local_file_path = "Shot/" + save_name
if os.path.exists(local_file_path):
qiniu_help.upload_file(local_file_path, save_name)
else:
utils.record_log(local_file_path + "file is not exist")
def select_driver_adapter(self, driver_name=""):
select_driver_name = "chrome" if driver_name == "" else driver_name
if select_driver_name.lower() == "firefox":
self.driver = webdriver.Firefox()
elif select_driver_name.lower() == "chrome":
self.driver = webdriver.Chrome()
else:
utils.record_log("select web driver name is wrong")
self.driver = ""
def upload_file(file_path, save_name):
init_data()
q = Auth(access_key, secret_key)
token = q.upload_token(bucket_name, save_name, 3600)
ret, info = put_file(token, save_name, file_path)
if info.status_code == 200:
utils.record_log("qiniu upload file_path success")
else:
utils.record_log("qiniu upload file_path failed")
def select_driver_adapter(self, driver_name=""):
select_driver_name = "chrome" if driver_name == "" else driver_name
if select_driver_name.lower() == "firefox":
self.driver = webdriver.Firefox()
elif select_driver_name.lower() == "chrome":
from selenium.webdriver.chrome.options import Options
chrome_options = Options()
chrome_options.add_argument("--disable-extensions")
self.driver = webdriver.Chrome(chrome_options=chrome_options)
#self.driver = webdriver.Chrome()
elif select_driver_name.lower() == "phantomjs":
self.driver = webdriver.PhantomJS(
executable_path=r'D:\phantomjs\phantomjs-2.1.1-windows\bin\phantomjs.exe')
else:
utils.record_log("select web driver name is wrong")
self.driver = ""
def login_bi(self):
self.home_page_login()
login_user_field = self.driver.find_element_by_id(self.login_user_id)
login_user_field.send_keys(self.login_user)
login_pwd_field = self.driver.find_element_by_id(self.login_pwd_id)
login_pwd_field.send_keys(self.login_pwd)
submit_elem = self.driver.find_element_by_name("submit")
submit_elem.click()
if 'BI' in self.driver.title:
msg = "bi login success"
self.is_login = True
else:
self.is_login = False
msg = "bi login failed"
utils.record_log(msg)
def downloadfile(index, private_url, file_name):
time_start = time.time()
result = False
try:
r = requests.get(private_url)
if r.status_code == 200:
file = open(file_name, "wb")
file.write(r.content)
file.flush()
file.close()
result = True
except Exception as e:
pass
time_end = time.time()
elapsed = time_end - time_start
utils.record_log('download: (%s) -> %s, result: %s, time: %ss' %
(index, private_url, result, elapsed))
return result, index, private_url
def login_yama(self):
self.default_action()
login_elem = self.driver.find_element_by_id(self.login_btn_id)
login_elem.click()
login_user_field = self.driver.find_element_by_id(self.login_user_id)
login_user_field.send_keys(self.login_user)
login_pwd_field = self.driver.find_element_by_id(self.login_pwd_id)
login_pwd_field.send_keys(self.login_pwd)
submit_elem = self.driver.find_element_by_name("submit")
submit_elem.click()
if 'YaMa' in self.driver.title:
msg = "yama login success"
self.is_login = True
else:
msg = "yama login failed"
self.is_login = False
utils.record_log(msg)
def down_file(key, save_name=''):
init_data()
q = Auth(access_key, secret_key)
base_url = 'http://%(domain)s/%(key)s' % {"domain": bucket_domain, "key": key}
private_url = q.private_download_url(base_url, expires=3600)
r = requests.get(private_url)
if r.status_code == 200:
msg = private_url + " download success"
utils.record_log(msg)
if not save_name:
save_name = key
f = open("Shot/" + save_name, 'wb')
f.write(r.content)
f.close()
else:
msg = private_url + " download failed"
utils.record_log(msg)
def send_mail_with_snapshot(content,
subject='WarnNotice',
snapshot_imgs='',
attach_file=''):
cf = ConfigParser.ConfigParser()
cf.read('config.ini.bk')
send_from = cf.get('email_account', 'send_from')
send_pwd = cf.get('email_account', 'send_pwd')
send_to = cf.get('email_account', 'send_to')
send_ccs = cf.get('email_account', 'send_cc')
send_smtp_server = cf.get('email_account', 'smtp_server')
send_port = cf.get('email_account', 'smtp_port')
send_tos = send_to.split(';')
message = MIMEMultipart()
message['From'] = Header("EasyPass机器人", 'utf-8')
#message['To'] = Header(''接收组, 'utf-8')
message['Subject'] = Header(subject, 'utf-8')
message.attach(MIMEText(content, 'html', 'utf-8'))
if attach_file:
att = MIMEText(open(attach_file, 'rb').read(), 'base64', 'utf-8')
att["Content-Type"] = 'application/octet-stream'
att["Content-Disposition"] = 'attachment; filename="' + attach_file + '"'
message.attach(att)
message = __add_snapshot_img(message, snapshot_imgs)
try:
server = smtplib.SMTP(send_smtp_server, send_port)
server.login(send_from, send_pwd)
server.sendmail(send_from, send_tos, message.as_string())
server.quit()
utils.record_log("send email success")
return True
except smtplib.SMTPException, e:
utils.record_log("send email failed " + e.message)
return False
(index, private_url, result, elapsed))
return result, index, private_url
if __name__ == '__main__':
access_key = ''
secret_key = ''
bucket_domain = ''
bucket_name = ''
path = 'download/'
if not os.path.exists(path):
os.makedirs(path)
q = Auth(access_key, secret_key)
bucket = BucketManager(q)
items = listfiles(bucket, bucket_name)
utils.record_log('total files: %s' % len(items))
results = []
jobs = []
p = Pool(cpu_count() * 2)
time_start = time.time()
for index, item in enumerate(items):
key = item['key']
if '/' in key:
key_path = path + key[:key.rfind('/')]
if not os.path.exists(key_path):
os.makedirs(key_path)
base_url = 'http://%(domain)s/%(key)s' % {
"domain": bucket_domain,
"key": key
}
private_url = q.private_download_url(base_url, expires=3600)
