def visit_url_action(self): self.check_login() self.driver.get(self.siteurl + "daily_report/url_action") if "URL" in self.driver.page_source: msg = "daily url action work success" else: msg = "daily url action work failed" utils.record_log(msg)
def run_jobs(self): self.visit_24hours_url_view() self.save_and_uload_snapshot() utils.record_log('upload yama 24 hours success') self.visit_url_action() self.save_and_uload_snapshot() utils.record_log('upload yama 24 hours success')
def visit_24hours_url_view(self): self.check_login() self.driver.get(self.siteurl + "daily_report/hour_url_action") if "用户量".decode("utf-8") in self.driver.page_source: msg = "24hours url work success" else: msg = "24hours url work failed" utils.record_log(msg)
def search_php(self): page_info = self.baidu_search("python") if "No results found" in page_info: msg = "search python success" else: msg = "search php success" utils.record_log(msg)
def cnblog_search_nosql(self): page_info = self.cnblog_search('nosql') if 'nosql' in page_info: msg = "cnblog search work succes" else: msg = "cnblog search work failed" utils.record_log(msg)
def save_snapshot(self, save_name="", width=1000, height=1000): #self.driver.set_window_size(width, height) if not save_name: save_name = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time())) save_path = "Shot/" + save_name + ".png" utils.record_log(save_path + " save success") self.driver.save_screenshot(save_path) return save_name + ".png"
def cnblog_search_golang(self): self.select_driver_adapter() page_info = self.cnblog_search('golang') if 'golang' in page_info: msg = "cnblog search work succes" else: msg = "cnblog search work failed" utils.record_log(msg)
def save_and_uload_snapshot(self, save_name="", width=1000, height=1000): auto_save_file_name = self.save_snapshot(save_name, width, height) if not save_name: save_name = auto_save_file_name local_file_path = "Shot/" + save_name if os.path.exists(local_file_path): qiniu_help.upload_file(local_file_path, save_name) else: utils.record_log(local_file_path + "file is not exist")
def select_driver_adapter(self, driver_name=""): select_driver_name = "chrome" if driver_name == "" else driver_name if select_driver_name.lower() == "firefox": self.driver = webdriver.Firefox() elif select_driver_name.lower() == "chrome": self.driver = webdriver.Chrome() else: utils.record_log("select web driver name is wrong") self.driver = ""
def upload_file(file_path, save_name): init_data() q = Auth(access_key, secret_key) token = q.upload_token(bucket_name, save_name, 3600) ret, info = put_file(token, save_name, file_path) if info.status_code == 200: utils.record_log("qiniu upload file_path success") else: utils.record_log("qiniu upload file_path failed")
def select_driver_adapter(self, driver_name=""): select_driver_name = "chrome" if driver_name == "" else driver_name if select_driver_name.lower() == "firefox": self.driver = webdriver.Firefox() elif select_driver_name.lower() == "chrome": from selenium.webdriver.chrome.options import Options chrome_options = Options() chrome_options.add_argument("--disable-extensions") self.driver = webdriver.Chrome(chrome_options=chrome_options) #self.driver = webdriver.Chrome() elif select_driver_name.lower() == "phantomjs": self.driver = webdriver.PhantomJS( executable_path=r'D:\phantomjs\phantomjs-2.1.1-windows\bin\phantomjs.exe') else: utils.record_log("select web driver name is wrong") self.driver = ""
def login_bi(self): self.home_page_login() login_user_field = self.driver.find_element_by_id(self.login_user_id) login_user_field.send_keys(self.login_user) login_pwd_field = self.driver.find_element_by_id(self.login_pwd_id) login_pwd_field.send_keys(self.login_pwd) submit_elem = self.driver.find_element_by_name("submit") submit_elem.click() if 'BI' in self.driver.title: msg = "bi login success" self.is_login = True else: self.is_login = False msg = "bi login failed" utils.record_log(msg)
def downloadfile(index, private_url, file_name): time_start = time.time() result = False try: r = requests.get(private_url) if r.status_code == 200: file = open(file_name, "wb") file.write(r.content) file.flush() file.close() result = True except Exception as e: pass time_end = time.time() elapsed = time_end - time_start utils.record_log('download: (%s) -> %s, result: %s, time: %ss' % (index, private_url, result, elapsed)) return result, index, private_url
def login_yama(self): self.default_action() login_elem = self.driver.find_element_by_id(self.login_btn_id) login_elem.click() login_user_field = self.driver.find_element_by_id(self.login_user_id) login_user_field.send_keys(self.login_user) login_pwd_field = self.driver.find_element_by_id(self.login_pwd_id) login_pwd_field.send_keys(self.login_pwd) submit_elem = self.driver.find_element_by_name("submit") submit_elem.click() if 'YaMa' in self.driver.title: msg = "yama login success" self.is_login = True else: msg = "yama login failed" self.is_login = False utils.record_log(msg)
def down_file(key, save_name=''): init_data() q = Auth(access_key, secret_key) base_url = 'http://%(domain)s/%(key)s' % {"domain": bucket_domain, "key": key} private_url = q.private_download_url(base_url, expires=3600) r = requests.get(private_url) if r.status_code == 200: msg = private_url + " download success" utils.record_log(msg) if not save_name: save_name = key f = open("Shot/" + save_name, 'wb') f.write(r.content) f.close() else: msg = private_url + " download failed" utils.record_log(msg)
def send_mail_with_snapshot(content, subject='WarnNotice', snapshot_imgs='', attach_file=''): cf = ConfigParser.ConfigParser() cf.read('config.ini.bk') send_from = cf.get('email_account', 'send_from') send_pwd = cf.get('email_account', 'send_pwd') send_to = cf.get('email_account', 'send_to') send_ccs = cf.get('email_account', 'send_cc') send_smtp_server = cf.get('email_account', 'smtp_server') send_port = cf.get('email_account', 'smtp_port') send_tos = send_to.split(';') message = MIMEMultipart() message['From'] = Header("EasyPass机器人", 'utf-8') #message['To'] = Header(''接收组, 'utf-8') message['Subject'] = Header(subject, 'utf-8') message.attach(MIMEText(content, 'html', 'utf-8')) if attach_file: att = MIMEText(open(attach_file, 'rb').read(), 'base64', 'utf-8') att["Content-Type"] = 'application/octet-stream' att["Content-Disposition"] = 'attachment; filename="' + attach_file + '"' message.attach(att) message = __add_snapshot_img(message, snapshot_imgs) try: server = smtplib.SMTP(send_smtp_server, send_port) server.login(send_from, send_pwd) server.sendmail(send_from, send_tos, message.as_string()) server.quit() utils.record_log("send email success") return True except smtplib.SMTPException, e: utils.record_log("send email failed " + e.message) return False
(index, private_url, result, elapsed)) return result, index, private_url if __name__ == '__main__': access_key = '' secret_key = '' bucket_domain = '' bucket_name = '' path = 'download/' if not os.path.exists(path): os.makedirs(path) q = Auth(access_key, secret_key) bucket = BucketManager(q) items = listfiles(bucket, bucket_name) utils.record_log('total files: %s' % len(items)) results = [] jobs = [] p = Pool(cpu_count() * 2) time_start = time.time() for index, item in enumerate(items): key = item['key'] if '/' in key: key_path = path + key[:key.rfind('/')] if not os.path.exists(key_path): os.makedirs(key_path) base_url = 'http://%(domain)s/%(key)s' % { "domain": bucket_domain, "key": key } private_url = q.private_download_url(base_url, expires=3600)