class Spys: def __init__(self): self.baseUrl = 'http://spys.one/en/https-ssl-proxy/' self.logger = Logger(name=self.__class__.__name__) self.payload = "xpp=2&xf1=0&xf4=2&xf5=0" self.headers = { 'content-type': "application/x-www-form-urlencoded", 'cache-control': "no-cache", } self.selector = { 'table': 'tr[onmouseover*="this"] td[colspan="1"]:nth-child(1)', 'ip': 'font.spy14', 'port': 'font.spy14 font' } def proxy_parser(self, html): result = [] try: tables = parser(html, self.selector['table']) for table in tables: ip = parser(table, self.selector['ip']).remove('script').text() port = 8080 result.append({'address': ip, 'port': port}) except Exception as e: self.logger.log(msg=str(e), level='error') return result def main(self, path): html = get_html(self.baseUrl, headers=self.headers, payloads=self.payload) checker = self.proxy_parser(html) self.logger.log('get {} proxies'.format(len(checker))) save_proxy(proxies=proxy_checker(checker), location=path)
def __init__(self, printCFG=True): self.url = "https://www.indiacashandcarry.com/login" self.default_zip = '94040' self.slots_result = '' self.addr_list = [] self.pickup_status = {} self.firstInstance = True self.delivery_status = {} self.icc_to_check = settings.ICC_STORES_TO_CHECK.upper() self.icc_option = settings.ICC_STATUS_TO_CHECK.upper() self.supported_store_list = ['SUNNYVALE', 'SAN JOSE', 'FREMONT', 'ALL'] self.logger = Logger() self.browser = Browser(driverpath=settings.CHROME_DRIVER_PATH, logger=self.logger) signal(SIGINT, self.handler) self.__validate_store__() self.email = SendEmail(settings.SENDER_GMAIL_ID, settings.SENDER_GMAIL_PASS) if printCFG: self.log_msg('\n##########################################') self.log_msg('Input Config:') self.log_msg('\tChrome driver path: {}'.format( settings.CHROME_DRIVER_PATH)) self.log_msg('\tStores List: {}'.format(self.icc_to_check)) self.log_msg('\tPickup/Delivery: {}'.format(self.icc_option)) self.log_msg('\tICC Login: {}'.format(settings.ICC_LOGIN_EMAIL)) self.log_msg('\tSEND_EMAIL report: {}'.format(settings.SEND_EMAIL)) self.log_msg('\n##########################################')
class Hidester: def __init__(self): self.logger = Logger(name=self.__class__.__name__) def get_data(self): results = None try: url = "https://hidester.com/proxydata/php/data.php" querystring = { "mykey": "data", "offset": "0", "limit": "500", "orderBy": "latest_check", "sortOrder": "DESC", "country": "", "port": "", "type": "http", "anonymity": "undefined", "ping": "undefined", "gproxy": "2" } headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36', 'X-Requested-With': 'XMLHttpRequest', 'Host': 'hidester.com', 'Referer': 'https://hidester.com/proxylist/' } response = requests.request("GET", url, headers=headers, params=querystring) if response.status_code == 200: results = response.json() results = [x for x in results if x['type'] == 'http'] except Exception as e: raise e return results def proxy_parser(self, data): results = [] try: for result in data: result['address'] = result['IP'] result['port'] = result['PORT'] results.append(result) except Exception as e: raise e return results def main(self, path): html_dict = self.get_data() checker = self.proxy_parser(html_dict) self.logger.log('get {} proxies'.format(len(checker))) save_proxy(proxies=proxy_checker(checker), location=path)
def __init__(self): self.baseUrl = 'https://www.free-proxy-list.net/' self.logger = Logger(name=self.__class__.__name__) self.selector = { 'table': '#list', 'item': 'tr:has("td")', 'ip': 'td:nth-child(1)', 'port': 'td:nth-child(2)', 'type': 'td:nth-child(7)', }
def __init__(self): self.baseUrl = 'http://spys.one/en/https-ssl-proxy/' self.logger = Logger(name=self.__class__.__name__) self.payload = "xpp=2&xf1=0&xf4=2&xf5=0" self.headers = { 'content-type': "application/x-www-form-urlencoded", 'cache-control': "no-cache", } self.selector = { 'table': 'tr[onmouseover*="this"] td[colspan="1"]:nth-child(1)', 'ip': 'font.spy14', 'port': 'font.spy14 font' }
class Proxy_list: def __init__(self): self.baseUrl = 'https://www.free-proxy-list.net/' self.logger = Logger(name=self.__class__.__name__) self.selector = { 'table': '#list', 'item': 'tr:has("td")', 'ip': 'td:nth-child(1)', 'port': 'td:nth-child(2)', 'type': 'td:nth-child(7)', } def proxy_parser(self, html): result = [] try: table = parser(html, self.selector['table']) raw_content = parser(table, self.selector['item']) if raw_content: proxys = raw_content.remove('.active') for proxy in proxys: tmp = dict() ip = parser(proxy, self.selector['ip']).text() port = parser(proxy, self.selector['port']).text() type = parser(proxy, self.selector['type']).text().lower() if type == 'yes': tmp['address'] = ip tmp['port'] = port result.append(tmp) except Exception as e: self.logger.log(str(e), level='error') return result def main(self, path): html = get_html(Proxy_list().baseUrl) checker = self.proxy_parser(html) self.logger.log('get {} proxies'.format(len(checker))) save_proxy(proxies=proxy_checker(checker), location=path)
class HyperparameterLogger: def __init__(self, dict, name="", detailed_name=""): self.dict = dict self.logger = Logger(name) self.detailed_logger = Logger(detailed_name) def log(self, result): self.logger.log(";".join(map(str, [datetime.now(), self.dict, result]))) def log_all_epochs(self, epochs): self.detailed_logger.log("***********") self.detailed_logger.log(";".join(map(str, [datetime.now(), self.dict]))) for epoch in epochs: self.detailed_logger.log(epoch)
def post(): Logger().set_request(request,'clear_error') data = rest.request.get_json(silent=True) if data is None or data['module'] is None: response = jsonify({'data': {'success': False, 'code': 403, 'message': 'bad module name'}}) response.status_code = 403 return response module_addr = data['module'] try: SqLite().set_error_clear(module_addr) return {'data': {'success': True}} except Exception as e: response = jsonify({'data': {'success': False, 'code': 500, 'message': e}}) response.status_code = 500 return response
def post(): Logger().set_request(request, 'set_module') data = rest.request.get_json(silent=True) if data is None or data['module'] is None: response = jsonify({ 'data': { 'success': False, 'code': 403, 'message': 'bad module name' } }) response.status_code = 403 return response module_addr = data['module'] p1 = data['p1'] p2 = data['p2'] power = data['power'] try: if module_addr == 0: all_module = SqLite().get_states() for module in all_module: if module[7] == 1: Modules().write_module(module[0], module[1], module[2], power) return {'data': {'success': True}} Modules().write_module(module_addr, p1, p2, power) if power == 1: Modules().write_module(0, 0, 0, 1) else: all_module = SqLite().get_states() all_calc = 0 for module in all_module: if module[7] == 1 and module[0] != 0: all_calc += module[4] if all_calc == 0: Modules().write_module(0, 0, 0, 0) return {'data': {'success': True}} except Exception as e: response = jsonify( {'data': { 'success': False, 'code': 500, 'message': e }}) response.status_code = 500 return response
def post(self): Logger().set_request(request, 'power') data = rest.request.get_json(silent=True) if data is None or data['power'] is None or data['modules'] is None: response = jsonify({ 'data': { 'success': False, 'code': 403, 'message': 'bad module name' } }) response.status_code = 403 return response power = data['power'] module_state = data['modules'] try: if power == 1: Modules().write_module(0, 0, 0, 1) for key in module_state: Modules().write_module(key['module'], key['p1'], key['p2'], 1) elif power == 0: for key in module_state: Modules().write_module(key['module'], key['p1'], key['p2'], 0) work = Thread(target=self.wait_closer) work.start() return {'data': {'success': 'true'}} except Exception as e: response = jsonify( {'data': { 'success': False, 'code': 500, 'message': e }}) response.status_code = 500 return response
# coding=utf-8 import time from selenium.common.exceptions import NoSuchElementException import os.path from helper.logger import Logger import sys from imp import reload reload(sys) # sys.setdefaultencoding('utf8') # 如果不添加以上三行代码,xpath如果表达式包括中文,就会报错,python 2.x 默认 # string 类型是assic类型,在xpath拆分的时候,报codec can't decode byte 0xe4 in position 17: ordinal not in range(128) # create a logger instance logger = Logger(logger="BasePage").getlog() class BasePage(object): """ 定义一个页面基类,让所有页面都继承这个类,封装一些常用的页面操作方法到这个类 """ def __init__(self, driver): self.driver = driver # quit browser and end testing def quit_browser(self): self.driver.quit() # 浏览器前进操作 def forward(self): self.driver.forward() logger.info("Click forward on current page.")
#!/usr/bin/python2.7 # -*- coding: utf-8 -*- import os import requests from bs4 import BeautifulSoup from pyquery import PyQuery from json import dump from helper.logger import Logger current_path = os.path.dirname(os.path.abspath(__file__)) root = os.path.abspath(os.path.join(current_path, os.pardir)) logger = Logger(__name__) def get_html(url, payloads=None, headers=''): result = None if payloads: try: response = requests.request(method='POST', url=url, data=payloads, headers=headers) if response.status_code == 200: result = response.text except Exception as e: raise e else: try: response = requests.get(url) if response.status_code == 200:
# -*- coding:utf-8 -*- import configparser import os.path from selenium import webdriver from helper.logger import Logger logger = Logger(logger="BrowserEngine").getlog() class BrowserEngine(object): chrome_driver_path = os.path.join(os.path.abspath('.'), 'tools/chromedriver') firefox_driver_path = os.path.join(os.path.abspath('.'), 'tools/geckodriver') ie_driver_path = os.path.join(os.path.abspath('.'), 'tools/iedriver') # print(chrome_driver_path,firefox_driver_path) def __init__(self, driver): self.driver = driver # read the browser type from config.ini file, return the driver def open_browser(self, driver): config = configparser.ConfigParser() file_path = os.path.join(os.path.abspath('.'), 'config/config.ini') # print(file_path) config.read(file_path) browser = config.get("browserType", "browserName") logger.info("You had select %s browser." % browser)
def __init__(self, dict, name="", detailed_name=""): self.dict = dict self.logger = Logger(name) self.detailed_logger = Logger(detailed_name)
def __init__(self): self.logger = Logger(name=self.__class__.__name__)
def post(self): Logger().set_request(request) data = rest.request.get_json(silent=True) if data is None or data['action'] is None: response = jsonify({ 'data': { 'success': False, 'code': 403, 'message': 'bad module name' } }) response.status_code = 403 return response action = data['action'] if action == "restart": work = Thread(target=self.reboot) work.start() return {'data': {'success': True}} elif action == "shutdown": work = Thread(target=self.shutdown) work.start() return {'data': {'success': True}} elif action == "hostname": data = os.system('hostnamectl set-hostname ' + data['hostname']) return { 'data': { 'success': True, "message": "Sistem Adı Değiştirildi" } } elif action == "save_module": module_id = data['module_id'] module_name = data['module_name'] is_active = data['is_active'] if is_active == 0: Modules().write_module(module_id, 0, 0, 0) SqLite().update_state(module_id, module_name, is_active) return { 'data': { 'success': True, "message": "Modül Bilgileri Güncellendi" } } elif action == "set_pg_db": pg_ip = data['pg_ip'] pg_port = data['pg_port'] pg_username = data['pg_username'] pg_password = data['pg_password'] SqLite().set_system_value('pg_ip', pg_ip) SqLite().set_system_value('pg_port', pg_port) SqLite().set_system_value('pg_username', pg_username) SqLite().set_system_value('pg_password', pg_password) return { 'data': { 'success': True, "message": "Veritabanı Bilgileri Güncellendi" } } elif action == "set_ip": if 'eth_ip' in data and 'eth_gateway' in data: eth_ip = data['eth_ip'] eth_gateway = data['eth_gateway'] ip_temp = "interface eth0 \n" \ "static ip_address=|eth_ip|\n" \ "static routers=|eth_gateway|\n" \ "static domain_name_servers=8.8.8.8\n" \ "\n" \ "interface wlan0\n" \ "static ip_address=192.168.88.240/24\n" \ "static routers=192.168.88.1\n" \ "static domain_name_servers=8.8.8.8" ip_temp = ip_temp.replace("|eth_ip|", str(eth_ip)) ip_temp = ip_temp.replace("|eth_gateway|", str(eth_gateway)) SqLite().set_system_value('eth_ip', eth_ip) SqLite().set_system_value('eth_gateway', eth_gateway) f = open("/etc/dhcpcd.conf", "w") f.write(ip_temp) f.close() work = Thread(target=self.reboot) work.start() return { 'data': { 'success': True, "message": "Ip Adresi Güncellendi. Sistem yeniden başlatılıyor", 'uri': 'http://' + eth_ip.split('/')[0] + '/settings.html' } } else: response = jsonify({ 'data': { 'success': False, 'code': 403, 'message': 'Hatalı Adres Bilgisi' } }) response.status_code = 403
class ICCSlotFinder: def __init__(self, printCFG=True): self.url = "https://www.indiacashandcarry.com/login" self.default_zip = '94040' self.slots_result = '' self.addr_list = [] self.pickup_status = {} self.firstInstance = True self.delivery_status = {} self.icc_to_check = settings.ICC_STORES_TO_CHECK.upper() self.icc_option = settings.ICC_STATUS_TO_CHECK.upper() self.supported_store_list = ['SUNNYVALE', 'SAN JOSE', 'FREMONT', 'ALL'] self.logger = Logger() self.browser = Browser(driverpath=settings.CHROME_DRIVER_PATH, logger=self.logger) signal(SIGINT, self.handler) self.__validate_store__() self.email = SendEmail(settings.SENDER_GMAIL_ID, settings.SENDER_GMAIL_PASS) if printCFG: self.log_msg('\n##########################################') self.log_msg('Input Config:') self.log_msg('\tChrome driver path: {}'.format( settings.CHROME_DRIVER_PATH)) self.log_msg('\tStores List: {}'.format(self.icc_to_check)) self.log_msg('\tPickup/Delivery: {}'.format(self.icc_option)) self.log_msg('\tICC Login: {}'.format(settings.ICC_LOGIN_EMAIL)) self.log_msg('\tSEND_EMAIL report: {}'.format(settings.SEND_EMAIL)) self.log_msg('\n##########################################') def handler(self, signal_received, frame): msg = 'SIGINT or CTRL-C detected. Exiting gracefully' self.log_msg(msg) self.close() sys.exit(0) def get_url(self, url, delay=3): self.browser.load_url(url, delay) def send_notification(self, subject, body): if settings.SEND_EMAIL and self.email: self.email.send_email(settings.SENDER_GMAIL_ID, settings.RECEIVER_EMAIL_ID, subject, body) @custom_decorator def start(self, withHead=False): try: self.browser.start(headless=(not withHead)) self.get_url(self.url, delay=5) except Exception as err: self.log_msg('browser CANT LAUNCH, err {}\n'.format(err)) self.close() sys.exit(0) self.log_msg('Attempting to login...') try: self.__login_icc_account__() except Exception as err: self.log_msg( 'LOGIN ERROR, CHECK CREDENTIALS, err: {}\n'.format(err)) self.close() sys.exit(0) else: self.log_msg('Login succeeded') def __validate_store__(self): if self.icc_to_check not in self.supported_store_list: self.log_msg("Invalid option for ICC_STORES_TO_CHECK - {}".format( self.icc_to_check)) self.close() self.exit(0) @custom_decorator def __login_icc_account__(self): xpath = '//*[@id="email"]' self.browser.perform_by_xpath(xpath, action='clear', delay=0.5) self.browser.perform_by_xpath(xpath, action='send_text', text=settings.ICC_LOGIN_EMAIL, delay=0.5) xpath = '/html/body/div[2]/div/div/div/div/div/div/div/form/div[2]/input' self.browser.perform_by_xpath(xpath, action='clear', delay=0.5) self.browser.perform_by_xpath(xpath, action='send_text', text=settings.ICC_LOGIN_PASS, delay=0.5) xpath = '/html/body/div[2]/div/div/div/div/div/div/div/form/div[3]/button' self.browser.perform_by_xpath(xpath, action='click', delay=3) try: xpath = '/html/body/div[2]/div/div/div/div/div/div/div/form/div[3]/div' status = self.browser.perform_by_xpath(xpath, action='read_text') if 'Incorrect username' in status: self.log_msg('LOGIN ERROR, CHECK CREDENTIALS') self.close() sys.exit(0) except Exception: pass try: xpath = '//*[@id="email-error"]' status = self.browser.perform_by_xpath(xpath, action='read_text') self.log_msg('LOGIN ERROR, CHECK EMAIL ID') self.close() sys.exit(0) except Exception: pass def log_msg(self, msg): self.logger.log(msg) @custom_decorator def __pass_zip__(self, inp_zip): xpath = '//*[@id="autocomplete"]' self.browser.perform_by_xpath(xpath, action='clear', delay=0.5) self.browser.perform_by_xpath(xpath, action='send_text', text=inp_zip, delay=0.5) self.browser.perform_by_xpath(xpath, action='send_text', text=Keys.DOWN, delay=0.5) self.browser.perform_by_xpath(xpath, action='send_text', text=Keys.RETURN, delay=0.5) @custom_decorator def __find_icc_store_list__(self): self.__pass_zip__(self.default_zip) out = [ x.text for (i, x) in enumerate( self.browser.driver.find_elements_by_tag_name('li')) if x.text and len(x.text.split(" ")) > 3 ] self.addr_list = [(self.default_zip, out[0])] for addr in out[1:]: self.addr_list.append((addr[-5:], addr)) @custom_decorator def __find_actual_icc_list__(self): store_map = { 'SUNNYVALE': '94087', 'SAN JOSE': '95129', 'FREMONT': '94538' } if self.icc_to_check == "ALL": try: self.__find_icc_store_list__() except TimeoutError: self.log_msg("TIMEOUT ERROR WITH FIND ICC STORE LIST") self.close() raise Exception("Restart program - Timeout ERR") except Exception as err: self.log_msg('RUNTIME ERROR WITH ICC STORE LIST, \ Err: {}'.format(err)) self.close() raise Exception("Restart program") else: self.addr_list = [(store_map[self.icc_to_check], self.icc_to_check) ] @custom_decorator def __switch_store__(self, new_zip): url = "https://www.indiacashandcarry.com/" self.get_url(url, delay=3) self.browser.perform_by_classname('header-location', action='click', delay=1) try: xpath = '//*[@id="price-list-0"]/p/a' self.browser.perform_by_xpath(xpath, action='click', delay=2) self.__pass_zip__(new_zip) return True except Exception: pass try: xpath = '//*[@id="price-list-1"]/p/a' self.browser.perform_by_xpath(xpath, action='click', delay=2) self.__pass_zip__(new_zip) return True except Exception: pass try: xpath = '//*[@id="price-list-2"]/p/a' self.browser.perform_by_xpath(xpath, action='click', delay=2) self.__pass_zip__(new_zip) return True except Exception as err: pass return False @custom_decorator def __select_store__(self): try: time.sleep(2) self.browser.perform_by_linktext('Select This Location', action='click', delay=2) xpath = '//*[@id="price-list-0"]/ul/li[1]/p' self.browser.perform_by_xpath(xpath, action='click', delay=2) except Exception as err: raise Exception(err) @custom_decorator def __check_pickup_slot__(self): try: xpath = '/html/body/div[2]/section/div/div/div[3]/div/ul/li[1]/div' self.browser.perform_by_xpath(xpath, action='click') except: xpath = '/html/body/div[2]/section/div/div/div[2]/div/ul/li[1]/div/h3' self.browser.perform_by_xpath(xpath, action='click') time.sleep(0.5) try: xpath = '/html/body/div[2]/section/div/div/div[1]' store_msg = self.browser.perform_by_xpath(xpath, action='read_text') if 'Checkout Continue Shopping' in store_msg: xpath = '/html/body/div[2]/section/div/div/div[2]/div/div/div' store_msg = self.browser.perform_by_xpath(xpath, action='read_text') return False else: return True except Exception as err: print( "Execption found with finding PICKUP slot, err : {}\n".format( err)) @custom_decorator def __check_delivery_slot__(self): if not self.__min_order_met__(): if self.icc_option == "BOTH": return False else: self.close() self.log_msg("\nProgram Ended\n") sys.exit(0) try: xpath = '/html/body/div[2]/section/div/div/div[3]/div/ul/li[2]/div/h3' self.browser.perform_by_xpath(xpath, action='click') except: xpath = '/html/body/div[2]/section/div/div/div[2]/div/ul/li[2]/div' self.browser.perform_by_xpath(xpath, action='click') time.sleep(0.5) try: xpath = '/html/body/div[2]/section/div/div/div[1]' store_msg = self.browser.perform_by_xpath(xpath, action='read_text') if 'Delivery is not available' in store_msg or 'Checkout Continue Shopping' in store_msg: return False else: return True except Exception as err: print("Execption found with finding DELIVERY slot, err : {}\n". format(err)) @custom_decorator def __is_cart_empty__(self): try: xpath = '/html/body/div[2]/section/div/div/div/div/h4' status = self.browser.perform_by_xpath(xpath, action='read_text') if 'Your cart is empty' in status: self.log_msg("EMPTY_CART ERROR, CAN't CHECK SLOT") self.close() sys.exit(0) except Exception: pass @custom_decorator def __min_order_met__(self): try: xpath = '/html/body/div[2]/section/div/div/div[1]/div/p' status = self.browser.perform_by_xpath(xpath, action='read_text') if 'All Delivery orders must be' in status: self.log_msg( "MIN_CART_NOT_MET ERROR, CAN't CHECK DELIVERY SLOT") return False else: return True except Exception: return True @custom_decorator def find_slots(self): self.__find_actual_icc_list__() for (zp, address) in self.addr_list: if self.icc_to_check == "ALL": selected_store = address[:address.index(' Select')] else: selected_store = address if any(('DE ANZA' in selected_store, 'SAN JOSE' in selected_store)): store = 'SAN JOSE' else: store = selected_store self.log_msg("Attempting to find slot for Store : {}".format( selected_store)) if not self.firstInstance: if not self.__switch_store__(zp): raise Exception("RUNTIME ERR IN SWITCH STORE") else: self.firstInstance = False self.__select_store__() url = 'https://www.indiacashandcarry.com/cart/checkout' self.get_url(url, delay=5) self.__is_cart_empty__() if any((self.icc_option == "BOTH", self.icc_option == "DELIVERY")): self.delivery_status[store] = False status = self.__check_delivery_slot__() self.delivery_status[store] = status if any((self.icc_option == "BOTH", self.icc_option == "PICKUP")): self.pickup_status[store] = False status = self.__check_pickup_slot__() self.pickup_status[store] = status @custom_decorator def chk_slot_status(self): is_slot_found = False self.slots_result = '' self.log_msg("\nSlot status:") self.slots_result += "\nSlot Status:\n" for (store, status) in self.delivery_status.items(): self.log_msg("\t{} Delivery Slot : {}".format(store, status)) self.slots_result += "\t{} Delivery Slot : {}\n".format( store, status) if status: is_slot_found = True self.slots_result += "\n" for (store, status) in self.pickup_status.items(): self.log_msg("\t{} Pickup Slot : {}".format(store, status)) self.slots_result += "\t{} PickUp Slot : {}\n".format( store, status) if status: is_slot_found = True self.log_msg("\n") self.slots_result += "\n" return is_slot_found def refresh_browser(self): if self.browser: self.browser.refresh() time.sleep(1) def close(self): if self.browser: self.chk_slot_status() self.browser.close() self.email = None self.browser = None self.log_msg('\n##########################################')