def __init__(self, query, random_agent=False, debug=False, lang="en", tld="com", re_search_strings=None):
self.query = query
self.debug = debug
self.browser = Browser(debug=debug)
self.results_info = None
self.eor = False # end of results
self._page = 0
self._first_indexed_in_previous = None
self._filetype = None
self._last_search_url = None
self._results_per_page = 10
self._last_from = 0
self._lang = lang
self._tld = tld
if re_search_strings:
self._re_search_strings = re_search_strings
elif lang == "de":
self._re_search_strings = ("Ergebnisse", "von", u"ungefähr")
elif lang == "es":
self._re_search_strings = ("Resultados", "de", "aproximadamente")
# add more localised versions here
else:
self._re_search_strings = ("Results", "of", "about")
if random_agent:
self.browser.set_random_user_agent()
def __init__(self, proxies=None, auth=None):
"""
Set up a browser-imitated slurper to slurp_raw Dogpile search results.
:type proxies: dict
:type auth: requests.auth.HTTPProxyAuth
"""
Browser.__init__(self, proxies, auth)
def __init__(self, config_path="config.yaml"):
self.browser = Browser()
self.dutys = ""
self.refresh_time = ''
self.verify_url = "http://www.114yygh.com/web/verify"
self.login_url = "http://www.114yygh.com/web/login"
self.send_code_url = "http://www.114yygh.com/web/getVerifyCode"
self.duty_url = "http://www.114yygh.com/web/product/detail"
self.confirm_url = "http://www.114yygh.com/web/order/saveOrder"
self.patient_id_url = "http://www.114yygh.com/order/confirm/"
self.query_hospital_url = "http://www.114yygh.com/web/queryHospitalById"
self.calendar = "http://www.114yygh.com/web/product/list"
self.order_patient_list = "http://www.114yygh.com/web/patient/orderPatientList"
self.appoint_info_url = "http://www.114yygh.com/web/order/getAppointInfo"
self.config = Config(config_path) # config对象
if self.config.useIMessage == 'true':
# 按需导入 imessage.py
import imessage
self.imessage = imessage.IMessage()
else:
self.imessage = None
if self.config.useQPython3 == 'true':
try: # Android QPython3 验证
# 按需导入 qpython3.py
import qpython3
self.qpython3 = qpython3.QPython3()
except ModuleNotFoundError:
self.qpython3 = None
else:
self.qpython3 = None
def __init__(self, proxies=None, auth=None):
"""
Set up a browser-imitated slurper to slurp_raw Dogpile search results.
:type proxies: dict
:type auth: requests.auth.HTTPProxyAuth
"""
Browser.__init__(self, proxies, auth)
def scrape():
browser = Browser()
data = {
"sportId": "4",
"leagueIds": "63154,63156,63158,63160,63162,63164,63166,63168",
"page": "0"
}
try:
# Post na pagina de odds
# contents = browser.get("https://sports.sportingbet.com/pt-br/sports#eventId=&leagueIds=63154,63156,63158,63160,63162,63164,63166,63168&marketGroupId=&page=0&sportId=4")
contents = browser.post(
"https://sports.sportingbet.com/en/sports/indexmultileague", data)
page = contents.read()
# pega os grupos e processa
doc = BeautifulSoup(page, "html.parser")
groups = doc.find_all("div", class_="marketboard-event-group")
data = [data_for_group(g) for g in groups]
# desfaz a estrutura grupos->subgrupos->eventos, já que so estamos interessados
# nos eventos (jogos)
flattened_data = [e for g in data for sg in g for e in sg]
# Transforma em dataframe e converte colunas numericas
df = pd.DataFrame(flattened_data,
columns=[
"datetime", "match_id", "team_a", "odds_a",
"odds_tie", "team_b", "odds_b"
])
df[["odds_a", "odds_tie",
"odds_b"]] = df[["odds_a", "odds_tie", "odds_b"]].astype(float)
df["match_id"] = df["match_id"].astype(int)
return df
except HTTPError as e:
print(e.msg, e.code, e.hdrs)
def start(self):
if self.count > 1 and self.url is None:
print("Cannot provide page count with no URL given")
exit(1)
if self.remove_params and self.url is None:
print(
"WARNING: Must have Internet connection to remove unneeded parameters"
)
# Scan for all APIs
if self.url:
os.makedirs(self.har_directory, exist_ok=True)
self.delete_existing_hars()
self.browser = Browser(
"chromedriver/chromedriver",
"browsermob-proxy-2.1.4/bin/browsermob-proxy",
self.har_directory,
cookies=self.cookies)
if self.search_string is not None:
print("Searching URL " + self.url + " for string " +
self.search_string)
# Move recursively through the site
api_calls = self.crawling_scan(self.url)
# Scan directory of har files
else:
print("Parsing existing directory of har files")
har_parser = HarParser(self.har_directory, self.search_string,
self.remove_params)
api_calls = har_parser.parse_multiple_hars()
if self.browser is not None:
self.browser.close()
return api_calls
def __init__(self):
self.browser = Browser()
self.parser = None
self.history = None
self.fen_history = FenGame()
self.pgn_history = PgnGame()
self.notation = AlgebraicNotation()
self.params = DEFAULT_LEELA_PARAMS
self.params['MultiPV'] = 3
self.engine = None
self.new_data = None
self.last_data = None
self.best_move = list()
self.best_lines = list()
self.scores = list()
self.new_analysis = False
self.parser_lock = threading.Lock()
self.engine_lock = threading.Lock()
self.setup_engine()
def __browser_view(self, *args):
self.vboxBrowser.remove(self.browser)
self.browser = Browser()
self.browser.show()
self.vboxBrowser.add(self.browser)
self.__change_current_view(self.vboxBrowser)
self.browser.load_uri('http://www.sbennel.es')
def create_grid(self):
grid = Gtk.Grid()
self.add(grid)
# from terminal.py class Terminal
self.vte = Terminal()
notebook, event_exit = self.vte.create_terminal()
# For closing application
event_exit.connect("button_press_event", self.app_close)
grid.add(notebook)
# From browser.py class Browser
self.webbrowser = Browser()
# Creating fist browser page
browser = self.webbrowser.create_browser()
grid.attach(browser, 1, 0, 1, 1)
# From folderview.py class Folderview
self.folder = Folderview()
# Create Folder view HOME
folderview = self.folder.create_foldeview()
grid.attach_next_to(folderview, notebook, Gtk.PositionType.BOTTOM, 1,
1)
# Filevew section
self.file = Fileview()
fileview = self.file.create_fileview()
grid.attach_next_to(fileview, folderview, Gtk.PositionType.RIGHT, 1, 1)
def setUp(self):
self.driver = Browser()
self.driver.goToFacebook()
self.driver.login('*****@*****.**', 'selab1623')
time.sleep(2)
self.driver.goToFacebookPhotoManagement()
time.sleep(3)
def getKickass(self): browser = Browser() zdata = browser.open(self.kickass_link) uzip_data = self.decomp.decompress(zdata) data = re.findall(self.kickass_regex, uzip_data) return data
def main():
parser = argparse.ArgumentParser(
description=
'This tool lets you invite people in bulk to your Facebook group')
parser.add_argument('-e',
'--email',
help='Your personal Facebook account email',
required=True)
parser.add_argument('-p',
'--password',
help='Your password in plain text',
required=True)
parser.add_argument('-g',
'--group',
help='The Facebook group name',
required=True)
parser.add_argument('-f',
'--file',
help='The csv file to load email addresses from',
default='emails.csv')
args = vars(parser.parse_args())
email_loader = EmailLoader(filename=args['file'])
browser = Browser()
browser.navigate(url='https://www.facebook.com',
wait_for='facebook',
error='Unable to load the Facebook website')
browser.enter_login_details(email=args['email'], password=args['password'])
browser.navigate(url='https://www.facebook.com/groups/' + args['group'],
wait_for='pagelet_group_',
error='Couldn\'t navigate to the group\'s members page')
browser.import_members(emails=email_loader.all_emails)
print(
str(len(email_loader.all_emails)) +
' email addresses successfully imported, you may now close Firefox')
def all(environment='ppm'):
# Create browsers.
user_browser = Browser('chrome')
admin_browser = Browser('firefox')
# Get the path
path = os.path.dirname(__file__)
# Get the URLs
urls = URLs.load(os.path.join(path, 'urls.yml'), environment)
accounts = Accounts.load(os.path.join(path, 'accounts.yml'))[environment]
# Get a user
user = NEERUser(urls, email_address='*****@*****.**', password='******')
user.log_in_to_p2m2(user_browser)
# Get an admin
admin = Administrator(urls,
email_address='*****@*****.**',
password='******')
admin.log_in_to_p2m2_admin(admin_browser)
email = Email(urls['inbox'])
user.verify_email(user_browser, email)
return user_browser, user, admin_browser, admin, email
def __init__(self, config_path="config.yaml"):
self.browser = Browser()
self.dutys = ""
self.refresh_time = ''
self.login_url = "http://www.bjguahao.gov.cn/quicklogin.htm"
self.send_code_url = "http://www.bjguahao.gov.cn/v/sendorder.htm"
self.get_doctor_url = "http://www.bjguahao.gov.cn/dpt/partduty.htm"
self.confirm_url = "http://www.bjguahao.gov.cn/order/confirm.htm"
self.patient_id_url = "http://www.bjguahao.gov.cn/order/confirm/"
self.department_url = "http://www.bjguahao.gov.cn/dpt/appoint/"
self.config = Config(config_path) # config对象
if self.config.useIMessage == 'true':
# 按需导入 imessage.py
import imessage
self.imessage = imessage.IMessage()
else:
self.imessage = None
if self.config.useQPython3 == 'true':
try: # Android QPython3 验证
# 按需导入 qpython3.py
import qpython3
self.qpython3 = qpython3.QPython3()
except ModuleNotFoundError:
self.qpython3 = None
else:
self.qpython3 = None
def __init__(self, items, random_agent=False, debug=False):
self.items = items
self.debug = debug
self.browser = Browser(debug=debug)
if random_agent:
self.browser.set_random_user_agent()
def updateDisplayLoop(self):
done = False
while not done:
for event in pygame.event.get():
if event.type == pygame.QUIT:
done = True
break
if event.type == pygame.KEYDOWN and (event.key == pygame.K_ESCAPE or event.key == pygame.K_x or event.key == pygame.K_q):
done = True
break
#logger.info("gui-event: %s", str(event))
if event.type == pygame.KEYDOWN and (event.key == pygame.K_o):
blueiris = BlueIris()
blueiris.login()
camName = "fdoor"
camUrl = blueiris.getCamURL(camName)
logger.info("CAM URL: %s", camUrl)
self.browser = Browser()
self.browser.openBrowserThread(camUrl)
if event.type == pygame.KEYDOWN and (event.key == pygame.K_c):
self.browser.close()
self.screen.fill((0,0,0))
self.updateTime(10,60)
pygame.display.update()
# Update interval (fps)
self.clock.tick(2)
class Game:
def __init__(self):
self.browser = Browser()
self.app = QApplication(sys.argv)
self.timer = QTimer()
self.timer.timeout.connect(self.updateEvent)
self.ai = AI()
self.last_score = 1
def run(self, delay):
self.timer.start(delay)
self.browser.run()
return self.app.exec_()
def updateEvent(self):
self.timer.stop()
(score, board) = self.browser.extractData()
if self.last_score > score:
self.ai.restart_game()
self.last_score = score
self.browser.sendMove(self.ai.get_move(score, board))
self.timer.start() # restart the event loop
class Game:
def __init__(self):
self.browser = Browser()
self.app = QApplication(sys.argv)
self.timer = QTimer()
self.timer.timeout.connect(self.updateEvent)
self.ai = AI()
self.last_score = 1
def run(self, delay):
self.timer.start(delay)
self.browser.run()
return self.app.exec_()
def updateEvent(self):
self.timer.stop()
(score, board) = self.browser.extractData()
if self.last_score > score:
self.ai.restart_game()
self.last_score = score
self.browser.sendMove(self.ai.get_move(score, board))
self.timer.start() # restart the event loop
def _init(self):
self.main_loop = GLib.MainLoop()
self.client = WebRTCClient()
self._connect_client_observer()
self.signalling = WebRTCSignallingClient(self.server, self.id)
self.remote_client = RemoteWebRTCObserver(self.signalling)
self._connect_signalling_observer()
actions = ActionObserver()
actions.register_action_types()
self._connect_actions(actions)
# wait for the signalling server to start up before creating the browser
self.signalling.wait_for_states([SignallingState.OPEN])
self.signalling.hello()
self.browser = Browser(create_driver(self.browser_name))
self.browser.open(self.html_source)
browser_id = self.browser.get_peer_id()
assert browser_id == self.peerid
self.signalling.create_session(self.peerid)
test_name = self.test_name if self.test_name else self.scenario_name
self.remote_client.set_title(test_name)
self._init_validate(self.scenario_name)
def wrapper(arguments: List[str]):
app_args = parse_app_arguments(arguments)
context = get_context(app_args.context_path)
current_session = None
if app_args.session_uuid:
current_session = next(
filter(lambda session: session.id == app_args.session_uuid,
context.sessions), None)
browser = Browser(current_session)
user_creds = UserCredentials(app_args.username, app_args.password,
None)
try:
func(browser, context, user_creds)
except Exception as ex:
logging.exception('Browser error.', ex)
current_session = None
context.sessions = [
session for session in context.sessions
if session.id != current_session.id
]
browser.dispose()
finally:
if current_session:
context.sessions.append(current_session)
save_context(context, app_args.context_path)
def __init__(self, settings: object):
self._browser = Browser()
b = self._browser
self._session_cookie_file_path = 'ada_session_cookie'
self._session_cookie_name = 'ASP.NET_SessionId'
s = settings
# URL to the certificate search page.
self._search_url = '%s://%s:%s@%s/%s?%s' % (
s.scheme, s.username, s.password, s.host, s.path, s.query_string)
# Ada may not work when a previous session is still active. If it
# works, store the new session cookie for future use. If not, look for
# an existing session cookie, load it, and try again.
if self._is_working():
self._store_session_cookie()
else:
try:
self._load_session_cookie()
except:
raise Exception('Could not open Ada in a new session, ' +
'and no old session cookie was found')
if not self._is_working():
raise Exception('Could not open Ada in a new session, ' +
'and the old session cookie does not work')
def __init__(self, window):
super().__init__()
self.setupUi(window)
self.window = window
self.settings = Settings()
self.window.closeEvent = self.closeEvent
self.server = Server()
self.downman = DownloadManager()
self.browser = Browser()
# snapshot updater is to be started on exchange connect
self.xchgClient = ExchangeClient()
self.lastKnownDir = "/tmp"
self.destPrefix = ''
self.userlist = None
self.di_list = []
self.addEventListeners()
self.browserTable.setColumnHidden(0, True)
self.userListTable.setColumnHidden(0, True)
self.tabWidget.setCurrentIndex(0)
self.urlFrame.setVisible(False)
self.window.setWindowIcon(QIcon(":/images/favicon.ico"))
self.window.setWindowTitle("21Lane")
self.makeMenuBar()
self.setupSystemTray()
self.loadSettings()
self.window.show()
def fetch(self):
url = "http://blog.sina.com.cn/u/1696709200"
b = Browser()
page = b.get_page(url)
be = BeautifulSoup(page)
div = be.find('div', {'class': 'diywidget'})
return div
def fetch(self):
url = "http://blog.sina.com.cn/u/1696709200"
b = Browser()
page = b.get_page(url)
be = BeautifulSoup(page)
div = be.find('div', {'class': 'diywidget'})
return div
def __init__(self):
# Instance of Config Util
self.config = Config()
# Instance of Browser Util
self.browser = Browser(self.config)
# AutoLogin when Init
self.login()
def search_internal(cls, search_terms, max_results):
"Use browser to get all case URLs matching the given search terms"
search_terms = cls.sanitize_search_terms(search_terms)
browser = Browser()
next_search_page = cls.format_internal_search_url(search_terms)
out = []
n_results = 0
while next_search_page:
print("using search page", next_search_page)
browser.get(next_search_page)
wait(10)
results = list(cls.extract_results_from_search_page(browser))
if cls.is_end_of_results(browser):
print("End of results")
break
if not results:
print("Please resolve the captcha.")
input()
results = list(cls.extract_results_from_search_page(browser))
n_results += len(results)
for result in results:
yield cls.get(result)
if n_results > max_results:
print("Over max results")
break
next_search_page = cls.get_next_search_page(browser)
def start_browser(self):
"""The crawler process tries to start browser using this function.
Presumably every browser needs hooks before starting. Firefox needs
a profile and extension, set-up by this function.
"""
self.browser_inst = Browser(self.browser_name, self.logger)
self.extension_inst = self.browser_inst.start(self.ext_port)
return self.extension_inst
def __init__(self):
self.browser = Browser()
self.app = QApplication(sys.argv)
self.timer = QTimer()
self.timer.timeout.connect(self.updateEvent)
self.ai = AI()
self.last_score = 1
def __init__(self):
self.browser = Browser()
self.dutys = ""
self.login_url = "http://www.bjguahao.gov.cn/quicklogin.htm"
self.send_code_url = "http://www.bjguahao.gov.cn/v/sendorder.htm"
self.get_doctor_url = "http://www.bjguahao.gov.cn/dpt/partduty.htm"
self.confirm_url = "http://www.bjguahao.gov.cn/order/confirm.htm"
self.patient_id_url = "http://www.bjguahao.gov.cn/order/confirm/"
self.department_url = "http://www.bjguahao.gov.cn/dpt/appoint/"
def do_browse_work(url):
# Scrape the child urls first without going through the proxy.
_,_,_,child_urls = no_proxy_browser.visit(url, timeout=args.timeout)
workq.extend(child_urls)
info("Added %d urls to workq."%(len(child_urls)))
# Now generate load for the proxy.
if args.proxy:
br = Browser()
target_url = "%s/?q="%(args.proxy) + url
br.visit(target_url, timeout=args.timeout)
def __init__(self,
email,
password,
url='https://www.sit.no/trening/hall',
silent=True):
self.browser = Browser(silent)
self.email = email
self.password = password
self.url = url
def process_scraping(self, text):
if (text is None):
current_path = 'https://clinicaltrials.gov/ct2/show/record/?cond=COVID&draw=3&rank=1&view=record'
else:
current_path = 'https://clinicaltrials.gov/ct2/show/record/?cond=' + text + '&draw=3&rank=1&view=record'
print('scraping the url:', current_path)
browser = Browser(current_path)
browser.start()
more_pages = True
while (more_pages):
try:
print('processing page:', len(self.lines) + 1, ' ...')
self.study_scraper = StudyScraper(browser.get_current_page())
study = self.study_scraper.get_study()
self.lines.append(study)
print('page:', study.id, 'processed')
time.sleep(1)
browser.navigate()
except:
print('no more pages to scrap')
more_pages = False
browser.stop()
def test_status(caplog):
caplog.setLevel(logging.CRITICAL, logger='selenium')
caplog.setLevel(logging.DEBUG, logger='opp')
b = Browser(use_virtual_display=VDISPLAY)
src = 'file://'+testdir+'/umsu.html'
b.goto(src)
assert b.status == 200
src = 'file://'+testdir+'/xxx.html'
b.goto(src)
assert b.status == 404
del b
def __init__(self, query, random_agent=False, debug=False):
self.query = query
self.debug = debug
self.browser = Browser(debug=debug)
self._page = 0
self.eor = False
self.results_info = None
self._results_per_page = 10
if random_agent:
self.browser.set_random_user_agent()
def __init__(self, query, random_agent=False, debug=False):
self._page = 1
self._timeout = 40
socket.setdefaulttimeout(self._timeout)
self._beforemd5 = 0
self._results_per_page = 10
self._query = query
self.browser = Browser(debug=debug)
if random_agent:
self.browser.set_random_user_agent()
def test_xvfb(caplog):
caplog.setLevel(logging.CRITICAL, logger='selenium')
caplog.setLevel(logging.DEBUG, logger='opp')
b = Browser(use_virtual_display=True)
src = 'file://'+testdir+'/umsu.html'
b.goto(src)
del b
ps = subprocess.Popen(('ps', 'aux'), stdout=subprocess.PIPE)
output = ps.communicate()[0]
for line in output.decode('ascii').split('\n'):
assert 'Xfvb' not in line
def translator(title=None, imdb_id=None, language=None, extra=True):
"""
Translate title
:param title: Title to translate
:type title: str
:param imdb_id: IMDB id from the title
:type imdb_id: str
:param language: requested language for the translation
:type language: str
:param extra: extra information
:type extra: bool
:return: translated title
"""
keywords = {
'en': '',
'de': '',
'es': 'espa',
'fr': 'french',
'it': 'italian',
'pt': 'portug'
}
if imdb_id:
url_themoviedb = "http://api.themoviedb.org/3/find/%s?api_key=8d0e4dca86c779f4157fc2c469c372ca&language=%s" \
"&external_source=imdb_id" % (imdb_id, language)
if Browser.open(url_themoviedb):
results = json.loads(Browser.content)
if len(results['movie_results']) > 0:
title = results['movie_results'][0]['title'].encode('utf-8')
original_title = results['movie_results'][0][
'original_title'].encode('utf-8')
elif len(results['tv_results']) > 0:
title = results['tv_results'][0]['name'].encode('utf-8')
original_title = results['tv_results'][0][
'original_name'].encode('utf-8')
else:
title = ""
original_title = ""
if title == original_title and extra:
title += ' ' + keywords[language]
else:
title = 'Pas de communication avec le themoviedb.org'
else:
url_themoviedb = "http://api.themoviedb.org/3/search/tv?api_key=8d0e4dca86c779f4157fc2c469c372ca" \
"&query=%s&language=%s" % (title.replace(' ', '+'), language)
if Browser.open(url_themoviedb):
results = json.loads(Browser.content)
if len(results['results']) > 0:
title = results['results'][0]['name']
original_title = results['results'][0]['original_name']
if title == original_title and extra:
title += ' ' + keywords[language]
else:
title = 'Pas de communication avec le themoviedb.org'
return Filtering.safe_name(title.rstrip())
class Base(object):
def __init__(self):
self.browser = Browser()
self.start_browser()
self.logged_in = False
def start_browser(self):
self.browser.start()
def close_browser(self):
self.browser.close()
class Main():
def __init__(self):
self.WALLPAPER_KEY = "org.gnome.desktop.background"
self.IMAGE_PATH = Gio.Settings.new(self.WALLPAPER_KEY).get_string("picture-uri")
self.SERVER_PORT = 8112
DBusGMainLoop(set_as_default=True)
self.bus = dbus.SessionBus()
self.bus.add_signal_receiver(self.dbus_listener,None,None,None,None,)
self.window = Gtk.Window()
self.webview = Browser()
self.window.add(self.webview)
self.refresh()
self.webview.load_uri('file://%s/res/desktop.html' % os.getcwd())
self.load_plugins()
self.show()
self.main()
def dbus_listener(self,section=None,arg2=None,arg3=None,arg4=None,arg5=None,arg6=None):
if "/org/gnome/desktop/background/" in str(section):
self.refresh()
def refresh(self):
import SocketServer
self.IMAGE_PATH = Gio.Settings.new(self.WALLPAPER_KEY).get_string("picture-uri")
stylesheet = open("%s/res/background.css" % os.getcwd() ,mode="w")
stylesheet.write("body { background: url('%s') no-repeat; background-size: 100%%;}" % self.IMAGE_PATH);
stylesheet.close();
self.webview.reload()
def load_plugins(self):
for filename in os.listdir("%s/plugins" % os.getcwd()):
if filename == "__init__.py" or filename == "":
break
try:
plugin = __import__("plugins."+filename[:-3])
except:
pass
pass
def show(self):
self.window.set_type_hint(Gdk.WindowTypeHint.DESKTOP)
self.window.connect("delete-event", Gtk.main_quit)
self.window.fullscreen()
self.window.show_all()
def main(self):
Gtk.main()
class User(object):
def __init__(self, username):
self.browser = Browser()
self.browser.debug_mode = False
self._username = username
self._token = 'AAAF9OVgazyEBALYio6jD06kmh1w9tXXeS0wDFuUE057dkh32IbgVcMqwQhOZAp0KW66BZAPKQlpqOPCxfqbCwSlk5gjQKDKdrEt1irBlwqJiCtuyZAP'
self._url = 'https://graph.facebook.com/' + self._username + '?fields=%s&access_token=' + self._token
def get_oncoming_events(self):
response = self.browser.get(self._url % 'events')
print self._url % 'events'
pprint(response)
events = json.loads(str(response.content))
today = datetime.datetime.today()
upcomingEvents = []
for key, value in events.iteritems():
if key == 'events':
for key2, data in value.iteritems():
if key2 == 'data':
for val in data:
#Porównanie dni, ale to sie zrobi pozniej bo nie ma czasu :0
# eventTime = re.sub('\+\d+$', '', val['start_time'])
# if datetime.datetime.strptime(eventTime, "%Y-%m-%dT%H:%M:%S") > today:
upcomingEvents.append(val)
return upcomingEvents
def get_friends_with_birthdays(self):
response = self.browser.get(self._url % 'friends.limit(500).fields(first_name,last_name,birthday)')
jsonData = json.loads(str(response.content))
today = datetime.datetime.today().strftime('%m/%d')
friends = []
for key, value in jsonData.iteritems():
if key == 'friends':
for key2, data in value.iteritems():
if key2 == 'data':
for val in data:
if 'birthday' in val:
if len(val['birthday']) > 5:
bday = val['birthday'][:5]
else:
bday = val['birthday']
if bday == today:
friends.append(val)
return friends
def get_info(self):
response = self.browser.get(self._url % 'first_name,last_name,birthday')
jsonData = json.loads(str(response.content))
pprint(friends)
def test_linkcontext(page, link, context, caplog):
caplog.setLevel(logging.CRITICAL, logger='selenium')
caplog.setLevel(logging.DEBUG, logger='opp')
scraper.debuglevel(5)
curpath = os.path.abspath(os.path.dirname(__file__))
testdir = os.path.join(curpath, 'sourcepages')
browser = Browser(reuse_browser=True, use_virtual_display=VDISPLAY)
src = 'file://'+testdir+'/'+page
browser.goto(src)
el = browser.find_elements_by_xpath('//a[@href="{}"]'.format(link))[0]
li = scraper.Link(element=el)
res = li.html_context()
assert res == context
class PageManager(QObject):
def __init__(self, view):
super(QObject, self).__init__()
self.main_xid = view.winId().__int__()
self.movie_store_page = Browser("http://pianku.xmp.kankan.com/moviestore_index.html")
self.movie_search_page = Browser("http://search.xmp.kankan.com/lndex4xmp.shtml")
@pyqtSlot(str, int, int, int, int)
def show_page(self, page_name, x, y, width, height):
self.hide_page()
x += 1
width -= 2
height -= 1
if page_name == "movie_store":
self.movie_store_page.show_with_parent(self.main_xid, x, y, width, height)
elif page_name == "movie_search":
self.movie_search_page.show_with_parent(self.main_xid, x, y, width, height)
@pyqtSlot()
def hide_page(self):
self.movie_store_page.hide()
self.movie_search_page.hide()
class PageManager(QObject):
def __init__(self, view):
super(QObject, self).__init__()
self.main_xid = view.winId().__int__()
self.movie_store_page = Browser("http://dy.yunfan.com")
self.movie_search_page = Browser("http://www.yunfan.com/qs")
@pyqtSlot(str, int, int, int, int)
def show_page(self, page_name, x, y, width, height):
self.hide_page()
x += 3
width -= 2
height -= 2
if page_name == "movie_store":
self.movie_store_page.show_with_parent(self.main_xid,
x, y, width, height)
elif page_name == "movie_search":
self.movie_search_page.show_with_parent(self.main_xid,
x, y, width, height)
@pyqtSlot()
def hide_page(self):
self.movie_store_page.hide()
self.movie_search_page.hide()
class Player:
extra_links = []
def __init__(self):
self.browser = Browser()
self.html = Html()
self.builder = Gtk.Builder()
self.builder.add_from_file(os.path.join(os.getcwd(), '../gui/src/player.glade'))
self.win = self.builder.get_object('mainWindow')
self.win.set_size_request(860, 600)
self.win.set_resizable(False)
self.mainBox = self.builder.get_object('mainBox')
self.mainBox.set_size_request(800, 600)
self.left = self.builder.get_object('left')
self.win.connect("window-state-event", self.on_window_state_event)
self.win.connect('destroy', Gtk.main_quit)
self.win.connect('delete-event', Gtk.main_quit)
self.win.set_icon_from_file(Helper.get_resource_path("icon.png"))
self.left_handler()
def on_window_state_event(self, widget, event, data=None):
mask = Gdk.WindowState.FULLSCREEN
fullscreen = widget.get_window().get_state() & mask == mask
if fullscreen:
self.mainBox.set_homogeneous(True)
else:
self.mainBox.set_homogeneous(False)
def set_title(self, title):
self.win.set_title(title)
def set_links(self, links):
self.extra_links = links[:]
def left_handler(self):
browser = self.browser.get()
self.left.add(browser)
def link_clicked(self, widget, link):
self.browser.open(link)
def open(self, url):
movie = self.html.create(url)
self.browser.open(movie)
self.win.show_all()
class LanguageDetector(object):
detect_url = "http://ajax.googleapis.com/ajax/services/language/detect?v=1.0&q=%(message)s"
def __init__(self):
self.browser = Browser()
def detect(self, message):
"""
Given a 'message' detects its language.
Returns Language object.
"""
message = quote_plus(message)
real_url = LanguageDetector.detect_url % { 'message': message }
try:
detection = self.browser.get_page(real_url)
data = json.loads(detection)
if data['responseStatus'] != 200:
raise DetectionError, "Failed detecting language: %s" % data['responseDetails']
rd = data['responseData']
return Language(rd['language'], rd['confidence'], rd['isReliable'])
except BrowserError, e:
raise DetectionError, "Failed detecting language (getting %s failed): %s" % (e.url, e.error)
except ValueError, e:
raise DetectionErrro, "Failed detecting language (json failed): %s" % e.message
class Translator(object):
translate_url = "http://ajax.googleapis.com/ajax/services/language/translate?v=1.0&q=%(message)s&langpair=%(from)s%%7C%(to)s"
def __init__(self):
self.browser = Browser()
def translate(self, message, lang_to='en', lang_from=''):
"""
Given a 'message' translate it from 'lang_from' to 'lang_to'.
If 'lang_from' is empty, auto-detects the language.
Returns the translated message.
"""
if lang_to not in _languages:
raise TranslationError, "Language %s is not supported as lang_to." % lang_to
if lang_from not in _languages and lang_from != '':
raise TranslationError, "Language %s is not supported as lang_from." % lang_from
message = quote_plus(message)
real_url = Translator.translate_url % { 'message': message,
'from': lang_from,
'to': lang_to }
try:
translation = self.browser.get_page(real_url)
data = json.loads(translation)
if data['responseStatus'] != 200:
raise TranslationError, "Failed translating: %s" % data['responseDetails']
return data['responseData']['translatedText']
except BrowserError, e:
raise TranslationError, "Failed translating (getting %s failed): %s" % (e.url, e.error)
except ValueError, e:
raise TranslationError, "Failed translating (json failed): %s" % e.message
def start(self):
if self.count > 1 and self.url is None:
print("Cannot provide page count with no URL given")
exit(1)
if self.removeParams and self.url is None:
print("WARNING: Must have Internet connection to remove unneeded parameters")
#Scan for all APIs
if self.url:
os.makedirs(self.harDirectory,exist_ok=True)
self.deleteExistingHars()
self.browser = Browser("chromedriver/chromedriver", "browsermob-proxy-2.1.4/bin/browsermob-proxy", self.harDirectory, cookies=self.cookies)
if self.searchString is not None:
print("Searching URL "+self.url+" for string "+self.searchString)
#Move recursively through the site
apiCalls = self.crawlingScan(self.url)
#Scan directory of har files
else:
print("Parsing existing directory of har files")
harParser = HarParser(self.harDirectory, self.searchString, self.removeParams)
apiCalls = harParser.parseMultipleHars()
if self.browser is not None:
self.browser.close()
return apiCalls
def __init__(self, items, random_agent=False, debug=False):
self.items = items
self.debug = debug
self.browser = Browser(debug=debug)
if random_agent:
self.browser.set_random_user_agent()
def __init__(self, config_path="config.yaml"):
self.browser = Browser()
self.dutys = ""
self.refresh_time = ''
self.login_url = "http://www.bjguahao.gov.cn/quicklogin.htm"
self.send_code_url = "http://www.bjguahao.gov.cn/v/sendorder.htm"
self.get_doctor_url = "http://www.bjguahao.gov.cn/dpt/partduty.htm"
self.confirm_url = "http://www.bjguahao.gov.cn/order/confirm.htm"
self.patient_id_url = "http://www.bjguahao.gov.cn/order/confirm/"
self.department_url = "http://www.bjguahao.gov.cn/dpt/appoint/"
self.config = Config(config_path) # config对象
if self.config.useIMessage == 'true':
# 按需导入 imessage.py
import imessage
self.imessage = imessage.IMessage()
else:
self.imessage = None
if self.config.useQPython3 == 'true':
try: # Android QPython3 验证
# 按需导入 qpython3.py
import qpython3
self.qpython3 = qpython3.QPython3()
except ModuleNotFoundError:
self.qpython3 = None
else:
self.qpython3 = None
def setUp(self):
self.driver = Browser()
self.driver.goToFacebook()
self.driver.login('*****@*****.**', 'selab1623')
time.sleep(2)
self.driver.goToFacebookPhotoManagement()
time.sleep(3)
def initWidgets(self):
# create subwidgets
self.player = Player(self)
self.playlist = Playlist(self)
self.browser = Browser(self)
if have_maemo:
# build Maemo stack hierarchy
self.playlist.setParent(self.player)
self.browser.setParent(self.player)
for w in [ self.player, self.playlist, self.browser ]:
w.setAttribute( Qt.WA_Maemo5StackedWindow)
w.setWindowFlags( w.windowFlags() | Qt.Window)
# add menu bar
menu = QMenuBar(self.player)
menu.addAction(self.actionPlaylist)
menu.addAction(self.actionBrowser)
menu.addAction(self.actionStats)
menu.addAction(self.actionOutputs)
menu.addAction(self.actionPrefs)
menu.addAction(self.actionConnect)
else:
self.stack = QStackedWidget()
for w in [ self.player, self.playlist, self.browser ]:
w.setParent(self.stack)
self.stack.addWidget(w)
def __init__(self, query, random_agent=False, debug=False, lang="en", tld="com", re_search_strings=None):
self.query = query
self.debug = debug
self.browser = Browser(debug=debug)
self.results_info = None
self.eor = False # end of results
self._page = 0
self._first_indexed_in_previous = None
self._filetype = None
self._last_search_url = None
self._results_per_page = 10
self._last_from = 0
self._lang = lang
self._tld = tld
if re_search_strings:
self._re_search_strings = re_search_strings
elif lang == "de":
self._re_search_strings = ("Ergebnisse", "von", u"ungefähr")
elif lang == "es":
self._re_search_strings = ("Resultados", "de", "aproximadamente")
# add more localised versions here
else:
self._re_search_strings = ("Results", "of", "about")
if random_agent:
self.browser.set_random_user_agent()
def __init__(self, userAgentString):
self.userAgentString = userAgentString
self.browser = Browser.parseUserAgentString(userAgentString)
self.operatingSystem = OperatingSystem.UNKNOWN
if browser is not Browser.BOT:
self.operatingSystem = OperatingSystem.parseUserAgentString(userAgentString)
class TestPhoto(unittest.TestCase):
def setUp(self):
self.driver = Browser()
self.driver.goToFacebook()
self.driver.login('*****@*****.**', 'selab1623')
time.sleep(2)
self.driver.goToFacebookPhotoManagement()
time.sleep(3)
def test_delete_photo(self):
self.driver.deletePhotos()
time.sleep(3)
def tearDown(self):
self.driver.close()
def find_imdb(title=None, storage_path=""):
imdb = storage.Storage(ttl=None, storage_path=storage_path)['imdb']
if title is None:
imdb.clear()
result = ''
else:
title = title.lower()
result = imdb.get(title, '')
if result == '':
Browser.open('https://www.google.ca/search?q=%s+imdb' % title.replace(' ', '+'))
data = Browser.content
s_imdb = search('(tt[0-9]+)', data)
if s_imdb:
result = s_imdb.group(1)
imdb[title] = result
imdb.sync()
return result
def __init__(self):
self.browser = Browser()
self.app = QApplication(sys.argv)
self.timer = QTimer()
self.timer.timeout.connect(self.updateEvent)
self.ai = AI()
self.last_score = 1
def start_browser(self):
"""The crawler process tries to start browser using this function.
Presumably every browser needs hooks before starting. Firefox needs
a profile and extension, set-up by this function.
"""
self.browser_inst = Browser(self.browser_name, self.logger)
self.extension_inst = self.browser_inst.start(self.ext_port)
return self.extension_inst
def runup(self):
b = Browser()
for i in xrange(100):
print i
b.get('http://wmtake.ru/m.base/bonus.php')
captcha = b.js('$("#scode-pic img")[0].src')
b.save(captcha, '/home/polzuka/inspirado/captcha/wmtake/%02d.gif' % i)
t = Transformer()
t.load('orig', b.image(captcha))
t.resizeby('resize', t['orig'], 2, 2)
t.grayscale('grayscale', t['resize'], 2)
t.binarize('binarize', t['grayscale'], 150, CV_THRESH_BINARY_INV)
'''
radius = 3
kernel = cvCreateStructuringElementEx(radius * 2 + 1, radius * 2 + 1, radius, radius, CV_SHAPE_ELLIPSE)
t.morphology('morphology', t['binarize'], 1, 1, kernel)
'''
t.contourSplit('breaksplit', t['binarize'], 0.001)
if len(t.symbols) != self.symbolqty:
continue
t.normolize('origsplit', 'breaksplit', 20, 30)
t.savesymbols('origsplit', '/home/polzuka/inspirado/symbols/wmtake', '%02d' % i)
del t
class GoogleSets(object):
URL_LARGE = "http://labs.google.com/sets?hl=en&q1=%s&q2=%s&q3=%s&q4=%s&q5=%s&btn=Large+Set"
URL_SMALL = "http://labs.google.com/sets?hl=en&q1=%s&q2=%s&q3=%s&q4=%s&q5=%s&btn=Small+Set+(15+items+or+fewer)"
def __init__(self, items, random_agent=False, debug=False):
self.items = items
self.debug = debug
self.browser = Browser(debug=debug)
if random_agent:
self.browser.set_random_user_agent()
def get_results(self, set_type=SMALL_SET):
page = self._get_results_page(set_type)
results = self._extract_results(page)
return results
def _maybe_raise(self, cls, *arg):
if self.debug:
raise cls(*arg)
def _get_results_page(self, set_type):
if set_type == LARGE_SET:
url = GoogleSets.URL_LARGE
else:
url = GoogleSets.URL_SMALL
safe_items = [urllib.quote_plus(i) for i in self.items]
blank_items = 5 - len(safe_items)
if blank_items > 0:
safe_items += ['']*blank_items
safe_url = url % tuple(safe_items)
try:
page = self.browser.get_page(safe_url)
except BrowserError(e):
raise GSError("Failed getting %s: %s" % (e.url, e.error))
return BeautifulSoup(page)
def _extract_results(self, soup):
a_links = soup.findAll('a', href=re.compile('/search'))
ret_res = [a.string for a in a_links]
return ret_res
def show_browser(self, handle, url):
# when calling load, url will be quoted? Seems yes.
if handle in self.browser_list:
self.browser_list[handle].load(QUrl(url))
self.browser_list[handle].show()
self.browser_list[handle].raise_()
self.browser_list[handle].activateWindow()
else:
web = Browser()
web.load(QUrl(url))
# print "pyqt opening: ", url
# web.load(QUrl("http://baidu.com"))
self.browser_list[handle] = web
web.show()
web.raise_()
web.activateWindow()
