def all(environment='ppm'):
# Create browsers.
user_browser = Browser('chrome')
admin_browser = Browser('firefox')
# Get the path
path = os.path.dirname(__file__)
# Get the URLs
urls = URLs.load(os.path.join(path, 'urls.yml'), environment)
accounts = Accounts.load(os.path.join(path, 'accounts.yml'))[environment]
# Get a user
user = NEERUser(urls, email_address='*****@*****.**', password='******')
user.log_in_to_p2m2(user_browser)
# Get an admin
admin = Administrator(urls,
email_address='*****@*****.**',
password='******')
admin.log_in_to_p2m2_admin(admin_browser)
email = Email(urls['inbox'])
user.verify_email(user_browser, email)
return user_browser, user, admin_browser, admin, email
def parsers(path, b = None):
base_path = f'{path}\\base'
if os.path.exists(base_path):
return b
for p in os.listdir(base_path):
os.remove(f'{base_path}\\{p}')
else:
os.mkdir(base_path)
with open(f'{path}\\result.json', encoding='UTF-8') as f: base = json.load(f)
if b is None:
b = Browser()
b.set_show()
b.run()
for item in base.get('items'):
html = ''
for i in range(3):
try:
b.get(item['url'])
html = b.driver.page_source
break
except Exception as e:
print(e)
b = Browser()
b.set_show()
b.run()
with open(f'{base_path}\\pos_{item["position"]}.html', 'w', encoding='UTF-8') as f: f.write(html)
return b
def runup(self):
b = Browser()
symbolqty = 5
for i in xrange(100):
print i
b.get('http://wmtake.ru/m.base/bonus.php')
captcha = b.js('$("#scode-pic img")[0].src')
b.save(captcha, '/home/polzuka/inspirado/captcha/wmtake/%02d.gif' % i)
t = Transformer()
t.load('orig', b.image(captcha))
t.resizeby('resize', t['orig'], 2, 2)
t.grayscale('grayscale', t['resize'], 2)
t.binarize('binarize', t['grayscale'], 150, CV_THRESH_BINARY_INV)
'''
radius = 3
kernel = cvCreateStructuringElementEx(radius * 2 + 1, radius * 2 + 1, radius, radius, CV_SHAPE_ELLIPSE)
t.morphology('morphology', t['binarize'], 1, 1, kernel)
'''
t.contourSplit('breaksplit', t['binarize'], 0.001)
if len(t.symbols) != symbolqty:
continue
t.normolize('origsplit', 'breaksplit', 20, 30)
t.savesymbols('origsplit', '/home/polzuka/inspirado/symbols/wmtake', '%02d' % i)
del t
def __init__(self, query, random_agent=False, debug=False, lang="en", tld="com", re_search_strings=None):
self.query = query
self.debug = debug
self.browser = Browser(debug=debug)
self.results_info = None
self.eor = False # end of results
self._page = 0
self._first_indexed_in_previous = None
self._filetype = None
self._last_search_url = None
self._results_per_page = 10
self._last_from = 0
self._lang = lang
self._tld = tld
if re_search_strings:
self._re_search_strings = re_search_strings
elif lang == "de":
self._re_search_strings = ("Ergebnisse", "von", u"ungefähr")
elif lang == "es":
self._re_search_strings = ("Resultados", "de", "aproximadamente")
# add more localised versions here
else:
self._re_search_strings = ("Results", "of", "about")
if random_agent:
self.browser.set_random_user_agent()
def __init__(self):
self.browser = Browser()
self.parser = None
self.history = None
self.fen_history = FenGame()
self.pgn_history = PgnGame()
self.notation = AlgebraicNotation()
self.params = DEFAULT_LEELA_PARAMS
self.params['MultiPV'] = 3
self.engine = None
self.new_data = None
self.last_data = None
self.best_move = list()
self.best_lines = list()
self.scores = list()
self.new_analysis = False
self.parser_lock = threading.Lock()
self.engine_lock = threading.Lock()
self.setup_engine()
def initialize():
global _browser
# Initialize reusable parts of the browser infrastructure.
tmlParser = parser_generator.makeParser(grammar_parser.parseFile(tmlGrm))
print >> sys.stderr, 'TML parser loaded.'
cs164Parser = parser_generator.makeParser(
grammar_parser.parseFile(cs164Grm))
print >> sys.stderr, 'CS164 parser loaded.'
# TODO: Currently, the interpreter "object" is the module itself. Later,
# expose an API for constructing interpreters.
renderer = Renderer(interpreter)
print >> sys.stderr, 'Renderer initialized.'
_browser = Browser(tmlParser, cs164Parser, interpreter, renderer)
interpreter.ExecGlobal(cs164Parser.parse(open(cs164Lib).read()))
print >> sys.stderr, 'Browser initialized.'
print >> sys.stderr, 'Loading the layout engine modules...'
# Load the layout engine, which is implemented in 164.
dependencies = [
'object', 'node', 'window', 'box', 'hbox', 'vbox', 'link', 'word',
'img', 'script', 'rx', 'bquery', 'layout', 'browser'
]
for dependency in dependencies:
print >> sys.stderr, '\tloading ' + dependency
ast = cs164Parser.parse(
open('./browser/{0}.164'.format(dependency)).read())
interpreter.ExecGlobal(ast)
print >> sys.stderr, 'Done.'
def jre_detalle(url):
"""Sacar los datos de los podcast de la pagina correspondiente"""
jre = Browser(url)
# Si cargamos desde la web
jre.obtener_html()
jre.preparar_html()
# Si cargamos desde un fichero en local
#jre.preparar_fichero()
parseador = Parser(jre.html)
parseador.datos_podcast()
bbdd = bd.BD()
bbdd.bd_jre()
for p in parseador.lista_podcasts:
#print p.invitado
print p.numero
#print p.twitter
#print p.fecha
#print p.descripcion
#print '*****************************************************'
# Guardar en base de datos
bbdd.guardar_podcast(bbdd.documento_podcast(p))
def parser_url(url, b = None):
if b is None:
b = Browser()
b.run()
b.get(url)
#with open(f'data_metrics\\data_url.html', 'w', encoding='UTF-8') as f: f.write(b.driver.page_source) #
return b.driver.page_source
def __init__(self, query, engine="google", conf="", random_agent=True, debug=False, lang="en", tld="com.hk", re_search_strings=None):
# read ini
self.cf = ConfigParser.RawConfigParser()
self.conf = conf
self.cf.read( self.conf )
self.query = query
self.debug = debug
self.engine = engine
self.browser = Browser(debug=debug)
self.results_info = None
self.eor = False # end of results
self._page = 0
self._first_indexed_in_previous = None
self._filetype = None
self._last_search_url = None
self._results_per_page = self.cf.getint(self.engine, "page_nums")
self._last_from = 0
self._lang = lang
self._tld = tld
self._encoding = 'utf8'
try:
self._encoding = self.cf.get(self.engine, "encoding")
except Exception, e:
pass
def collect(self):
b = Browser()
for i in xrange(200):
log.info('LOAD PAGE WITH CAPTCHA')
b.get('http://sms-webmoney.ru/')
captcha = 'http://sms-webmoney.ru/img.php'
b.save(captcha, CAPTCHA_PATH + self.site + '/%02d.png' % i)
t = Transformer()
t.load('orig', b.image(captcha))
t.resizeby('resize', t['orig'], 3, 3)
t.grayscale('grayscale', t['resize'], 2)
t.binarize('binarize', t['grayscale'], 200, CV_THRESH_BINARY_INV)
radius = 2
kernel = cvCreateStructuringElementEx(radius * 2 + 1,
radius * 2 + 1, radius,
radius, CV_SHAPE_ELLIPSE)
t.morphology('morphology', t['binarize'], 0, 1, kernel)
t.contourSplit('breaksplit', t['morphology'], 0.01)
if len(t.symbols) != self.symqty:
log.debug(colorize('INCORRECT SYMBOL NUMBER', RED))
continue
t.normolize('origsplit', 'breaksplit', self.symsize)
t.savesymbols('origsplit', SYMBOLS_PATH + self.site, '%02d' % i)
del t
def process_scraping(self, text):
if (text is None):
current_path = 'https://clinicaltrials.gov/ct2/show/record/?cond=COVID&draw=3&rank=1&view=record'
else:
current_path = 'https://clinicaltrials.gov/ct2/show/record/?cond=' + text + '&draw=3&rank=1&view=record'
print('scraping the url:', current_path)
browser = Browser(current_path)
browser.start()
more_pages = True
while (more_pages):
try:
print('processing page:', len(self.lines) + 1, ' ...')
self.study_scraper = StudyScraper(browser.get_current_page())
study = self.study_scraper.get_study()
self.lines.append(study)
print('page:', study.id, 'processed')
time.sleep(1)
browser.navigate()
except:
print('no more pages to scrap')
more_pages = False
browser.stop()
def __init__(self):
# Instance of Config Util
self.config = Config()
# Instance of Browser Util
self.browser = Browser(self.config)
# AutoLogin when Init
self.login()
def fetch(self):
url = "http://blog.sina.com.cn/u/1696709200"
b = Browser()
page = b.get_page(url)
be = BeautifulSoup(page)
div = be.find('div', {'class': 'diywidget'})
return div
def before_all(context):
context.browser = Browser()
PORT = 8000
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
context.server = SocketServer.TCPServer(("", PORT), Handler)
context.thread = threading.Thread(target=context.server.serve_forever)
context.thread.start()
def pickup(self):
b = Browser()
b.show()
b.get('http://2ip.ru/')
ip = b.js("$('div.ip big').text()")
print ip
self.quit()
def main():
parser = argparse.ArgumentParser(
description=
'This tool lets you invite people in bulk to your Facebook group')
parser.add_argument('-e',
'--email',
help='Your personal Facebook account email',
required=True)
parser.add_argument('-p',
'--password',
help='Your password in plain text',
required=True)
parser.add_argument('-g',
'--group',
help='The Facebook group name',
required=True)
parser.add_argument('-f',
'--file',
help='The csv file to load email addresses from',
default='emails.csv')
args = vars(parser.parse_args())
email_loader = EmailLoader(filename=args['file'])
browser = Browser()
browser.navigate(url='https://www.facebook.com',
wait_for='facebook',
error='Unable to load the Facebook website')
browser.enter_login_details(email=args['email'], password=args['password'])
browser.navigate(url='https://www.facebook.com/groups/' + args['group'],
wait_for='pagelet_group_',
error='Couldn\'t navigate to the group\'s members page')
browser.import_members(emails=email_loader.all_emails)
print(
str(len(email_loader.all_emails)) +
' email addresses successfully imported, you may now close Firefox')
def user(environment='ppm'):
# Create browsers.
user_browser = Browser('chrome')
# Get the path
path = os.path.dirname(__file__)
# Get the URLs
urls = URLs.load(os.path.join(path, 'urls.yml'), environment)
accounts = Accounts.load(os.path.join(path, 'accounts.yml'))[environment]
# Get a user
user = AutismUser(urls,
email_address='*****@*****.**',
password='******',
independent=False)
user.sign_up_to_p2m2(user_browser)
email = Email(urls['inbox'])
user.verify_email(user_browser, email)
user.register(user_browser)
user.consent(user_browser)
return user_browser, user, email
def _init(self):
self.main_loop = GLib.MainLoop()
self.client = WebRTCClient()
self._connect_client_observer()
self.signalling = WebRTCSignallingClient(self.server, self.id)
self.remote_client = RemoteWebRTCObserver(self.signalling)
self._connect_signalling_observer()
actions = ActionObserver()
actions.register_action_types()
self._connect_actions(actions)
# wait for the signalling server to start up before creating the browser
self.signalling.wait_for_states([SignallingState.OPEN])
self.signalling.hello()
self.browser = Browser(create_driver(self.browser_name))
self.browser.open(self.html_source)
browser_id = self.browser.get_peer_id()
assert browser_id == self.peerid
self.signalling.create_session(self.peerid)
test_name = self.test_name if self.test_name else self.scenario_name
self.remote_client.set_title(test_name)
self._init_validate(self.scenario_name)
def get_image_urls_from_current_page(browser):
"When on a Radiopaedia case, find and visit pages listing image URLs"
case_name = filename_from_url(browser.current_url)
study_ids = [
element.get_attribute("data-study-id") for element in
browser.find_elements_by_xpath("//div[@data-study-id]")
]
json_objs = []
for study_id in study_ids:
print(study_id)
stacks_url = "https://radiopaedia.org/studies/{}/stacks?lang=us".format(
study_id)
stacks_browser = Browser()
stacks_browser.get_local(SimpleMHTMLCache.get(stacks_url))
json_text = stacks_browser.find_element_by_xpath("/*").text
json_obj = json.loads(json_text)
json_objs.append(json_obj)
stacks_browser.close()
wait(4)
viewers = []
for viewer, study_id in zip(json_objs, study_ids):
slides = []
for index, slide in enumerate(viewer):
if slide["modality"] == "X-ray":
slides.append(slide)
elif slide["modality"] == "CT":
pass
viewers.append(slides)
return viewers
def search_internal(cls, search_terms, max_results):
"Use browser to get all case URLs matching the given search terms"
search_terms = cls.sanitize_search_terms(search_terms)
browser = Browser()
next_search_page = cls.format_internal_search_url(search_terms)
out = []
n_results = 0
while next_search_page:
print("using search page", next_search_page)
browser.get(next_search_page)
wait(10)
results = list(cls.extract_results_from_search_page(browser))
if cls.is_end_of_results(browser):
print("End of results")
break
if not results:
print("Please resolve the captcha.")
input()
results = list(cls.extract_results_from_search_page(browser))
n_results += len(results)
for result in results:
yield cls.get(result)
if n_results > max_results:
print("Over max results")
break
next_search_page = cls.get_next_search_page(browser)
def start(self):
if self.count > 1 and self.url is None:
print("Cannot provide page count with no URL given")
exit(1)
if self.remove_params and self.url is None:
print(
"WARNING: Must have Internet connection to remove unneeded parameters"
)
# Scan for all APIs
if self.url:
os.makedirs(self.har_directory, exist_ok=True)
self.delete_existing_hars()
self.browser = Browser(
"chromedriver/chromedriver",
"browsermob-proxy-2.1.4/bin/browsermob-proxy",
self.har_directory,
cookies=self.cookies)
if self.search_string is not None:
print("Searching URL " + self.url + " for string " +
self.search_string)
# Move recursively through the site
api_calls = self.crawling_scan(self.url)
# Scan directory of har files
else:
print("Parsing existing directory of har files")
har_parser = HarParser(self.har_directory, self.search_string,
self.remove_params)
api_calls = har_parser.parse_multiple_hars()
if self.browser is not None:
self.browser.close()
return api_calls
def main(num_agents, urls):
# The list of URLs for agents to visit.
if urls == []:
with open('urls.txt') as f:
for line in f.readlines():
if line.startswith('#') or line.startswith(';'):
continue
urls.append(line.strip())
# A list of browsing-agent threads
agents = []
for i in range(1, num_agents + 1):
try:
print('{}: Spawning browser agent #{}'.format(
time.strftime("%H:%M:%S"), i))
agents.append(Browser(urls, i))
agents[-1].start()
except Exception as e:
print('Error: unable to start thread for browser agent #{}: {}'.
format(i, e))
while True:
try:
time.sleep(30)
except KeyboardInterrupt:
print('{}: Received kill signal, so exiting'.format(
time.strftime("%H:%M:%S")))
for agent in agents:
agent.stop()
for agent in agents:
agent.join()
sys.exit(0)
def updateDisplayLoop(self):
done = False
while not done:
for event in pygame.event.get():
if event.type == pygame.QUIT:
done = True
break
if event.type == pygame.KEYDOWN and (event.key == pygame.K_ESCAPE or event.key == pygame.K_x or event.key == pygame.K_q):
done = True
break
#logger.info("gui-event: %s", str(event))
if event.type == pygame.KEYDOWN and (event.key == pygame.K_o):
blueiris = BlueIris()
blueiris.login()
camName = "fdoor"
camUrl = blueiris.getCamURL(camName)
logger.info("CAM URL: %s", camUrl)
self.browser = Browser()
self.browser.openBrowserThread(camUrl)
if event.type == pygame.KEYDOWN and (event.key == pygame.K_c):
self.browser.close()
self.screen.fill((0,0,0))
self.updateTime(10,60)
pygame.display.update()
# Update interval (fps)
self.clock.tick(2)
def create_grid(self):
grid = Gtk.Grid()
self.add(grid)
# from terminal.py class Terminal
self.vte = Terminal()
notebook, event_exit = self.vte.create_terminal()
# For closing application
event_exit.connect("button_press_event", self.app_close)
grid.add(notebook)
# From browser.py class Browser
self.webbrowser = Browser()
# Creating fist browser page
browser = self.webbrowser.create_browser()
grid.attach(browser, 1, 0, 1, 1)
# From folderview.py class Folderview
self.folder = Folderview()
# Create Folder view HOME
folderview = self.folder.create_foldeview()
grid.attach_next_to(folderview, notebook, Gtk.PositionType.BOTTOM, 1,
1)
# Filevew section
self.file = Fileview()
fileview = self.file.create_fileview()
grid.attach_next_to(fileview, folderview, Gtk.PositionType.RIGHT, 1, 1)
def __init__(self, window):
super().__init__()
self.setupUi(window)
self.window = window
self.settings = Settings()
self.window.closeEvent = self.closeEvent
self.server = Server()
self.downman = DownloadManager()
self.browser = Browser()
# snapshot updater is to be started on exchange connect
self.xchgClient = ExchangeClient()
self.lastKnownDir = "/tmp"
self.destPrefix = ''
self.userlist = None
self.di_list = []
self.addEventListeners()
self.browserTable.setColumnHidden(0, True)
self.userListTable.setColumnHidden(0, True)
self.tabWidget.setCurrentIndex(0)
self.urlFrame.setVisible(False)
self.window.setWindowIcon(QIcon(":/images/favicon.ico"))
self.window.setWindowTitle("21Lane")
self.makeMenuBar()
self.setupSystemTray()
self.loadSettings()
self.window.show()
def __browser_view(self, *args):
self.vboxBrowser.remove(self.browser)
self.browser = Browser()
self.browser.show()
self.vboxBrowser.add(self.browser)
self.__change_current_view(self.vboxBrowser)
self.browser.load_uri('http://www.sbennel.es')
def __init__(self, items, random_agent=False, debug=False):
self.items = items
self.debug = debug
self.browser = Browser(debug=debug)
if random_agent:
self.browser.set_random_user_agent()
def __init__(self, config_path="config.yaml"):
self.browser = Browser()
self.dutys = ""
self.refresh_time = ''
self.login_url = "http://www.bjguahao.gov.cn/quicklogin.htm"
self.send_code_url = "http://www.bjguahao.gov.cn/v/sendorder.htm"
self.get_doctor_url = "http://www.bjguahao.gov.cn/dpt/partduty.htm"
self.confirm_url = "http://www.bjguahao.gov.cn/order/confirm.htm"
self.patient_id_url = "http://www.bjguahao.gov.cn/order/confirm/"
self.department_url = "http://www.bjguahao.gov.cn/dpt/appoint/"
self.config = Config(config_path) # config对象
if self.config.useIMessage == 'true':
# 按需导入 imessage.py
import imessage
self.imessage = imessage.IMessage()
else:
self.imessage = None
if self.config.useQPython3 == 'true':
try: # Android QPython3 验证
# 按需导入 qpython3.py
import qpython3
self.qpython3 = qpython3.QPython3()
except ModuleNotFoundError:
self.qpython3 = None
else:
self.qpython3 = None
def __init__(self, config_path="config.yaml"):
self.browser = Browser()
self.dutys = ""
self.refresh_time = ''
self.verify_url = "http://www.114yygh.com/web/verify"
self.login_url = "http://www.114yygh.com/web/login"
self.send_code_url = "http://www.114yygh.com/web/getVerifyCode"
self.duty_url = "http://www.114yygh.com/web/product/detail"
self.confirm_url = "http://www.114yygh.com/web/order/saveOrder"
self.patient_id_url = "http://www.114yygh.com/order/confirm/"
self.query_hospital_url = "http://www.114yygh.com/web/queryHospitalById"
self.calendar = "http://www.114yygh.com/web/product/list"
self.order_patient_list = "http://www.114yygh.com/web/patient/orderPatientList"
self.appoint_info_url = "http://www.114yygh.com/web/order/getAppointInfo"
self.config = Config(config_path) # config对象
if self.config.useIMessage == 'true':
# 按需导入 imessage.py
import imessage
self.imessage = imessage.IMessage()
else:
self.imessage = None
if self.config.useQPython3 == 'true':
try: # Android QPython3 验证
# 按需导入 qpython3.py
import qpython3
self.qpython3 = qpython3.QPython3()
except ModuleNotFoundError:
self.qpython3 = None
else:
self.qpython3 = None
def wrapper(arguments: List[str]):
app_args = parse_app_arguments(arguments)
context = get_context(app_args.context_path)
current_session = None
if app_args.session_uuid:
current_session = next(
filter(lambda session: session.id == app_args.session_uuid,
context.sessions), None)
browser = Browser(current_session)
user_creds = UserCredentials(app_args.username, app_args.password,
None)
try:
func(browser, context, user_creds)
except Exception as ex:
logging.exception('Browser error.', ex)
current_session = None
context.sessions = [
session for session in context.sessions
if session.id != current_session.id
]
browser.dispose()
finally:
if current_session:
context.sessions.append(current_session)
save_context(context, app_args.context_path)
