def DetachFromBrowser(): """Close the socket and kill the process if necessary.""" if Iterate.s: Iterate.s.close() Iterate.s = None if Iterate.proc: if not windowing.WaitForProcessExit(Iterate.proc, 0): try: windowing.EndProcess(Iterate.proc) windowing.WaitForProcessExit(Iterate.proc, 0) except pywintypes.error: # Exception here most likely means the process died on its own pass Iterate.proc = None
def Time(urls, size, timeout, **kwargs): """Measure how long it takes to load each of a series of URLs Args: urls: list of URLs to time size: size of browser window to use timeout: amount of time to wait for page to load kwargs: miscellaneous keyword args Returns: A list of tuples (url, time). "time" can be "crashed" or "timeout" """ if "path" in kwargs and kwargs["path"]: path = kwargs["path"] else: path = DEFAULT_PATH proc = None # Visit each URL we're given if type(urls) in types.StringTypes: urls = [urls] ret = [] for url in urls: try: # Invoke the browser if necessary if not proc: (wnd, proc, address_bar, render_pane, tab_window) = InvokeBrowser(path) # Resize and reposition the frame windowing.MoveAndSizeWindow(wnd, (0, 0), size, render_pane) # Double-click in the address bar, type the name, and press Enter mouse.DoubleClickInWindow(address_bar) keyboard.TypeString(url) keyboard.TypeString("\n") # Wait for the page to finish loading load_time = windowing.WaitForThrobber(tab_window, (6, 8, 22, 24), timeout) timedout = load_time < 0 if timedout: load_time = "timeout" # Send an alt-F4 to make the browser close; if this times out, # we've probably got a crash keyboard.TypeString(r"{\4}", use_modifiers=True) if not windowing.WaitForProcessExit(proc, timeout): windowing.EndProcess(proc) load_time = "crashed" proc = None except pywintypes.error: load_time = "crashed" proc = None ret.append((url, load_time)) # Send an alt-F4 to make the browser close; if this times out, # we've probably got a crash if proc: keyboard.TypeString(r"{\4}", use_modifiers=True) if not windowing.WaitForProcessExit(proc, timeout): windowing.EndProcess(proc) return ret
def Scrape(urls, outdir, size, pos, timeout, kwargs): """Invoke a browser, send it to a series of URLs, and save its output. Args: urls: list of URLs to scrape outdir: directory to place output size: size of browser window to use pos: position of browser window timeout: amount of time to wait for page to load kwargs: miscellaneous keyword args Returns: None if success, else an error string """ if "path" in kwargs and kwargs["path"]: path = kwargs["path"] else: path = DEFAULT_PATH (wnd, proc, address_bar, render_pane) = InvokeBrowser(path) # Resize and reposition the frame windowing.MoveAndSizeWindow(wnd, pos, size, render_pane) # Visit each URL we're given if type(urls) in types.StringTypes: urls = [urls] timedout = False for url in urls: # Double-click in the address bar, type the name, and press Enter mouse.ClickInWindow(address_bar) keyboard.TypeString(url, 0.1) keyboard.TypeString("\n") # Wait for the page to finish loading load_time = windowing.WaitForThrobber(wnd, (20, 16, 36, 32), timeout) timedout = load_time < 0 if timedout: break # Scrape the page image = windowing.ScrapeWindow(render_pane) # Save to disk if "filename" in kwargs: if callable(kwargs["filename"]): filename = kwargs["filename"](url) else: filename = kwargs["filename"] else: filename = windowing.URLtoFilename(url, outdir, ".bmp") image.save(filename) if proc: windowing.SetForegroundWindow(wnd) # Send Alt-F4, then wait for process to end keyboard.TypeString(r"{\4}", use_modifiers=True) if not windowing.WaitForProcessExit(proc, timeout): windowing.EndProcess(proc) return "crashed" if timedout: return "timeout" return None