def ssid_test(ssid, session_mode): # Silence output from Twill commands f = open(os.devnull, "w") twill.set_output(f) # Generate File names for diff file_name = ssid + '.html' generated_html_path = 'output/ssid/' + file_name expected_html_path = 'expected/ssidForm/' + file_name # Start with a fresh page every time go(url) print '\n**Testing SSID of ' + ssid + '**' code(200) # Fill the HTML forms with test values and submit fv("1", "ssid", ssid) fv("1", "session_mode", session_mode) submit('0') save_html(generated_html_path) # Diff with HTML page we know should 'come back' command = 'diff {0} {1}'.format(generated_html_path, expected_html_path) result = subprocess.call(command.split(), shell=False) if result is not 0: print 'Test failed' else: print 'Test Passed'
def _grab_remote_html(url): global base_url, data_output, screen_output twill.commands.clear_cookies() twill.commands.agent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36') twill.commands.go(base_url + url) # Ensure we get a 200 http status code back try: code_response = twill.commands.code(200) except: code_response = "" # Step into the html and extract the links if code_response is None: # Reset buffer data_output.seek(0) data_output.truncate(0) # Change sys output to capture output in a variable sys.stdout = data_output twill.set_output(data_output) # Grab the HTML data which will be stored in data_reponse twill.commands.show() # Change the sys output back to the screen, now we have captured the data sys.stdout = screen_output twill.set_output(screen_output) return data_output
def __init__(self): Web2UnitTest.__init__(self) self.b = get_browser() self.b_data = StringIO() set_output(self.b_data) # list of links that return a http_code other than 200 # with the key being the URL and the value the http code self.brokenLinks = dict() # List of links visited (key) with the depth self.urlList = dict() # List of urls for each model self.model_url = dict() # This string must exist in the URL for it to be followed # Useful to avoid going to linked sites self.homeURL = self.url # Tuple of strings that if in the URL will be ignored # Useful to avoid dynamic URLs that trigger the same functionality self.include_ignore = ( "_language=", "/admin/default/", ) # tuple of strings that should be removed from the URL before storing # Typically this will be some variables passed in via the URL self.strip_url = ("?_next=", ) self.maxDepth = 2 # sanity check
def setAgent(self, agentAcronym): # Decide on the agent that will be used to power the smoke test if agentAcronym == "g": self.agent = "Ghost" try: from ghost import Ghost self.ghost = Ghost(wait_timeout = 360) except ImportError: raise NameError("Ghost not installed") from using_ghost import login, visit else: self.agent = "Twill" try: from twill import get_browser from twill import set_output except ImportError: raise NameError("Twill not installed") try: import mechanize except ImportError: raise NameError("Mechanize not installed") self.b = get_browser() self.b_data = StringIO() set_output(self.b_data) from using_twill import login, visit self.visit = MethodType(visit, self) self.login = MethodType(login, self)
def __init__(self): self.twill_browser = twill.get_browser() if not settings.get('verbose', True): twill.set_output(open(os.devnull, 'w')) #twill.browser.OUT = open(os.devnull, 'w') # Handle HTTP authentication if settings.get('http_auth_username', None) and settings.get('http_auth_password', None): base64string = base64.encodestring('%s:%s' % (settings['http_auth_username'], settings['http_auth_password'])).replace('\n', '') twill.commands.add_auth("wiki", settings['mediawiki_url'], settings['http_auth_username'], settings['http_auth_password']) #self.twill_browser._session.headers.update([("Authorization", "Basic %s" % base64string)]) twill.commands.add_extra_header("Authorization", "Basic %s" % base64string) # Handle Mediawiki authentication if settings.get('mediawiki_username', None) and settings.get('mediawiki_password', None): login_url = urlparse.urljoin(settings['mediawiki_url'], '/index.php?title=Special:UserLogin') self.openurl(login_url) self._set_form_value('userlogin', 'wpName', settings.get('mediawiki_username')) self._set_form_value('userlogin', 'wpPassword', settings.get('mediawiki_password')) self.twill_browser.submit() self.openurl(settings['mediawiki_url'])
def __init__(self): Web2UnitTest.__init__(self) self.b = get_browser() self.b_data = StringIO() set_output(self.b_data) self.clearRecord() # This string must exist in the URL for it to be followed # Useful to avoid going to linked sites self.homeURL = self.url # Link used to identify a URL to a ticket self.url_ticket = "/admin/default/ticket/" # Tuple of strings that if in the URL will be ignored # Useful to avoid dynamic URLs that trigger the same functionality self.include_ignore = ("_language=", "logout", "appadmin", "admin", "delete", ) # tuple of strings that should be removed from the URL before storing # Typically this will be some variables passed in via the URL self.strip_url = ("?_next=", ) self.reportOnly = False self.maxDepth = 16 # sanity check self.setThreshold(10) self.setUser("[email protected]/eden") self.total_visited = 0 self.broken_links_count = 0
def add_class(self, unique_number): class_url = self.url + '/' + unique_number tc.go(class_url) html = StringIO.StringIO() twill.set_output(html) tc.show() soup = BeautifulSoup(html.getvalue()) table = soup.find('table') for row in table.findAll('tr')[1:]: columns = row.findAll('td') unique = columns[0].string days = [d.text for d in columns[1].findAll('span')] hour = [d.text for d in columns[2].findAll('span')] room = [d.text for d in columns[3].findAll('span')] instructor = columns[4].span.text new_course = Course(unique, days, hour, room, instructor) if self._check_planner_to_add(new_course): self.course_set.add(new_course) days_to_add = new_course.parse_days() hours_to_add = new_course.parse_hours() for d in range(len(days_to_add)): for h in range(hours_to_add[d][0], hours_to_add[d][1]): for day in days_to_add[d]: self.grid[h][day] = new_course print("Course successfully added.")
def __init__(self): Web2UnitTest.__init__(self) self.b = get_browser() self.b_data = StringIO() set_output(self.b_data) # list of links that return a http_code other than 200 # with the key being the URL and the value the http code self.brokenLinks = dict() # List of links visited (key) with the depth self.urlList = dict() # List of urls for each model self.model_url = dict() # This string must exist in the URL for it to be followed # Useful to avoid going to linked sites self.homeURL = self.url # Tuple of strings that if in the URL will be ignored # Useful to avoid dynamic URLs that trigger the same functionality self.include_ignore = ("_language=", "/admin/default/", ) # tuple of strings that should be removed from the URL before storing # Typically this will be some variables passed in via the URL self.strip_url = ("?_next=", ) self.maxDepth = 2 # sanity check
def wiki_lookup(self): wiki = twill.commands out = StringIO() twill.set_output(out) res = wiki.go('http://en.wikipedia.org/wiki/Main_Page') res = wiki.formvalue(1, "searchInput", self.name) res = wiki.tidy_ok() res = wiki.submit(0) data = (wiki.get_browser().get_html()) out = open('tmp', 'a') i = 0 go = False for line in data.split('\n'): soep = BeautifulSoup(line) for line in soep.popTag(): try: for div in line.findAll('div').pop(): if (div.encode().strip().startswith('may refer to:')): desc = str(line.findAll('div')[0]).split( '>')[13].split('<')[0].strip() link = str(line.findAll('a')[0]).replace( '/wiki/', 'http://en.wikipedia.org/wiki/') return (desc, link) except: pass return (False)
def __init__(self): Web2UnitTest.__init__(self) self.b = get_browser() self.b_data = StringIO() set_output(self.b_data) self.clearRecord() # This string must exist in the URL for it to be followed # Useful to avoid going to linked sites self.homeURL = self.url # Link used to identify a URL to a ticket self.url_ticket = "/admin/default/ticket/" # Tuple of strings that if in the URL will be ignored # Useful to avoid dynamic URLs that trigger the same functionality self.include_ignore = ( "_language=", "logout", "appadmin", "admin", "delete", ) # tuple of strings that should be removed from the URL before storing # Typically this will be some variables passed in via the URL self.strip_url = ("?_next=", ) self.reportOnly = False self.maxDepth = 16 # sanity check self.setThreshold(10) self.setUser("[email protected]/eden")
def timeout_test(timeout, time_unit): # Silence output from Twill commands f = open(os.devnull, "w") twill.set_output(f) # Generate File names for diff # e.x. output/10seconds.html, expected/sessionsForm/10seconds.html file_name = ` timeout ` + time_unit + '.html' generated_html_path = 'output/timeout/' + file_name expected_html_path = 'expected/sessionsForm/' + file_name # Start with a fresh page every time go(url) print '\n**Testing timeout of ' + ` timeout ` + ' ' + time_unit + '**' code(200) # Fill the HTML forms with test values and submit fv("1", "timeout", ` timeout `) fv("1", "timeUnit", time_unit) submit('0') save_html(generated_html_path) # Diff with HTML page we know should 'come back' command = 'diff {0} {1}'.format(generated_html_path, expected_html_path) result = subprocess.call(command.split(), shell=False) if result is not 0: print 'Test failed' else: print 'Test Passed'
def tearDown(self): import twill import twill.commands twill.commands.reset_browser() twill.remove_wsgi_intercept('localhost', 6543) twill.set_output(None) testing.tearDown()
def setUp(self): '''Create the app''' test_path = os.path.abspath(os.path.dirname(__file__)) testpath_command = "setglobal test_path " + test_path twill.execute_string(testpath_command) fixtures = os.path.join(test_path, 'fixtures') for to_delete in [fname for fname in os.listdir(fixtures) if fname.startswith('Data.fs') or fname in ['blobs']]: _rm(os.path.join(fixtures, to_delete)) os.mkdir(os.path.join(fixtures, 'blobs')) wsgi_app = get_app(os.path.join(test_path, 'fixtures', 'karl.ini'), 'main') def build_app(): return wsgi_app twill.add_wsgi_intercept('localhost', 6543, build_app) # XXX How do we suppress the annoying "AT LINE: " output? twill.set_output(open('/dev/null', 'wb')) twill.execute_string("extend_with karl.twillcommands") # mostly the same as karl3.conf without extending with flunc # and few other adjustments. twill.execute_string("runfile '" + os.path.abspath(os.path.dirname(__file__)) + "/test_twill_wsgi_karl3.conf'")
def ssid_test(ssid, session_mode): # Silence output from Twill commands f = open(os.devnull,"w") twill.set_output(f) # Generate File names for diff file_name = ssid + '.html' generated_html_path = 'output/ssid/' + file_name expected_html_path = 'expected/ssidForm/' + file_name # Start with a fresh page every time go(url) print '\n**Testing SSID of ' + ssid + '**' code(200) # Fill the HTML forms with test values and submit fv("1","ssid",ssid) fv("1","session_mode",session_mode) submit('0') save_html(generated_html_path) # Diff with HTML page we know should 'come back' command = 'diff {0} {1}'.format(generated_html_path, expected_html_path) result = subprocess.call(command.split(), shell=False) if result is not 0: print 'Test failed' else: print 'Test Passed'
def timeout_test(timeout, time_unit): # Silence output from Twill commands f = open(os.devnull,"w") twill.set_output(f) # Generate File names for diff # e.x. output/10seconds.html, expected/sessionsForm/10seconds.html file_name = `timeout` + time_unit + '.html' generated_html_path = 'output/timeout/' + file_name expected_html_path = 'expected/sessionsForm/' + file_name # Start with a fresh page every time go(url) print '\n**Testing timeout of ' + `timeout` + ' ' + time_unit + '**' code(200) # Fill the HTML forms with test values and submit fv("1","timeout",`timeout`) fv("1","timeUnit",time_unit) submit('0') save_html(generated_html_path) # Diff with HTML page we know should 'come back' command = 'diff {0} {1}'.format(generated_html_path, expected_html_path) result = subprocess.call(command.split(), shell=False) if result is not 0: print 'Test failed' else: print 'Test Passed'
def __enter__(self): twill.set_output(StringIO.StringIO()) twill.commands.clear_cookies() twill.add_wsgi_intercept(self.host, self.port, lambda: self.app) return self
def _pre_setup(self): super(TwillTestCase, self)._pre_setup() twill.set_output(StringIO.StringIO()) twill.commands.clear_cookies() twill.add_wsgi_intercept(self.twill_host, self.twill_port, lambda: self.app) self.browser = twill.get_browser()
def reset_output(): """ >> reset_output Reset twill output to go to the screen. """ import twill twill.set_output(None)
def redirect_output(filename): """ >> redirect_output <filename> Append all twill output to the given file. """ import twill fp = open(filename, 'a') twill.set_output(fp)
def __init__(self): self.username = USERNAME self.password = PASSWORD self.resourses = defaultdict(int) self.fields = defaultdict(list) self.farms = [] # suppress twill output f = open(os.devnull, "w") set_output(f)
def execute_script(self): """ Executes twill script. Returns a tuple status, output """ out = StringIO() # execute the twill, catching any exceptions try: twill.set_errout(out) twill.set_output(out) twill.parse._execute_script(self.watch.script.split("\n")) status = STATUS_OK except Exception, e: status = STATUS_FAILED
def tearDown(self): # remove intercept twill.remove_wsgi_intercept('localhost', 6543) twill.set_output(None) test_path = os.path.abspath(os.path.dirname(__file__)) fixtures = os.path.join(test_path, 'fixtures') for to_delete in [fname for fname in os.listdir(fixtures) if fname.startswith('Data.fs') or fname in ['blobs', 'mail_queue']]: _rm(os.path.join(fixtures, to_delete))
def before_all(context): context.baseurl = 'http://127.0.0.1:8000' twill.set_output(StringIO.StringIO()) twill.commands.clear_cookies() context.app = beer_app.create_app() context.app.config['TESTING'] = True # fn=(lambda : context.app), it help create the WSGI app object only once, # because function passed into wsgi_intercept is called # once for each intercepted connection # more here: http://ivory.idyll.org/articles/twill-and-wsgi_intercept.html twill.add_wsgi_intercept('127.0.0.1', 8000, lambda : context.app) context.browser = twill.get_browser()
def fetch_disease_model(id): from twill import set_output set_output(open('/dev/null', 'w')) dismod_server_login() twc.go(DISMOD_DOWNLOAD_URL % id) result_json = twc.show() twc.get_browser()._browser._response.close() # end the connection, so that apache doesn't get upset dm = DiseaseJson(result_json) return dm
def scrape(url): global start_count, end_count, data_output, screen_output, query_url, page_links, sleep_time # Start configuring twill twill.commands.clear_cookies() twill.commands.agent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36') twill.commands.go(url) # Ensure we get a 200 http status code back try: code_response = twill.commands.code(200) except: code_response = "" # Step into the html and extract the links if code_response is None: # Change sys output to capture output in a variable sys.stdout = data_output twill.set_output(data_output) # Grab the HTML data which will be stored in data_reponse twill.commands.showlinks() # Change the sys output back to the screen, now we have captured the data sys.stdout = screen_output twill.set_output(screen_output) # Split data up using new line char page_links_raw = data_output.getvalue().split("\n") # Loop through each row and look for a hyperlink for item in page_links_raw: # Find http in string httpString = item.find(query_url) # Add url to the array if not already if httpString is not -1: page_links.append(item[httpString:]) # Goto the next page url start_count = start_count + 10; if start_count <= end_count: # Wait "sleep_time" seconds before visiting the next page sleep(sleep_time) # Recursive call, visit the next page scrape(base_url + str(start_count))
def setUp(self): super(AdminSurveyTwillTest, self).setUp() self.old_propagate = settings.DEBUG_PROPAGATE_EXCEPTIONS settings.DEBUG_PROPAGATE_EXCEPTIONS = True signals.request_finished.disconnect(close_connection) twill.set_output(StringIO()) twill.add_wsgi_intercept(TWILL_TEST_HOST, 80, WSGIHandler) self.browser = twill.get_browser() self.browser.go(reverse_for_twill('admin:index')) twill.commands.formvalue(1, 'username', self.username) twill.commands.formvalue(1, 'password', self.pw) self.browser.submit() twill.commands.find('Welcome')
def fetch_disease_model(id): from twill import set_output set_output(open("/dev/null", "w")) dismod_server_login() twc.go(DISMOD_DOWNLOAD_URL % id) result_json = twc.show() twc.get_browser()._browser._response.close() # end the connection, so that apache doesn't get upset dm = DiseaseJson(result_json) return dm
def setUp(self): import sys import twill from pyramid.configuration import Configurator config = Configurator(root_factory=self.root_factory) config.load_zcml(self.config) twill.add_wsgi_intercept('localhost', 6543, config.make_wsgi_app) if sys.platform is 'win32': # pragma: no cover out = open('nul:', 'wb') else: out = open('/dev/null', 'wb') twill.set_output(out) testing.setUp(registry=config.registry)
def enable_debug(debug): if isinstance(debug, bool) or debug is None: if debug: twillout=sys.stderr else: twillout=StringIO.StringIO() elif isinstance(debug, str): twillout=file(debug, 'a') else: raise ValueError, debug import twill twill.set_output(twillout) twill.set_errout(twillout)
def download_with_login(url, login_url, login=None, password=None, ext='', username_field='username', password_field='password', form_id=1): ''' Download a URI from a website using Django by loging-in first 1. Logs in using supplied login & password (if provided) 2. Create a temp file on disk using extension if provided 3. Write content of URI into file ''' # log-in to Django site if login and password: tw.go(login_url) tw.formvalue('%s' % form_id, username_field, login) tw.formvalue('%s' % form_id, password_field, password) tw.submit() # retrieve URI try: tw.go(url) tw.code('200') except TwillAssertionError: code = get_browser().get_code() # ensure we don't keep credentials tw.reset_browser() raise DownloadFailed(u"Unable to download %(url)s. " u"Received HTTP #%(code)s." % { 'url': url, 'code': code }) buff = StringIO.StringIO() twill.set_output(buff) try: tw.show() finally: twill.set_output(None) tw.reset_browser() # write file on disk suffix = '.%s' % ext if ext else '' fileh, filename = tempfile.mkstemp(suffix=suffix) os.write(fileh, buff.getvalue()) os.close(fileh) buff.close() return filename
def enable_debug(debug): if isinstance(debug, bool) or debug is None: if debug: twillout = sys.stderr else: twillout = StringIO.StringIO() elif isinstance(debug, str): twillout = file(debug, 'a') else: raise ValueError, debug import twill twill.set_output(twillout) twill.set_errout(twillout)
def setUp(self): webapp.web.db.open_database("sqlite://") cherrypy.config.update( { "environment": "embedded", "global" : { "tools.auth.on" : True, "tools.sessions.on" : True, } }) wsgiApp = cherrypy.tree.mount(webapp.web.root.Root()) cherrypy.server.start() twill.add_wsgi_intercept('localhost', 8080, lambda : wsgiApp) self.outp = StringIO() twill.set_output(self.outp)
def __init__(self, *args, **kwargs): self.db_connection = db.Connection( host=settings.SETTINGS['db_host'], port=settings.SETTINGS['db_port'] ) self.db = self.db_connection[settings.SETTINGS['db_name']] self.tc = tc self.tc.browser.set_agent_string( 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8) ' 'Gecko/20051111 Firefox/1.5 BAVM/1.0.0' ) if not self.debug: twill.set_output(StringIO()) return super(BasicTest, self).__init__(*args, **kwargs)
def pageinfo(self): buf = StringIO() twill.set_output(buf) twillc.info() info = {} for line in buf.getvalue().split('\n'): if len(line.strip()) > 0: parts = line.strip().split(': ') if len(parts) > 1: info[parts[0].strip()] = string.join(parts[1:], ': ') else: info[parts[0].strip()] = '' return info
def setUp(self): """Setup an authorized user so we can test app functionality""" # Handles setting the state once for the all tests in this class # http://stackoverflow.com/questions/402483/caching-result-of-setup-using-python-unittest/402492#402492 if not self.browser: twill.set_output(StringIO()) twill.commands.clear_cookies() twill.add_wsgi_intercept(TWILLHOST, 80, lambda:app) self.__class__.browser = twill.get_browser() # authorize user against Gmail for our app self.__class__.browser.go(url_for_twill('/')) twill.commands.formvalue(1, 'search', 'test') self.__class__.browser.submit() twill.commands.formvalue(1, 'Passwd', secrets.TEST_GOOGLE_PASSWORD) twill.commands.formvalue(1, 'Email', secrets.TEST_GOOGLE_EMAIL) self.__class__.browser.submit() self.__class__.browser.submit('allow')
def download_with_login(url, login_url, login=None, password=None, ext='', username_field='username', password_field='password', form_id=1): ''' Download a URI from a website using Django by loging-in first 1. Logs in using supplied login & password (if provided) 2. Create a temp file on disk using extension if provided 3. Write content of URI into file ''' # log-in to Django site if login and password: tw.go(login_url) tw.formvalue('%s' % form_id, username_field, login) tw.formvalue('%s' % form_id, password_field, password) tw.submit() # retrieve URI try: tw.go(url) tw.code('200') except TwillAssertionError: code = get_browser().get_code() # ensure we don't keep credentials tw.reset_browser() raise DownloadFailed(u"Unable to download %(url)s. " u"Received HTTP #%(code)s." % {'url': url, 'code': code}) buff = StringIO.StringIO() twill.set_output(buff) try: tw.show() finally: twill.set_output(None) tw.reset_browser() # write file on disk suffix = '.%s' % ext if ext else '' fileh, filename = tempfile.mkstemp(suffix=suffix) os.write(fileh, buff.getvalue()) os.close(fileh) buff.close() return filename
def getResults(opts): # Twill prints a lot of shit, we don't want that. twill.set_output(StringIO.StringIO()) # Log into studweb tc.go( "https://studweb.uio.no/as/WebObjects/studentweb2.woa/3/wa/default?inst=UiO" ) tc.fv("2", "fodselsnr", opts["fnr"]) tc.fv("2", "pinkode", opts["pin"]) tc.submit() # Navigating to grades. Bad localization for NN users. No loc. for EN :( try: tc.follow('Se opplysninger om deg') except te.TwillAssertionError: try: # Note: wildcard in linkname. # Problems with special NO chars (æøå). tc.follow('Sj. opplysningar om deg') except te.TwillAssertionError: print "Error: Unknown language on site?" sys.exit(1) tc.follow('Resultater') # Storing HTML in var. data = tc.show() tc.follow('Logg ut') # Pulling coursecode and grade out of HTML, converting flunking to # 'F', and putting it in an array. # Ex. result: [["INF1000", "B"], ["INF1040", "E"]] res = re.findall('<tr class="pysj\d">(.*?)</tr>', data) ans = {} for i in res: if not re.search("Ikkje møtt|Ikke møtt", i): tmp = re.findall("<td.*?>(.*?)</td>", i) if not re.search("[A-E]", tmp[7]): tmp[7] = "F" if (not ans.has_key(tmp[1])) or (ans.has_key(tmp[1]) and ans[tmp[1]] == "F"): ans[tmp[1]] = tmp[7] return reduce(lambda x, y: x + [[y, ans[y]]], ans, [])
def lookup(sname, syear): imdb = twill.commands out = StringIO() twill.set_output(out) res = imdb.go("http://www.imdb.com/find?s=all&q=") res = imdb.formvalue(1, "q", sname) res = imdb.tidy_ok() res = imdb.submit(0) soep = BeautifulSoup(imdb.get_browser().get_html()) u = unicode(sname + " (" + syear + ")") if soep.title.contents[0] == u: print("superb!") links = soep.findAll('a') for l in links: line = str(l) if (line.find("directorlist") > -1): print "*************" + line print l.contents
def getResults(opts): # Twill prints a lot of shit, we don't want that. twill.set_output(StringIO.StringIO()) # Log into studweb tc.go("https://studweb.uio.no/as/WebObjects/studentweb2.woa/3/wa/default?inst=UiO") tc.fv("2", "fodselsnr", opts["fnr"]) tc.fv("2", "pinkode", opts["pin"]) tc.submit() # Navigating to grades. Bad localization for NN users. No loc. for EN :( try: tc.follow('Se opplysninger om deg') except te.TwillAssertionError: try: # Note: wildcard in linkname. # Problems with special NO chars (æøå). tc.follow('Sj. opplysningar om deg') except te.TwillAssertionError: print "Error: Unknown language on site?" sys.exit(1) tc.follow('Resultater') # Storing HTML in var. data = tc.show() tc.follow('Logg ut') # Pulling coursecode and grade out of HTML, converting flunking to # 'F', and putting it in an array. # Ex. result: [["INF1000", "B"], ["INF1040", "E"]] res = re.findall('<tr class="pysj\d">(.*?)</tr>', data) ans = {} for i in res: if not re.search("Ikkje møtt|Ikke møtt", i): tmp = re.findall("<td.*?>(.*?)</td>", i) if not re.search("[A-E]", tmp[7]): tmp[7] = "F" if (not ans.has_key(tmp[1])) or (ans.has_key(tmp[1]) and ans[tmp[1]]== "F"): ans[tmp[1]] = tmp[7] return reduce(lambda x, y: x + [[y, ans[y]]], ans, [])
def annotate(params, proteins, \ url="http://signalfind.org/tatfind.html", force=False): """ Interfaces with the TatFind web service at (http://signalfind.org/tatfind.html) to predict if protein sequences contain Twin-Arginine Translocation (Tat) signal peptides. """ # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) outfn = 'tatfind.out' log_stderr("# TatFind(web) %s > %s" % (params['fasta'], outfn)) if not force and os.path.isfile(outfn): log_stderr("# -> skipped: %s already exists" % outfn) fh = open(outfn, 'r') proteins = parse_tatfind_output(fh, proteins) fh.close() return proteins # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go(url) if __DEBUG__: showforms() formfile("1", "seqFile", params["fasta"]) submit() if __DEBUG__: show() tatfind_output = show() if __DEBUG__: log_stderr(tatfind_output) # write raw TatFind output to a file fh = open(outfn, 'w') fh.write(tatfind_output) fh.close() proteins = parse_tatfind_output(tatfind_output.split("\n"), proteins) return proteins
def setup(): if not os.path.exists(AUTHENTIC_SRCDIR): print >> sys.stderr, 'Authentic source dir (%s) does not exist' % AUTHENTIC_SRCDIR print >> sys.stderr, 'Create it or edit tests/config.py to match your local installation' sys.exit(1) silent = os.environ.get('NO_SILENT') is None twill.commands.reset_browser() twill.set_output(file('/dev/null', 'w')) base = [] if os.environ.get('VALGRIND') is '1' and os.path.exists( '/usr/bin/valgrind'): base = ['./valgrind-wrapper.sh', 'python'] os.mkdir('/tmp/.tests') authentic_command = base + [ AUTHENTICCTL, 'start', '--app-dir', '/tmp/.tests/authentictests', '--data-dir', AUTHENTIC_DATADIR, '--extra', os.path.join(AUTHENTIC_SRCDIR, 'extra', 'conformance'), '--port', '10001', '--http' ] if silent: authentic_command.append('--silent') sp = subprocess.Popen(authentic_command) pids.append(sp.pid) lcs_command = base + [ LCSCTL, 'start', '--app-dir', '/tmp/.tests/lcstests', '--data-dir', LCS_DATADIR, '--port', '10002', '--http' ] if silent: lcs_command.append('--silent') sp = subprocess.Popen(lcs_command) pids.append(sp.pid) # Wait for the daemons to load themselves starttime = time.time() waitforport(10001, starttime) waitforport(10002, starttime)
def imdb_lookup(self): imdb = twill.commands out = StringIO() twill.set_output(out) res = imdb.go("http://www.imdb.com/find?s=all&q=") res = imdb.formvalue(1, "q", self.new_name) res = imdb.tidy_ok() res = imdb.submit(0) data = (imdb.get_browser().get_html()) out = open('tmp', 'w') i = 0 for line in data.split('\n'): if len(line.strip()) > 1: out.write(line.strip() + '\n') if line.find('/title/tt') > -1: if i < 10: i += 1 soup = BeautifulSoup(line) imdb_name = soup.findAll('a')[0] try: return (unescape(imdb_name.contents.decode(), { "&": '&', "'": "'", '"': '' })) except: pass soup = BeautifulSoup(line) imdb_name = soup.findAll('td') try: return (unescape(imdb_name.contents, { "&": '&', "'": "'", '"': '' })) except: return (False) out.close()
def setUp(self): "Run before all tests in this class, sets the output to the console" twill.set_output(StringIO())
def __init__(self): #ignore twill output, until we need it twill.set_output(StringIO())
def setUp(self): app = AdminMediaHandler(WSGIHandler()) twill.add_wsgi_intercept(self.HOST, self.PORT, lambda: app) twill.set_output(StringIO()) self.command = twill.commands
from xml.etree import ElementTree # Be sure to use Galaxy's vanilla pyparsing instead of the older version # imported by twill. import pyparsing # noqa: F401 import twill import twill.commands as tc from six import string_types, StringIO from six.moves.urllib.parse import urlencode, urlparse from twill.other_packages._mechanize_dist import ClientForm from base.testcase import FunctionalTestCase # noqa: I100 # Force twill to log to a buffer -- FIXME: Should this go to stdout and be captured by nose? buffer = StringIO() twill.set_output(buffer) tc.config('use_tidy', 0) # Dial ClientCookie logging down (very noisy) logging.getLogger("ClientCookie.cookies").setLevel(logging.WARNING) log = logging.getLogger(__name__) DEFAULT_TOOL_TEST_WAIT = os.environ.get("GALAXY_TEST_DEFAULT_WAIT", 86400) class TwillTestCase(FunctionalTestCase): """Class of FunctionalTestCase geared toward HTML interactions using the Twill library.""" def check_for_strings(self, strings_displayed=[], strings_not_displayed=[]): if strings_displayed:
path = "dx/" convertDocxToText(path) from pylab import * import urllib2 import twill from twill.commands import * import re import os import magic import sys from docx import * # Set up dummy non-existent file to suppress twill output f = open(os.devnull, 'w') twill.set_output(f) # Navigate to the website and authenticate url = 'http://linguistlist.org/confservices/EasyAbs/login.cfm' go(url) fv('2', 'emailaddress', '[INSERT EMAIL]') fv('2', 'password', '[INSERT PASSWORD]') submit(0) follow('View/Assign Abstracts') # Grab the paper IDs (6 digit numbers) pagetext = show() allIDs = [ pagetext[num.start() + 1:num.start() + 7] for num in list(re.finditer('t19', pagetext)) ]
def annotate(params, proteins, \ force=False): """ DEPRECATED: The TMB-HUNT server appears to be permanently offline. Uses the TMB-HUNT web service (http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi) to predict if proteins are outer membrane beta-barrels. NOTE: In my limited testing, TMB-HUNT tends to perform very poorly in terms of false positives and false negetives. I'd suggest using only BOMP. """ # TODO: automatically split large sets into multiple jobs # TMB-HUNT will only take 10000 seqs at a time if len(proteins) >= 10000: log_stderr( "# ERROR: TMB-HUNT(web): can't take more than 10,000 sequences.") return # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) out = 'tmbhunt.out' log_stderr("# TMB-HUNT(web) %s > %s" % (params['fasta'], out)) if not force and os.path.isfile(out): log_stderr("# -> skipped: %s already exists" % out) return parse_tmbhunt(proteins, out) # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi" ) if __DEBUG__: showforms() # read up the FASTA format seqs fh = open(params['fasta'], 'r') fasta_seqs = fh.read() fh.close() # fill out the form fv("1", "sequences", fasta_seqs) submit() if __DEBUG__: showlinks() # small jobs will lead us straight to the results, big jobs # go via a 'waiting' page which we skip past if we get it job_id = None try: # we see this with big jobs result_table_url = follow( "http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output.*.html" ) job_id = result_table_url.split('tmp_output')[-1:][0].split('.')[0] except: # small jobs take us straight to the html results table pass # parse the job_id from the url, since due to a bug in # TMB-HUNT the link on the results page from large jobs is wrong if not job_id: job_id = \ follow("Full results").split('/')[-1:][0].split('.')[0] log_stderr( "# TMB-HUNT(web) job_id is: %s <http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output%s.html>" % (job_id, job_id)) # polling until TMB-HUNT finishes # TMB-HUNT advises that 4000 sequences take ~10 mins # we poll a little faster than that polltime = (len(proteins) * 0.1) + 2 while True: log_stderr("# TMB-HUNT(web): waiting another %i sec ..." % (polltime)) time.sleep(polltime) try: go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/%s.txt" % (job_id)) break except: polltime = polltime * 2 if polltime >= 7200: # 2 hours log_stderr("# TMB-HUNT error: Taking too long.") return txt_out = show() # write raw TMB-HUNT results fh = open(out, 'w') fh.write(txt_out) fh.close() return parse_tmbhunt(proteins, out)
def _twill_quiet(): # suppress normal output of twill.. You don't want to # call this if you want an interactive session twill.set_output(StringIO())
def twill_quiet(): # suppress normal output of twill.. You don't want to # call this if you want an interactive session if testlib.TWILL_QUIET: twill.set_output(StringIO())
def annotate(params, proteins, \ url="http://psfs.cbrc.jp/tmbeta-net/", \ category='OM(barrel)', force=False): """ Uses the TMBETA-NET web service (http://psfs.cbrc.jp/tmbeta-net/) to predict strands of outer membrane beta-barrels. By default, category='BARREL' means prediction will only be run on proteins in the set with this category property. To process all proteins, change category to None. These keys are added to the proteins dictionary: 'tmbeta_strands' - a list of lists with paired start and end residues of each predicted strand. (eg [[3,9],[14,21], ..etc ]) """ # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) outfile = 'tmbeta_net.out' log_stderr("# TMBETA-NET(web) %s > %s" % (params['fasta'], outfile)) tmbeta_strands = {} if not force and os.path.isfile(outfile): log_stderr("# -> skipped: %s already exists" % outfile) fh = open(outfile, 'r') tmbeta_strands = json.loads(fh.read()) fh.close() for seqid in tmbeta_strands: proteins[seqid]['tmbeta_strands'] = tmbeta_strands[seqid] return tmbeta_strands # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) for seqid in proteins: # only run on sequences which match the category filter if force or \ (category == None) or \ (dict_get(proteins[seqid], 'category') == category): pass else: continue go(url) if __DEBUG__: showforms() fv("1", "sequence", proteins[seqid]['seq']) submit() log_stderr("# TMBETA-NET: Predicting strands for %s - %s\n" \ % (seqid, proteins[seqid]['name'])) out = show() time.sleep(1) if ("Some query is already running. Please try again." in out): log_stderr("# TMBETA-NET(web) error: %s" % (out)) return {} # parse the web page returned, extract strand boundaries proteins[seqid]['tmbeta_strands'] = [] for l in out.split('\n'): if __DEBUG__: log_stderr("## " + l) if "<BR>Segment " in l: i, j = l.split(":")[1].split("to") i = int(i.strip()[1:]) j = int(j.strip()[1:]) proteins[seqid]['tmbeta_strands'].append([i, j]) if __DEBUG__: log_stderr("# TMBETA-NET(web) segments: %s, %s" % (i, j)) tmbeta_strands[seqid] = proteins[seqid]['tmbeta_strands'] # we store the parsed strand boundaries in JSON format fh = open(outfile, 'w') fh.write(json.dumps(tmbeta_strands, separators=(',', ':\n'))) fh.close() return tmbeta_strands