def test(): url = twilltestlib.get_url() # test empty page get_title namespaces.new_local_dict() twill.commands.reset_browser() browser = twill.get_browser() try: browser.get_title() assert 0, "should never get here" except TwillException: pass ### now test a few special cases commands.go(url) commands.go('/login') # test no matching forms try: commands.fv('2', 'submit', '1') assert 0 except TwillAssertionError: pass # test regexp match commands.fv('1', '.*you', '1') # test ambiguous match to value commands.go('/testform') commands.fv('1', 'selecttest', 'val') commands.fv('1', 'selecttest', 'value1') commands.fv('1', 'selecttest', 'selvalue1') commands.formclear('1') commands.showforms() try: commands.fv('1', 'selecttest', 'value') assert 0 except TwillException: pass # test ambiguous match to name commands.go('/testform') try: commands.fv('1', 'item_', 'value') assert 0 except Exception: pass try: commands.formfile('1', 'selecttest', 'null') assert 0 except Exception: pass commands.go('http://www.google.com/') browser.get_title() # test the twill script. twilltestlib.execute_twill_script('test-form.twill', initial_url=url)
def login(): if s218: page = 'http://129.21.142.218:8008/securesync/login/' username = '******' password = '******' facility = '0939bcf9d5fe59ff8fde46b5a729a232' else: page = 'http://129.21.142.118:8008/securesync/login/' username = '******' password = '******' facility = 'dbae7005f9b45ce082b5fe0a0985946a' print 'Logging In...' go(page) print "Forms:" showforms() try: # Force try using the first form found on a page. formclear('2') fv("2", "username", username) fv("2", "password", password) fv("2", "facility", facility) #fv("1", "csrfmiddlewaretoken", '3F1bMuIIM9ERzcp6ceEyFxlT51yJKsK6') submit('0') content = showSilently() print 'debug twill post content:', content except urllib2.HTTPError, e: sys.exit("%d: %s" % (e.code, e.msg))
def __create_session(self): browser = get_browser() browser.go('http://www.erstenachhilfe.de/user?destination=node%2F767') tw.fv('2', 'edit-name', LOGIN) tw.fv('2', 'edit-pass', PASSWORD) tw.showforms() tw.submit('op') return browser
def admin_create_teacher(**kwargs): """ create teacher in admin with given properties and return id """ passwd = kwargs['passwd'] del kwargs['passwd'] teacher_id = _django_admin_change_object( '/YP-admin/admin/Teacher/teacher/add/', **kwargs) cmd.go('/YP-admin/admin/Teacher/teacher/%s/password/' % teacher_id) cmd.showforms() cmd.fv(1, 'password', passwd) cmd.fv(1, 'retype_password', passwd) cmd.submit() return teacher_id
def annotate(params, proteins, \ url="http://signalfind.org/tatfind.html", force=False): """ Interfaces with the TatFind web service at (http://signalfind.org/tatfind.html) to predict if protein sequences contain Twin-Arginine Translocation (Tat) signal peptides. """ # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) outfn = 'tatfind.out' log_stderr("# TatFind(web) %s > %s" % (params['fasta'], outfn)) if not force and os.path.isfile(outfn): log_stderr("# -> skipped: %s already exists" % outfn) fh = open(outfn, 'r') proteins = parse_tatfind_output(fh, proteins) fh.close() return proteins # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go(url) if __DEBUG__: showforms() formfile("1", "seqFile", params["fasta"]) submit() if __DEBUG__: show() tatfind_output = show() if __DEBUG__: log_stderr(tatfind_output) # write raw TatFind output to a file fh = open(outfn, 'w') fh.write(tatfind_output) fh.close() proteins = parse_tatfind_output(tatfind_output.split("\n"), proteins) return proteins
def main(): try: username = sys.argv[1] password = sys.argv[2] except IndexError: print "Please supply username password" sys.exit(1) browser = twill.get_browser() login(username, password) serial = time.time() pagename = "Test-%s.png" % serial filenames = make_files(pagename) upload_list(browser, pagename, filenames[0:2]) # try it again with two replacement files. # pagename = "Test-%sA.png" % serial # filenames = make_files(pagename) # upload_list(browser, pagename, filenames) t.showforms() t.save_html("/tmp/testabcd")
def wyloguj(commands): forms = commands.showforms() # Pobranie formularzy naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form for form in forms: # Petla po formualrzach if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/logOutUser.do': # Jesli akcja danego formularza przenosi do szczegolow naszForm = form # To zapisujemy znaleziony formularz #print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki for ct in ctrl: if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit commands.get_browser().clicked(naszForm, ct.attrs['name']) # klikam na ten przycisk commands.get_browser().submit()
def wyloguj(commands): forms = commands.showforms() # Pobranie formularzy naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form for form in forms: # Petla po formualrzach if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/logOutUser.do': # Jesli akcja danego formularza przenosi do szczegolow naszForm = form # To zapisujemy znaleziony formularz #print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki for ct in ctrl: if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit commands.get_browser().clicked( naszForm, ct.attrs['name']) # klikam na ten przycisk commands.get_browser().submit()
def goplurk(): from twill.commands import go, showforms, fv, submit, agent, showlinks,save_html from twill import get_browser b=get_browser() b._browser._factory.is_html = True browser = b agent('moz17') go("http://www.plurk.com/") showforms() fv("1","nick_name","PUT_YOUR_NICKNAME") fv("1","password","PUT_YOU_PASSWORD") submit('') go("http://www.plurk.com/m") showforms() fv("1","content",ccontent) submit('submit_plurk') showforms() return
def annotate(params, proteins, \ force=False): """ Uses the TMB-HUNT web service (http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi) to predict if proteins are outer membrane beta-barrels. NOTE: In my limited testing, TMB-HUNT tends to perform very poorly in terms of false positives and false negetives. I'd suggest using only BOMP. """ # TODO: automatically split large sets into multiple jobs # TMB-HUNT will only take 10000 seqs at a time if len(proteins) >= 10000: log_stderr("# ERROR: TMB-HUNT(web): can't take more than 10,000 sequences.") return # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) out = 'tmbhunt.out' log_stderr("# TMB-HUNT(web) %s > %s" % (params['fasta'], out)) if not force and os.path.isfile(out): log_stderr("# -> skipped: %s already exists" % out) return parse_tmbhunt(proteins, out) # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi") if __DEBUG__: showforms() # read up the FASTA format seqs fh = open(params['fasta'], 'r') fasta_seqs = fh.read() fh.close() # fill out the form fv("1", "sequences", fasta_seqs) submit() if __DEBUG__: showlinks() # small jobs will lead us straight to the results, big jobs # go via a 'waiting' page which we skip past if we get it job_id = None try: # we see this with big jobs result_table_url = follow("http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output.*.html") job_id = result_table_url.split('tmp_output')[-1:][0].split('.')[0] except: # small jobs take us straight to the html results table pass # parse the job_id from the url, since due to a bug in # TMB-HUNT the link on the results page from large jobs is wrong if not job_id: job_id = follow("Full results").split('/')[-1:][0].split('.')[0] log_stderr("# TMB-HUNT(web) job_id is: %s <http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output%s.html>" % (job_id, job_id)) # polling until TMB-HUNT finishes # TMB-HUNT advises that 4000 sequences take ~10 mins # we poll a little faster than that polltime = (len(proteins)*0.1)+2 while True: log_stderr("# TMB-HUNT(web): waiting another %i sec ..." % (polltime)) time.sleep(polltime) try: go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/%s.txt" % (job_id)) break except: polltime = polltime * 2 if polltime >= 7200: # 2 hours log_stderr("# TMB-HUNT error: Taking too long.") return txt_out = show() # write raw TMB-HUNT results fh = open(out, 'w') fh.write(txt_out) fh.close() return parse_tmbhunt(proteins, out)
def pobierzPlan(user, password): tablicaHTMLow = [] commands.add_extra_header( 'User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.6') commands.clear_cookies() # Czyszczenie ciastek commands.reset_browser() #restart przegladarki commands.reset_output() #commands.config('use_BeautifulSoup', '0') commands.go("https://edukacja.pwr.wroc.pl/EdukacjaWeb/studia.do" ) # Przechodzimy do edukacji commands.showlinks() # DO USUNIECIA! Pokazuje linki commands.formclear('1') # Czysci formularz logowania commands.formvalue('1', 'login', user) # Podaje login commands.formvalue('1', 'password', password) # Podaje hasło commands.submit('0') # Klika zaloguj print("Linki po submit") # DO USUNIECIA! Pokazuje informacje commands.showlinks() # DO USUNIECIA! Pokazuje linki czyBledneLogowanie = sprawdzCzyBledneLogowanie( commands.show()) # Sprawdza czy na stronie wystapil blad czyLogowanieWInnejSesji = sprawdzCzyLogowanieWInnejSesji(commands.show()) if (czyBledneLogowanie == True): return "Bledne dane logowania" if (czyLogowanieWInnejSesji == True): return "zalogowany w innej sesji" commands.follow("zapisy.do") # Przechodzi linkiem na stronę zapisów # DO USUNIECIA! Pokazuje stronę po kliknięciu zapisy! print("Co po kliknieciu Zapisy") #commands.show() commands.showlinks() print "Forms:" formsy = commands.showforms() links = commands.showlinks() # Pobiera linki z danej strony control = None values = None select_options = None try: control = commands.browser.get_form("1").find_control('ineSluId', type="select") except: print("Nie ma selecta.") #control = commands.browser.get_form("1").find_control('ineSluId', type="select") if (control != None): # Jesli na stronie jest select values = pobierzElementySelect( commands.show()) # Pobieram parserem wartosci selecta select_options = utworzTabeleSelect( values) # Tworze nowy select podmieniajac stary select_attrs = {'id': 'some_id'} # Tworze atrybuty nowego selecta for v in values: # Lece petla po wartosciach selecta form = commands.get_browser().get_form("1") # Pobieram formularz ct = commands.browser.get_form("1").find_control( 'ineSluId', type="select") # Pobieram kontrolke z selectem add_select_to_form(form, 'ineSluId', select_attrs, select_options) # Tworze nowego selecta form.fixup( ) # Sprawdzam czy cos nowego nie zostało dodanego do form commands.showforms() commands.formvalue("1", ct.name, v.strip()) # Podaje wartosc dla selecta commands.submit('0') # Klikam submit html = pobierzZajecia(commands) # Pobieram zajecia commands.follow("zapisy.do") # Wracam do strony zapisów #commands.sleep(6) if ( html != "skreslony z kierunku" ): # Jesli funkcja zwrocila ze jest ktos skreslony to nie dodaje htmlu tablicaHTMLow.append( html ) # Jeli nie zwrocila takiego komunikatu to dodajemy ten html do tablicy else: html = pobierzZajecia( commands ) # Jesli nie ma selecta to pobieramy zajeci z tego kierunku i juz :D if (html != "skreslony z kierunku"): tablicaHTMLow.append(html) #print control.name, control.value, control.type #item = control.get("172748") #print item.name, item.selected, item.id, item.attrs #new_options #commands.formclear('1') # # #form = commands.get_browser().get_form("1") # #print('TO JEST TEN FORM:') #print(form) #print(len(form.controls)) # #notIsSelect = True #for ct in form.controls: # #print(ct) # if ct.name == 'ineSluId': # notIsSelect = False # print('JESTEM') # commands.sleep(3) # # select_attrs = {'id': 'some_id'} # values = pobierzElementySelect(commands.show()) # select_options = utworzTabeleSelect(values) # print(values) # print(select_options) # # # for v in values: # #form.fixup() # add_select_to_form(form, 'ineSluId', select_attrs, select_options) # form.fixup() # #ct.get(v).selected = True # print(ct) # print(form) # print(v) # commands.showforms() # # commands.formvalue("1", ct.name, v.strip()) # Podaje login # print("JEEDEFE") # commands.submit('0') # html = pobierzZajecia(commands) # commands.follow("zapisy.do") # print("JEEEEEEEEESSSSSTTTTTEEEEEMMM") # # commands.sleep(6) # if (html != "skreslony z kierunku"): # tablicaHTMLow.append(html) #ct.get(v).selected = False #for ct2 in form.controls: # if ct2.type == 'submit': # szukam wsrod niej tej co ma typ submit # commands.get_browser().clicked(form, ct2.attrs['name']) # klikam na ten przycisk # commands.get_browser().submit() #links = commands.showlinks() #commands.back() #commands.showforms() #return "no" #if (notIsSelect == True): # html = pobierzZajecia(commands) # if (html != "skreslony z kierunku"): # tablicaHTMLow.append(html) wyloguj(commands) commands.browser.clear_cookies() #usuwanie ciasteczek commands.reset_browser() #restart przegladarki commands.reset_output() return tablicaHTMLow
def pobierzZajecia(commands): print("Po wybraniu") links = commands.showlinks() # Pobiera linki z danej strony commands.showforms() # Szuka w linkach tego przenoszącego na odpowiedni semestr. dateToday = datetime.date.today() firstOctober = datetime.date(dateToday.year, 10, 1) if dateToday > firstOctober: # Jesli dzisiaj jest wiekszy niz 1 Pazdziernika #Semestr zimowy for link in links: ktory = 0 if link.text == '' + str(dateToday.year) + '/' + str( dateToday.year + 1) + '': # Szukamy linka o tytule (rok/rok+1) ktory = ktory + 1 if ktory == 1: # Znalazł! commands.go( link.url ) # Przechodzimy pod url'sa który się kryje pod tym rokiem else: #Semest letni for link in links: ktory = 0 if link.text == '' + str(dateToday.year) + '/' + str( dateToday.year + 1) + '': # Szukamy linka o tytule (rok/rok+1) ktory = ktory + 1 if ktory == 2: # Znalazł! commands.go( link.url ) # Przechodzimy pod url'sa który się kryje pod tym rokiem # DO USUNIECIA! Pokazuje stronę po kliknięciu danego semestru! print("Co po kliknieciu semestru:") commands.showlinks() print "Forms:" commands.showforms() # Szuka w formularzach tego odpowiadającego za pokazanie zapisanych kursów. forms = commands.showforms() # Pobranie formularzy naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form for form in forms: # Petla po formualrzach if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/zapisy.do?href=#hrefZapisySzczSlu': # Jesli akcja danego formularza przenosi do szczegolow naszForm = form # To zapisujemy znaleziony formularz print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz if ( naszForm != None ): # To znaczy ze znalazlo taki rok i ze jeszcze istnieje student na tym kierunku ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki for ct in ctrl: if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit commands.get_browser().clicked( naszForm, ct.attrs['name']) # klikam na ten przycisk commands.get_browser().submit() print("Co po kliknieciu szczegoly zajec") commands.showlinks() #commands.sleep(5) print "Forms:" #commands.showforms() content = commands.show() #content ="" #commands.browser.clear_cookies() #usuwanie ciasteczek #print('PRZZEEEEEENIOSLEM') #commands.sleep(5) return content else: #commands.follow("zapisy.do") #print('PRZZEEEEEENIOSLEM') return ("skreslony z kierunku")
def showforms(self): """Shows form, helpful for debugging new tests""" return tc.showforms()
else: return 'unknown', 'unknown', 'unknown' ###################################################################################################### if __name__ == '__main__': min_price_list = [] max_price_list = [] avg_price_list = [] delay_time = 1 cbrw = com.get_browser() cbrw.go('your website') com.showforms() com.fv("loginform", "login_id_F", "your id") com.fv("loginform", "password_F", "your passwd") com.submit() ### Access Excel Entries #### wb = open_workbook('filename.xls') for sheet in wb.sheets(): if sheet.number == 0: item_names = sheet.col_values(5, start_rowx=1) bottle_volumes = sheet.col_values(9, start_rowx=1) vintages = sheet.col_values(13, start_rowx=1) data_len = len(item_names) intab = "a"
def annotate(params, proteins, \ url="http://services.cbu.uib.no/tools/bomp/", force=False): """ Uses the BOMP web service (http://services.cbu.uib.no/tools/bomp/) to predict if proteins are outer membrane beta-barrels. """ # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane/%s)" % (python_version, inmembrane.__version__)) bomp_out = 'bomp.out' log_stderr("# BOMP(web) %s > %s" % (params['fasta'], bomp_out)) if not force and os.path.isfile(bomp_out): log_stderr("# -> skipped: %s already exists" % bomp_out) bomp_categories = {} fh = open(bomp_out, 'r') for l in fh: words = l.split() bomp_category = int(words[-1:][0]) seqid = parse_fasta_header(l)[0] proteins[seqid]['bomp'] = bomp_category bomp_categories[seqid] = bomp_category fh.close() return bomp_categories # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go(url) if __DEBUG__: showforms() formfile("1", "queryfile", params["fasta"]) submit() if __DEBUG__: show() # extract the job id from the page links = showlinks() job_id = None for l in links: if l.url.find("viewOutput") != -1: # grab job id from "viewOutput?id=16745338" job_id = int(l.url.split("=")[1]) if __DEBUG__: log_stderr("BOMP job id: %d" % job_id) if not job_id: # something went wrong log_stderr("# BOMP error: Can't find job id") return # parse the HTML table and extract categories go("viewOutput?id=%i" % (job_id)) polltime = 10 log_stderr("# Waiting for BOMP to finish .") while True: try: find("Not finished") log_stderr(".") except: # Finished ! Pull down the result page. log_stderr(". done!\n") go("viewOutput?id=%i" % (job_id)) if __DEBUG__: log_stderr(show()) break # Not finished. We keep polling for a time until # we give up time.sleep(polltime) polltime = polltime * 2 if polltime >= 7200: # 2 hours log_stderr("# BOMP error: Taking too long.") return go("viewOutput?id=%i" % (job_id)) if __DEBUG__: log_stderr(show()) bomp_html = show() if __DEBUG__: log_stderr(bomp_html) # Results are in the only <table> on this page, formatted like: # <tr><th>gi|107836852|gb|ABF84721.1<th>5</tr> soup = BeautifulSoup(bomp_html) bomp_categories = {} # dictionary of {name, category} pairs for tr in soup.findAll('tr')[1:]: n, c = tr.findAll('th') name = parse_fasta_header(n.text.strip())[0] category = int(c.text) bomp_categories[name] = category # write BOMP results to a tab delimited file fh = open(bomp_out, 'w') for k, v in bomp_categories.iteritems(): fh.write("%s\t%i\n" % (k, v)) fh.close() if __DEBUG__: log_stderr(str(bomp_categories)) # label proteins with bomp classification (int) or False for name in proteins: if "bomp" not in proteins[name]: if name in bomp_categories: category = int(bomp_categories[name]) proteins[name]['bomp'] = category else: proteins[name]['bomp'] = False if __DEBUG__: log_stderr(str(proteins)) return bomp_categories """
def __call__(self, url): browser.go(url) browser.showforms() browser.fv('2', 'UserName', self._user) browser.fv('2', 'password', self._password) browser.submit('5')
def pobierzPlan(user, password): tablicaHTMLow = [] commands.add_extra_header('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.6') commands.clear_cookies() # Czyszczenie ciastek commands.reset_browser() #restart przegladarki commands.reset_output() #commands.config('use_BeautifulSoup', '0') commands.go("https://edukacja.pwr.wroc.pl/EdukacjaWeb/studia.do") # Przechodzimy do edukacji commands.showlinks() # DO USUNIECIA! Pokazuje linki commands.formclear('1') # Czysci formularz logowania commands.formvalue('1', 'login', user) # Podaje login commands.formvalue('1', 'password', password) # Podaje hasło commands.submit('0') # Klika zaloguj print("Linki po submit") # DO USUNIECIA! Pokazuje informacje commands.showlinks() # DO USUNIECIA! Pokazuje linki czyBledneLogowanie = sprawdzCzyBledneLogowanie(commands.show()) # Sprawdza czy na stronie wystapil blad czyLogowanieWInnejSesji = sprawdzCzyLogowanieWInnejSesji(commands.show()) if (czyBledneLogowanie == True): return "Bledne dane logowania" if (czyLogowanieWInnejSesji == True): return "zalogowany w innej sesji" commands.follow("zapisy.do") # Przechodzi linkiem na stronę zapisów # DO USUNIECIA! Pokazuje stronę po kliknięciu zapisy! print("Co po kliknieciu Zapisy") #commands.show() commands.showlinks() print "Forms:" formsy = commands.showforms() links = commands.showlinks() # Pobiera linki z danej strony control = None values = None select_options = None try: control = commands.browser.get_form("1").find_control('ineSluId', type="select") except: print("Nie ma selecta.") #control = commands.browser.get_form("1").find_control('ineSluId', type="select") if(control != None): # Jesli na stronie jest select values = pobierzElementySelect(commands.show()) # Pobieram parserem wartosci selecta select_options = utworzTabeleSelect(values) # Tworze nowy select podmieniajac stary select_attrs = {'id': 'some_id'} # Tworze atrybuty nowego selecta for v in values: # Lece petla po wartosciach selecta form = commands.get_browser().get_form("1") # Pobieram formularz ct = commands.browser.get_form("1").find_control('ineSluId', type="select") # Pobieram kontrolke z selectem add_select_to_form(form, 'ineSluId', select_attrs, select_options) # Tworze nowego selecta form.fixup() # Sprawdzam czy cos nowego nie zostało dodanego do form commands.showforms() commands.formvalue("1", ct.name, v.strip()) # Podaje wartosc dla selecta commands.submit('0') # Klikam submit html = pobierzZajecia(commands) # Pobieram zajecia commands.follow("zapisy.do") # Wracam do strony zapisów #commands.sleep(6) if (html != "skreslony z kierunku"): # Jesli funkcja zwrocila ze jest ktos skreslony to nie dodaje htmlu tablicaHTMLow.append(html) # Jeli nie zwrocila takiego komunikatu to dodajemy ten html do tablicy else: html = pobierzZajecia(commands) # Jesli nie ma selecta to pobieramy zajeci z tego kierunku i juz :D if (html != "skreslony z kierunku"): tablicaHTMLow.append(html) #print control.name, control.value, control.type #item = control.get("172748") #print item.name, item.selected, item.id, item.attrs #new_options #commands.formclear('1') # # #form = commands.get_browser().get_form("1") # #print('TO JEST TEN FORM:') #print(form) #print(len(form.controls)) # #notIsSelect = True #for ct in form.controls: # #print(ct) # if ct.name == 'ineSluId': # notIsSelect = False # print('JESTEM') # commands.sleep(3) # # select_attrs = {'id': 'some_id'} # values = pobierzElementySelect(commands.show()) # select_options = utworzTabeleSelect(values) # print(values) # print(select_options) # # # for v in values: # #form.fixup() # add_select_to_form(form, 'ineSluId', select_attrs, select_options) # form.fixup() # #ct.get(v).selected = True # print(ct) # print(form) # print(v) # commands.showforms() # # commands.formvalue("1", ct.name, v.strip()) # Podaje login # print("JEEDEFE") # commands.submit('0') # html = pobierzZajecia(commands) # commands.follow("zapisy.do") # print("JEEEEEEEEESSSSSTTTTTEEEEEMMM") # # commands.sleep(6) # if (html != "skreslony z kierunku"): # tablicaHTMLow.append(html) #ct.get(v).selected = False #for ct2 in form.controls: # if ct2.type == 'submit': # szukam wsrod niej tej co ma typ submit # commands.get_browser().clicked(form, ct2.attrs['name']) # klikam na ten przycisk # commands.get_browser().submit() #links = commands.showlinks() #commands.back() #commands.showforms() #return "no" #if (notIsSelect == True): # html = pobierzZajecia(commands) # if (html != "skreslony z kierunku"): # tablicaHTMLow.append(html) wyloguj(commands) commands.browser.clear_cookies() #usuwanie ciasteczek commands.reset_browser() #restart przegladarki commands.reset_output() return tablicaHTMLow
def pobierzZajecia(commands): print("Po wybraniu") links = commands.showlinks() # Pobiera linki z danej strony commands.showforms() # Szuka w linkach tego przenoszącego na odpowiedni semestr. dateToday = datetime.date.today() firstOctober = datetime.date(dateToday.year, 10, 1) if dateToday > firstOctober: # Jesli dzisiaj jest wiekszy niz 1 Pazdziernika #Semestr zimowy for link in links: ktory = 0 if link.text=='' + str(dateToday.year) + '/' + str(dateToday.year+1) + '': # Szukamy linka o tytule (rok/rok+1) ktory = ktory + 1 if ktory == 1: # Znalazł! commands.go(link.url) # Przechodzimy pod url'sa który się kryje pod tym rokiem else: #Semest letni for link in links: ktory = 0 if link.text=='' + str(dateToday.year)+ '/' + str(dateToday.year+1) + '': # Szukamy linka o tytule (rok/rok+1) ktory = ktory + 1 if ktory == 2: # Znalazł! commands.go(link.url) # Przechodzimy pod url'sa który się kryje pod tym rokiem # DO USUNIECIA! Pokazuje stronę po kliknięciu danego semestru! print("Co po kliknieciu semestru:") commands.showlinks() print "Forms:" commands.showforms() # Szuka w formularzach tego odpowiadającego za pokazanie zapisanych kursów. forms = commands.showforms() # Pobranie formularzy naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form for form in forms: # Petla po formualrzach if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/zapisy.do?href=#hrefZapisySzczSlu': # Jesli akcja danego formularza przenosi do szczegolow naszForm = form # To zapisujemy znaleziony formularz print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz if(naszForm != None): # To znaczy ze znalazlo taki rok i ze jeszcze istnieje student na tym kierunku ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki for ct in ctrl: if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit commands.get_browser().clicked(naszForm, ct.attrs['name']) # klikam na ten przycisk commands.get_browser().submit() print("Co po kliknieciu szczegoly zajec") commands.showlinks() #commands.sleep(5) print "Forms:" #commands.showforms() content = commands.show() #content ="" #commands.browser.clear_cookies() #usuwanie ciasteczek #print('PRZZEEEEEENIOSLEM') #commands.sleep(5) return content else: #commands.follow("zapisy.do") #print('PRZZEEEEEENIOSLEM') return("skreslony z kierunku")
def annotate(params, proteins, \ url="http://psfs.cbrc.jp/tmbeta-net/", \ category='OM(barrel)', force=False): """ Uses the TMBETA-NET web service (http://psfs.cbrc.jp/tmbeta-net/) to predict strands of outer membrane beta-barrels. By default, category='BARREL' means prediction will only be run on proteins in the set with this category property. To process all proteins, change category to None. These keys are added to the proteins dictionary: 'tmbeta_strands' - a list of lists with paired start and end residues of each predicted strand. (eg [[3,9],[14,21], ..etc ]) """ # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) outfile = 'tmbeta_net.out' log_stderr("# TMBETA-NET(web) %s > %s" % (params['fasta'], outfile)) tmbeta_strands = {} if not force and os.path.isfile(outfile): log_stderr("# -> skipped: %s already exists" % outfile) fh = open(outfile, 'r') tmbeta_strands = json.loads(fh.read()) fh.close() for seqid in tmbeta_strands: proteins[seqid]['tmbeta_strands'] = tmbeta_strands[seqid] return tmbeta_strands # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) for seqid in proteins: # only run on sequences which match the category filter if force or \ (category == None) or \ (dict_get(proteins[seqid], 'category') == category): pass else: continue go(url) if __DEBUG__: showforms() fv("1","sequence",proteins[seqid]['seq']) submit() log_stderr("# TMBETA-NET: Predicting strands for %s - %s\n" \ % (seqid, proteins[seqid]['name'])) out = show() time.sleep(1) if ("Some query is already running. Please try again." in out): log_stderr("# TMBETA-NET(web) error: %s" % (out)) return {} # parse the web page returned, extract strand boundaries proteins[seqid]['tmbeta_strands'] = [] for l in out.split('\n'): if __DEBUG__: log_stderr("## " + l) if "<BR>Segment " in l: i,j = l.split(":")[1].split("to") i = int(i.strip()[1:]) j = int(j.strip()[1:]) proteins[seqid]['tmbeta_strands'].append([i,j]) if __DEBUG__: log_stderr("# TMBETA-NET(web) segments: %s, %s" % (i, j)) tmbeta_strands[seqid] = proteins[seqid]['tmbeta_strands'] # we store the parsed strand boundaries in JSON format fh = open(outfile, 'w') fh.write(json.dumps(tmbeta_strands, separators=(',',':\n'))) fh.close() return tmbeta_strands
def annotate(params, proteins, \ url="http://psfs.cbrc.jp/tmbeta-net/", \ category='OM(barrel)', force=False): """ Uses the TMBETA-NET web service (http://psfs.cbrc.jp/tmbeta-net/) to predict strands of outer membrane beta-barrels. By default, category='BARREL' means prediction will only be run on proteins in the set with this category property. To process all proteins, change category to None. These keys are added to the proteins dictionary: 'tmbeta_strands' - a list of lists with paired start and end residues of each predicted strand. (eg [[3,9],[14,21], ..etc ]) """ # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) outfile = 'tmbeta_net.out' log_stderr("# TMBETA-NET(web) %s > %s" % (params['fasta'], outfile)) tmbeta_strands = {} if not force and os.path.isfile(outfile): log_stderr("# -> skipped: %s already exists" % outfile) fh = open(outfile, 'r') tmbeta_strands = json.loads(fh.read()) fh.close() for seqid in tmbeta_strands: proteins[seqid]['tmbeta_strands'] = tmbeta_strands[seqid] return tmbeta_strands # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) for seqid in proteins: # only run on sequences which match the category filter if force or \ (category == None) or \ (dict_get(proteins[seqid], 'category') == category): pass else: continue go(url) if __DEBUG__: showforms() fv("1", "sequence", proteins[seqid]['seq']) submit() log_stderr("# TMBETA-NET: Predicting strands for %s - %s\n" \ % (seqid, proteins[seqid]['name'])) out = show() time.sleep(1) if ("Some query is already running. Please try again." in out): log_stderr("# TMBETA-NET(web) error: %s" % (out)) return {} # parse the web page returned, extract strand boundaries proteins[seqid]['tmbeta_strands'] = [] for l in out.split('\n'): if __DEBUG__: log_stderr("## " + l) if "<BR>Segment " in l: i, j = l.split(":")[1].split("to") i = int(i.strip()[1:]) j = int(j.strip()[1:]) proteins[seqid]['tmbeta_strands'].append([i, j]) if __DEBUG__: log_stderr("# TMBETA-NET(web) segments: %s, %s" % (i, j)) tmbeta_strands[seqid] = proteins[seqid]['tmbeta_strands'] # we store the parsed strand boundaries in JSON format fh = open(outfile, 'w') fh.write(json.dumps(tmbeta_strands, separators=(',', ':\n'))) fh.close() return tmbeta_strands
def annotate(params, proteins, \ force=False): """ DEPRECATED: The TMB-HUNT server appears to be permanently offline. Uses the TMB-HUNT web service (http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi) to predict if proteins are outer membrane beta-barrels. NOTE: In my limited testing, TMB-HUNT tends to perform very poorly in terms of false positives and false negetives. I'd suggest using only BOMP. """ # TODO: automatically split large sets into multiple jobs # TMB-HUNT will only take 10000 seqs at a time if len(proteins) >= 10000: log_stderr( "# ERROR: TMB-HUNT(web): can't take more than 10,000 sequences.") return # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) out = 'tmbhunt.out' log_stderr("# TMB-HUNT(web) %s > %s" % (params['fasta'], out)) if not force and os.path.isfile(out): log_stderr("# -> skipped: %s already exists" % out) return parse_tmbhunt(proteins, out) # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi" ) if __DEBUG__: showforms() # read up the FASTA format seqs fh = open(params['fasta'], 'r') fasta_seqs = fh.read() fh.close() # fill out the form fv("1", "sequences", fasta_seqs) submit() if __DEBUG__: showlinks() # small jobs will lead us straight to the results, big jobs # go via a 'waiting' page which we skip past if we get it job_id = None try: # we see this with big jobs result_table_url = follow( "http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output.*.html" ) job_id = result_table_url.split('tmp_output')[-1:][0].split('.')[0] except: # small jobs take us straight to the html results table pass # parse the job_id from the url, since due to a bug in # TMB-HUNT the link on the results page from large jobs is wrong if not job_id: job_id = \ follow("Full results").split('/')[-1:][0].split('.')[0] log_stderr( "# TMB-HUNT(web) job_id is: %s <http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output%s.html>" % (job_id, job_id)) # polling until TMB-HUNT finishes # TMB-HUNT advises that 4000 sequences take ~10 mins # we poll a little faster than that polltime = (len(proteins) * 0.1) + 2 while True: log_stderr("# TMB-HUNT(web): waiting another %i sec ..." % (polltime)) time.sleep(polltime) try: go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/%s.txt" % (job_id)) break except: polltime = polltime * 2 if polltime >= 7200: # 2 hours log_stderr("# TMB-HUNT error: Taking too long.") return txt_out = show() # write raw TMB-HUNT results fh = open(out, 'w') fh.write(txt_out) fh.close() return parse_tmbhunt(proteins, out)
def annotate(params, proteins, \ url="http://rbf.bioinfo.tw/"+ "~sachen/OMPpredict/"+ "TMBETADISC-RBF-Content.html", force=False): """ Interfaces with the TatFind web service at (http://rbf.bioinfo.tw/~sachen/OMPpredict/TMBETADISC-RBF.php) to predict if protein sequence is likely to be an outer membrane beta-barrel. Note that the default URL we use it different to the regular form used by web browsers, since we need to bypass some AJAX fun. """ # TODO: automatically split large sets into multiple jobs # since TMBETADISC seems to not like more than take # ~5000 seqs at a time if len(proteins) >= 5000: log_stderr("# ERROR: TMBETADISC-RBF(web): tends to fail with > ~5000 sequences.") return # set the user-agent so web services can block us if they want ... :/ python_version = sys.version.split()[0] agent("Python-urllib/%s (twill; inmembrane)" % python_version) outfn = 'tmbetadisc-rbf.out' log_stderr("# TMBETADISC-RBF(web) %s > %s" % (params['fasta'], outfn)) if not force and os.path.isfile(outfn): log_stderr("# -> skipped: %s already exists" % outfn) fh = open(outfn, 'r') proteins = parse_tmbetadisc_output(fh.read(), proteins) fh.close() return proteins # dump extraneous output into this blackhole so we don't see it if not __DEBUG__: twill.set_output(StringIO.StringIO()) go(url) if __DEBUG__: showforms() formfile("1", "userfile", params["fasta"]) fv("1", "format", "file") # set the user defined method method_map = {"aa":"Amino Acid Composition", "dp":"Depipetide Composition", "aadp":"Amino Acid & Depipetide Composition", "pssm":"PSSM"} if dict_get(params, 'tmbetadisc_rbf_method'): try: method = method_map[params['tmbetadisc_rbf_method']] except KeyError: log_stderr("# ERROR: Invalid setting from tmbetadisc_rbf_method. \ Must be set to aa, dp, aadp or pssm.") sys.exit() #fv("1", "select", "Amino Acid Composition") #fv("1", "select", "Depipetide Composition") #fv("1", "select", "Amino Acid & Depipetide Composition") #fv("1", "select", "PSSM") fv("1", "select", method) submit() waiting_page = show() if __DEBUG__: log_stderr(waiting_page) for l in waiting_page.split('\n'): if l.find("TMBETADISC-RBF-action.php?UniqueName=") != -1: result_url = l.split("'")[1] time.sleep(5) go(result_url) output = show() if __DEBUG__: log_stderr(output) # write raw output to a file fh = open(outfn, 'w') fh.write(output) fh.close() proteins = parse_tmbetadisc_output(output, proteins) return proteins
def test(): url = twilltestlib.get_url() # test empty page get_title namespaces.new_local_dict() twill.commands.reset_browser() browser = twill.get_browser() try: browser.get_title() assert 0, "should never get here" except BrowserStateError: pass ### now test a few special cases commands.go(url) commands.go('/login') commands.showforms() # test no matching forms try: commands.fv('2', 'submit', '1') assert 0 except TwillAssertionError: pass # test regexp match commands.fv('1', '.*you', '1') # test ambiguous match to value commands.go('/testform') commands.fv('1', 'selecttest', 'val') commands.fv('1', 'selecttest', 'value1') commands.fv('1', 'selecttest', 'selvalue1') commands.formclear('1') try: commands.fv('1', 'selecttest', 'value') assert 0 except ClientForm.ItemNotFoundError: pass # test ambiguous match to name commands.go('/testform') try: commands.fv('1', 'item_', 'value') assert 0 except Exception: pass try: commands.formfile('1', 'selecttest', 'null') assert 0 except Exception: pass commands.go('http://www.google.com/') browser.get_title() # test the twill script. twilltestlib.execute_twill_script('test-form.twill', initial_url=url)