def test():
url = twilltestlib.get_url()
# test empty page get_title
namespaces.new_local_dict()
twill.commands.reset_browser()
browser = twill.get_browser()
try:
browser.get_title()
assert 0, "should never get here"
except TwillException:
pass
### now test a few special cases
commands.go(url)
commands.go('/login')
# test no matching forms
try:
commands.fv('2', 'submit', '1')
assert 0
except TwillAssertionError:
pass
# test regexp match
commands.fv('1', '.*you', '1')
# test ambiguous match to value
commands.go('/testform')
commands.fv('1', 'selecttest', 'val')
commands.fv('1', 'selecttest', 'value1')
commands.fv('1', 'selecttest', 'selvalue1')
commands.formclear('1')
commands.showforms()
try:
commands.fv('1', 'selecttest', 'value')
assert 0
except TwillException:
pass
# test ambiguous match to name
commands.go('/testform')
try:
commands.fv('1', 'item_', 'value')
assert 0
except Exception:
pass
try:
commands.formfile('1', 'selecttest', 'null')
assert 0
except Exception:
pass
commands.go('http://www.google.com/')
browser.get_title()
# test the twill script.
twilltestlib.execute_twill_script('test-form.twill', initial_url=url)
def login():
if s218:
page = 'http://129.21.142.218:8008/securesync/login/'
username = '******'
password = '******'
facility = '0939bcf9d5fe59ff8fde46b5a729a232'
else:
page = 'http://129.21.142.118:8008/securesync/login/'
username = '******'
password = '******'
facility = 'dbae7005f9b45ce082b5fe0a0985946a'
print 'Logging In...'
go(page)
print "Forms:"
showforms()
try:
# Force try using the first form found on a page.
formclear('2')
fv("2", "username", username)
fv("2", "password", password)
fv("2", "facility", facility)
#fv("1", "csrfmiddlewaretoken", '3F1bMuIIM9ERzcp6ceEyFxlT51yJKsK6')
submit('0')
content = showSilently()
print 'debug twill post content:', content
except urllib2.HTTPError, e:
sys.exit("%d: %s" % (e.code, e.msg))
def __create_session(self):
browser = get_browser()
browser.go('http://www.erstenachhilfe.de/user?destination=node%2F767')
tw.fv('2', 'edit-name', LOGIN)
tw.fv('2', 'edit-pass', PASSWORD)
tw.showforms()
tw.submit('op')
return browser
def __create_session(self):
browser = get_browser()
browser.go('http://www.erstenachhilfe.de/user?destination=node%2F767')
tw.fv('2', 'edit-name', LOGIN)
tw.fv('2', 'edit-pass', PASSWORD)
tw.showforms()
tw.submit('op')
return browser
def admin_create_teacher(**kwargs):
""" create teacher in admin with given properties and return id """
passwd = kwargs['passwd']
del kwargs['passwd']
teacher_id = _django_admin_change_object(
'/YP-admin/admin/Teacher/teacher/add/', **kwargs)
cmd.go('/YP-admin/admin/Teacher/teacher/%s/password/' % teacher_id)
cmd.showforms()
cmd.fv(1, 'password', passwd)
cmd.fv(1, 'retype_password', passwd)
cmd.submit()
return teacher_id
def annotate(params, proteins, \
url="http://signalfind.org/tatfind.html", force=False):
"""
Interfaces with the TatFind web service at (http://signalfind.org/tatfind.html)
to predict if protein sequences contain Twin-Arginine Translocation (Tat)
signal peptides.
"""
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
outfn = 'tatfind.out'
log_stderr("# TatFind(web) %s > %s" % (params['fasta'], outfn))
if not force and os.path.isfile(outfn):
log_stderr("# -> skipped: %s already exists" % outfn)
fh = open(outfn, 'r')
proteins = parse_tatfind_output(fh, proteins)
fh.close()
return proteins
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go(url)
if __DEBUG__: showforms()
formfile("1", "seqFile", params["fasta"])
submit()
if __DEBUG__: show()
tatfind_output = show()
if __DEBUG__: log_stderr(tatfind_output)
# write raw TatFind output to a file
fh = open(outfn, 'w')
fh.write(tatfind_output)
fh.close()
proteins = parse_tatfind_output(tatfind_output.split("\n"), proteins)
return proteins
def annotate(params, proteins, \
url="http://signalfind.org/tatfind.html", force=False):
"""
Interfaces with the TatFind web service at (http://signalfind.org/tatfind.html)
to predict if protein sequences contain Twin-Arginine Translocation (Tat)
signal peptides.
"""
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
outfn = 'tatfind.out'
log_stderr("# TatFind(web) %s > %s" % (params['fasta'], outfn))
if not force and os.path.isfile(outfn):
log_stderr("# -> skipped: %s already exists" % outfn)
fh = open(outfn, 'r')
proteins = parse_tatfind_output(fh, proteins)
fh.close()
return proteins
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go(url)
if __DEBUG__: showforms()
formfile("1", "seqFile", params["fasta"])
submit()
if __DEBUG__: show()
tatfind_output = show()
if __DEBUG__: log_stderr(tatfind_output)
# write raw TatFind output to a file
fh = open(outfn, 'w')
fh.write(tatfind_output)
fh.close()
proteins = parse_tatfind_output(tatfind_output.split("\n"), proteins)
return proteins
def main():
try:
username = sys.argv[1]
password = sys.argv[2]
except IndexError:
print "Please supply username password"
sys.exit(1)
browser = twill.get_browser()
login(username, password)
serial = time.time()
pagename = "Test-%s.png" % serial
filenames = make_files(pagename)
upload_list(browser, pagename, filenames[0:2])
# try it again with two replacement files.
# pagename = "Test-%sA.png" % serial
# filenames = make_files(pagename)
# upload_list(browser, pagename, filenames)
t.showforms()
t.save_html("/tmp/testabcd")
def main():
try:
username = sys.argv[1]
password = sys.argv[2]
except IndexError:
print "Please supply username password"
sys.exit(1)
browser = twill.get_browser()
login(username, password)
serial = time.time()
pagename = "Test-%s.png" % serial
filenames = make_files(pagename)
upload_list(browser, pagename, filenames[0:2])
# try it again with two replacement files.
# pagename = "Test-%sA.png" % serial
# filenames = make_files(pagename)
# upload_list(browser, pagename, filenames)
t.showforms()
t.save_html("/tmp/testabcd")
def wyloguj(commands):
forms = commands.showforms() # Pobranie formularzy
naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form
for form in forms: # Petla po formualrzach
if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/logOutUser.do': # Jesli akcja danego formularza przenosi do szczegolow
naszForm = form # To zapisujemy znaleziony formularz
#print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz
ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki
for ct in ctrl:
if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit
commands.get_browser().clicked(naszForm, ct.attrs['name']) # klikam na ten przycisk
commands.get_browser().submit()
def wyloguj(commands):
forms = commands.showforms() # Pobranie formularzy
naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form
for form in forms: # Petla po formualrzach
if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/logOutUser.do': # Jesli akcja danego formularza przenosi do szczegolow
naszForm = form # To zapisujemy znaleziony formularz
#print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz
ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki
for ct in ctrl:
if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit
commands.get_browser().clicked(
naszForm, ct.attrs['name']) # klikam na ten przycisk
commands.get_browser().submit()
def goplurk():
from twill.commands import go, showforms, fv, submit, agent, showlinks,save_html
from twill import get_browser
b=get_browser()
b._browser._factory.is_html = True
browser = b
agent('moz17')
go("http://www.plurk.com/")
showforms()
fv("1","nick_name","PUT_YOUR_NICKNAME")
fv("1","password","PUT_YOU_PASSWORD")
submit('')
go("http://www.plurk.com/m")
showforms()
fv("1","content",ccontent)
submit('submit_plurk')
showforms()
return
def annotate(params, proteins, \
force=False):
"""
Uses the TMB-HUNT web service
(http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi) to
predict if proteins are outer membrane beta-barrels.
NOTE: In my limited testing, TMB-HUNT tends to perform very poorly in
terms of false positives and false negetives. I'd suggest using only
BOMP.
"""
# TODO: automatically split large sets into multiple jobs
# TMB-HUNT will only take 10000 seqs at a time
if len(proteins) >= 10000:
log_stderr("# ERROR: TMB-HUNT(web): can't take more than 10,000 sequences.")
return
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
out = 'tmbhunt.out'
log_stderr("# TMB-HUNT(web) %s > %s" % (params['fasta'], out))
if not force and os.path.isfile(out):
log_stderr("# -> skipped: %s already exists" % out)
return parse_tmbhunt(proteins, out)
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi")
if __DEBUG__: showforms()
# read up the FASTA format seqs
fh = open(params['fasta'], 'r')
fasta_seqs = fh.read()
fh.close()
# fill out the form
fv("1", "sequences", fasta_seqs)
submit()
if __DEBUG__: showlinks()
# small jobs will lead us straight to the results, big jobs
# go via a 'waiting' page which we skip past if we get it
job_id = None
try:
# we see this with big jobs
result_table_url = follow("http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output.*.html")
job_id = result_table_url.split('tmp_output')[-1:][0].split('.')[0]
except:
# small jobs take us straight to the html results table
pass
# parse the job_id from the url, since due to a bug in
# TMB-HUNT the link on the results page from large jobs is wrong
if not job_id: job_id = follow("Full results").split('/')[-1:][0].split('.')[0]
log_stderr("# TMB-HUNT(web) job_id is: %s <http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output%s.html>" % (job_id, job_id))
# polling until TMB-HUNT finishes
# TMB-HUNT advises that 4000 sequences take ~10 mins
# we poll a little faster than that
polltime = (len(proteins)*0.1)+2
while True:
log_stderr("# TMB-HUNT(web): waiting another %i sec ..." % (polltime))
time.sleep(polltime)
try:
go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/%s.txt" % (job_id))
break
except:
polltime = polltime * 2
if polltime >= 7200: # 2 hours
log_stderr("# TMB-HUNT error: Taking too long.")
return
txt_out = show()
# write raw TMB-HUNT results
fh = open(out, 'w')
fh.write(txt_out)
fh.close()
return parse_tmbhunt(proteins, out)
def pobierzPlan(user, password):
tablicaHTMLow = []
commands.add_extra_header(
'User-agent',
'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.6')
commands.clear_cookies() # Czyszczenie ciastek
commands.reset_browser() #restart przegladarki
commands.reset_output()
#commands.config('use_BeautifulSoup', '0')
commands.go("https://edukacja.pwr.wroc.pl/EdukacjaWeb/studia.do"
) # Przechodzimy do edukacji
commands.showlinks() # DO USUNIECIA! Pokazuje linki
commands.formclear('1') # Czysci formularz logowania
commands.formvalue('1', 'login', user) # Podaje login
commands.formvalue('1', 'password', password) # Podaje hasło
commands.submit('0') # Klika zaloguj
print("Linki po submit") # DO USUNIECIA! Pokazuje informacje
commands.showlinks() # DO USUNIECIA! Pokazuje linki
czyBledneLogowanie = sprawdzCzyBledneLogowanie(
commands.show()) # Sprawdza czy na stronie wystapil blad
czyLogowanieWInnejSesji = sprawdzCzyLogowanieWInnejSesji(commands.show())
if (czyBledneLogowanie == True):
return "Bledne dane logowania"
if (czyLogowanieWInnejSesji == True):
return "zalogowany w innej sesji"
commands.follow("zapisy.do") # Przechodzi linkiem na stronę zapisów
# DO USUNIECIA! Pokazuje stronę po kliknięciu zapisy!
print("Co po kliknieciu Zapisy")
#commands.show()
commands.showlinks()
print "Forms:"
formsy = commands.showforms()
links = commands.showlinks() # Pobiera linki z danej strony
control = None
values = None
select_options = None
try:
control = commands.browser.get_form("1").find_control('ineSluId',
type="select")
except:
print("Nie ma selecta.")
#control = commands.browser.get_form("1").find_control('ineSluId', type="select")
if (control != None): # Jesli na stronie jest select
values = pobierzElementySelect(
commands.show()) # Pobieram parserem wartosci selecta
select_options = utworzTabeleSelect(
values) # Tworze nowy select podmieniajac stary
select_attrs = {'id': 'some_id'} # Tworze atrybuty nowego selecta
for v in values: # Lece petla po wartosciach selecta
form = commands.get_browser().get_form("1") # Pobieram formularz
ct = commands.browser.get_form("1").find_control(
'ineSluId', type="select") # Pobieram kontrolke z selectem
add_select_to_form(form, 'ineSluId', select_attrs,
select_options) # Tworze nowego selecta
form.fixup(
) # Sprawdzam czy cos nowego nie zostało dodanego do form
commands.showforms()
commands.formvalue("1", ct.name,
v.strip()) # Podaje wartosc dla selecta
commands.submit('0') # Klikam submit
html = pobierzZajecia(commands) # Pobieram zajecia
commands.follow("zapisy.do") # Wracam do strony zapisów
#commands.sleep(6)
if (
html != "skreslony z kierunku"
): # Jesli funkcja zwrocila ze jest ktos skreslony to nie dodaje htmlu
tablicaHTMLow.append(
html
) # Jeli nie zwrocila takiego komunikatu to dodajemy ten html do tablicy
else:
html = pobierzZajecia(
commands
) # Jesli nie ma selecta to pobieramy zajeci z tego kierunku i juz :D
if (html != "skreslony z kierunku"):
tablicaHTMLow.append(html)
#print control.name, control.value, control.type
#item = control.get("172748")
#print item.name, item.selected, item.id, item.attrs
#new_options
#commands.formclear('1')
#
#
#form = commands.get_browser().get_form("1")
#
#print('TO JEST TEN FORM:')
#print(form)
#print(len(form.controls))
#
#notIsSelect = True
#for ct in form.controls:
# #print(ct)
# if ct.name == 'ineSluId':
# notIsSelect = False
# print('JESTEM')
# commands.sleep(3)
#
# select_attrs = {'id': 'some_id'}
# values = pobierzElementySelect(commands.show())
# select_options = utworzTabeleSelect(values)
# print(values)
# print(select_options)
#
#
# for v in values:
# #form.fixup()
# add_select_to_form(form, 'ineSluId', select_attrs, select_options)
# form.fixup()
# #ct.get(v).selected = True
# print(ct)
# print(form)
# print(v)
# commands.showforms()
#
# commands.formvalue("1", ct.name, v.strip()) # Podaje login
# print("JEEDEFE")
# commands.submit('0')
# html = pobierzZajecia(commands)
# commands.follow("zapisy.do")
# print("JEEEEEEEEESSSSSTTTTTEEEEEMMM")
#
# commands.sleep(6)
# if (html != "skreslony z kierunku"):
# tablicaHTMLow.append(html)
#ct.get(v).selected = False
#for ct2 in form.controls:
# if ct2.type == 'submit': # szukam wsrod niej tej co ma typ submit
# commands.get_browser().clicked(form, ct2.attrs['name']) # klikam na ten przycisk
# commands.get_browser().submit()
#links = commands.showlinks()
#commands.back()
#commands.showforms()
#return "no"
#if (notIsSelect == True):
# html = pobierzZajecia(commands)
# if (html != "skreslony z kierunku"):
# tablicaHTMLow.append(html)
wyloguj(commands)
commands.browser.clear_cookies() #usuwanie ciasteczek
commands.reset_browser() #restart przegladarki
commands.reset_output()
return tablicaHTMLow
def pobierzZajecia(commands):
print("Po wybraniu")
links = commands.showlinks() # Pobiera linki z danej strony
commands.showforms()
# Szuka w linkach tego przenoszącego na odpowiedni semestr.
dateToday = datetime.date.today()
firstOctober = datetime.date(dateToday.year, 10, 1)
if dateToday > firstOctober: # Jesli dzisiaj jest wiekszy niz 1 Pazdziernika
#Semestr zimowy
for link in links:
ktory = 0
if link.text == '' + str(dateToday.year) + '/' + str(
dateToday.year +
1) + '': # Szukamy linka o tytule (rok/rok+1)
ktory = ktory + 1
if ktory == 1: # Znalazł!
commands.go(
link.url
) # Przechodzimy pod url'sa który się kryje pod tym rokiem
else:
#Semest letni
for link in links:
ktory = 0
if link.text == '' + str(dateToday.year) + '/' + str(
dateToday.year +
1) + '': # Szukamy linka o tytule (rok/rok+1)
ktory = ktory + 1
if ktory == 2: # Znalazł!
commands.go(
link.url
) # Przechodzimy pod url'sa który się kryje pod tym rokiem
# DO USUNIECIA! Pokazuje stronę po kliknięciu danego semestru!
print("Co po kliknieciu semestru:")
commands.showlinks()
print "Forms:"
commands.showforms()
# Szuka w formularzach tego odpowiadającego za pokazanie zapisanych kursów.
forms = commands.showforms() # Pobranie formularzy
naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form
for form in forms: # Petla po formualrzach
if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/zapisy.do?href=#hrefZapisySzczSlu': # Jesli akcja danego formularza przenosi do szczegolow
naszForm = form # To zapisujemy znaleziony formularz
print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz
if (
naszForm != None
): # To znaczy ze znalazlo taki rok i ze jeszcze istnieje student na tym kierunku
ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki
for ct in ctrl:
if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit
commands.get_browser().clicked(
naszForm, ct.attrs['name']) # klikam na ten przycisk
commands.get_browser().submit()
print("Co po kliknieciu szczegoly zajec")
commands.showlinks()
#commands.sleep(5)
print "Forms:"
#commands.showforms()
content = commands.show()
#content =""
#commands.browser.clear_cookies() #usuwanie ciasteczek
#print('PRZZEEEEEENIOSLEM')
#commands.sleep(5)
return content
else:
#commands.follow("zapisy.do")
#print('PRZZEEEEEENIOSLEM')
return ("skreslony z kierunku")
def showforms(self):
"""Shows form, helpful for debugging new tests"""
return tc.showforms()
def showforms(self):
"""Shows form, helpful for debugging new tests"""
return tc.showforms()
else:
return 'unknown', 'unknown', 'unknown'
######################################################################################################
if __name__ == '__main__':
min_price_list = []
max_price_list = []
avg_price_list = []
delay_time = 1
cbrw = com.get_browser()
cbrw.go('your website')
com.showforms()
com.fv("loginform", "login_id_F", "your id")
com.fv("loginform", "password_F", "your passwd")
com.submit()
### Access Excel Entries ####
wb = open_workbook('filename.xls')
for sheet in wb.sheets():
if sheet.number == 0:
item_names = sheet.col_values(5, start_rowx=1)
bottle_volumes = sheet.col_values(9, start_rowx=1)
vintages = sheet.col_values(13, start_rowx=1)
data_len = len(item_names)
intab = "a"
def annotate(params, proteins, \
url="http://services.cbu.uib.no/tools/bomp/", force=False):
"""
Uses the BOMP web service (http://services.cbu.uib.no/tools/bomp/) to
predict if proteins are outer membrane beta-barrels.
"""
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane/%s)" %
(python_version, inmembrane.__version__))
bomp_out = 'bomp.out'
log_stderr("# BOMP(web) %s > %s" % (params['fasta'], bomp_out))
if not force and os.path.isfile(bomp_out):
log_stderr("# -> skipped: %s already exists" % bomp_out)
bomp_categories = {}
fh = open(bomp_out, 'r')
for l in fh:
words = l.split()
bomp_category = int(words[-1:][0])
seqid = parse_fasta_header(l)[0]
proteins[seqid]['bomp'] = bomp_category
bomp_categories[seqid] = bomp_category
fh.close()
return bomp_categories
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go(url)
if __DEBUG__: showforms()
formfile("1", "queryfile", params["fasta"])
submit()
if __DEBUG__: show()
# extract the job id from the page
links = showlinks()
job_id = None
for l in links:
if l.url.find("viewOutput") != -1:
# grab job id from "viewOutput?id=16745338"
job_id = int(l.url.split("=")[1])
if __DEBUG__: log_stderr("BOMP job id: %d" % job_id)
if not job_id:
# something went wrong
log_stderr("# BOMP error: Can't find job id")
return
# parse the HTML table and extract categories
go("viewOutput?id=%i" % (job_id))
polltime = 10
log_stderr("# Waiting for BOMP to finish .")
while True:
try:
find("Not finished")
log_stderr(".")
except:
# Finished ! Pull down the result page.
log_stderr(". done!\n")
go("viewOutput?id=%i" % (job_id))
if __DEBUG__: log_stderr(show())
break
# Not finished. We keep polling for a time until
# we give up
time.sleep(polltime)
polltime = polltime * 2
if polltime >= 7200: # 2 hours
log_stderr("# BOMP error: Taking too long.")
return
go("viewOutput?id=%i" % (job_id))
if __DEBUG__: log_stderr(show())
bomp_html = show()
if __DEBUG__: log_stderr(bomp_html)
# Results are in the only <table> on this page, formatted like:
# <tr><th>gi|107836852|gb|ABF84721.1<th>5</tr>
soup = BeautifulSoup(bomp_html)
bomp_categories = {} # dictionary of {name, category} pairs
for tr in soup.findAll('tr')[1:]:
n, c = tr.findAll('th')
name = parse_fasta_header(n.text.strip())[0]
category = int(c.text)
bomp_categories[name] = category
# write BOMP results to a tab delimited file
fh = open(bomp_out, 'w')
for k, v in bomp_categories.iteritems():
fh.write("%s\t%i\n" % (k, v))
fh.close()
if __DEBUG__: log_stderr(str(bomp_categories))
# label proteins with bomp classification (int) or False
for name in proteins:
if "bomp" not in proteins[name]:
if name in bomp_categories:
category = int(bomp_categories[name])
proteins[name]['bomp'] = category
else:
proteins[name]['bomp'] = False
if __DEBUG__: log_stderr(str(proteins))
return bomp_categories
"""
def __call__(self, url):
browser.go(url)
browser.showforms()
browser.fv('2', 'UserName', self._user)
browser.fv('2', 'password', self._password)
browser.submit('5')
def pobierzPlan(user, password):
tablicaHTMLow = []
commands.add_extra_header('User-agent', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.6')
commands.clear_cookies() # Czyszczenie ciastek
commands.reset_browser() #restart przegladarki
commands.reset_output()
#commands.config('use_BeautifulSoup', '0')
commands.go("https://edukacja.pwr.wroc.pl/EdukacjaWeb/studia.do") # Przechodzimy do edukacji
commands.showlinks() # DO USUNIECIA! Pokazuje linki
commands.formclear('1') # Czysci formularz logowania
commands.formvalue('1', 'login', user) # Podaje login
commands.formvalue('1', 'password', password) # Podaje hasło
commands.submit('0') # Klika zaloguj
print("Linki po submit") # DO USUNIECIA! Pokazuje informacje
commands.showlinks() # DO USUNIECIA! Pokazuje linki
czyBledneLogowanie = sprawdzCzyBledneLogowanie(commands.show()) # Sprawdza czy na stronie wystapil blad
czyLogowanieWInnejSesji = sprawdzCzyLogowanieWInnejSesji(commands.show())
if (czyBledneLogowanie == True):
return "Bledne dane logowania"
if (czyLogowanieWInnejSesji == True):
return "zalogowany w innej sesji"
commands.follow("zapisy.do") # Przechodzi linkiem na stronę zapisów
# DO USUNIECIA! Pokazuje stronę po kliknięciu zapisy!
print("Co po kliknieciu Zapisy")
#commands.show()
commands.showlinks()
print "Forms:"
formsy = commands.showforms()
links = commands.showlinks() # Pobiera linki z danej strony
control = None
values = None
select_options = None
try:
control = commands.browser.get_form("1").find_control('ineSluId', type="select")
except:
print("Nie ma selecta.")
#control = commands.browser.get_form("1").find_control('ineSluId', type="select")
if(control != None): # Jesli na stronie jest select
values = pobierzElementySelect(commands.show()) # Pobieram parserem wartosci selecta
select_options = utworzTabeleSelect(values) # Tworze nowy select podmieniajac stary
select_attrs = {'id': 'some_id'} # Tworze atrybuty nowego selecta
for v in values: # Lece petla po wartosciach selecta
form = commands.get_browser().get_form("1") # Pobieram formularz
ct = commands.browser.get_form("1").find_control('ineSluId', type="select") # Pobieram kontrolke z selectem
add_select_to_form(form, 'ineSluId', select_attrs, select_options) # Tworze nowego selecta
form.fixup() # Sprawdzam czy cos nowego nie zostało dodanego do form
commands.showforms()
commands.formvalue("1", ct.name, v.strip()) # Podaje wartosc dla selecta
commands.submit('0') # Klikam submit
html = pobierzZajecia(commands) # Pobieram zajecia
commands.follow("zapisy.do") # Wracam do strony zapisów
#commands.sleep(6)
if (html != "skreslony z kierunku"): # Jesli funkcja zwrocila ze jest ktos skreslony to nie dodaje htmlu
tablicaHTMLow.append(html) # Jeli nie zwrocila takiego komunikatu to dodajemy ten html do tablicy
else:
html = pobierzZajecia(commands) # Jesli nie ma selecta to pobieramy zajeci z tego kierunku i juz :D
if (html != "skreslony z kierunku"):
tablicaHTMLow.append(html)
#print control.name, control.value, control.type
#item = control.get("172748")
#print item.name, item.selected, item.id, item.attrs
#new_options
#commands.formclear('1')
#
#
#form = commands.get_browser().get_form("1")
#
#print('TO JEST TEN FORM:')
#print(form)
#print(len(form.controls))
#
#notIsSelect = True
#for ct in form.controls:
# #print(ct)
# if ct.name == 'ineSluId':
# notIsSelect = False
# print('JESTEM')
# commands.sleep(3)
#
# select_attrs = {'id': 'some_id'}
# values = pobierzElementySelect(commands.show())
# select_options = utworzTabeleSelect(values)
# print(values)
# print(select_options)
#
#
# for v in values:
# #form.fixup()
# add_select_to_form(form, 'ineSluId', select_attrs, select_options)
# form.fixup()
# #ct.get(v).selected = True
# print(ct)
# print(form)
# print(v)
# commands.showforms()
#
# commands.formvalue("1", ct.name, v.strip()) # Podaje login
# print("JEEDEFE")
# commands.submit('0')
# html = pobierzZajecia(commands)
# commands.follow("zapisy.do")
# print("JEEEEEEEEESSSSSTTTTTEEEEEMMM")
#
# commands.sleep(6)
# if (html != "skreslony z kierunku"):
# tablicaHTMLow.append(html)
#ct.get(v).selected = False
#for ct2 in form.controls:
# if ct2.type == 'submit': # szukam wsrod niej tej co ma typ submit
# commands.get_browser().clicked(form, ct2.attrs['name']) # klikam na ten przycisk
# commands.get_browser().submit()
#links = commands.showlinks()
#commands.back()
#commands.showforms()
#return "no"
#if (notIsSelect == True):
# html = pobierzZajecia(commands)
# if (html != "skreslony z kierunku"):
# tablicaHTMLow.append(html)
wyloguj(commands)
commands.browser.clear_cookies() #usuwanie ciasteczek
commands.reset_browser() #restart przegladarki
commands.reset_output()
return tablicaHTMLow
def pobierzZajecia(commands):
print("Po wybraniu")
links = commands.showlinks() # Pobiera linki z danej strony
commands.showforms()
# Szuka w linkach tego przenoszącego na odpowiedni semestr.
dateToday = datetime.date.today()
firstOctober = datetime.date(dateToday.year, 10, 1)
if dateToday > firstOctober: # Jesli dzisiaj jest wiekszy niz 1 Pazdziernika
#Semestr zimowy
for link in links:
ktory = 0
if link.text=='' + str(dateToday.year) + '/' + str(dateToday.year+1) + '': # Szukamy linka o tytule (rok/rok+1)
ktory = ktory + 1
if ktory == 1: # Znalazł!
commands.go(link.url) # Przechodzimy pod url'sa który się kryje pod tym rokiem
else:
#Semest letni
for link in links:
ktory = 0
if link.text=='' + str(dateToday.year)+ '/' + str(dateToday.year+1) + '': # Szukamy linka o tytule (rok/rok+1)
ktory = ktory + 1
if ktory == 2: # Znalazł!
commands.go(link.url) # Przechodzimy pod url'sa który się kryje pod tym rokiem
# DO USUNIECIA! Pokazuje stronę po kliknięciu danego semestru!
print("Co po kliknieciu semestru:")
commands.showlinks()
print "Forms:"
commands.showforms()
# Szuka w formularzach tego odpowiadającego za pokazanie zapisanych kursów.
forms = commands.showforms() # Pobranie formularzy
naszForm = None # Zmienna do ktorej zapiszemy znaleziony przez nas form
for form in forms: # Petla po formualrzach
if form.action == 'https://edukacja.pwr.wroc.pl/EdukacjaWeb/zapisy.do?href=#hrefZapisySzczSlu': # Jesli akcja danego formularza przenosi do szczegolow
naszForm = form # To zapisujemy znaleziony formularz
print(naszForm) # DO USUNIECIA! Wypisuje znaleziony formularz
if(naszForm != None): # To znaczy ze znalazlo taki rok i ze jeszcze istnieje student na tym kierunku
ctrl = naszForm.controls # pobieram ze znalezionego formularza wszystkie kontrolki
for ct in ctrl:
if ct.type == 'submit': # szukam wsrod niej tej co ma typ submit
commands.get_browser().clicked(naszForm, ct.attrs['name']) # klikam na ten przycisk
commands.get_browser().submit()
print("Co po kliknieciu szczegoly zajec")
commands.showlinks()
#commands.sleep(5)
print "Forms:"
#commands.showforms()
content = commands.show()
#content =""
#commands.browser.clear_cookies() #usuwanie ciasteczek
#print('PRZZEEEEEENIOSLEM')
#commands.sleep(5)
return content
else:
#commands.follow("zapisy.do")
#print('PRZZEEEEEENIOSLEM')
return("skreslony z kierunku")
def annotate(params, proteins, \
url="http://psfs.cbrc.jp/tmbeta-net/", \
category='OM(barrel)',
force=False):
"""
Uses the TMBETA-NET web service (http://psfs.cbrc.jp/tmbeta-net/) to
predict strands of outer membrane beta-barrels.
By default, category='BARREL' means prediction will only be run
on proteins in the set with this category property. To process all
proteins, change category to None.
These keys are added to the proteins dictionary:
'tmbeta_strands' - a list of lists with paired start and end
residues of each predicted strand.
(eg [[3,9],[14,21], ..etc ])
"""
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
outfile = 'tmbeta_net.out'
log_stderr("# TMBETA-NET(web) %s > %s" % (params['fasta'], outfile))
tmbeta_strands = {}
if not force and os.path.isfile(outfile):
log_stderr("# -> skipped: %s already exists" % outfile)
fh = open(outfile, 'r')
tmbeta_strands = json.loads(fh.read())
fh.close()
for seqid in tmbeta_strands:
proteins[seqid]['tmbeta_strands'] = tmbeta_strands[seqid]
return tmbeta_strands
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
for seqid in proteins:
# only run on sequences which match the category filter
if force or \
(category == None) or \
(dict_get(proteins[seqid], 'category') == category):
pass
else:
continue
go(url)
if __DEBUG__: showforms()
fv("1","sequence",proteins[seqid]['seq'])
submit()
log_stderr("# TMBETA-NET: Predicting strands for %s - %s\n" \
% (seqid, proteins[seqid]['name']))
out = show()
time.sleep(1)
if ("Some query is already running. Please try again." in out):
log_stderr("# TMBETA-NET(web) error: %s" % (out))
return {}
# parse the web page returned, extract strand boundaries
proteins[seqid]['tmbeta_strands'] = []
for l in out.split('\n'):
if __DEBUG__: log_stderr("## " + l)
if "<BR>Segment " in l:
i,j = l.split(":")[1].split("to")
i = int(i.strip()[1:])
j = int(j.strip()[1:])
proteins[seqid]['tmbeta_strands'].append([i,j])
if __DEBUG__: log_stderr("# TMBETA-NET(web) segments: %s, %s" % (i, j))
tmbeta_strands[seqid] = proteins[seqid]['tmbeta_strands']
# we store the parsed strand boundaries in JSON format
fh = open(outfile, 'w')
fh.write(json.dumps(tmbeta_strands, separators=(',',':\n')))
fh.close()
return tmbeta_strands
def annotate(params, proteins, \
url="http://psfs.cbrc.jp/tmbeta-net/", \
category='OM(barrel)',
force=False):
"""
Uses the TMBETA-NET web service (http://psfs.cbrc.jp/tmbeta-net/) to
predict strands of outer membrane beta-barrels.
By default, category='BARREL' means prediction will only be run
on proteins in the set with this category property. To process all
proteins, change category to None.
These keys are added to the proteins dictionary:
'tmbeta_strands' - a list of lists with paired start and end
residues of each predicted strand.
(eg [[3,9],[14,21], ..etc ])
"""
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
outfile = 'tmbeta_net.out'
log_stderr("# TMBETA-NET(web) %s > %s" % (params['fasta'], outfile))
tmbeta_strands = {}
if not force and os.path.isfile(outfile):
log_stderr("# -> skipped: %s already exists" % outfile)
fh = open(outfile, 'r')
tmbeta_strands = json.loads(fh.read())
fh.close()
for seqid in tmbeta_strands:
proteins[seqid]['tmbeta_strands'] = tmbeta_strands[seqid]
return tmbeta_strands
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
for seqid in proteins:
# only run on sequences which match the category filter
if force or \
(category == None) or \
(dict_get(proteins[seqid], 'category') == category):
pass
else:
continue
go(url)
if __DEBUG__: showforms()
fv("1", "sequence", proteins[seqid]['seq'])
submit()
log_stderr("# TMBETA-NET: Predicting strands for %s - %s\n" \
% (seqid, proteins[seqid]['name']))
out = show()
time.sleep(1)
if ("Some query is already running. Please try again." in out):
log_stderr("# TMBETA-NET(web) error: %s" % (out))
return {}
# parse the web page returned, extract strand boundaries
proteins[seqid]['tmbeta_strands'] = []
for l in out.split('\n'):
if __DEBUG__: log_stderr("## " + l)
if "<BR>Segment " in l:
i, j = l.split(":")[1].split("to")
i = int(i.strip()[1:])
j = int(j.strip()[1:])
proteins[seqid]['tmbeta_strands'].append([i, j])
if __DEBUG__:
log_stderr("# TMBETA-NET(web) segments: %s, %s" % (i, j))
tmbeta_strands[seqid] = proteins[seqid]['tmbeta_strands']
# we store the parsed strand boundaries in JSON format
fh = open(outfile, 'w')
fh.write(json.dumps(tmbeta_strands, separators=(',', ':\n')))
fh.close()
return tmbeta_strands
def annotate(params, proteins, \
force=False):
"""
DEPRECATED: The TMB-HUNT server appears to be permanently offline.
Uses the TMB-HUNT web service
(http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi) to
predict if proteins are outer membrane beta-barrels.
NOTE: In my limited testing, TMB-HUNT tends to perform very poorly in
terms of false positives and false negetives. I'd suggest using only
BOMP.
"""
# TODO: automatically split large sets into multiple jobs
# TMB-HUNT will only take 10000 seqs at a time
if len(proteins) >= 10000:
log_stderr(
"# ERROR: TMB-HUNT(web): can't take more than 10,000 sequences.")
return
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
out = 'tmbhunt.out'
log_stderr("# TMB-HUNT(web) %s > %s" % (params['fasta'], out))
if not force and os.path.isfile(out):
log_stderr("# -> skipped: %s already exists" % out)
return parse_tmbhunt(proteins, out)
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/aaTMB_Hunt.cgi"
)
if __DEBUG__: showforms()
# read up the FASTA format seqs
fh = open(params['fasta'], 'r')
fasta_seqs = fh.read()
fh.close()
# fill out the form
fv("1", "sequences", fasta_seqs)
submit()
if __DEBUG__: showlinks()
# small jobs will lead us straight to the results, big jobs
# go via a 'waiting' page which we skip past if we get it
job_id = None
try:
# we see this with big jobs
result_table_url = follow(
"http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output.*.html"
)
job_id = result_table_url.split('tmp_output')[-1:][0].split('.')[0]
except:
# small jobs take us straight to the html results table
pass
# parse the job_id from the url, since due to a bug in
# TMB-HUNT the link on the results page from large jobs is wrong
if not job_id: job_id = \
follow("Full results").split('/')[-1:][0].split('.')[0]
log_stderr(
"# TMB-HUNT(web) job_id is: %s <http://www.bioinformatics.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/tmp_output%s.html>"
% (job_id, job_id))
# polling until TMB-HUNT finishes
# TMB-HUNT advises that 4000 sequences take ~10 mins
# we poll a little faster than that
polltime = (len(proteins) * 0.1) + 2
while True:
log_stderr("# TMB-HUNT(web): waiting another %i sec ..." % (polltime))
time.sleep(polltime)
try:
go("http://bmbpcu36.leeds.ac.uk/~andy/betaBarrel/AACompPred/tmp/%s.txt"
% (job_id))
break
except:
polltime = polltime * 2
if polltime >= 7200: # 2 hours
log_stderr("# TMB-HUNT error: Taking too long.")
return
txt_out = show()
# write raw TMB-HUNT results
fh = open(out, 'w')
fh.write(txt_out)
fh.close()
return parse_tmbhunt(proteins, out)
def annotate(params, proteins, \
url="http://rbf.bioinfo.tw/"+
"~sachen/OMPpredict/"+
"TMBETADISC-RBF-Content.html", force=False):
"""
Interfaces with the TatFind web service at
(http://rbf.bioinfo.tw/~sachen/OMPpredict/TMBETADISC-RBF.php)
to predict if protein sequence is likely to be an outer membrane beta-barrel.
Note that the default URL we use it different to the regular form used
by web browsers, since we need to bypass some AJAX fun.
"""
# TODO: automatically split large sets into multiple jobs
# since TMBETADISC seems to not like more than take
# ~5000 seqs at a time
if len(proteins) >= 5000:
log_stderr("# ERROR: TMBETADISC-RBF(web): tends to fail with > ~5000 sequences.")
return
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane)" % python_version)
outfn = 'tmbetadisc-rbf.out'
log_stderr("# TMBETADISC-RBF(web) %s > %s" % (params['fasta'], outfn))
if not force and os.path.isfile(outfn):
log_stderr("# -> skipped: %s already exists" % outfn)
fh = open(outfn, 'r')
proteins = parse_tmbetadisc_output(fh.read(), proteins)
fh.close()
return proteins
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go(url)
if __DEBUG__: showforms()
formfile("1", "userfile", params["fasta"])
fv("1", "format", "file")
# set the user defined method
method_map = {"aa":"Amino Acid Composition",
"dp":"Depipetide Composition",
"aadp":"Amino Acid & Depipetide Composition",
"pssm":"PSSM"}
if dict_get(params, 'tmbetadisc_rbf_method'):
try:
method = method_map[params['tmbetadisc_rbf_method']]
except KeyError:
log_stderr("# ERROR: Invalid setting from tmbetadisc_rbf_method. \
Must be set to aa, dp, aadp or pssm.")
sys.exit()
#fv("1", "select", "Amino Acid Composition")
#fv("1", "select", "Depipetide Composition")
#fv("1", "select", "Amino Acid & Depipetide Composition")
#fv("1", "select", "PSSM")
fv("1", "select", method)
submit()
waiting_page = show()
if __DEBUG__: log_stderr(waiting_page)
for l in waiting_page.split('\n'):
if l.find("TMBETADISC-RBF-action.php?UniqueName=") != -1:
result_url = l.split("'")[1]
time.sleep(5)
go(result_url)
output = show()
if __DEBUG__: log_stderr(output)
# write raw output to a file
fh = open(outfn, 'w')
fh.write(output)
fh.close()
proteins = parse_tmbetadisc_output(output, proteins)
return proteins
def test():
url = twilltestlib.get_url()
# test empty page get_title
namespaces.new_local_dict()
twill.commands.reset_browser()
browser = twill.get_browser()
try:
browser.get_title()
assert 0, "should never get here"
except BrowserStateError:
pass
### now test a few special cases
commands.go(url)
commands.go('/login')
commands.showforms()
# test no matching forms
try:
commands.fv('2', 'submit', '1')
assert 0
except TwillAssertionError:
pass
# test regexp match
commands.fv('1', '.*you', '1')
# test ambiguous match to value
commands.go('/testform')
commands.fv('1', 'selecttest', 'val')
commands.fv('1', 'selecttest', 'value1')
commands.fv('1', 'selecttest', 'selvalue1')
commands.formclear('1')
try:
commands.fv('1', 'selecttest', 'value')
assert 0
except ClientForm.ItemNotFoundError:
pass
# test ambiguous match to name
commands.go('/testform')
try:
commands.fv('1', 'item_', 'value')
assert 0
except Exception:
pass
try:
commands.formfile('1', 'selecttest', 'null')
assert 0
except Exception:
pass
commands.go('http://www.google.com/')
browser.get_title()
# test the twill script.
twilltestlib.execute_twill_script('test-form.twill', initial_url=url)
def annotate(params, proteins, \
url="http://services.cbu.uib.no/tools/bomp/", force=False):
"""
Uses the BOMP web service (http://services.cbu.uib.no/tools/bomp/) to
predict if proteins are outer membrane beta-barrels.
"""
# set the user-agent so web services can block us if they want ... :/
python_version = sys.version.split()[0]
agent("Python-urllib/%s (twill; inmembrane/%s)" %
(python_version, inmembrane.__version__))
bomp_out = 'bomp.out'
log_stderr("# BOMP(web) %s > %s" % (params['fasta'], bomp_out))
if not force and os.path.isfile(bomp_out):
log_stderr("# -> skipped: %s already exists" % bomp_out)
bomp_categories = {}
fh = open(bomp_out, 'r')
for l in fh:
words = l.split()
bomp_category = int(words[-1:][0])
seqid = parse_fasta_header(l)[0]
proteins[seqid]['bomp'] = bomp_category
bomp_categories[seqid] = bomp_category
fh.close()
return bomp_categories
# dump extraneous output into this blackhole so we don't see it
if not __DEBUG__: twill.set_output(StringIO.StringIO())
go(url)
if __DEBUG__: showforms()
formfile("1", "queryfile", params["fasta"])
submit()
if __DEBUG__: show()
# extract the job id from the page
links = showlinks()
job_id = None
for l in links:
if l.url.find("viewOutput") != -1:
# grab job id from "viewOutput?id=16745338"
job_id = int(l.url.split("=")[1])
if __DEBUG__: log_stderr("BOMP job id: %d" % job_id)
if not job_id:
# something went wrong
log_stderr("# BOMP error: Can't find job id")
return
# parse the HTML table and extract categories
go("viewOutput?id=%i" % (job_id))
polltime = 10
log_stderr("# Waiting for BOMP to finish .")
while True:
try:
find("Not finished")
log_stderr(".")
except:
# Finished ! Pull down the result page.
log_stderr(". done!\n")
go("viewOutput?id=%i" % (job_id))
if __DEBUG__: log_stderr(show())
break
# Not finished. We keep polling for a time until
# we give up
time.sleep(polltime)
polltime = polltime * 2
if polltime >= 7200: # 2 hours
log_stderr("# BOMP error: Taking too long.")
return
go("viewOutput?id=%i" % (job_id))
if __DEBUG__: log_stderr(show())
bomp_html = show()
if __DEBUG__: log_stderr(bomp_html)
# Results are in the only <table> on this page, formatted like:
# <tr><th>gi|107836852|gb|ABF84721.1<th>5</tr>
soup = BeautifulSoup(bomp_html)
bomp_categories = {} # dictionary of {name, category} pairs
for tr in soup.findAll('tr')[1:]:
n, c = tr.findAll('th')
name = parse_fasta_header(n.text.strip())[0]
category = int(c.text)
bomp_categories[name] = category
# write BOMP results to a tab delimited file
fh = open(bomp_out, 'w')
for k, v in bomp_categories.iteritems():
fh.write("%s\t%i\n" % (k, v))
fh.close()
if __DEBUG__: log_stderr(str(bomp_categories))
# label proteins with bomp classification (int) or False
for name in proteins:
if "bomp" not in proteins[name]:
if name in bomp_categories:
category = int(bomp_categories[name])
proteins[name]['bomp'] = category
else:
proteins[name]['bomp'] = False
if __DEBUG__: log_stderr(str(proteins))
return bomp_categories
"""
