def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except (TwillAssertionError, TwillException), e:
field_error = False
# Ignore all ``TwillException`` errors, but "no field matches"
if e.__class__ == TwillException:
message = e.args[0]
if not message.startswith('no field matches "'):
raise
field_error = True
cmd = field_error and showforms or show
saved, showed = False, False
if "TWILL_ERROR_DIR" in os.environ:
dirname = os.environ["TWILL_ERROR_DIR"]
if not os.path.isdir(dirname):
try:
os.mkdir(dirname)
except:
print("Cannot create directory at %r.") % dirname
cmd()
showed = True
if not showed:
timestamp = int(time.time())
if clsname:
filename = "%s.%s.%s" % (func.__module__, clsname, func.__name__)
else:
filename = "%s.%s" % (func.__module__, func.__name__)
format = field_error and "txt" or "html"
filename = "%s-%d.%s" % (filename, timestamp, format)
filename = os.path.abspath(os.path.join(dirname, filename))
try:
if field_error:
save_forms(filename)
else:
save_html(filename)
except:
print("Cannot write to file at %r.") % filename
cmd()
showed = True
else:
print("Output saved to %r.") % filename
saved = True
if not saved and not showed:
cmd()
raise
def send(self, msg, *send_to):
"""
@todo: make use of native vodafone multi-recipients functionality
"""
for contact in send_to:
web.follow(self.SERVICE_URL)
try:
web.find("/myv/messaging/webtext/Challenge.shtml")
except twill.errors.TwillAssertionError, e:
pass
else:
web.go("/myv/messaging/webtext/Challenge.shtml")
with tempfile.NamedTemporaryFile(suffix=".jpeg") as captcha:
web.save_html(captcha.name)
web.back()
os.system("open %s " % captcha.name)
web.formvalue("WebText", "jcaptcha_response", raw_input("Captcha: "))
web.formvalue("WebText", "message", msg)
to = getattr(contact, "mobile", contact)
web.formvalue("WebText", "recipient_0", to)
web.sleep(2)
web.submit()
web.code(200)
web.find("Message sent!")
def send(self, msg, *send_to):
web.go(self.SERVICE_URL)
self._retry_find("editableSmsComposeForm", 5)
try:
page = web.get_browser().get_html()
web.notfind("inputCaptcha")
except twill.errors.TwillAssertionError, e:
found = re.search("(/composer/public/jcaptcha\?id=.*)", page)
assert found
web.go(found.groups()[0])
with tempfile.NamedTemporaryFile(suffix=".jpeg") as captcha:
web.save_html(captcha.name)
web.back()
os.system("open %s " % captcha.name)
web.formvalue("editableSmsComposeForm", "inputCaptcha", raw_input("Captcha: "))
def test_wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except TwillAssertionError:
saved, showed = False, False
if 'TWILL_ERROR_DIR' in os.environ:
dirname = os.environ['TWILL_ERROR_DIR']
if not os.path.isdir(dirname):
try:
os.mkdir(dirname)
except:
print 'Cannot create directory at %r.' % dirname
show()
showed = True
if not showed:
timestamp = int(time.time())
if clsname is None:
filename = '%s.%s' % (func.__module__, func.__name__)
else:
filename = '%s.%s.%s' % (func.__module__,
clsname,
func.__name__)
filename = '%s-%d.html' % (filename, timestamp)
filename = os.path.join(dirname, filename)
try:
save_html(filename)
except:
print 'Cannot write to file at %r.' % filename
show()
showed = True
else:
print 'Output saved to %r.' % filename
saved = True
if not saved and not showed:
show()
raise
def get_bank(cert, get_bhc=False):
try:
if not completed_certs.has_key(cert):
completed_certs[cert] = True
go('http://www2.fdic.gov/idasp/confirmation.asp?inCert1=%s&AsOf=9/30/2008'
% (cert))
html = get_browser().get_html()
bhc_links = bhc_cert_href.findall(html)
if bhc_links is not None:
for bhc_link in bhc_links:
pending_certs.append(bhc_link)
save_html('%s_bank.html' % (cert))
fv('1', 'ReportName', '99')
submit()
fv('2', 'ReportName', '99')
submit()
save_html('%s_report.html' % (cert))
go('http://www2.fdic.gov/sod/sodInstBranchRpt.asp?rCert=%s&baritem=1&ryear=2008'
% (cert))
save_html('%s_sod.html' % (cert))
except Exception, e:
print e
def main():
try:
username = sys.argv[1]
password = sys.argv[2]
except IndexError:
print "Please supply username password"
sys.exit(1)
browser = twill.get_browser()
login(username, password)
serial = time.time()
pagename = "Test-%s.png" % serial
filenames = make_files(pagename)
upload_list(browser, pagename, filenames[0:2])
# try it again with two replacement files.
# pagename = "Test-%sA.png" % serial
# filenames = make_files(pagename)
# upload_list(browser, pagename, filenames)
t.showforms()
t.save_html("/tmp/testabcd")
def main():
try:
username = sys.argv[1]
password = sys.argv[2]
except IndexError:
print "Please supply username password"
sys.exit(1)
browser = twill.get_browser()
login(username, password)
serial = time.time()
pagename = "Test-%s.png" % serial
filenames = make_files(pagename)
upload_list(browser, pagename, filenames[0:2])
# try it again with two replacement files.
# pagename = "Test-%sA.png" % serial
# filenames = make_files(pagename)
# upload_list(browser, pagename, filenames)
t.showforms()
t.save_html("/tmp/testabcd")
def get_bhc(cert):
try:
if not completed_certs.has_key(cert):
completed_certs[cert] = True
go('http://www2.fdic.gov/idasp/BHClist_cert.asp?inCert1=%s&inSortIndex=0'
% (cert))
html = get_browser().get_html()
save_html('%s_hier.html' % (cert))
cert_links = cert_href.findall(html)
if cert_links is not None:
for cert_link in cert_links:
pending_certs.append(cert_link)
except Exception, e:
print e
b = a.split('\n')[2]
jobid = b.split('of')[1].split('</span')[0].strip()
# Constructing the url with the results...
url2go = 'http://www.cbs.dtu.dk//cgi-bin/webface2.fcgi?jobid=' + jobid
print 'Our results are in ' + url2go + '\n\n'
# Waiting a bit for the results...
print 'Waiting some moments for the server to complete our request!'
time.sleep(5*60)
# This timing has actually two purposes: while the results are being
# calculated, the script will be quiet. But also, there will be some
# time after the results are ready so that the server won't go overloaded.
# Please, do not overload the server using this script!!
# Going after the results...
commands.go(url2go)
# Following the link for the result file
commands.follow('processed fasta entries')
# Saving the result in the hard drive
commands.save_html( fastafilename.split('.fasta')[0] + '_RESULT.fasta' )
# This is the end, my friend.
def download_trans(print_output=False,
days_ago=None):
"""
Download transactions according to configuration file 'account_info.ini', using saved
keyring credentials (if available). If not, user will be asked for username and password
print_output: boolean
switch to control whether or not information is printed to the console
days_ago: int or None
number of days back for which to download transactions.
If None, will be read from configuration file
Returns:
--------
files: list of str
list of file names that were downloaded
"""
global print_out
print_out = print_output
files = []
user_dict = read_config_section('account_info.ini', 'UserInfo')
acc_dict = read_config_section('account_info.ini', 'AccountInfo')
# Figure out number of days
if days_ago is None:
days_ago = str(user_dict['days'])
# Get username and password either from the keyring, or from the user
try:
import keyring
user = user_dict['user']
passwd = keyring.get_password(user_dict['keyring_name'], user)
if passwd is None:
raise ValueError("Password not found in keyring")
except ValueError as e:
print(e)
user = raw_input("Betterment username: "******"Betterment password: "******"Betterment username: "******"Betterment password: "******"Could not import twill library. "
"Please check installation and try again")
sys.exit(1)
# Disable twill output if not desired
if not print_out:
import os
f = open(os.devnull, "w")
set_output(f)
else:
set_output(None)
# Login to betterment:
go('https://www.betterment.com/')
fv("2", "userName", user)
fv("2", "password", passwd)
got_in = False
for i in range(4):
if not got_in:
try:
submit(str(i))
got_in = True
except:
log(str(i) + "th submit failed, trying again")
# Get account group ID from config file and delete it
acc_group_id = acc_dict['account_group_id']
del acc_dict['account_group_id']
# Loop through the accounts defined in the config file,
# downloading and saving the csv file of transactions for each one
for account_name in acc_dict:
account_number = acc_dict[account_name]
# Download accounts:
go("https://wwws.betterment.com/transactions.csv?accountGroupId=" +
acc_group_id + "&account=" + account_number + "&startDaysAgo=" +
days_ago + "&format=csv")
fname = "transactions_" + account_name + ".csv"
save_html(fname)
# show()
log("Saved " + fname)
files.append(fname)
return files
def twill_handler(self, url, links, login_func):
"""
This function uses twill to download the files defined in links
and passing them to the magic handler to be processed.
"""
self.log.debug('twill_handler(%s)' % locals())
parsed_url = urlparse(url)
hostname = parsed_url[1].replace('www.', '')
parent = Entity.by_name(hostname)
if not parent:
parent = Entity(name=hostname)
DBSession.add(parent)
root = Entity.by_name(u'CIVX')
if not root:
root = Entity(name=u'CIVX')
DBSession.add(root)
parent.parent = root
# See if this entity already exists
entity = Entity.by_name(url)
if entity:
self.log.info('Entity(%r) already exists; skipping.' % url)
return
#DBSession.flush()
for category, link_list in links.items():
dest = [self.config['git_dir'], hostname]
if len(links) == 1:
entity = parent
else:
entity = Entity(name=category)
entity[u'url'] = url
entity[u'repo'] = hostname
entity.parent = parent
dest.append(entity.name)
DBSession.add(entity)
DBSession.flush()
dest = os.path.join(*dest)
if not os.path.isdir(dest):
os.makedirs(dest)
b = self.get_browser()
for link in link_list:
# We might have timed out, try to log in again.
login_func()
b.go(link['href'])
# Try to pick out the filename if there is a query
if link['href'].find('=') >= 0:
filename = urlparse(link['href'])[-2].split('=')[1]
else:
filename = link['href'].split('/')[-1]
filename = os.path.join(dest, filename)
save_html(filename)
file_entity = Entity(name=link.contents[0])
file_entity[u'filename'] = filename
file_entity[u'repo'] = hostname
DBSession.add(file_entity)
file_entity.parent = entity
self.log.debug("Created entity %r (parent %r)" % (
file_entity.name, file_entity.parent.name))
magic = self.call_magic_handler(filename, file_entity)
file_entity[u'magic'] = magic
DBSession.flush()
