Beispiel #1
0
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except (TwillAssertionError, TwillException), e:
            field_error = False

            # Ignore all ``TwillException`` errors, but "no field matches"
            if e.__class__ == TwillException:
                message = e.args[0]

                if not message.startswith('no field matches "'):
                    raise

                field_error = True

            cmd = field_error and showforms or show
            saved, showed = False, False

            if "TWILL_ERROR_DIR" in os.environ:
                dirname = os.environ["TWILL_ERROR_DIR"]

                if not os.path.isdir(dirname):
                    try:
                        os.mkdir(dirname)
                    except:
                        print("Cannot create directory at %r.") % dirname
                        cmd()

                        showed = True

                if not showed:
                    timestamp = int(time.time())

                    if clsname:
                        filename = "%s.%s.%s" % (func.__module__, clsname, func.__name__)
                    else:
                        filename = "%s.%s" % (func.__module__, func.__name__)

                    format = field_error and "txt" or "html"
                    filename = "%s-%d.%s" % (filename, timestamp, format)
                    filename = os.path.abspath(os.path.join(dirname, filename))

                    try:
                        if field_error:
                            save_forms(filename)
                        else:
                            save_html(filename)
                    except:
                        print("Cannot write to file at %r.") % filename
                        cmd()

                        showed = True
                    else:
                        print("Output saved to %r.") % filename
                        saved = True

            if not saved and not showed:
                cmd()

            raise
Beispiel #2
0
    def send(self, msg, *send_to):
        """
        @todo: make use of native vodafone multi-recipients functionality
        """
        for contact in send_to:
            web.follow(self.SERVICE_URL)

            try:
                web.find("/myv/messaging/webtext/Challenge.shtml")
            except twill.errors.TwillAssertionError, e:
                pass
            else:
                web.go("/myv/messaging/webtext/Challenge.shtml")
                with tempfile.NamedTemporaryFile(suffix=".jpeg") as captcha:
                    web.save_html(captcha.name)
                    web.back()
                    os.system("open %s " % captcha.name)
                    web.formvalue("WebText", "jcaptcha_response", raw_input("Captcha: "))

            web.formvalue("WebText", "message", msg)
            to = getattr(contact, "mobile", contact)
            web.formvalue("WebText", "recipient_0", to)

            web.sleep(2)
            web.submit()
            web.code(200)
            web.find("Message sent!")
Beispiel #3
0
    def send(self, msg, *send_to):
        web.go(self.SERVICE_URL)
        self._retry_find("editableSmsComposeForm", 5)

        try:
            page = web.get_browser().get_html()
            web.notfind("inputCaptcha")
        except twill.errors.TwillAssertionError, e:
            found = re.search("(/composer/public/jcaptcha\?id=.*)", page)
            assert found
            web.go(found.groups()[0])
            with tempfile.NamedTemporaryFile(suffix=".jpeg") as captcha:
                web.save_html(captcha.name)
                web.back()
                os.system("open %s " % captcha.name)
                web.formvalue("editableSmsComposeForm", "inputCaptcha", raw_input("Captcha: "))
Beispiel #4
0
    def test_wrapper(*args, **kwargs):
        try:
            func(*args, **kwargs)
        except TwillAssertionError:
            saved, showed = False, False

            if 'TWILL_ERROR_DIR' in os.environ:
                dirname = os.environ['TWILL_ERROR_DIR']

                if not os.path.isdir(dirname):
                    try:
                        os.mkdir(dirname)
                    except:
                        print 'Cannot create directory at %r.' % dirname
                        show()

                        showed = True

                if not showed:
                    timestamp = int(time.time())

                    if clsname is None:
                        filename = '%s.%s' % (func.__module__, func.__name__)
                    else:
                        filename = '%s.%s.%s' % (func.__module__,
                                                 clsname,
                                                 func.__name__)

                    filename = '%s-%d.html' % (filename, timestamp)
                    filename = os.path.join(dirname, filename)

                    try:
                        save_html(filename)
                    except:
                        print 'Cannot write to file at %r.' % filename
                        show()

                        showed = True
                    else:
                        print 'Output saved to %r.' % filename
                        saved = True

            if not saved and not showed:
                show()

            raise
Beispiel #5
0
def get_bank(cert, get_bhc=False):

    try:

        if not completed_certs.has_key(cert):

            completed_certs[cert] = True

            go('http://www2.fdic.gov/idasp/confirmation.asp?inCert1=%s&AsOf=9/30/2008'
               % (cert))

            html = get_browser().get_html()

            bhc_links = bhc_cert_href.findall(html)

            if bhc_links is not None:
                for bhc_link in bhc_links:
                    pending_certs.append(bhc_link)

            save_html('%s_bank.html' % (cert))

            fv('1', 'ReportName', '99')
            submit()
            fv('2', 'ReportName', '99')
            submit()

            save_html('%s_report.html' % (cert))

            go('http://www2.fdic.gov/sod/sodInstBranchRpt.asp?rCert=%s&baritem=1&ryear=2008'
               % (cert))
            save_html('%s_sod.html' % (cert))

    except Exception, e:
        print e
Beispiel #6
0
def main():
    try:
        username = sys.argv[1]
        password = sys.argv[2]
    except IndexError:
        print "Please supply username password"
        sys.exit(1)
    browser = twill.get_browser()
    login(username, password)

    serial = time.time()
    pagename = "Test-%s.png" % serial
    filenames = make_files(pagename)
    upload_list(browser, pagename, filenames[0:2])

    # try it again with two replacement files.
    #    pagename = "Test-%sA.png" % serial
    #    filenames = make_files(pagename)
    #    upload_list(browser, pagename, filenames)

    t.showforms()
    t.save_html("/tmp/testabcd")
Beispiel #7
0
def main():
    try:
        username = sys.argv[1]
        password = sys.argv[2]
    except IndexError:
        print "Please supply username password"
        sys.exit(1)
    browser = twill.get_browser()
    login(username, password)

    serial = time.time()
    pagename = "Test-%s.png" % serial
    filenames = make_files(pagename)
    upload_list(browser, pagename, filenames[0:2])

    # try it again with two replacement files.
    #    pagename = "Test-%sA.png" % serial
    #    filenames = make_files(pagename)
    #    upload_list(browser, pagename, filenames)

    t.showforms()
    t.save_html("/tmp/testabcd")
Beispiel #8
0
def get_bhc(cert):

    try:

        if not completed_certs.has_key(cert):

            completed_certs[cert] = True

            go('http://www2.fdic.gov/idasp/BHClist_cert.asp?inCert1=%s&inSortIndex=0'
               % (cert))

            html = get_browser().get_html()

            save_html('%s_hier.html' % (cert))

            cert_links = cert_href.findall(html)

            if cert_links is not None:
                for cert_link in cert_links:
                    pending_certs.append(cert_link)

    except Exception, e:
        print e
    b = a.split('\n')[2]
    jobid = b.split('of')[1].split('</span')[0].strip()
    
    # Constructing the url with the results...
    url2go = 'http://www.cbs.dtu.dk//cgi-bin/webface2.fcgi?jobid=' + jobid
    print 'Our results are in ' + url2go + '\n\n'
    
    # Waiting a bit for the results...
    print 'Waiting some moments for the server to complete our request!'
    time.sleep(5*60)
    
    # This timing has actually two purposes: while the results are being 
    # calculated, the script will be quiet. But also, there will be some 
    # time after the results are ready so that the server won't go overloaded.
    # Please, do not overload the server using this script!!
    
    # Going after the results...
    commands.go(url2go)
    
    
    # Following the link for the result file
    commands.follow('processed fasta entries')
    
    # Saving the result in the hard drive
    commands.save_html( fastafilename.split('.fasta')[0] + '_RESULT.fasta' )




# This is the end, my friend.
def download_trans(print_output=False,
                   days_ago=None):
    """
    Download transactions according to configuration file 'account_info.ini', using saved
    keyring credentials (if available). If not, user will be asked for username and password

    print_output: boolean
        switch to control whether or not information is printed to the console
    days_ago: int or None
        number of days back for which to download transactions.
        If None, will be read from configuration file

    Returns:
    --------
    files: list of str
        list of file names that were downloaded
    """
    global print_out
    print_out = print_output
    files = []

    user_dict = read_config_section('account_info.ini', 'UserInfo')
    acc_dict = read_config_section('account_info.ini', 'AccountInfo')

    # Figure out number of days
    if days_ago is None:
        days_ago = str(user_dict['days'])

    # Get username and password either from the keyring, or from the user
    try:
        import keyring
        user = user_dict['user']
        passwd = keyring.get_password(user_dict['keyring_name'], user)
        if passwd is None:
            raise ValueError("Password not found in keyring")
    except ValueError as e:
        print(e)
        user = raw_input("Betterment username: "******"Betterment password: "******"Betterment username: "******"Betterment password: "******"Could not import twill library. "
              "Please check installation and try again")
        sys.exit(1)

    # Disable twill output if not desired
    if not print_out:
        import os
        f = open(os.devnull, "w")
        set_output(f)
    else:
        set_output(None)

    # Login to betterment:
    go('https://www.betterment.com/')
    fv("2", "userName", user)
    fv("2", "password", passwd)

    got_in = False
    for i in range(4):
        if not got_in:
            try:
                submit(str(i))
                got_in = True
            except:
                log(str(i) + "th submit failed, trying again")
            
    # Get account group ID from config file and delete it
    acc_group_id = acc_dict['account_group_id']
    del acc_dict['account_group_id']

    # Loop through the accounts defined in the config file,
    # downloading and saving the csv file of transactions for each one
    for account_name in acc_dict:
        account_number = acc_dict[account_name]

        # Download accounts:
        go("https://wwws.betterment.com/transactions.csv?accountGroupId=" +
           acc_group_id + "&account=" + account_number + "&startDaysAgo=" +
           days_ago + "&format=csv")
        fname = "transactions_" + account_name + ".csv"
        save_html(fname)
        # show()

        log("Saved " + fname)
        files.append(fname)

    return files
Beispiel #11
0
    def twill_handler(self, url, links, login_func):
        """
        This function uses twill to download the files defined in links
        and passing them to the magic handler to be processed.
        """

        self.log.debug('twill_handler(%s)' % locals())

        parsed_url = urlparse(url)
        hostname = parsed_url[1].replace('www.', '')

        parent = Entity.by_name(hostname)
        if not parent:
            parent = Entity(name=hostname)
            DBSession.add(parent)
            root = Entity.by_name(u'CIVX')
            if not root:
                root = Entity(name=u'CIVX')
                DBSession.add(root)
            parent.parent = root

        # See if this entity already exists
        entity = Entity.by_name(url)
        if entity:
            self.log.info('Entity(%r) already exists; skipping.' % url)
            return

        #DBSession.flush()
        for category, link_list in links.items():

            dest = [self.config['git_dir'], hostname]

            if len(links) == 1:
                entity = parent
            else:
                entity = Entity(name=category)
                entity[u'url'] = url
                entity[u'repo'] = hostname
                entity.parent = parent
                dest.append(entity.name)

                DBSession.add(entity)
                DBSession.flush()

            dest = os.path.join(*dest)
            if not os.path.isdir(dest):
                os.makedirs(dest)

            b = self.get_browser()
            for link in link_list:
                # We might have timed out, try to log in again.
                login_func()
                b.go(link['href'])
                # Try to pick out the filename if there is a query
                if link['href'].find('=') >= 0:
                    filename = urlparse(link['href'])[-2].split('=')[1]
                else:
                    filename = link['href'].split('/')[-1]
                filename = os.path.join(dest, filename)
                save_html(filename)

                file_entity = Entity(name=link.contents[0])
                file_entity[u'filename'] = filename
                file_entity[u'repo'] = hostname
                DBSession.add(file_entity)
                file_entity.parent = entity
                self.log.debug("Created entity %r (parent %r)" % (
                    file_entity.name, file_entity.parent.name))

                magic = self.call_magic_handler(filename, file_entity)
                file_entity[u'magic'] = magic

                DBSession.flush()