예제 #1
0
def pdbfixretrieve(joblink):
	browser=RoboBrowser(history=True)
	browser.open(joblink)
	stdout=browser.get_links('stdout')
	while not stdout:
		time.sleep(5)
		browser.open(joblink)
		stdout=browser.get_links('stdout')
	pdbout=browser.get_links('outpdb')
	browser.follow_link(stdout[0])
	stdcontent=browser.response.content
	browser.follow_link(pdbout[0])
	pdbcontent=browser.response.content
	pdb.set_trace()
	return stdcontent, pdbcontent
예제 #2
0
def _download_rib(dir, date):
    url = "http://archive.routeviews.org/route-views.wide/bgpdata/"

    dt_web = date.strftime("%Y") + "." + date.strftime("%m") + "/"

    print("Looking for RIB file...")
    br = RoboBrowser()
    br.open(url)

    link_date = br.get_link(dt_web)
    br.follow_link(link_date)

    link_rib = br.get_link("RIBS/")
    br.follow_link(link_rib)

    elem = "rib." + date.strftime("%Y") + date.strftime("%m") + date.strftime(
        "%d")
    _dt_web = date.strftime("%Y") + "." + date.strftime("%m") + "/"
    links = br.get_links(elem)
    one_link = links[0]

    file = (str(one_link).split('"'))[1]
    url_dw = "http://archive.routeviews.org/route-views.wide/bgpdata/" + _dt_web + "RIBS/" + file
    filename = dir + file

    r = requests.get(url_dw)
    with open(filename, "wb") as code:
        code.write(r.content)

    rib = _decompress_rib(filename)

    return rib
예제 #3
0
 def pushedbutton(self, b):
     account = self.lineEdit.text()
     pasw = self.lineEdit_3.text()
     #use robobrowser module to manipulate web page
     browser = RoboBrowser(history=True)
     browser.open('http://web1.cmu.edu.tw/stdinfo/login.asp')
     form1 = browser.get_form(id='form1')
     form1['f_id'].value = account
     form1['f_pwd'].value = pasw
     browser.submit_form(form1)
     if browser.state.url == "http://web1.cmu.edu.tw/stdinfo/loginerr.asp":
         self.lineEdit_2.setText('帳號密碼錯了?')
     else:
         link_one = browser.get_link(text=re.compile('.意見調查'))
         browser.follow_link(link_one)
         list = []
         for l in browser.get_links(text=re.compile('.填寫.')):
             list.append(l)
         list.pop(0)
         for li in list:
             browser.follow_link(li)
             form2 = browser.get_form(id='thisform')
             form2['Cos_Q1'].value = '1'
             browser.submit_form(form2)
         self.lineEdit_2.setText('Done!')
예제 #4
0
 def pushedbutton(self,b):
     account = self.lineEdit.text()
     pasw = self.lineEdit_3.text()
     #use robobrowser module to manipulate web page 
     browser = RoboBrowser(history = True)
     browser.open('http://web1.cmu.edu.tw/stdinfo/login.asp')
     form1 = browser.get_form(id = 'form1')
     form1['f_id'].value = account
     form1['f_pwd'].value = pasw
     browser.submit_form(form1)
     if browser.state.url == "http://web1.cmu.edu.tw/stdinfo/loginerr.asp":
         self.lineEdit_2.setText('帳號密碼錯了?')
     else:
         link_one = browser.get_link(text = re.compile('.意見調查'))
         browser.follow_link(link_one)
         list = []
         for l in browser.get_links(text = re.compile('.填寫.')): 
             list.append(l)
         list.pop(0)
         for li in list:
             browser.follow_link(li)
             form2 = browser.get_form(id = 'thisform')
             form2['Cos_Q1'].value = '1'
             browser.submit_form(form2)
         self.lineEdit_2.setText('Done!')
예제 #5
0
def pdbfixsubmit(pdbfile):
	"Submit structure to pdb_fix online tool"
	url='http://lorentz.dynstr.pasteur.fr/pdb/frozen_submission.php'
	browser=RoboBrowser(history=True)
	browser.open(url)
	upload_form=browser.get_form()
	upload_form['job_name'].value = pdbfile.split("/")[0]+'_atomfix'
	upload_form['inpdb'].value = open(pdbfile, 'r')
	browser.submit_form(upload_form)
	print 'submitted ', pdbfile.split("/")[0]
	joblink=browser.get_links('jobs')
	return joblink[0]['href']
예제 #6
0
class infs_brsr:
    """This browser will have functions useful to someone
    browsing the Infusionsoft front end programatically.
    """

    def __init__(self, appname, username, password, *args, **kwargs):
        self.loggedin=False
        self.browser=RoboBrowser(history=True)
        self.appname=appname
        self.username=username
        self.password=password
        self.baseurl = 'https://' + self.appname + '.infusionsoft.com'

    def openbase(self):
        self.browser.open(self.baseurl)

    def login(self):
        self.openbase()
        loginform = self.browser.get_form()
        loginform.fields['username'].value = self.username
        loginform.fields['password'].value = self.password
        self.browser.submit_form(loginform)
        # This next step is probably a bad idea.  It needs
        # some form of control
        self.browser.follow_link(self.browser.get_links()[1])
        self.loggedin=True

    def getapikey(self):
        if not self.loggedin:
            self.login()
        self.browser.open(self.baseurl + 'app/miscSetting/itemWrapper?systemId=nav.admin&settingModuleName=Application&settingTabName=Application')
        pageSoup = BeautifulSoup(self.browser.response.content, 'html.parser')
        self.apikey=pageSoup.findAll(id='Application_Encrypted_Key:_data')[0].text
        return self.apikey

    def importContactCSV(self, pathToCSV='/home/jlmarks/importme.csv'):
        if not self.loggedin:
            self.login()
        importURL = "https://" + self.appname + ".infusionsoft.com/Import/jumpToWizard.jsp?update=false&profileClass=com.infusion.crm.db.importer.profiles.ContactProfile"
        self.browser.open(importURL)
        frms = self.browser.get_forms()
        for eachform in frms:
            if 'id' in eachform.fields.keys():
                self.thisimportid=eachform['id'].value
                correctform = eachform
        correctform.fields.pop('Back')
        correctform.fields['importFile'].value=open(pathToCSV, 'rb')
        self.browser.submit_form(correctform)
예제 #7
0
    def gather(self):
        browser = RoboBrowser()
        page = 0
        browser.open(self.url)

        while (page < self.max_pages):
            links = browser.get_links()

            if (page == 0):
                for link in links:
                    self.list.append(link)
            else:
                for link in links:
                    if (self.isInTheList(link)):
                        self.list.append(link)

            browser.follow_link(self.list[page])
            page += 1
예제 #8
0
    def gather(self):
        browser = RoboBrowser()
        page = 0
        browser.open(self.url)

        while (page < self.max_pages):
            links = browser.get_links()

            if(page == 0):
                for link in links:
                    self.list.append(link)
            else:
                for link in links:
                     if(self.isInTheList(link)):
                        self.list.append(link)

            browser.follow_link(self.list[page])
            page+=1
예제 #9
0
def _login():

    username = input('User ID: ')
    password = getpass('Password: '******'html.parser')

    browser.open('http://online.lloydsbank.co.uk/personal/logon.login.jsp')

    form = browser.get_form('frmLogin')
    form['frmLogin:strCustomerLogin_userID'] = username
    form['frmLogin:strCustomerLogin_pwd'] = password
    browser.submit_form(form)

    mem_info = getpass('Memorable information: ').lower()

    form_name = 'frmentermemorableinformation1'
    option_name = ':strEnterMemorableInformation_memInfo{}'
    form = browser.get_form(form_name)
    indices = re.findall('Character (\d+) :', form.parsed.text)
    indices = [int(x) for x in indices]

    for i, idx in enumerate(indices):
        form[form_name +
             option_name.format(i + 1)] = '&nbsp;' + mem_info[idx - 1]
    browser.submit_form(form)

    assert 'Lloyds Bank - Personal Account Overview' in browser.parsed.title

    accounts = {}
    for link in browser.get_links():
        if 'lnkAccName' in link.attrs.get('id', ''):
            accounts[link.text] = link

    print('Accounts:', list(accounts))

    account = input('Account: ')

    browser.follow_link(accounts[account])
    export_link = browser.get_link(title='Export')
    browser.follow_link(export_link)

    return browser
예제 #10
0
def download_internal(user_id, from_date, to_date):
    """Download the csv files for the transaction between the given dates"""
    # Create the browser and open the lloyds login page
    browser = RoboBrowser(parser='html5lib')
    browser.open(
        'https://online.lloydsbank.co.uk/personal/logon/login.jsp?WT.ac=hpIBlogon'
    )

    while 'Enter Memorable Information' not in browser.parsed.title.text:
        print(browser.parsed.title.text)
        form = browser.get_form(id='frmLogin')
        form['frmLogin:strCustomerLogin_userID'] = str(user_id)
        form['frmLogin:strCustomerLogin_pwd'] = prompt('Enter password: '******'re logged in, now enter memorable information
    print(browser.parsed.title.text)
    form = browser.get_form(id='frmentermemorableinformation1')
    field = 'frmentermemorableinformation1:strEnterMemorableInformation_memInfo{0}'

    for i in range(1, 4):
        label = browser.find("label", {"for": field.format(i)})
        form[field.format(i)] = '&nbsp;' + prompt(label.text.strip())
    browser.submit_form(form)

    # hopefully now we're logged in...
    print(browser.parsed.title.text)
    links = []
    for link in browser.get_links("View statement"):
        if link.text == "View statement":
            links.append(link)

    # loop through all accounts
    for index, link in enumerate(links):
        acc_name = link['data-wt-ac'].split(" resource")[0]
        print(acc_name)
        print(browser.parsed.title)
        browser.follow_link(link)
        yield acc_name, download_account_internal(browser, from_date, to_date)
        browser.back()
예제 #11
0
def main(competition, username, password):

    browser = RoboBrowser(history=True, parser="html.parser")
    base = 'https://www.kaggle.com'
    browser.open('/'.join([base, 'account/login']))

    login_form = browser.get_form(action='/account/login')
    login_form['UserName'] = username
    login_form['Password'] = password
    browser.submit_form(login_form)

    browser.open('/'.join([base, 'c', competition, 'data']))
    files = []
    for a_href in browser.get_links():
        if '.zip' in a_href.text:
            files.append(a_href)

    print('...downloading {0} files...'.format(len(files)))
    for f in files:
        request = browser.session.get(base + f.attrs['href'], stream=True)
        with open(f.attrs['name'], "wb") as zip_file:
            zip_file.write(request.content)
예제 #12
0
def download(competition, username, password):

    browser = RoboBrowser(history=True, parser="html.parser")
    base = 'https://www.kaggle.com'
    browser.open('/'.join([base, 'account/login']))

    login_form = browser.get_form(action='/account/login')
    login_form['UserName'] = username
    login_form['Password'] = password
    browser.submit_form(login_form)

    browser.open('/'.join([base, 'c', competition, 'data']))
    files = []
    for a_href in browser.get_links():
        if '.zip' in a_href.text:
            files.append(a_href)

    print('...downloading {0} files...'.format(len(files)))
    for f in files:
        request = browser.session.get(base + f.attrs['href'], stream=True)
        with open(f.attrs['name'], "wb") as zip_file:
            zip_file.write(request.content)
예제 #13
0
    def pushedbutton(self, b):
        account = self.lineEdit.text()
        pasw = self.lineEdit_3.text()
        #use robobrowser module to manipulate web page
        browser = RoboBrowser(history=True)
        browser.open('http://web1.cmu.edu.tw/stdinfo/login.asp')
        form1 = browser.get_form(id='form1')
        form1['f_id'].value = account
        form1['f_pwd'].value = pasw
        browser.submit_form(form1)
        if browser.state.url == "http://web1.cmu.edu.tw/stdinfo/loginerr.asp":
            self.lineEdit_2.setText('帳號密碼錯了?')
        else:
            self.lineEdit_2.setText('成功登入,填寫中....')
            link_one = browser.get_link(text='教師教學意見調查')
            browser.follow_link(link_one)
            list = []
            for l in browser.get_links(text='填寫'):
                list.append(l)
            list.pop(0)
            for li in list:
                browser.follow_link(li)
                form2 = browser.get_form(id='thisform')
                form2['CH_1'].value = '3'
                form2['CH_2'].value = '3'
                form2['CH_3'].value = '3'
                form2['CH_4'].value = '3'
                form2['CH_5'].value = '3'
                form2['CH_6'].value = '3'
                form2['CH_7'].value = '3'
                form2['CH_8'].value = '3'
                form2['CH_9'].value = '3'
                form2['CH_10'].value = '3'

                browser.submit_form(form2)
            self.lineEdit_2.setText('Done!')
예제 #14
0
 def pushedbutton(self,b):
     account = self.lineEdit.text()
     pasw = self.lineEdit_3.text()
     #use robobrowser module to manipulate web page 
     browser = RoboBrowser(history = True)
     browser.open('http://web1.cmu.edu.tw/stdinfo/login.asp')
     form1 = browser.get_form(id = 'form1')
     form1['f_id'].value = account
     form1['f_pwd'].value = pasw
     browser.submit_form(form1)
     if browser.state.url == "http://web1.cmu.edu.tw/stdinfo/loginerr.asp":
         self.lineEdit_2.setText('帳號密碼錯了?')
     else:
         self.lineEdit_2.setText('成功登入,填寫中....')
         link_one = browser.get_link(text = '教師教學意見調查')
         browser.follow_link(link_one)
         list = []
         for l in browser.get_links(text = '填寫'): 
             list.append(l)
         list.pop(0)
         for li in list:
             browser.follow_link(li)
             form2 = browser.get_form(id = 'thisform')
             form2['CH_1'].value = '3'
             form2['CH_2'].value = '3'
             form2['CH_3'].value = '3'
             form2['CH_4'].value = '3'
             form2['CH_5'].value = '3'
             form2['CH_6'].value = '3'
             form2['CH_7'].value = '3'
             form2['CH_8'].value = '3'
             form2['CH_9'].value = '3'
             form2['CH_10'].value = '3'
                 
             browser.submit_form(form2)
         self.lineEdit_2.setText('Done!')
예제 #15
0
if __name__ == "__main__":
    sessions_page = sys.argv[1]

    if len(sys.argv) > 1:
        download_folder = sys.argv[2]

        if not os.path.exists(download_folder):
            os.mkdir(download_folder)
    else:
        download_folder = None

    browser = RoboBrowser()
    browser.open(sessions_page)

    lecture_links = browser.get_links(href=re.compile("lecture-"))

    for link in lecture_links:
        lecture_url = link["href"]
        # ensure absolute url:
        lecture_url = urljoin(sessions_page, lecture_url)

        mp3_url = get_mp3_url(lecture_url)

        if not mp3_url:
            print("No .mp3 at {}".format(lecture_url))
            continue

        try:
            filename = download_file(mp3_url, output_dir=download_folder)
            print("Downloaded {}".format(filename))
예제 #16
0
 def add_valid_links(self, url):
     browser = RoboBrowser(parser=PARSER, history=True)
     browser.open(url)
     links = browser.get_links()
     for link in links:
         self.add_valid_link(browser, link)
예제 #17
0
def _download_traces(dates, period):

    url = "http://archive.routeviews.org/route-views.wide/bgpdata/"

    MRAI_BIN_DIR = "/srv/agarcia/TFM/"  #path to save files
    if (period == 0):
        dir = MRAI_BIN_DIR + "ONE_DAY_BGP_TRACES_BZ2/"
    elif (period == 1):
        dir = MRAI_BIN_DIR + "THREE_DAY_BGP_TRACES_BZ2/"
    else:
        dir = MRAI_BIN_DIR + "BGP_TRACES_BZ2/"

    if not os.path.exists(dir):
        os.makedirs(dir)

    for _dt in dates:

        dt = datetime.datetime.strptime(_dt, "%Y-%m-%d")
        dt_web = dt.strftime("%Y") + "." + dt.strftime("%m") + "/"

        print("Opening browser...")
        br = RoboBrowser()
        br.open(url)

        #Buscamos la fecha que queremos y hacemos click
        link_date = br.get_link(dt_web)
        br.follow_link(link_date)

        #Buscamos el link UPDATES y hacemos click
        link_update = br.get_link("UPDATES/")
        br.follow_link(link_update)

        #Obtenemos los 2 DIAS antes y despues de la fecha deseada
        #(5 dias en total)
        days = []
        days.append(dt)
        if (period != 0):
            for day_p in range(1, period + 1):
                d_before = dt - datetime.timedelta(days=day_p)
                d_after = dt + datetime.timedelta(days=day_p)
                days.insert(0, d_before)
                days.append(d_after)

        print(days)

        #Para cada dia descargamos todos los BGP update traces
        for day in days:
            print("Downloading files of day " + day.strftime("%Y-%m-%d") +
                  "\n")
            elem = "updates." + day.strftime("%Y") + day.strftime(
                "%m") + day.strftime("%d")
            _dt_web = day.strftime("%Y") + "." + day.strftime("%m") + "/"
            br.back()
            br.back()
            br.follow_link(br.get_link(dt_web))
            br.follow_link(br.get_link("UPDATES/"))
            links = br.get_links(elem)
            for link in links:
                file = (str(link).split('"'))[1]
                url_dw = "http://archive.routeviews.org/route-views.wide/bgpdata/" + _dt_web + "UPDATES/" + file
                filename = dir + file

                r = requests.get(url_dw)
                with open(filename, "wb") as code:
                    code.write(r.content)
예제 #18
0
r.url
r.status_code
r.headers["location"]
requests.request?
import robobrowser
from robobrowser import RoboBrowser
b = RoboBrowser(parser="lxml.html")
b
b.open("http://www.chandrashekar.info")
b.url
b.contents
b.response
b.response.status_code
b.links
dir(b)
b.get_links()
b = RoboBrowser(parser="lxml")
b.open("http://www.chandrashekar.info")
b.get_links()
b.get_links()
b.forms
dir(b)
b.get_links()
b.get_links()[-3]
l = b.get_links()[-3]
b.follow_link(l)
b.url
b.back()
b.url
b.forward()
b.url
예제 #19
0
# -*- coding: utf-8 -*-
import re
import sys
from robobrowser import RoboBrowser
from getpass import getpass

account = input('account:')
password = getpass('password:'******'http://web1.cmu.edu.tw/stdinfo/login.asp')
form1 = browser.get_form(id='form1')
form1['f_id'].value = account
form1['f_pwd'].value = password
browser.submit_form(form1)

link_one = browser.get_link(text='期中網路教學意見調查')
browser.follow_link(link_one)
list = []
for l in browser.get_links(text=re.compile('填寫')):
    list.append(l)
list.pop(0)
for li in list:
    browser.follow_link(li)
    form2 = browser.get_form(id='thisform')
    form2['Cos_Q1'].value = '1'
    browser.submit_form(form2)
예제 #20
0
파일: poster.py 프로젝트: Z-Foster/rs-tools
class MarketPoster:
    def __init__(self):
        self.login_url = 'http://forums.zybez.net/index.php?' \
                         'app=curseauth&module=global&section=login'
        self.browser = RoboBrowser(history=False, parser='html.parser')
        self.logged_in = False

    def login(self, login_name, login_password):
        login_name = login_name
        login_password = login_password
        self.browser.open(self.login_url)

        sign_in_form = self.browser.get_form(class_='authentication-box')
        sign_in_form['ips_username'].value = login_name
        sign_in_form['ips_password'].value = login_password
        self.browser.submit_form(sign_in_form)

        correct_url = 'http://forums.zybez.net/index.php'
        if self.browser.url == correct_url:
            self.logged_in = True
            return True
        else:
            return False

    def deleteItemPosts(self, post):
        item_url = self.getItemURL(post.item_name)
        self.browser.open(item_url)
        items_to_delete = self.browser.get_links(href=re.compile(
                "do=trade-delete"))
        for i in items_to_delete:
            self.browser.follow_link(i)
        self.browser.open(item_url)

    def deleteAllPosts(self):
        self.browser.open('http://forums.zybez.net/runescape-2007-prices')
        delete_button = self.browser.get_link('Remove all active offers')
        if delete_button is not None:
            self.browser.follow_link(delete_button)

    def getItemURL(self, item_name):
        item_name = item_name.split()
        item_name = '+'.join(item_name)

        item_data_url = 'http://forums.zybez.net' \
                        '/runescape-2007-prices/api/item/' + item_name

        item_data_dict = self.browser.session.get(item_data_url).json()
        item_id = item_data_dict['id']
        item_url = 'http://forums.zybez.net/runescape-2007-prices/' \
                   + str(item_id) + '-' + item_name
        return item_url

    def postItem(self, post):
        price = post.price
        quantity = post.quantity
        note = post.note
        offer_type = post.offer_type
        contact_method = post.contact_method

        self.deleteItemPosts(post)

        post_item_form = self.browser.find(
            action='http://forums.zybez.net/index.php?app=priceguide&module='
                   'public&section=action&do=trade-add')
        post_item_form = self.browser.get_form(post_item_form)

        # Fill out and submit form
        post_item_form['type'].value = str(int(offer_type))
        post_item_form['qty'].value = quantity
        post_item_form['price'].value = price
        post_item_form['notes'].value = note
        post_item_form['contact'].value = str(int(contact_method))

        self.browser.submit_form(post_item_form)
예제 #21
0
import re
from robobrowser import RoboBrowser

browser = RoboBrowser()
browser.open("https://duckduckgo.com")
# Must find the proper id in the html
form = browser.get_form(id="search_form_homepage")
form
form["q"].value = "python"
browser.submit_form(form)
links = browser.get_links()
for link in links:
    print(link)
예제 #22
0
# -*- coding: utf-8 -*-
import re
import sys
from robobrowser import RoboBrowser
from getpass import getpass

account = input('account:')
password = getpass('password:'******'http://web1.cmu.edu.tw/stdinfo/login.asp')
form1 = browser.get_form(id = 'form1')
form1['f_id'].value = account
form1['f_pwd'].value = password
browser.submit_form(form1)

link_one = browser.get_link(text = '期中網路教學意見調查')
browser.follow_link(link_one)
list = []
for l in browser.get_links(text = re.compile('填寫')): 
    list.append(l)
list.pop(0)
for li in list:
    browser.follow_link(li)
    form2 = browser.get_form(id = 'thisform')
    form2['Cos_Q1'].value = '1'
    browser.submit_form(form2)
    

예제 #23
0
class KadenzeClient(object):
    def __init__(self):
        self.conf = Settings()
        self.base_url = "https://www.kadenze.com"
        self.session = Session()
        self.browser = RoboBrowser(history=True, session=self.session, parser="lxml", allow_redirects=False)

    def execute_login(self):
        print("Signing in www.kadenze.com ...")
        self.browser.open(self.base_url)
        signup_form = self.browser.get_form(id="login_user")
        signup_form['user[email]'].value = self.conf.login
        signup_form['user[password]'].value = self.conf.password
        self.browser.session.headers["Referer"] = self.base_url
        self.browser.submit_form(signup_form)

    def list_courses(self):
        self.browser.open(self.base_url)
        response = self.browser.parsed()[0].text
        courses = helpers.get_courses_from_json(response)
        return courses

    def list_sessions(self, course):
        sessions_url = "/".join((self.base_url, "courses", course, "sessions"))
        self.browser.open(sessions_url)
        links = self.browser.get_links()
        sessions = helpers.get_sessions_from_links(course, links)
        return sessions

    def list_videos(self, url):
        self.browser.open(self.base_url + url)
        response = self.browser.select("#video_json")[0]["value"]
        videos = helpers.get_videos_from_json(response, self.conf.video_format)
        return videos

    def download_videos_per_session(self, course, session, session_videos):
        print("Parsing session: {0}".format(session))
        for i, video_url in enumerate(session_videos):
            filename = helpers.extract_filename(video_url)
            if i == 0:
                session_prefix = helpers.extract_session_prefix(filename)
                session_prefixed = session_prefix + "-" + session
                full_path = self.conf.path + "/" + course + "/" + session_prefixed
                os.makedirs(full_path, exist_ok=True)
            helpers.write_video(video_url, full_path, filename)

    def download_course_videos(self, course):
        sessions = self.list_sessions(course)
        videos = [self.list_videos(url) for url in sessions]
        videos_per_sessions = zip(sessions, videos)
        for session_data, session_videos in videos_per_sessions:
            session_data = session_data.replace("courses/", "").replace("sessions/", "")
            course, session = session_data.split("/")[-2], session_data.split("/")[-1]
            self.download_videos_per_session(course, session, session_videos)

    def download_all_courses_videos(self):
        self.execute_login()
        enrolled_courses = [helpers.format_course(course) for course in self.list_courses()]
        courses = set(self.conf.courses).intersection(enrolled_courses)
        for course in courses:
            print("Parsing course: {0}".format(course))
            self.download_course_videos(course)
예제 #24
0
forecast_url
weather_url
weather_url.format(CITY="Bengaluru", APIKEY=api_key)
q = weather_url.format(CITY="Bengaluru", APIKEY=api_key)
r = requests.get(q)
r
r.ok
r.json()
from robobrowser import RoboBrowser
br = RoboBrowser(parser="lxml")
br
br.open("http://pypi.org/")
br.response
br.response.ok
br.url
br.get_links()
br.get_link("Register)
br.get_link("Register")
br.get_link(text="Register")
br.get_link()
get_ipython().run_line_magic('pinfo', 'br.get_link')
br.get_link(text="Register")
br.get_link(text_re="Register")
br.get_forms()
br.get_form()
f = br.get_form()
f["q"]
f["q"] = "xml"
br.submit_form(f)
br.url
br.response.ok
예제 #25
0
import re
from robobrowser import RoboBrowser

browser = RoboBrowser()
browser.open("https://duckduckgo.com")
# Must find the proper id in the html
form = browser.get_form(id = "search_form_homepage")
form
form["q"].value = "python"
browser.submit_form(form)
links = browser.get_links()
for link in links:
	print(link)
예제 #26
0
from robobrowser import RoboBrowser
import os

GOOGLE_USER = os.getenv('INGRESS_GOOGLE_USER')
GOOGLE_PASS = os.getenv('INGRESS_GOOGLE_PASS')

s = requests.Session()
s.cookies = cookielib.LWPCookieJar()
# cookiejar = cookielib.LWPCookieJar()
# browser = mechanize.Browser()

browser = RoboBrowser(user_agent='TestBot', history=True, session=s)
# browser.set_cookiejar(cookiejar)

browser.open('http://ingress.com/intel')
for link in browser.get_links(url_regex='ServiceLogin'):
    browser.follow_link(link)
    browser.select_form(nr=0)
    browser.form['Email'] = GOOGLE_USER
    browser.form['Passwd'] = GOOGLE_PASS
    browser.submit()

    # req = mechanize.Request('http://www.ingress.com/rpc/dashboard.getGameScore', '{"method": "dashboard.getGameScore"}')
    s2 = requests.Session()
    s2.headers['method'] = 'dashboard.getGameScore'
    for cookie in s.cookies:
        if cookie.name == 'csrftoken':
            # req.add_header('X-CSRFToken', cookie.value)
            s2.headers['X-CSRFToken'] = cookie.value
    s.cookies.add_cookie_header(s2)
    browser = RoboBrowser(session=s2)
예제 #27
0
class ISServer:
    def __init__(self):
        global pw
        self.pw = pw
        self.startingpath = os.path.abspath(os.curdir)
        self.infusionsoftapp=self.getappname()
        self.baseurl = 'https://' + self.infusionsoftapp + '.infusionsoft.com/'
        self.infusionsoftAPIKey=self.getapikey()
        self.appurl = "https://" + self.infusionsoftapp + ".infusionsoft.com:443/api/xmlrpc"
        self.connection = xmlrpclib.ServerProxy(self.appurl)

    def getappname(self):
        return raw_input("Please enter appname:").strip('\n \t')
    def getapikey(self):
        global pw
        username = self.pw['username']
        password = self.pw['password']
        #Basically:
        #    #Add username and password to your global variables.
        self.browser = RoboBrowser(history=True)
        self.browser.open(self.baseurl)
        logform = self.browser.get_form()
        logform.fields['username'].value = username
        logform.fields['password'].value = password
        self.browser.submit_form(logform)
        self.browser.follow_link(self.browser.get_links()[1])
        self.browser.open(self.baseurl + 'app/miscSetting/itemWrapper?systemId=nav.admin&settingModuleName=Application&settingTabName=Application')
        pageSoup = BeautifulSoup(self.browser.response.content, 'html.parser')
        return pageSoup.findAll(id='Application_Encrypted_Key:_data')[0].text
    ########################################################
    ## Methods to get records from various tables
    ##
    ##
    def getMatchingRecords(self, tableName, criteria, desiredFields=None, orderedBy=None):
        """Search at table by criteria
        """
        return self.getAllRecords(tableName, searchCriteria=criteria, interestingData=desiredFields, orderedBy=orderedBy)
    def getTagCats(self):
        return self.getAllRecords("ContactGroupCategory")
    def getAllTags(self):
        return self.getAllRecords("ContactGroup")
    def getAllProductCats(self):
        return self.getAllRecords("ProductCategory")
    def getAllProducts(self):
        return self.getAllRecords("Product")
    def getAllRecords(self, tableName, interestingData=None, searchCriteria=None, orderedBy=None):
        if interestingData is None:
            interestingData = tables[tableName]
        if searchCriteria is None:
            searchCriteria={}
        if orderedBy is None:
            orderedBy = interestingData[0]
        records = []
        p=0
        while True:
            listOfDicts = self.connection.DataService.query(self.infusionsoftAPIKey, tableName, 1000, p, searchCriteria, interestingData, orderedBy, True)
            for each in listOfDicts:
                thisRecord={}
                for eachbit in interestingData:   # this should be records.append(zip(interestingData, each)) perhaps
                    if not each.has_key(eachbit):   # TODO: research THIS
                        each[eachbit]=None
                    thisRecord[eachbit] = each[eachbit]
                records.append(thisRecord)
            if not(len(listOfDicts)==1000):
                break
            p+=1
        return records
    def incrementlyGetRecords(self, tableName, interestingData=None, searchCriteria=None, orderedBy=None):
        if interestingData is None:
            interestingData = tables[tableName]
        if searchCriteria is None:
            searchCriteria={}
        if orderedBy is None:
            orderedBy = interestingData[0]
        records = []
        p=0
        while True:
            print tableName, p
            print "trying!"
            try:
                listOfDicts = self.connection.DataService.query(self.infusionsoftAPIKey, tableName, 1000, p, searchCriteria, interestingData, orderedBy, True)
            except Exception, e:
                print e ,p
            for each in listOfDicts:
                thisRecord={}
                for eachbit in interestingData:   # this should be records.append(zip(interestingData, each)) perhaps
                    if not each.has_key(eachbit):   # TODO: research THIS
                        each[eachbit]=None
                    thisRecord[eachbit] = each[eachbit]
                records.append(thisRecord)
            if not(len(listOfDicts)==1000):
                break
            p+=1
            if p%10==0:
                fname = tableName + "%010d" %(p) + ".csv"
                print 'writing', p, fname
                with open(fname, 'wb') as outfile:
                    thisWriter = csv.DictWriter(outfile, records[0])
                    thisWriter.writeheader()
                    thisWriter.writerows(records)
                records=[]
        fname = tableName + "%010d" %(p) + ".csv"
        print 'writing', p, fname
        with open(fname, 'wb') as outfile:
            thisWriter = csv.DictWriter(outfile, records[0])
            thisWriter.writeheader()
            thisWriter.writerows(records)
예제 #28
0
class tat:
    global pw

    def __init__(self, appname=None):
        self.startingpath = os.path.abspath(os.curdir)
        if not appname:
            self.appname = self.getappname()
        else:
            self.appname = appname
        self.apppath = os.path.join(self.startingpath, self.appname)
        if not os.path.exists(self.apppath):
            os.mkdir(self.apppath)
        os.chdir(self.apppath)
        self.mapping = {}
        self.mapping["Contact"] = -1
        self.mapping["Affiliate"] = -3
        self.mapping["ContactAction"] = -5
        self.mapping["Company"] = -6
        self.mapping["OrderItem"] = -9

        self.menu()

    def menu(self, context="initial"):
        if context is "initial":
            self.baseurl = "https://" + self.appname + ".infusionsoft.com/"
            self.apikey = self.getapikey()
            self.svr = ISServer.ISServer(self.appname, self.apikey)
            if not os.path.exists(self.apppath):
                os.mkdir(self.apppath)
            os.chdir(self.apppath)
            if not os.path.exists("files"):
                os.mkdir("files")
            os.chdir("files")
            self.usermenu = {}
            self.usermenu["downloadAPITables"] = "apit"
            self.usermenu["play"] = "play"
            self.usermenu["reports"] = "rpts"
        # for eachitem in self.usermenu.keys():
        #     print eachitem + ":\t" + self.usermenu[eachitem]
        # thisChoice = raw_input('please make a choice: ').strip(' \n\t')
        thisChoice = "play"
        if thisChoice == "apit":
            self.handleAPItables()
        elif thisChoice == "play":
            self.play()
        elif thisChoice == "rpts":
            self.downloadAllReports()
        else:
            self.inchandlefiles()

    def handlefiles(self):
        os.chdir(self.startingpath)
        if not os.path.exists("files"):
            os.mkdir("files")
        os.chdir("files")
        allfiles = self.svr.getAllRecords("FileBox")
        for eachfile in allfiles:
            downloadurl = self.baseurl + "Download?Id=" + str(eachfile["Id"])
            self.browser.open(downloadurl)
            fileoutpath = os.path.join(self.startingpath, "files", eachfile["ContactId"], eachfile["FileName"])
            if not os.path.exists(os.path.dirname(fileoutpath)):
                os.makedirs(fileoutpath)
            fout = open(fileoutpath, "wb")
            fout.write(self.browser.response.content)
            fout.close()

    def inchandleAPItables(self):
        apidata = {}
        self.customfields = self.svr.getAllRecords("DataFormField")
        for eachtable in ISServer.tables.keys():
            if eachtable not in [
                "LeadSourceExpense",
                "DataFormTab",
                "GroupAssign",
                "AffResource",
                "InvoiceItem",
                "UserGroup",
                "CProgram",
                "ActionSequence",
                "Template",
                "LeadSource",
                "Status",
                "Campaignee",
                "DataFormField",
                "OrderItem",
                "DataFormGroup",
                "ProductOptValue",
                "ContactGroup",
                "Company",
                "TicketStage",
                "ProductCategoryAssign",
                "ContactGroupAssign",
            ]:
                print "starting " + eachtable
                if eachtable not in self.mapping.keys():
                    self.mapping[eachtable] = 99
                fields = ISServer.tables[eachtable] + [
                    "_" + fld["Name"] for fld in self.customfields if fld["FormId"] is self.mapping[eachtable]
                ]
                self.svr.incrementlyGetRecords(eachtable, interestingData=fields)
                print "done writing " + eachtable
            else:
                print "already completed " + eachtable
        self.apidata = apidata

    def inchandleAPItable(self, tablename):
        self.customfields = self.svr.getAllRecords("DataFormField")
        if tablename not in self.mapping.keys():
            self.mapping[tablename] = 99
        fields = ISServer.tables[tablename] + [
            "_" + fld["Name"] for fld in self.customfields if fld["FormId"] is self.mapping[tablename]
        ]
        self.svr.incrementlyGetRecords(tablename, interestingData=fields)
        print "done writing " + tablename

    def inchandlefiles(self):
        os.chdir(self.startingpath)
        self.svr.incgetfiles(self.browser)

    def downloadContact0files(self, numberofmostrecentfilestodownload):
        thesefiles = self.svr.getAllRecords("FileBox", searchCriteria={"ContactId": 0})
        for eachfile in thesefiles[-int(numberofmostrecentfilestodownload) :]:
            print "doing " + str(eachfile)
            self.svr.getfile(self.browser, eachfile)

    def play(self):
        print "she's all yours captain!"

    def downloadAReport(self, reportname):
        self.browser.open(self.baseurl + "Reports/exportResults.jsp?reportClass=" + reportname)
        reportForm = [eachform for eachform in self.browser.get_forms() if eachform.action == "qbExport.jsp"]
        if len(reportForm) > 0:
            self.browser.submit_form(reportForm[0], submit=reportForm[0].submit_fields["process"])
            with open(reportname + ".csv", "wb") as outfile:
                outfile.write(self.browser.response.content)
        else:
            print "no " + reportname

    def downloadAllReports(self):
        for reportname in [
            "AffiliateActivitySummary",
            "AffiliateLedger",
            "AffiliateRedirectActivity",
            "AffiliateReferral",
            "AffPayout",
            "AllOrders",
            "AllSales",
            "AllSalesItemized",
            "ARAgingReport",
            "CampaigneeBasic",
            "CampaigneeByDay",
            "CampaignProductConversion",
            "ClickThroughPercentage",
            "ClickThroughPercentageByEmail",
            "ContactDistributed",
            "CProgramRevenueSummary",
            "CreditCard",
            "CreditsIssued",
            "CustomerLifetimeValue",
            "DailyPayments",
            "DailyReceivables",
            "DailySalesTotals",
            "DashboardCampaign",
            "DashboardEmail",
            "DashboardLeads",
            "DashboardOrders",
            "DashboardUsers",
            "DigitalProductKey",
            "EmailBatchSearch",
            "EmailBroadcastConversionReport",
            "EmailConversion",
            "EmailSentSearch",
            "FailedCharge",
            "FaxBatchSearch",
            "FollowUpSequenceConversionReport",
            "FunnelFlowRecipient",
            "FunnelFlowRecipientWaiting",
            "FunnelGoalAchieved",
            "FunnelQueuedFlowItem",
            "FunnelUniqueContacts",
            "GroupAdds",
            "HabeasDetail",
            "InvoiceNetIncome",
            "LeadSourceConversion",
            "LeadSourceIncome",
            "LeadSourceROI",
            "LeadSourceROIByCategory",
            "MonthlyPayments",
            "MonthlyReceivables",
            "MonthlySalesTotals",
            "MonthlySalesTotalsByProduct",
            "OptOutSearch",
            "PaymentsReport",
            "PieceResponse",
            "ProductNetIncome",
            "Receivables",
            "RevenueForecastReport",
            "TaskSearch",
            "VoiceBatchSearch",
            "VoiceOptOutSearch",
            "WebformActivitySummary",
            "WebFormTracking",
        ]:
            self.downloadAReport(reportname)

    def getFilePath(self):
        return tkFileDialog.askopenfilename()

    def getFolderPath(self):
        return tkFileDialog.askdirectory()

    def getappname(self):
        return raw_input("Please enter appname:").strip("\n \t")

    def getapikey(self):
        global pw
        username = pw["username"]
        password = pw["password"]
        # Basically:
        #    #Add username and password to your global variables.
        self.browser = RoboBrowser(history=True)
        self.browser.open(self.baseurl)
        logform = self.browser.get_form()
        logform.fields["username"].value = username
        logform.fields["password"].value = password
        self.browser.submit_form(logform)
        self.browser.follow_link(self.browser.get_links()[1])
        self.browser.open(
            self.baseurl
            + "app/miscSetting/itemWrapper?systemId=nav.admin&settingModuleName=Application&settingTabName=Application"
        )
        pageSoup = BeautifulSoup(self.browser.response.content, "html.parser")
        return pageSoup.findAll(id="Application_Encrypted_Key:_data")[0].text

    def handleAPItables(self):
        apidata = {}
        self.customfields = self.svr.getAllRecords("DataFormField")
        for eachtable in ISServer.tables.keys():
            print "starting " + eachtable
            if eachtable not in self.mapping.keys():
                self.mapping[eachtable] = 99
            fields = ISServer.tables[eachtable] + [
                "_" + fld["Name"] for fld in self.customfields if fld["FormId"] is self.mapping[eachtable]
            ]
            apidata[eachtable] = self.svr.getAllRecords(eachtable, interestingData=fields)
            with open(eachtable + ".csv", "wb") as outfile:
                writer = csv.DictWriter(outfile, fields)
                writer.writeheader()
                writer.writerows(apidata[eachtable])
            print "done writing " + eachtable
        self.apidata = apidata

    def handlewebforms(self):
        # for eachid
        # webformsubmissionpath="https://" + self.appname + ".infusionsoft.com/app/webformSubmission/contactTabDetails?customFormWebResultId=" + str(x)
        pass

    def creditCardsToCSV(self):
        ccs = self.svr.getAllRecords(
            "CreditCard",
            interestingData=[
                "Id",
                "ContactId",
                "CardType",
                "Last4",
                "ExpirationMonth",
                "ExpirationYear",
                "Email",
                "StartDateMonth",
                "StartDateYear",
                "Status",
            ],
        )
        os.chdir(self.startingpath)
        if not os.path.exists("pyDatas"):
            os.mkdir("pyDatas")
        os.chdir("pyDatas")
        with open("ccs.csv", "wb") as outfile:
            thiswriter = csv.DictWriter(outfile, ccs[0].keys())
            thiswriter.writeheader()
            thiswriter.writerows(ccs)
        print "File written to " + str(os.path.abspath(os.curdir))
        os.chdir(self.startingpath)

    def contactsToCSV(self):
        os.chdir(self.startingpath)
        self.customfields = self.svr.getAllRecords("DataFormField")
        fields = ISServer.tables["Contact"] + ["_" + fld["Name"] for fld in self.customfields if fld["FormId"] == -1]
        cons = self.svr.getAllRecords("Contact", interestingData=fields)
        if not os.path.exists("pyDatas"):
            os.mkdir("pyDatas")
        os.chdir("pyDatas")
        with open("contacts.csv", "wb") as outfile:
            thiswriter = csv.DictWriter(outfile, cons[0].keys())
            thiswriter.writeheader()
            thiswriter.writerows(cons)
예제 #29
0
# pip3 install robobrowser
from robobrowser import RoboBrowser

browser = RoboBrowser()
browser.open('https://cdn.hackerrank.com/hackerrank/static/contests/capture-the-flag/infinite/qds.html')
history = ['https://cdn.hackerrank.com/hackerrank/static/contests/capture-the-flag/infinite/qds.html']
font = browser.find_all('font')
links = browser.get_links()

while len(font) < 50000 and len(links) > 0:
    for link in links:
        if link not in history:
            browser.follow_link(link)
            font.extend(browser.find_all())
            links.extend(browser.get_links())
            history.append(link)
        links.remove(link)

print(font)
class fullexporter():
    global pw
    def __init__(self):
        self.startingpath = os.path.abspath(os.curdir)
        self.appname=self.getappname()
        self.mapping={}
        self.mapping['Contact']=-1
        self.mapping['Affiliate']=-3
        self.mapping['ContactAction']=-5
        self.mapping['Company']=-6
        self.mapping['OrderItem']=-9

        self.menu()

    def menu(self, context="initial"):
        if context is "initial":
            self.baseurl = 'https://' + self.appname + '.infusionsoft.com/'
            self.apikey=self.getapikey()
            self.svr = ISServer.ISServer(self.appname, self.apikey)
            self.apppath = os.path.join(self.startingpath, self.appname)
            if not os.path.exists(self.apppath):
                os.mkdir(self.apppath)
            os.chdir(self.apppath)
            if not os.path.exists('files'):
                os.mkdir('files')
            os.chdir('files')
            self.usermenu={}
            self.usermenu['downloadAPITables'] = 'apit'
            self.usermenu['play'] = 'play'
            self.usermenu['reports'] = 'rpts'
        for eachitem in self.usermenu.keys():
            print eachitem + ":\t" + self.usermenu[eachitem]
        thisChoice = raw_input('please make a choice: ').strip(' \n\t')
        if thisChoice == 'apit':
            self.handleAPItables()
        elif thisChoice == 'play':
            self.play()
        elif thisChoice == 'rpts':
            self.downloadAllReports()
        else:
            self.menu()

    def handlefiles(self):
        os.chdir(self.startingpath)
        if not os.path.exists('files'):
            os.mkdir('files')
        os.chdir('files')
        allfiles = self.svr.getAllRecords('FileBox')
        for eachfile in allfiles:
            downloadurl = self.baseurl+"Download?Id="+str(eachfile['Id'])
            self.browser.open(downloadurl)
            fileoutpath = os.path.join(self.startingpath, 'files', eachfile['ContactId'], eachfile['FileName'])
            if not os.path.exists(os.path.dirname(fileoutpath)):
                os.makedirs(fileoutpath)
            fout = open(fileoutpath, 'wb')
            fout.write(self.browser.response.content)
            fout.close()


    def play(self):
        print "she's all yours captain!"

    def downloadAReport(self, reportname):
        self.browser.open(self.baseurl + "Reports/exportResults.jsp?reportClass=" + reportname)
        reportForm = [eachform for eachform in self.browser.get_forms() if eachform.action == 'qbExport.jsp']
        if len(reportForm) > 0:
            self.browser.submit_form(reportForm[0], submit=reportForm[0].submit_fields['process'])
            with open(reportname+".csv", 'wb') as outfile:
                outfile.write(self.browser.response.content)
        else:
            print "no " + reportname

    def downloadAllReports(self):
        for reportname in [ "AffiliateActivitySummary", "AffiliateLedger", "AffiliateRedirectActivity", "AffiliateReferral", "AffPayout", "AllOrders", "AllSales", "AllSalesItemized", "ARAgingReport", "CampaigneeBasic", "CampaigneeByDay", "CampaignProductConversion", "ClickThroughPercentage", "ClickThroughPercentageByEmail", "ContactDistributed", "CProgramRevenueSummary", "CreditCard", "CreditsIssued", "CustomerLifetimeValue", "DailyPayments", "DailyReceivables", "DailySalesTotals", "DashboardCampaign", "DashboardEmail", "DashboardLeads", "DashboardOrders", "DashboardUsers", "DigitalProductKey", "EmailBatchSearch", "EmailBroadcastConversionReport", "EmailConversion", "EmailSentSearch", "FailedCharge", "FaxBatchSearch", "FollowUpSequenceConversionReport", "FunnelFlowRecipient", "FunnelFlowRecipientWaiting", "FunnelGoalAchieved", "FunnelQueuedFlowItem", "FunnelUniqueContacts", "GroupAdds", "HabeasDetail", "InvoiceNetIncome", "LeadSourceConversion", "LeadSourceIncome", "LeadSourceROI", "LeadSourceROIByCategory", "MonthlyPayments", "MonthlyReceivables", "MonthlySalesTotals", "MonthlySalesTotalsByProduct", "OptOutSearch", "PaymentsReport", "PieceResponse", "ProductNetIncome", "Receivables", "RevenueForecastReport", "TaskSearch", "VoiceBatchSearch", "VoiceOptOutSearch", "WebformActivitySummary", "WebFormTracking" ]:
            self.downloadAReport(reportname)

    def getFilePath(self):
        return tkFileDialog.askopenfilename()

    def getFolderPath(self):
        return tkFileDialog.askdirectory()

    def getappname(self):
        return raw_input("Please enter appname:").strip('\n \t')

    def getapikey(self):
        global pw
        username = pw['username']
        password = pw['password']
        #Basically:
        #    #Add username and password to your global variables.
        self.browser = RoboBrowser(history=True)
        self.browser.open(self.baseurl)
        logform = self.browser.get_form()
        logform.fields['username'].value = username
        logform.fields['password'].value = password
        self.browser.submit_form(logform)
        self.browser.follow_link(self.browser.get_links()[1])
        self.browser.open(self.baseurl + 'app/miscSetting/itemWrapper?systemId=nav.admin&settingModuleName=Application&settingTabName=Application')
        pageSoup = BeautifulSoup(self.browser.response.content, 'html.parser')
        return pageSoup.findAll(id='Application_Encrypted_Key:_data')[0].text

    def handleAPItables(self):
        apidata={}
        self.customfields=self.svr.getAllRecords('DataFormField')
        for eachtable in ISServer.tables.keys():
            print "starting " + eachtable
            if eachtable not in self.mapping.keys():
                self.mapping[eachtable]=99
            fields = ISServer.tables[eachtable] +  ['_'+fld['Name'] for fld in self.customfields if fld['FormId'] is self.mapping[eachtable]]
            apidata[eachtable] =  self.svr.getAllRecords(eachtable, interestingData=fields)
            with open(eachtable+".csv", 'wb') as outfile:
                writer=csv.DictWriter(outfile, fields)
                writer.writeheader()
                writer.writerows(apidata[eachtable])
            print "done writing " + eachtable
        self.apidata = apidata



    def handlewebforms(self):
        # for eachid
        # webformsubmissionpath="https://" + self.appname + ".infusionsoft.com/app/webformSubmission/contactTabDetails?customFormWebResultId=" + str(x)
        pass
예제 #31
0
    def scrap_process(self, storage):

        # You can iterate over ids, or get list of objects
        # from any API, or iterate throught pages of any site
        # Do not forget to skip already gathered data
        # Here is an example for you
        #     'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
        #     'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        #     'Accept-Language' : 'ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4',
        #     'Accept-Encoding' : 'gzip, deflate, sdch'}
        def auth():
            LoginForm = q.get_form(id="aspnetForm")

            LoginForm[
                'ctl00$ctl00$ModalLogin1$LoginTextBox'].value = 'avtostrada'
            LoginForm[
                'ctl00$ctl00$ModalLogin1$PasswordTextBox'].value = '5ed8f5'

            LoginForm.add_field(
                robobrowser.forms.fields.Input(
                    '\<input name="__EVENTARGUMENT" value="" \/\>'))
            LoginForm.add_field(
                robobrowser.forms.fields.Input(
                    '\<input name="__EVENTTARGET" value="ctl00$ctl00$ModalLogin1$ButtonFilter" \/\>'
                ))

            q.submit_form(
                LoginForm,
                submit=LoginForm['ctl00$ctl00$ModalLogin1$ButtonFilter'])

        def get_download_url(link):
            d = link.get('href')
            return 'http://brokenstone.ru/' + d
            #content=q.session.get(durl)

        def save_download(durl):
            file1 = q.session.get(durl)
            #if file1.headers['Content-Type'] == 'application/octet-stream':
            nameFile = (file1.headers['Content-Disposition']).split('=')
            xls_file = './{0}'.format(nameFile[1])
            logger.info(xls_file)
            with open(xls_file, "wb") as output:
                output.write(file1.content)
            return nameFile[1]

            #else:
            #    logger.error(u'Какая то хрень')

        url = storage
        logger.info("Начинаем загрузку")
        q = RoboBrowser()
        q.open(url)
        response = q.state

        if not response.response.ok:
            logger.error(response.response.text)
            # then continue process, or retry, or fix your code
        else:
            auth()
            a = q.get_links('Поставки по жд за')
            durls = list(map(get_download_url, a))
            logger.info(durls)
            #d=a[1].get('href')
            #durl='http://brokenstone.ru/'+ d
            #content=map(q.session.get, durls)

            downloadFales = list(map(save_download, durls))

            logger.info(downloadFales)

            dataframe = pd.concat(map(pd.read_excel, downloadFales))

            dataframe.to_pickle('data.pkl')

            logger.info(u'Данные сохранены')

            # Note: here json can be used as response.json
            #data = q.response

            # save scrapped objects here
            # you can save url to identify already scrapped objects
            #storage.write_data([url + '\t' + data.replace('\n', '')])


#%%
#Scrapper.scrap_process(Scrapper,'http://brokenstone.ru/supplyfileexport.aspx')
예제 #32
0
#procompiledRegex = re.compile(regex, re.MULTILINE | re.DOTALL)

# log in to FIT
browser.open('https://cas.fit.vutbr.cz')

form = browser.get_form(action='/cosign.cgi')
form['login'].value = config["login"]
form['password'].value = config["password"]
browser.submit_form(form)

# open video server
browser.open('https://video1.fit.vutbr.cz/av/records-categ.php?id=1')

# get links to semesters
links = browser.get_links(class_='link')
del links[:1]
# regex - get link names
# ^<a class="link" .+id=\d{1,4}">(.+)<\/a>
# but use of BeautifullSoup is advised instead

# BeatifulSoup(html_doc)
# titleName = soup.title.name

print("\nSemesters")
i = 0
for link in links:
    print("{} - {}".format(i, link.text))
    i += 1

# select semester