def set_category(self, category):
if category:
select = self.parser.select(self.document.getroot(),
'select#issue_category_id', 1)
for option in select.findall('option'):
if option.text and option.text.strip() == category:
self.browser['issue[category_id]'] = [
option.attrib['value']
]
return
value = None
if len(self.document.xpath('//a[@title="New category"]')) > 0:
value = self.browser.create_category(
self.get_project_name(), category,
self.get_authenticity_token())
if value:
control = self.browser.find_control('issue[category_id]')
mechanize.Item(control, {'name': category, 'value': value})
self.browser['issue[category_id]'] = [value]
else:
self.logger.warning('Category "%s" not found' % category)
else:
try:
self.browser['issue[category_id]'] = ['']
except mechanize.ControlNotFoundError:
self.logger.warning('Category item not found')
def __login(self):
""" Private member method to login into the website using login details
stored in configuration file.Returns True on success and False on
failure.Logs failure message.
"""
#Reinstate browser state
br = self.data['BROWSER_STATE']
#select form with user and password field
br.select_form(predicate=self.__form_with_fields("user", "password"))
br['user'] = self.config['COMMON.USER']
br['password'] = self.config['COMMON.PASSWORD']
#update site and zone if specified in config file
if 'COMMON.SITE' in self.config and 'COMMON.ZONE' in self.config:
try:
br.form.set_all_readonly(False)
br.form["shortsite"] = self.config['COMMON.SITE']
zone1 = br.form.find_control("zone")
mechanize.Item(zone1, {"contents": \
"lee10","value": "lee10"})
br.form["zone"] = ["lee10"]
except ControlNotFoundError, err:
logger = logging.getLogger(__name__)
logger.info(str(err))
def donate(donation_amount, tid):
logging.info('Donating ${} for tid {}'.format(donation_amount, tid))
donation_amount = str(donation_amount)
br = mechanize.Browser(factory=mechanize.RobustFactory())
br.addheaders = [(
'User-agent',
' Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'
)]
br.set_handle_robots(False)
br.open('https://www.propublica.org/donate/')
br.select_form('payment_form')
prices = br.find_control('UnitPrice1')
custom_price = mechanize.Item(prices, {
'contents': donation_amount,
'value': donation_amount
})
custom_price.selected = True
br.form['c_amount'] = donation_amount
br.form['BillingFirstName'] = billing['first']
br.form['BillingLastName'] = billing['last']
if billing['email'].endswith('+{}@gmail.com'):
br.form['BillingEmail'] = billing['email'].format(tid)
else:
br.form['BillingEmail'] = billing['email'].format(tid)
br.form['CardNumber'] = billing['cc']
br.form['ExpirationMonth'] = [billing['exp_mo']]
br.form['ExpirationYear'] = [billing['exp_yr']]
br.form['Cvv2'] = billing['cvv']
br.form['BillingAddress1'] = billing['street']
br.form['BillingCity'] = billing['city']
br.form['BillingStateProvince'] = [billing['state']]
br.form['BillingPostalCode'] = billing['zip']
br.form['BillingCountryCode'] = [billing['country']]
response = br.submit()
if 'Thank You.' in response.read():
logging.info('Donation success: ${} for tid {}'.format(
donation_amount, tid))
return True
else:
logging.warning(response.read())
logging.warning('Donation failed: ${} for tid {}'.format(
donation_amount, tid))
return False
def _post_article(self, browser, post_config):
import re, random, mechanize
pre_post_resp = browser.open(post_config['post_url'])
browser.select_form(post_config['form_name'])
browser[post_config['form_title']] = post_config['title']
browser[post_config['form_content']] = post_config['content']
if 'cat_id' in post_config['form_additional_data']:
cat_id = str(post_config['form_additional_data']['cat_id'])
else:
#fill form
pattern = re.compile('BokeCat_ID\[0\]=\[(.*)\]')
matchs = pattern.search( pre_post_resp.read() )
cat_ids = [str(id) for id in eval( matchs.groups()[0])]
cat_id = random.choice( cat_ids )
#create control for select cat_id
cat_item = mechanize.Item( browser.form.find_control('Catid'), { 'value': cat_id } )
browser['Catid'] = [cat_id]
# print cat_ids
# print form.find_control('Catid')
if 'sub_cat_id' in post_config['form_additional_data']:
browser['sCatID'] = [str(post_config['form_additional_data']['sub_cat_id'])]
else:
#choice for sCatID
items = browser.form.find_control('sCatID').get_items()
values = [ item.attrs['value'] for item in items ]
if -1 in values:
values.remove( -1 )
#print values
browser['sCatID'] = [ random.choice( values ) ]
if 'sub_type_id' in post_config['form_additional_data']:
browser['sType'] = [str(post_config['form_additional_data']['sub_type_id'])]
else:
browser['sType'] = ['0']
self._sleep('post')
post_resp = browser.submit()
# print post_resp.read()
return post_resp.code == 200
def post_ad():
# Step 1
br.open('http://auto.ria.ua/add_auto.html')
br.select_form(nr=1)
br['stateId'] = ['10-10'] # Регион Киев
# Тип транспорта
br.form.find_control("categoryId").readonly = False
br['categoryId'] = '1' # Легковые
# br['categoryId'] = '2' # Мото
# br['categoryId'] = '3' # Водный транспорт
# br['categoryId'] = '4' # Спецтехника
# br['categoryId'] = '5' # Прицепы
# br['categoryId'] = '6' # Грузовики
# br['categoryId'] = '7' # Автобусы
# br['categoryId'] = '8' # Автодома
# br['categoryId'] = '9' # Воздушный транспорт
br['markaId'] = ['48'] # Марка
mechanize.Item(br.form.find_control('modelId'), attrs={'value': '422'})
br['modelId'] = ['422'] # Модель
# br['version'] = 'Wtf' # Версия
# br['VIN'] = '123456' # VIN
# Коробка передач
br.form.find_control('gearId').set_value_by_label(['Автомат'])
# br.form.find_control('gearId').set_value_by_label(['Адаптивная'])
# br.form.find_control('gearId').set_value_by_label(['Вариатор'])
# br.form.find_control('gearId').set_value_by_label(['Ручная / Механика'])
# br.form.find_control('gearId').set_value_by_label(['Типтроник'])
# Тип привода
br.form.find_control('driveId').get(label='Задний').selected = True
# br.form.find_control('driveId').set_value_by_label(['Передний'])
# br.form.find_control('driveId').set_value_by_label(['Полный'])
br['yers'] = ['2010'] # Год выпуска
br['race'] = '10' # Пробег
br['price'] = '20000' # Цена
br.submit()
# Step 2
br.select_form(nr=1)
br.submit()
for item in form.find_control("cheeses").items:
print item.name
# To remove items from a list control, remove it from .items:
cheeses = form.find_control("cheeses")
curd = cheeses.get("curd")
del cheeses.items[cheeses.items.index(curd)]
# To add items to a list container, instantiate an Item with its control
# and attributes:
# Note that you are responsible for getting the attributes correct here,
# and these are not quite identical to the original HTML, due to
# defaulting rules and a few special attributes (e.g. Items that represent
# OPTIONs have a special "contents" key in their .attrs dict). In future
# there will be an explicitly supported way of using the parsing logic to
# add items and controls from HTML strings without knowing these details.
mechanize.Item(cheeses, {"contents": "mascarpone", "value": "mascarpone"})
# You can specify list items by label using set/get_value_by_label() and
# the label argument of the .get() method. Sometimes labels are easier to
# maintain than names, sometimes the other way around.
form.set_value_by_label(["Mozzarella", "Caerphilly"], "cheeses")
# Which items are present, selected, and successful?
# is the "parmesan" item of the "cheeses" control successful (selected
# and not disabled)?
print "parmesan" in form["cheeses"]
# is the "parmesan" item of the "cheeses" control selected?
print "parmesan" in [
item.name for item in form.find_control("cheeses").items if item.selected
]
# does cheeses control have a "caerphilly" item?
br.submit()
#Select the first form on the second page.
br.select_form(nr=1)
#Make the controls in the form not read-only; we need to back-fill some values.
br.form.set_all_readonly(False)
#This edits the year selection to be 2015.
br.form['Y'] = year
#Select the second form on the second page.
br.select_form(nr=2)
#The control for DC's "Counties" is empty, so create the right object in it.
item = mechanize.Item(br.form.find_control(name='C'), {'contents': '001000', 'value': '001000', 'label': '001000'})
br.form['C'] = ['001000']
# Store the html of the page to parse.
response2 = br.submit()
site3 = response2.read()
# The data we want is uniquely in a table with a border of 1
soup = BeautifulSoup(site3, 'html.parser')
table = soup.find("table", border=1)
# Scrape the relevant figures from the table and store as a record.
try:
for each in table.findAll('tr')[4:8]:
col = each.findAll('td')
bldgtype = col[1].string
