def get_claim_code_from_server():
time.sleep(5)
browser = Browser('phantomjs', service_args=['--ignore-ssl-errors=true'])
browser.visit(ROOT_ADDRESS + "/merchant-login")
browser.fill_form({"email": USER_NAME, "password": PASSWORD})
browser.find_by_id("loginButton")[0].click()
time.sleep(1)
browser.visit(ROOT_ADDRESS + "/api-tokens")
code = get_code_from_page(browser, "thiscodewillneverbevalid")
gopath = os.environ['GOPATH']
tempath = gopath + "/temp"
if not os.path.exists(tempath):
os.makedirs(tempath)
write_code_to_file(code, tempath + "/retrievecode.txt")
print(code)
time.sleep(10)
browser.reload()
code = get_code_from_page(browser, code)
write_code_to_file(code, tempath + "/paircode.txt")
print(code)
time.sleep(10)
browser.reload()
code = get_code_from_page(browser, code)
write_code_to_file(code, tempath + "/invoicecode.txt")
print(code)
return code
def get_claim_code_from_server():
browser = Browser('phantomjs', service_args=['--ignore-ssl-errors=true'])
browser.visit(ROOT_ADDRESS + "/merchant-login")
browser.fill_form({"email": USER_NAME, "password": PASSWORD})
browser.find_by_id("loginButton")[0].click()
time.sleep(5)
browser.visit(ROOT_ADDRESS + "/api-tokens")
browser.find_by_css(".token-access-new-button").find_by_css(".btn").find_by_css(".icon-plus")[0].click()
browser.find_by_id("token-new-form").find_by_css(".btn")[0].click()
return browser.find_by_css(".token-claimcode")[0].html
def main():
# Open new firefox browser window, login to indeed.com
br = Browser('firefox')
br.visit(LOGIN_URL)
br.fill_form({'email': INDEED_USERNAME, 'password': INDEED_PASSWORD})
br.find_by_css('.input_submit').first.click()
resume_links = []
resume_index = 0
while True:
br.visit(RESUMES_URL + '&start=%d' % resume_index)
links = br.find_by_css('.app_link')
if len(links) > 0:
for link in links:
# Rewrite this. The Browser object cannot find the href value for some reason...
resume_links.append('http://www.indeed.com' + re.search(
r'href=[\'"]?([^\'" >]+)', link.outer_html).group(1))
resume_index += 50
else:
resume_index = 1
break
print '[+] Resume Links Found: %d' % len(resume_links)
# Load the 1,000 most commonly used words into common_words
sys.stdout.write('[+] Loading Common Word List [ ]')
common_words = gen_comword_list()
sys.stdout.flush()
sys.stdout.write('\r[+] Loading Common Word List [DONE]\n')
for resume in resume_links:
br.visit(resume)
try:
#print '[+] Scanning %s\'s Resume...' % br.find_by_id("resume-contact").text
resume_words = parse_resume_words(br.html, common_words)
#print '\tUnique Words Found: %d' % len(resume_words)
for word in resume_words:
if not word in KEYWORDS:
KEYWORDS[word] = 1
else:
KEYWORDS[word] += 1
except:
pass
sys.stdout.write('[+] Sanning Resumes: {0:.0f}%\r'.format(
float(resume_index) / len(resume_links) * 100))
sys.stdout.flush()
resume_index += 1
print '\n[+] Total Unique Keywords Found: %d' % len(KEYWORDS.keys())
print '============\nTOP 20 WORDS\n============'
for pair in sorted(KEYWORDS.items(),
key=operator.itemgetter(1),
reverse=True)[:20]:
print '%s: %d' % (pair[0], pair[1])
def get_claim_code_from_server():
browser = Browser('phantomjs', service_args=['--ignore-ssl-errors=true'])
browser.visit(ROOT_ADDRESS + "/merchant-login")
browser.fill_form({"email": USER_NAME, "password": PASSWORD})
browser.find_by_id("loginButton")[0].click()
time.sleep(5)
browser.visit(ROOT_ADDRESS + "/api-tokens")
browser.find_by_css(".token-access-new-button").find_by_css(
".btn").find_by_css(".icon-plus")[0].click()
browser.find_by_id("token-new-form").find_by_css(".btn")[0].click()
return browser.find_by_css(".token-claimcode")[0].html
def testLoginWithWrongCredentialsFails():
browser = Browser()
browser.visit('http://127.0.0.1:8000')
browser.fill_form({'username': '******'})
browser.fill_form({'password': '******'})
browser.find_by_value('Anmelden').click()
if browser.is_text_present('Your login credentials are incorrect'):
print "Test passed"
else:
print "Test failed"
browser.quit()
def main():
# Open new firefox browser window, login to indeed.com
br = Browser('firefox')
br.visit(LOGIN_URL)
br.fill_form({'email':INDEED_USERNAME,'password':INDEED_PASSWORD})
br.find_by_css('.input_submit').first.click()
resume_links = []
resume_index = 0
while True:
br.visit(RESUMES_URL + '&start=%d' % resume_index)
links = br.find_by_css('.app_link')
if len(links) > 0:
for link in links:
# Rewrite this. The Browser object cannot find the href value for some reason...
resume_links.append('http://www.indeed.com' + re.search(r'href=[\'"]?([^\'" >]+)', link.outer_html).group(1))
resume_index += 50
else:
resume_index = 1
break
print '[+] Resume Links Found: %d' % len(resume_links)
# Load the 1,000 most commonly used words into common_words
sys.stdout.write('[+] Loading Common Word List [ ]')
common_words = gen_comword_list()
sys.stdout.flush()
sys.stdout.write('\r[+] Loading Common Word List [DONE]\n')
for resume in resume_links:
br.visit(resume)
try:
#print '[+] Scanning %s\'s Resume...' % br.find_by_id("resume-contact").text
resume_words = parse_resume_words(br.html, common_words)
#print '\tUnique Words Found: %d' % len(resume_words)
for word in resume_words:
if not word in KEYWORDS:
KEYWORDS[word] = 1
else:
KEYWORDS[word] += 1
except:
pass
sys.stdout.write('[+] Sanning Resumes: {0:.0f}%\r'.format(float(resume_index)/len(resume_links) * 100))
sys.stdout.flush()
resume_index += 1
print '\n[+] Total Unique Keywords Found: %d' % len(KEYWORDS.keys())
print '============\nTOP 20 WORDS\n============'
for pair in sorted(KEYWORDS.items(), key=operator.itemgetter(1), reverse=True)[:20]:
print '%s: %d' % (pair[0], pair[1])
def get_url_code(auth_url, username, password, login='******'):
b = Browser(driver_name='chrome')
b.visit(auth_url)
b.click_link_by_partial_href("/en/login")
if login == 'facebook':
b.click_link_by_partial_href("https://www.facebook.com")
b.fill_form({'email': username, 'pass': password})
b.click_link_by_id('loginbutton')
elif login == 'spotify':
b.fill_form({'username': username, 'password': password})
loginbutton = b.find_by_text('Log In')[0]
loginbutton.click()
b.visit(auth_url)
codeurl = b.url
code = codeurl.split("?code=")[1].split('&')[0]
b.quit()
return code
class SplinterTestCase(LiveServerTestCase):
username = '******'
email = '*****@*****.**'
password = '******'
is_anonymous = True
is_staff = False
is_logged_in = True
def setUp(self):
settings.DEBUG = True
super(SplinterTestCase, self).setUp()
self.user = None
self.base_url = URL(self.live_server_url)
self.browser = Browser(SPLINTER_WEBDRIVER)
if self.is_anonymous and not self.is_staff:
return
self.user = factories.UserFactory(
username=self.username,
email=self.email,
password=self.password,
is_staff=self.is_staff,
)
if self.is_logged_in:
self.goto(reverse('admin:index'))
self.browser.fill_form({
'username': self.username,
'password': self.password,
})
self.browser.find_by_css("input[type='submit']").first.click()
self.assertIn('Log out', self.browser.html)
def tearDown(self):
super(SplinterTestCase, self).tearDown()
self.browser.quit()
def goto(self, path):
url = self.base_url.path(path)
return self.browser.visit(url.as_string())
class SplinterTestCase(LiveServerTestCase):
username = '******'
email = '*****@*****.**'
password = '******'
is_anonymous = True
is_staff = False
is_logged_in = True
def setUp(self):
settings.DEBUG = True
super(SplinterTestCase, self).setUp()
self.user = None
self.base_url = URL(self.live_server_url)
self.browser = Browser(SPLINTER_WEBDRIVER)
if self.is_anonymous and not self.is_staff:
return
self.user = factories.UserFactory(
username=self.username,
email=self.email,
password=self.password,
is_staff=self.is_staff,
)
if self.is_logged_in:
self.goto(reverse('admin:index'))
self.browser.fill_form({
'username': self.username,
'password': self.password,
})
self.browser.find_by_css("input[type='submit']").first.click()
self.assertIn('Log out', self.browser.html)
def tearDown(self):
super(SplinterTestCase, self).tearDown()
self.browser.quit()
def goto(self, path):
url = self.base_url.path(path)
return self.browser.visit(url.as_string())
def filloutleads(contacturl, appurl):
'''Fills out lead forms specified in geturls()'''
#TODO: see if we can get addendumDict inside appDict.
formDict = dictBuilder(contacturl)
appDict = dictBuilder(appurl)
addendumDict = {
'ctlTypedSignature': 'test',
'ctlDateSigned': today.strftime('%m/%d/%Y')
}
browser = Browser()
browser.visit(contacturl)
browser.fill_form(formDict)
browser.find_by_text('Send').click()
browser.visit(appurl)
try:
browser.fill_form(appDict)
except:
# print('Failed to fill out credit app')
pass
page = requests.get(appurl)
soup = BeautifulSoup(page.text, 'lxml')
for menu in soup.findAll('select'):
if menu.has_attr('required'):
if 'CoApplicant' in str(menu.get('id')):
pass
elif 'State' in str(menu.get('id')):
browser.select(menu.get('name'), 'AL')
elif 'Years' in str(menu.get('id')):
browser.select(menu.get('name'), '1')
elif 'Month' in str(menu.get('id')):
browser.select(menu.get('name'), '1')
browser.fill_form(addendumDict)
browser.find_by_xpath(
'''/html/body/section/form/div/fieldset[7]/div[1]/div[4]/div/div/div/label/span'''
).click()
browser.find_by_text('Submit Your Application').click()
browser.quit()
class Session:
def __init__(self, browser, user):
self.browser = Browser(browser)
self.browser.visit('http://jizdenky.studentagency.cz/')
self.browser.fill_form({'passwordAccountCode': user['login'],
'password': user['password']})
self.browser.execute_script('window.scrollTo(0, 100)')
button = self.browser.find_by_value('Přihlásit').first
button.click()
self.user = user
self.log = logging.getLogger(__name__)
def go_search(self):
self.browser.visit('http://jizdenky.studentagency.cz/')
def search(self, task, date_return=None, is_open=False):
self.browser.find_by_id('hp_form_itinerar').first \
.find_by_xpath('div/input[@type="radio"]'
)[1 if date_return or is_open else 0].check()
for city, i in [(task.from_city, 1), (task.to_city, 2)]:
self.browser.find_by_css('input[tabindex="{}"]'.format(i)) \
.first.fill(city)
for item in self.browser.find_by_css('.ui-menu-item'):
link = item.find_by_tag('a')
if link.value.lower() == city.lower():
link.click()
break
self.browser.fill('departure:dateField', task.date)
if date_return:
self.browser.fill('returnDeparture:dateField', date_return)
if is_open:
self.browser.check('returnTicketOpen')
self.browser.find_option_by_text('ISIC').first.check()
self.browser.find_by_value('Vyhledat').first.click()
while self.browser.is_element_not_present_by_css('.left_column',
wait_time=1):
pass
items = self.browser.find_by_css('.left_column') \
.find_by_xpath('div/div/*')
connections = []
for item in items:
if item.tag_name == 'h2':
date_local = item.text.split(' ')[1]
elif item.tag_name == 'div' and item.has_class('routeSummary'):
assert date_local
if date_local != task.date:
break
connections.append(Connection(item))
return connections
def order_time(self, connection):
while True:
if connection.click():
self.browser
dialog = self.browser.find_by_css('[id^=_wicket_window]')
if dialog:
dialog.first.find_by_tag('button').click()
if self.browser.is_element_present_by_id('sumary_lines',
wait_time=1):
break
self.browser.find_by_id('sumary_lines') \
.first.find_by_tag('button') \
.first.click()
seats = {}
bus = self.browser.find_by_css('.seatsContainer')
if bus:
for seat in bus.first.find_by_css(
'.seatContainer:not([style*=blocked])'):
seats[int(seat.find_by_tag('div').first.html[:-1])] = seat
else:
bus = self.browser.find_by_css('.vehicle')
for seat in bus.first.find_by_css('.free, .selected'):
seats[int(seat.text[:-1])] = seat
return seats
def order_seat(self, seat):
if not seat.has_class('selected'):
seat.click()
for fs in self.browser.find_by_css('fieldset.topRoute'):
legend = fs.find_by_css('legend')
if legend and 'Pojištění' in legend[0].text:
for package in fs.find_by_css('.insurancePackageType'):
if 'nechci' in package.find_by_tag('label').text:
package.find_by_tag('input').click()
time.sleep(1)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
reserved = 'Rezervovat' in interaction_type
if not reserved:
submit.click()
time.sleep(1)
data = (self.user['first'],
self.user['last'],
self.user['email'],
self.user['phone'])
for item, value in zip(self.browser.find_by_id('passengerInfo')
.first.find_by_tag('input'),
data):
item.fill(value)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
assert 'Rezervovat' in interaction_type
agreement = self.browser.find_by_css('[name="bottomComponent:termsAgreementCont:termsAgreementCB"]')
if agreement:
agreement[0].check()
time.sleep(1)
submit.click()
with open('conf.yaml') as f:
conf = yaml.load(f)
if 'email' in conf:
email = conf['email']
while self.browser.is_element_not_present_by_id('ticketPage', wait_time=1):
pass
msg = MIMEText(self.browser.find_by_id('ticketPage').first.html, 'html')
msg['Subject'] = 'SA reservation'
msg['From'] = email['from']
msg['To'] = self.user['email']
username = email['username']
password = email['password']
server = smtplib.SMTP(email['server'])
server.starttls()
server.login(username, b64decode(password).decode())
server.sendmail(msg['From'], msg['To'], msg.as_string())
server.quit()
from splinter import Browser
browser = Browser('chrome')
def myClick_txt(url):
print 'myclick',url
while(browser.is_element_not_present_by_text(url)):
pass
browser.click_link_by_text(url)
# Visit URL
url = "http://weibo.com/login.php"
browser.visit(url)
# 登录
myClick_txt(u'帐号登录')
# data里填真实的username和password
data ={'username':'******','password':'******'}
browser.fill_form(data)
browser.find_by_css('.W_btn_a').first.click()
import time
browser.visit('http://weibo.com/message/history?uid=5175429989#_0')
question = browser.find_by_css('.bubble_r .page').last.text
lastAnswer = ''
answer = ''
j = 0
while True:
try:
if j % 100 == 0:
browser.reload()
while browser.is_element_not_present_by_css('.bubble_l .page'):
browser.reload()
j += 1
i = 0
class Session:
def __init__(self, browser, user):
self.browser = Browser(browser)
self.browser.visit('http://jizdenky.studentagency.cz/')
self.browser.fill_form({
'passwordAccountCode': user['login'],
'password': user['password']
})
self.browser.execute_script('window.scrollTo(0, 100)')
button = self.browser.find_by_value('Přihlásit').first
button.click()
self.user = user
self.log = logging.getLogger(__name__)
def go_search(self):
self.browser.visit('http://jizdenky.studentagency.cz/')
def search(self, task, date_return=None, is_open=False):
self.browser.find_by_id('hp_form_itinerar').first \
.find_by_xpath('div/input[@type="radio"]'
)[1 if date_return or is_open else 0].check()
for city, i in [(task.from_city, 1), (task.to_city, 2)]:
self.browser.find_by_css('input[tabindex="{}"]'.format(i)) \
.first.fill(city)
for item in self.browser.find_by_css('.ui-menu-item'):
link = item.find_by_tag('a')
if link.value.lower() == city.lower():
link.click()
break
self.browser.fill('departure:dateField', task.date)
if date_return:
self.browser.fill('returnDeparture:dateField', date_return)
if is_open:
self.browser.check('returnTicketOpen')
self.browser.find_option_by_text('ISIC').first.check()
self.browser.find_by_value('Vyhledat').first.click()
while self.browser.is_element_not_present_by_css('.left_column',
wait_time=1):
pass
items = self.browser.find_by_css('.left_column') \
.find_by_xpath('div/div/*')
connections = []
for item in items:
if item.tag_name == 'h2':
date_local = item.text.split(' ')[1]
elif item.tag_name == 'div' and item.has_class('routeSummary'):
assert date_local
if date_local != task.date:
break
connections.append(Connection(item))
return connections
def order_time(self, connection):
while True:
if connection.click():
self.browser
dialog = self.browser.find_by_css('[id^=_wicket_window]')
if dialog:
dialog.first.find_by_tag('button').click()
if self.browser.is_element_present_by_id('sumary_lines',
wait_time=1):
break
self.browser.find_by_id('sumary_lines') \
.first.find_by_tag('button') \
.first.click()
seats = {}
bus = self.browser.find_by_css('.seatsContainer')
if bus:
for seat in bus.first.find_by_css(
'.seatContainer:not([style*=blocked])'):
seats[int(seat.find_by_tag('div').first.html[:-1])] = seat
else:
bus = self.browser.find_by_css('.vehicle')
for seat in bus.first.find_by_css('.free, .selected'):
seats[int(seat.text[:-1])] = seat
return seats
def order_seat(self, seat):
if not seat.has_class('selected'):
seat.click()
for fs in self.browser.find_by_css('fieldset.topRoute'):
legend = fs.find_by_css('legend')
if legend and 'Pojištění' in legend[0].text:
for package in fs.find_by_css('.insurancePackageType'):
if 'nechci' in package.find_by_tag('label').text:
package.find_by_tag('input').click()
time.sleep(1)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
reserved = 'Rezervovat' in interaction_type
if not reserved:
submit.click()
time.sleep(1)
data = (self.user['first'], self.user['last'], self.user['email'],
self.user['phone'])
for item, value in zip(
self.browser.find_by_id('passengerInfo').first.find_by_tag(
'input'), data):
item.fill(value)
submit = self.browser.find_by_css('[name^=buttonContainer]').first
interaction_type = submit.text
assert 'Rezervovat' in interaction_type
agreement = self.browser.find_by_css(
'[name="bottomComponent:termsAgreementCont:termsAgreementCB"]')
if agreement:
agreement[0].check()
time.sleep(1)
submit.click()
with open('conf.yaml') as f:
conf = yaml.load(f)
if 'email' in conf:
email = conf['email']
while self.browser.is_element_not_present_by_id('ticketPage',
wait_time=1):
pass
msg = MIMEText(
self.browser.find_by_id('ticketPage').first.html, 'html')
msg['Subject'] = 'SA reservation'
msg['From'] = email['from']
msg['To'] = self.user['email']
username = email['username']
password = email['password']
server = smtplib.SMTP(email['server'])
server.starttls()
server.login(username, b64decode(password).decode())
server.sendmail(msg['From'], msg['To'], msg.as_string())
server.quit()
class SplinterTestCase(LiveServerTestCase):
username = '******'
email = '*****@*****.**'
password = '******'
is_anonymous = True
is_staff = False
is_logged_in = True
def setUp(self):
super(SplinterTestCase, self).setUp()
self.user = None
self.base_url = URL(self.live_server_url)
self.browser = Browser(SPLINTER_WEBDRIVER)
self.browser.driver.set_window_size(1280, 1024)
if self.is_anonymous and not self.is_staff:
return
self.user = factories.UserFactory(
username=self.username,
email=self.email,
password=self.password,
is_staff=self.is_staff,
)
if self.is_logged_in:
self.goto(reverse('admin:index'))
self.browser.fill_form({
'username': self.username,
'password': self.password,
})
self.browser.find_by_css("input[type='submit']").first.click()
exists = self.browser.is_text_present('Log out', wait_time=2)
self.assertTrue(exists)
def tearDown(self):
super(SplinterTestCase, self).tearDown()
if not os.getenv('SPLINTER_DEBUG'):
self.browser.quit()
def goto(self, path):
url = self.base_url.path(path)
return self.browser.visit(url.as_string())
def wait_for_editor_reload(self, wait_for=3):
time.sleep(wait_for)
def ensure_element(self, element_or_list, index=0):
"""
Selects either the element with *index* from the list of elements given
in *element_or_list* or returns the single element if it is not a list.
This make it possible to handle an element and a list of elements where
only a single element is required.
:param element: ``Element`` instance or ``ElementList``.
:parem int index: Index of element to be returned if a list.
(Default: 0)
:rtype: Element
"""
if isinstance(element_or_list, ElementList):
return element_or_list[index]
return element_or_list
def find_and_click_by_css(self, browser, selector, wait_time=3):
browser.is_element_present_by_css(selector, wait_time)
elem = self.ensure_element(browser.find_by_css(selector))
return elem.click()
class ESOArchive:
"""
Query the ESO raw data archive.
Parameters
----------
config : dict
Dictionary containing configuration values
"""
url = 'http://archive.eso.org/eso/eso_archive_main.html'
schema = Schema(
{
Optional('max_workers'): Str2Int,
Optional('rows'): Str2Int,
Optional('starttime'): Str2Int,
Optional('endtime'): Str2Int,
Optional('headless'): Str2Bool
},
extra=True)
def __init__(self, config, debug=False):
self.logger = logging.getLogger(__class__.__name__)
self.logger.setLevel(logging.DEBUG if debug else logging.INFO)
self._validate_config(config)
self._setup()
self._instrument = self._nightobs = self._target = None
self._coords = None
self._resolver = 'simbad'
self._endtime = self._startime = 12
self._rows = 200
self.instrument = self._config.get('instrument', None)
self.nightobs = self._config.get('nightobs', None)
self.target = self._config.get('target', None)
self.starttime = self._config.get('starttime', None)
self.endtime = self._config.get('endtime', None)
self.max_rows = self._config.get('rows', None)
self.output = self._config.get('output', None)
def _validate_config(self, config):
try:
self._config = self.schema(dict(config))
except Exception as err:
self.logger.error(err)
raise
def _setup(self):
"""Setup the connection.
"""
self.logger.debug(f'Setting URL {self.url}')
headless = self._config.get('headless', True)
self.browser = Browser('chrome', headless=headless)
self.browser.visit(self.url)
@property
def output(self):
return self._output.format(nightobs=self.nightobs,
instrument=self.instrument)
@output.setter
def output(self, out: str):
self._output = out
@property
def nightobs(self):
"""Night of observation to request (YYYYMMDD)
"""
try:
n = self._nightobs.replace(' ', '')
except Exception:
n = ''
return n
@nightobs.setter
def nightobs(self, nightobs: str):
"""Set the night to request.
Parameters
----------
nightobs : str
Night of observation to request, YYYYMMDD.
"""
if nightobs is None:
return
night = ' '.join(
[*map(lambda x: nightobs[slice(*x)], [(0, 4), (4, 6), (6, 8)])])
self.logger.debug(f'Selecting night {nightobs}')
self.browser.find_by_name('night').fill(night)
self._nightobs = night
@property
def target(self):
"""Name of the astronomical object you want to search for.
"""
return self._target
@target.setter
def target(self, target: str):
if target is None:
return
self.logger.debug(f'Selecting target {target}')
self.browser.find_by_name('target').fill(target)
self._target = target
@property
def resolver(self):
"""Resolver to use for target (simbad | ned | none).
"""
return self._resolver
@resolver.setter
def resolver(self, resolver: str = 'simbad'):
if resolver is None:
return
self.logger.debug(f'Selecting target {resolver}')
self.browser.find_by_name('resolver').select(resolver)
self._resolver = resolver
@property
def coords(self):
"""Coordinates of position to look for.
"""
return self._coords
@coords.setter
def coords(self, ra: str, dec: str):
if ra is None or dec is None:
return
self.browser.find_by_name('ra').fill(ra)
self.browser.find_by_name('dec').fill(dec)
self._coords = [ra, dec]
def _getlog(self):
self.browser.find_by_name('wdbo').first.select('ascii/display')
self.browser.find_by_id('search').click()
res = self.browser.html
self.browser.back()
self.browser.find_by_name('wdbo').first.select('html/display')
return res
@property
def instrument(self):
"""Instrument to query for data (e.g. VIRCAM)
"""
return self._instrument
@instrument.setter
def instrument(self, instrument: str = None):
"""Select instrument.
Parameters
----------
instrument : str
Instrument or list separated by commas
"""
if instrument is None:
return
self.logger.debug(f'Selecting instrument {instrument}')
for ins in instrument.split(','):
self.browser.find_by_value(ins).first.click()
self._instrument = instrument
@property
def starttime(self):
"""Start time of observations.
"""
return self._starttime
@starttime.setter
def starttime(self, hour: int = 12):
"""Set the start time.
Parameters
----------
hour : int, default: 12
Start time (0, 23)
"""
if hour is None:
return
self.browser.find_by_name('starttime').first.select(hour)
self._starttime = hour
@property
def endtime(self):
"""End time of observations.
"""
return self._endtime
@endtime.setter
def endtime(self, hour: int = 12):
"""Set the end time.
Parameters
----------
hour : int, default: 12
End time (0, 23)
"""
if hour is None:
return
self.browser.find_by_name('endtime').first.select(hour)
self._endtime = hour
@property
def max_rows(self):
"""Maximum number of records to retrieve.
"""
return self._rows
@max_rows.setter
def max_rows(self, rows: int = 200):
"""Set maximum number of rows to return.
Parameters
----------
rows : int, default: 200
Maximum number of rows to return
"""
if rows is None:
return
self.logger.debug(f'Selecting max rows {rows}')
self.browser.find_by_name('max_rows_returned').first.fill(rows)
self._rows = rows
def _login(self):
"""Login to the archive.
"""
if self.browser.is_element_not_present_by_text('Login'):
self.logger.warn('No login form found')
return
credentials = {
k: v
for k, v in self._config.items() if k in ['username', 'password']
}
self.browser.fill_form(credentials)
self.browser.find_by_text('Login')[1].click()
if self.browser.html.find('Login failed') > 0:
raise LoginUnsuccessful(
f'Login unsuccessfull for {credentials["username"]}')
else:
self.logger.debug(
f'Login successfull for {credentials["username"]}')
def _make_outdir(self):
if not os.access(self.output, os.X_OK):
os.makedirs(self.output)
def request(self):
"""Perform a request to the archive.
"""
self.logger.debug('Searching and requesting data')
# First page - fill in extras
for tab in [
'tab_origfile', 'tab_obs_name', 'tab_ob_id', 'tab_rel_date'
]:
self.browser.find_by_name(tab).first.click()
# Request ASCII table
self.logdata = self._getlog()
# First page - click on search
self.browser.find_by_id('search').click()
# Second page - check that there are some data returned
if self.browser.html.find('No data returned') > 0:
self.logger.error('No data returned')
return
# Second page - mark all rows
self.browser.find_by_id('ibmarkall').click()
# Second page - request datasets
self.browser.find_by_value('Request marked datasets').first.click()
# Third page - fill login credentials
try:
self._login()
except LoginUnsuccessful as err:
self.logger.error(err)
return
# Fourth page - submit
self.browser.find_by_name('submit').first.click()
# Fifth page - wait for completed
while self.browser.html.find('Completed') < 0:
time.sleep(2)
href = self.browser.find_link_by_partial_text('downloadRequest').first
url = re.compile('(https://.*)"').search(href.outer_html).group(1)
result = requests.get(url, cookies=self.browser.cookies.all())
script = result.content.decode('utf-8')
self._make_outdir()
with open(os.path.join(self.output, href.value), 'w') as fh:
fh.write(script)
self.logger.debug(f'{href.value} written')
with open(os.path.join(self.output, f'ESOLOG.{self.nightobs}'),
'w') as fh:
fh.write(self.logdata)
self.logger.debug(f'ESOLOG.{self.nightobs} written')
filelist = re.compile('https://.*[fz|txt]').findall(script)
allfiles = re.compile('VCAM.\S+').findall(self.logdata)
for f in allfiles:
if not any(map(lambda x: f in x, filelist)):
with open(os.path.join(self.output, f'{f}.dummy'), 'w') as fh:
fh.write('')
return filelist
def download(self, filelist: list):
"""Download a list of files.
Parameters
----------
filelist : list of str
URLs to download.
"""
max_workers = self._config.get('max_workers', 1)
self.logger.debug(f'Downloading using {max_workers} workers')
with ThreadPoolExecutor(max_workers=int(max_workers)) as executor:
future_to_url = {
executor.submit(self._downloadfile, url): url
for url in filelist
}
for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
self.logger.info('Saved %s', os.path.basename(url))
def _downloadfile(self, f):
r = requests.get(f,
stream=True,
auth=(self._config['username'],
self._config['password']))
filename = os.path.basename(f)
fullname = os.path.join(self.output, filename)
if os.access(fullname, os.F_OK):
oldsize = os.stat(fullname).st_size
newsize = int(r.headers['Content-Length'])
if oldsize == newsize:
return
with open(fullname, 'wb') as fh:
shutil.copyfileobj(r.raw, fh)
def close(self):
self.browser.quit()
class Client(object):
"""Proxy class which adds more functionatily to Splinter's Browser."""
def __init__(self, browser_name='firefox', url=None):
"""Initial parameters for a web browser client.
:param str browser_name: Name for the web browser type to instantiate.
:param str url: The base url for your server.
"""
self.browser = Browser(browser_name)
# If a `url` is provided...
if url is not None:
# ... use it ...
self.base_url = url
else:
# ... else, use the one from configuration file
self.base_url = conf.properties['url']
def login(self, username=None, password=None):
"""Performs a login using the provided credentials.
:param str username: A user name.
:param str password: A user password.
"""
if username is None:
username = conf.properties['username']
if password is None:
password = conf.properties['password']
# Fill the login form.
self.browser.fill_form(
{FLD_USERNAME: username, FLD_PASSWORD: password})
# Click the 'Login' button
self.browser.find_by_name(BTN_LOGIN).click()
def logout(self):
"""Performs a logout."""
# Position the mouse over the account menu to expose submenus.
self.browser.find_by_id(MENU_ACCOUNT).mouse_over()
# Then, click the logout menu
self.browser.find_by_id(MENU_LOGOUT).click()
def __getattr__(self, name):
"""Proxy attr lookup to self.browser.
Allows calling Splinter's Browser methods directly. If `name` is not
a method found in either `Browser` or `Client`, then ``AttributeError``
is raised.
:param str name: Name for a supported method.
:raises: ``AttributeError``
"""
attr = getattr(self.browser, name, None)
if attr is None:
super(Client, self).__getattribute__(name)
return attr
def __enter__(self):
return self
def __exit__(self, *exc):
self.browser.__exit__(*exc)
class AuthenticationRegisterBrowserTestCase(TestCase):
def setUp(self):
self.browser = Browser('django')
self.test_user_id = '*****@*****.**'
self.test_user_password = '******'
def test_register_page_ok(self):
self.browser.visit('/register')
assert '/register' in self.browser.url
def test_register_ok(self):
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert 'registeration completed' in self.browser.title.lower()
user = User.objects.get(email=self.test_user_id)
assert user.is_active is False
def test_activation_ok(self):
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
user = User.objects.get(email=self.test_user_id)
activation_key = UserActivation.objects.get(user=user).activation_key
assert user.is_active is False
assert UserActivation.objects.filter(activation_key=activation_key).exists() is True
self.browser.visit('/activation/' + activation_key)
assert 'activation completed' in self.browser.title.lower()
user = User.objects.get(email=self.test_user_id)
assert user.is_active is True
assert UserActivation.objects.filter(activation_key=activation_key).exists() is False
def test_too_long_email_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({'email': 'a' * 254 + '@pergenie.org',
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Ensure this value has at most 254')
def test_too_long_password_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': 'a' * 1025,
'password2': 'a' * 1025,
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Ensure this value has at most 1024')
def test_too_weak_password_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': 'weak',
'password2': 'weak',
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Passwords too short')
def test_not_match_passwords_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': self.test_user_password,
'password2': 'not-match-password',
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Passwords do not match')
def test_not_agree_with_terms_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': False,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Not read and accept about service')
def test_already_registerd_user_should_fail_register(self):
self.user = User.objects.create_user(self.test_user_id,
self.test_user_password)
self.user.save()
self.browser.visit('/register')
self.browser.fill_form({'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Already registered')
# TODO:
def test_email_sending_error_should_fail_register(self):
pass
def test_incorrect_activation_key_should_fail_activation(self):
self.browser.visit('/activation/' + 'a' * 40)
assert self.browser.status_code == 404
class TestRoutes(unittest.TestCase):
@classmethod
def setUpClass(cls):
# socketio.run(flapp, port = flapp.config['PORT'])
pass
# @init_db
def setUp(self):
self.browser = Browser()
self.username = uuid4().hex
self.userpassword = uuid4().hex
def tearDown(self):
self.browser.quit()
def go_home(self):
self.browser.visit( 'http://localhost:%s'%flapp.config['PORT'])
def login(self, user):
self.go_home()
self.browser.fill_form({'username': self.username, 'password':self.userpassword})
self.browser.find_by_value('Sign in').click()
def test_login_success_with_confirmed_user(self):
self.login(UserFactory.seed_confirmed_user(self.username, self.userpassword))
assert self.browser.is_text_present('Signed in as %s'%self.username)
def test_login_failure_with_nonconfirmed_user(self):
user = UserFactory.seed_nonconfirmed_user(self.username, self.userpassword)
self.login(user)
assert self.browser.is_text_not_present('Signed in as %s'%self.userpassword)
assert self.browser.is_text_present('Sign in')
def test_login_failure_with_nonexisting_user(self):
self.go_home()
fake_username = uuid4().hex
self.browser.fill_form({'username': fake_username, 'password':uuid4().hex})
self.browser.find_by_value('Sign in').click()
assert self.browser.is_text_not_present('Signed in as %s'%fake_username)
assert self.browser.is_text_present('Sign in')
def test_logout(self):
self.login(UserFactory.seed_confirmed_user(self.username, self.userpassword))
self.browser.click_link_by_text('Sign out')
assert self.browser.is_text_not_present('Signed in as %s'%self.username)
assert self.browser.is_text_present('Sign in')
# def test_index(self):
# r = flapp.get('/')
# assert r.okbr
# def test_login_with_id(self):
# self.login()
# print self.browser.html
# # print browser.html
# class TestRoutesHeadless(unittest.TestCase):
# def setUp(self):
# self.browser
class AuthenticationRegisterBrowserTestCase(TestCase):
def setUp(self):
self.browser = Browser('django')
self.test_user_id = '*****@*****.**'
self.test_user_password = '******'
def test_register_page_ok(self):
self.browser.visit('/register')
assert '/register' in self.browser.url
def test_register_ok(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert 'registeration completed' in self.browser.title.lower()
user = User.objects.get(email=self.test_user_id)
assert user.is_active is False
def test_activation_ok(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
user = User.objects.get(email=self.test_user_id)
activation_key = UserActivation.objects.get(user=user).activation_key
assert user.is_active is False
assert UserActivation.objects.filter(
activation_key=activation_key).exists() is True
self.browser.visit('/activation/' + activation_key)
assert 'activation completed' in self.browser.title.lower()
user = User.objects.get(email=self.test_user_id)
assert user.is_active is True
assert UserActivation.objects.filter(
activation_key=activation_key).exists() is False
def test_too_long_email_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': 'a' * 254 + '@pergenie.org',
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present(
'Ensure this value has at most 254')
def test_too_long_password_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': 'a' * 1025,
'password2': 'a' * 1025,
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present(
'Ensure this value has at most 1024')
def test_too_weak_password_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': 'weak',
'password2': 'weak',
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Passwords too short')
def test_not_match_passwords_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': self.test_user_password,
'password2': 'not-match-password',
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Passwords do not match')
def test_not_agree_with_terms_should_fail_register(self):
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': False,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present(
'Not read and accept about service')
def test_already_registerd_user_should_fail_register(self):
self.user = User.objects.create_user(self.test_user_id,
self.test_user_password)
self.user.save()
self.browser.visit('/register')
self.browser.fill_form({
'email': self.test_user_id,
'password1': self.test_user_password,
'password2': self.test_user_password,
'terms_ok_0': True,
'terms_ok_1': True
})
self.browser.find_by_name('submit').click()
assert self.browser.is_text_present('Already registered')
# TODO:
def test_email_sending_error_should_fail_register(self):
pass
def test_incorrect_activation_key_should_fail_activation(self):
self.browser.visit('/activation/' + 'a' * 40)
assert self.browser.status_code == 404
class SplinterTestCase(LiveServerTestCase):
username = '******'
email = '*****@*****.**'
password = '******'
is_anonymous = True
is_staff = False
is_logged_in = True
def setUp(self):
super(SplinterTestCase, self).setUp()
self.user = None
self.base_url = URL(self.live_server_url)
self.browser = Browser(SPLINTER_WEBDRIVER)
self.browser.driver.set_window_size(1280, 1024)
if self.is_anonymous and not self.is_staff:
return
self.user = factories.UserFactory(
username=self.username,
email=self.email,
password=self.password,
is_staff=self.is_staff,
)
if self.is_logged_in:
self.goto(reverse('admin:index'))
self.browser.fill_form({
'username': self.username,
'password': self.password,
})
self.browser.find_by_css("input[type='submit']").first.click()
exists = self.browser.is_text_present('Log out', wait_time=2)
self.assertTrue(exists)
def tearDown(self):
super(SplinterTestCase, self).tearDown()
if not os.getenv('SPLINTER_DEBUG'):
self.browser.quit()
def goto(self, path):
url = self.base_url.path(path)
return self.browser.visit(url.as_string())
def wait_for_editor_reload(self, wait_for=3):
time.sleep(wait_for)
def ensure_element(self, element_or_list, index=0):
"""
Selects either the element with *index* from the list of elements given
in *element_or_list* or returns the single element if it is not a list.
This make it possible to handle an element and a list of elements where
only a single element is required.
:param element: ``Element`` instance or ``ElementList``.
:parem int index: Index of element to be returned if a list.
(Default: 0)
:rtype: Element
"""
if isinstance(element_or_list, ElementList):
return element_or_list[index]
return element_or_list
def find_and_click_by_css(self, browser, selector, wait_time=3):
browser.is_element_present_by_css(selector, wait_time)
elem = self.ensure_element(browser.find_by_css(selector))
return elem.click()
