def get_bp1(self):
b = MyBrowser(head=False)
b.clinvar_path_nav(self.gene)
frameshift = b.browser.find_by_xpath(
'//*[@id="_molConseq"]/li/ul/li[1]/span').value
frameshift = int(frameshift[1:][:-1])
missense = b.browser.find_by_xpath(
'//*[@id="_molConseq"]/li/ul/li[2]/span').value
missense = int(missense[1:][:-1])
nonsense = b.browser.find_by_xpath(
'//*[@id="_molConseq"]/li/ul/li[3]/span').value
nonsense = int(nonsense[1:][:-1])
splice_site = b.browser.find_by_xpath(
'//*[@id="_molConseq"]/li/ul/li[4]/span').value
splice_site = int(splice_site[1:][:-1])
if (frameshift + nonsense + splice_site) / (
frameshift + missense + nonsense + splice_site) > 0.9:
return True
else:
return False
def meteopocasi():
browser = MyBrowser()
html = browser.get(
"http://www.meteo-pocasi.cz/maps/cz/praha/962-meteostanice-praha-lysolaje/"
)
soup = BeautifulSoup(html, "html.parser")
#with open('x.html', 'w') as f:
# f.write(html)
svalues = soup.find_all("div", "svalue")
print svalues[0].text
def __init__(self,password, proxy_port,controller_port,pass_function=None):
"""
password is for the set tor password
proxy_port is the port use by the given tor
pass_function is the function that given a html it will return True if not detected as ROBOT
"""
self._password = password
self._proxy_port = proxy_port
self._controller_port = controller_port
self._pass_function = pass_function
self._br = MyBrowser(self._proxy_port)
self._new_ip()
self._randtime()
def get_new_ip():
with Controller.from_port(port = self._controller_port) as controller:
controller.authenticate(self._password)
controller.signal(Signal.NEWNYM)
sleep(controller.get_newnym_wait())
controller.close()
# reset the browser and its settings by calling a new one
self._br.close()
self._br = MyBrowser(self._proxy_port)
#reset the time before next change is required
self._randtime()
def get_pp2(self):
b = MyBrowser(head=False)
b.clinvar_missense_nav(self.gene)
total_missense = b.browser.find_by_xpath(
'//*[@id="_molConseq"]/li/ul/li[2]/span').value
total_missense = int(total_missense[1:][:-1])
benign_missense = b.browser.find_by_xpath(
'//*[@id="_Properties"]/li/ul/li[2]/span').value
benign_missense = int(benign_missense[1:][:-1])
likely_benign_missense = b.browser.find_by_xpath(
'//*[@id="_Properties"]/li/ul/li[3]/span').value
likely_benign_missense = int(likely_benign_missense[1:][:-1])
if (likely_benign_missense + benign_missense) / total_missense < 0.1:
return True
else:
return False
variant = MyVariant(chr, pos, ref, alt, gene, transcript_id, c_dot,
mimNumber)
patient_path = r'U:\ASHITA\screenshots\\' + copath_no
variant_path = variant.gene + '_' + variant.chr + '_' + variant.pos + '_' + variant.ref + '_' + variant.alt
screenshot_path = patient_path + '\\' + variant_path
## Make directories for patient and variant if they don't already exist
if not os.path.exists(patient_path):
os.makedirs(patient_path)
if not os.path.exists(screenshot_path):
os.makedirs(screenshot_path)
# Open browser and take screenshots
b = MyBrowser()
# gnomAD
b.gnomad_nav(variant.chr, variant.pos, variant.ref, variant.alt)
b.take_screenshot(screenshot_path, 'gnomAD.jpg', 350, 200, 1600, 900)
# dbNSFP
b.dbnsfp_nav(variant.chr, variant.pos, variant.ref, variant.alt)
b.take_screenshot(screenshot_path, 'dbNSFP.jpg', 270, 170, 670, 1000)
# GERP
b.gerp_nav(variant.chr, variant.pos, variant.ref, variant.alt)
b.take_screenshot(screenshot_path, 'GERP.jpg', 270, 170, 1170, 600)
## ClinVar missense
b.clinvar_missense_nav(variant.gene)
class UseTor(object):
"""
Uses tor to change the IP so Zillow cant block the IP
Also, uses mechanize to minic browser
"""
def __init__(self,password, proxy_port,controller_port,pass_function=None):
"""
password is for the set tor password
proxy_port is the port use by the given tor
pass_function is the function that given a html it will return True if not detected as ROBOT
"""
self._password = password
self._proxy_port = proxy_port
self._controller_port = controller_port
self._pass_function = pass_function
self._br = MyBrowser(self._proxy_port)
self._new_ip()
self._randtime()
def _randtime(self):
"""
Sets the date time used to change IP ever so often
"""
##change every 30 to 60 mins
self._next_time = datetime.datetime.now() + datetime.timedelta(0,random.randint(1800,3600))
def _new_ip(self):
"""
Changes the IP address that tor uses.
It will check to make sure that the IP is not on Zillows Robot list or blocked list
If on list will repeat new IP till not on list
"""
def get_new_ip():
with Controller.from_port(port = self._controller_port) as controller:
controller.authenticate(self._password)
controller.signal(Signal.NEWNYM)
sleep(controller.get_newnym_wait())
controller.close()
# reset the browser and its settings by calling a new one
self._br.close()
self._br = MyBrowser(self._proxy_port)
#reset the time before next change is required
self._randtime()
get_new_ip()
if self._pass_function != None:
while(self._pass_function(self._br)==False):
##get_new till zillow doesn't fail robot test
get_new_ip()
def request(self,url):
"""
Will return html of the requested url
If needed it will change the Tor IP address to get the data
"""
#see if it is time to find a new IP address
if datetime.datetime.now() > self._next_time:
self._new_ip()
## try to connect to website and get the html
try:
r = self._br.open(url)
html = r.read()
except:
self._new_ip()
return self.request(url) #not successful try again
# if pass function given and html received see if it passes the test function
if self._pass_function != None:
if self._pass_function(self._br, html=html)==False:
self._new_ip()
return self.request(url) #not successful try again
# everything was successful return the html
return html
def agrobiologie():
browser = MyBrowser()
html = browser.get("http://meteostanice.agrobiologie.cz/")
soup = BeautifulSoup(html, "html.parser")
svalues = soup.find_all("span", "ramecek")
print svalues[0].text[:-3].replace(',', '.')
