def get_bp1(self): b = MyBrowser(head=False) b.clinvar_path_nav(self.gene) frameshift = b.browser.find_by_xpath( '//*[@id="_molConseq"]/li/ul/li[1]/span').value frameshift = int(frameshift[1:][:-1]) missense = b.browser.find_by_xpath( '//*[@id="_molConseq"]/li/ul/li[2]/span').value missense = int(missense[1:][:-1]) nonsense = b.browser.find_by_xpath( '//*[@id="_molConseq"]/li/ul/li[3]/span').value nonsense = int(nonsense[1:][:-1]) splice_site = b.browser.find_by_xpath( '//*[@id="_molConseq"]/li/ul/li[4]/span').value splice_site = int(splice_site[1:][:-1]) if (frameshift + nonsense + splice_site) / ( frameshift + missense + nonsense + splice_site) > 0.9: return True else: return False
def meteopocasi(): browser = MyBrowser() html = browser.get( "http://www.meteo-pocasi.cz/maps/cz/praha/962-meteostanice-praha-lysolaje/" ) soup = BeautifulSoup(html, "html.parser") #with open('x.html', 'w') as f: # f.write(html) svalues = soup.find_all("div", "svalue") print svalues[0].text
def __init__(self,password, proxy_port,controller_port,pass_function=None): """ password is for the set tor password proxy_port is the port use by the given tor pass_function is the function that given a html it will return True if not detected as ROBOT """ self._password = password self._proxy_port = proxy_port self._controller_port = controller_port self._pass_function = pass_function self._br = MyBrowser(self._proxy_port) self._new_ip() self._randtime()
def get_new_ip(): with Controller.from_port(port = self._controller_port) as controller: controller.authenticate(self._password) controller.signal(Signal.NEWNYM) sleep(controller.get_newnym_wait()) controller.close() # reset the browser and its settings by calling a new one self._br.close() self._br = MyBrowser(self._proxy_port) #reset the time before next change is required self._randtime()
def get_pp2(self): b = MyBrowser(head=False) b.clinvar_missense_nav(self.gene) total_missense = b.browser.find_by_xpath( '//*[@id="_molConseq"]/li/ul/li[2]/span').value total_missense = int(total_missense[1:][:-1]) benign_missense = b.browser.find_by_xpath( '//*[@id="_Properties"]/li/ul/li[2]/span').value benign_missense = int(benign_missense[1:][:-1]) likely_benign_missense = b.browser.find_by_xpath( '//*[@id="_Properties"]/li/ul/li[3]/span').value likely_benign_missense = int(likely_benign_missense[1:][:-1]) if (likely_benign_missense + benign_missense) / total_missense < 0.1: return True else: return False
variant = MyVariant(chr, pos, ref, alt, gene, transcript_id, c_dot, mimNumber) patient_path = r'U:\ASHITA\screenshots\\' + copath_no variant_path = variant.gene + '_' + variant.chr + '_' + variant.pos + '_' + variant.ref + '_' + variant.alt screenshot_path = patient_path + '\\' + variant_path ## Make directories for patient and variant if they don't already exist if not os.path.exists(patient_path): os.makedirs(patient_path) if not os.path.exists(screenshot_path): os.makedirs(screenshot_path) # Open browser and take screenshots b = MyBrowser() # gnomAD b.gnomad_nav(variant.chr, variant.pos, variant.ref, variant.alt) b.take_screenshot(screenshot_path, 'gnomAD.jpg', 350, 200, 1600, 900) # dbNSFP b.dbnsfp_nav(variant.chr, variant.pos, variant.ref, variant.alt) b.take_screenshot(screenshot_path, 'dbNSFP.jpg', 270, 170, 670, 1000) # GERP b.gerp_nav(variant.chr, variant.pos, variant.ref, variant.alt) b.take_screenshot(screenshot_path, 'GERP.jpg', 270, 170, 1170, 600) ## ClinVar missense b.clinvar_missense_nav(variant.gene)
class UseTor(object): """ Uses tor to change the IP so Zillow cant block the IP Also, uses mechanize to minic browser """ def __init__(self,password, proxy_port,controller_port,pass_function=None): """ password is for the set tor password proxy_port is the port use by the given tor pass_function is the function that given a html it will return True if not detected as ROBOT """ self._password = password self._proxy_port = proxy_port self._controller_port = controller_port self._pass_function = pass_function self._br = MyBrowser(self._proxy_port) self._new_ip() self._randtime() def _randtime(self): """ Sets the date time used to change IP ever so often """ ##change every 30 to 60 mins self._next_time = datetime.datetime.now() + datetime.timedelta(0,random.randint(1800,3600)) def _new_ip(self): """ Changes the IP address that tor uses. It will check to make sure that the IP is not on Zillows Robot list or blocked list If on list will repeat new IP till not on list """ def get_new_ip(): with Controller.from_port(port = self._controller_port) as controller: controller.authenticate(self._password) controller.signal(Signal.NEWNYM) sleep(controller.get_newnym_wait()) controller.close() # reset the browser and its settings by calling a new one self._br.close() self._br = MyBrowser(self._proxy_port) #reset the time before next change is required self._randtime() get_new_ip() if self._pass_function != None: while(self._pass_function(self._br)==False): ##get_new till zillow doesn't fail robot test get_new_ip() def request(self,url): """ Will return html of the requested url If needed it will change the Tor IP address to get the data """ #see if it is time to find a new IP address if datetime.datetime.now() > self._next_time: self._new_ip() ## try to connect to website and get the html try: r = self._br.open(url) html = r.read() except: self._new_ip() return self.request(url) #not successful try again # if pass function given and html received see if it passes the test function if self._pass_function != None: if self._pass_function(self._br, html=html)==False: self._new_ip() return self.request(url) #not successful try again # everything was successful return the html return html
def agrobiologie(): browser = MyBrowser() html = browser.get("http://meteostanice.agrobiologie.cz/") soup = BeautifulSoup(html, "html.parser") svalues = soup.find_all("span", "ramecek") print svalues[0].text[:-3].replace(',', '.')