class check_the_mangas(): def __init__(self,manga_name, db_conn): self.db_conn = db_conn self.manga_name = manga_name self.manga_oldnumber = sqlite_manager.get_manga_chapter( db_conn, manga_name) self.manga_nownumber = self.manga_oldnumber self.manga_olddate = sqlite_manager.get_manga_date( db_conn, manga_name) self.nowdate = self.today_date() self.br = URLopener() def today_date(self): return subprocess.check_output(["date","+%a-%b-%e"]).replace("\n","") #return 1 if the connection is working def test_connection(self): try: response = self.br.open(configuration.WEBSITE_TO_CHECK_CONNECTION).read() if configuration.KEYWORD in response: return 1 else: return 0 except: print "manga connection" return 0 def exec_cmd(self): pid = os.fork() os.umask(0) os.system(configuration.MANGA_NEW_CMD.replace("MANGA",self.manga_name)) def run(self): if( self.test_connection() ): last_chapter = False try: while(last_chapter==False): to_open = "http://www.mangareader.net/" + self.manga_name + "/" + str( int(self.manga_nownumber)+1 ) response = self.br.open( to_open).read() if "is not released yet" in response or "not published yet" in response or response == "": last_chapter = True if self.manga_nownumber != sqlite_manager.get_manga_chapter(self.db_conn, self.manga_name): print self.manga_name+":"+self.manga_nownumber+":"+self.nowdate sqlite_manager.update_manga(self.db_conn, self.manga_name, self.manga_nownumber, self.nowdate) else: self.manga_nownumber = str( int(self.manga_nownumber)+1 ) except Exception,e : if "is not released yet. If you liked" in response: if self.manga_nownumber != sqlite_manager.get_manga_chapter(self.db_conn,self.manga_name): print self.manga_name+":"+self.manga_nownumber+":"+self.nowdate sqlite_manager.update_manga(self.db_conn, self.manga_name, self.manga_nownumber, self.nowdate) pass
class check_the_mangas(): def __init__(self,manga_name): self.manga_name = manga_name self.myfile = open(configuration.DATA_FILE,'r').read() self.manga_oldnumber = self.get_number() self.manga_nownumber = self.manga_oldnumber self.manga_olddate = self.get_date () self.nowdate = self.today_date() self.br = URLopener() def get_number(self): return re.findall(self.manga_name+':([0-9]+):',self.myfile)[0] def get_date(self): return re.findall(self.manga_name+":"+str(self.manga_oldnumber)+':(.*)\n',self.myfile)[0] def today_date(self): return subprocess.check_output(["date","+%a-%b-%e"]).replace("\n","") #return 1 if the connection is working def test_connection(self): try: response = self.br.open(configuration.WEBSITE_TO_CHECK_CONNECTION).read() if configuration.KEYWORD in response: return 1 else: return 0 except: print "manga connection" return 0 def exec_cmd(self): pid = os.fork() os.umask(0) os.system(configuration.MANGA_NEW_CMD.replace("MANGA",self.manga_name)) def run(self): if( self.test_connection() ): last_chapter = False try: while(last_chapter==False): to_open = "http://www.mangareader.net/" + self.manga_name + "/" + str( int(self.manga_nownumber)+1 ) response = self.br.open( to_open).read() if "is not released yet" in response or "not published yet" in response or response == "": last_chapter = True if self.manga_name + ":" + str(self.manga_nownumber) not in open(configuration.DATA_FILE, "r").read(): Thread(target=self.exec_cmd).start() configuration.backup() open(configuration.DATA_FILE,'w').write(open(configuration.DATA_FILE+".bak", "r").read().replace(self.manga_name+":"+str(self.manga_oldnumber)+":"+ self.manga_olddate, self.manga_name+":"+str(self.manga_nownumber)+":"+self.nowdate)) else: print "not last chapter" self.manga_nownumber = str( int(self.manga_nownumber)+1 ) except Exception,e : print e print "manga run" if "is not released yet. If you liked" in response: if self.manga_name + ":" + str(self.manga_nownumber) not in open(configuration.DATA_FILE, "r").read(): configuration.backup() open(configuration.DATA_FILE,'w').write(open(configuration.DATA_FILE+".bak", "r").read().replace(self.manga_name+":"+str(self.manga_oldnumber)+":"+ self.manga_olddate, self.manga_name+":"+str(self.manga_nownumber)+":"+self.nowdate)) pass
def unshortenurl(short): from urllib import URLopener opener = URLopener() try: opener.open(short) except IOError, e: f = e
def unshortenurl(short): from urllib import URLopener opener = URLopener() try: opener.open(short) except IOError, e: f = e
class Updater: def __init__(self, server, infoFile): """ takes a server location and an info file as parameters in the constructor it will use this server to fetch the new information there should be a json/version and json/info.json dir on this server """ self._infoFile = infoFile self._serverJSON = server + self._infoFile self._serverDate = server + "json/version" if sys.version < '3': self.br = URLopener() else: self.br = request def hasNewInfo(self): """ hasNewInfo :: Boolean compare the local version tag with the one found on the server and returns true if the server version is newer """ jsonDate = open(location_manager.VERSION, 'r').read().strip() if sys.version < '3': servDate = self.br.open(self._serverDate).read().strip() else: servDate = self.br.urlopen(self._serverDate).read().strip() return (int(jsonDate) < int(servDate)) def generateTimeStamp(self): """ generateTimeStamp :: String returns a string that is used to timestamp old config backup files """ return open(location_manager.VERSION, 'r').read().strip() def fetchNewInfo(self): """ fetchNewInfo :: Void it will download the info file from the server use the timestamp to back it up and overwrite it """ # Fetching server's info.json if sys.version < '3': response = self.br.open(self._serverJSON).read() else: response = self.br.urlopen(self._serverJSON).read().decode("utf-8") oldInfo = open(self._infoFile, 'r').read() open(self._infoFile + "." + self.generateTimeStamp(), 'w').write(oldInfo) open(self._infoFile, 'w').write(response) # Fetching server's version if sys.version < '3': servDate = int(self.br.open(self._serverDate).read().strip()) else: servDate = int(self.br.urlopen(self._serverDate).read().strip()) open(location_manager.VERSION, 'w').write(str(servDate))
class Updater: def __init__(self, server, infoFile): """ takes a server location and an info file as parameters in the constructor it will use this server to fetch the new information there should be a json/version and json/info.json dir on this server """ self._infoFile = infoFile self._serverJSON = server + self._infoFile self._serverDate = server + "json/version" if sys.version < '3': self.br = URLopener() else: self.br = request def hasNewInfo(self): """ hasNewInfo :: Boolean compare the local version tag with the one found on the server and returns true if the server version is newer """ jsonDate = open(location_manager.VERSION , 'r').read().strip() if sys.version < '3': servDate = self.br.open(self._serverDate).read().strip() else: servDate = self.br.urlopen(self._serverDate).read().strip() return (int(jsonDate) < int(servDate)) def generateTimeStamp(self): """ generateTimeStamp :: String returns a string that is used to timestamp old config backup files """ return open(location_manager.VERSION, 'r').read().strip() def fetchNewInfo(self): """ fetchNewInfo :: Void it will download the info file from the server use the timestamp to back it up and overwrite it """ # Fetching server's info.json if sys.version < '3': response = self.br.open(self._serverJSON).read() else: response = self.br.urlopen(self._serverJSON).read().decode("utf-8") oldInfo = open(self._infoFile, 'r').read() open(self._infoFile + "." + self.generateTimeStamp(), 'w').write(oldInfo) open(self._infoFile, 'w').write(response) # Fetching server's version if sys.version < '3': servDate = int(self.br.open(self._serverDate).read().strip()) else: servDate = int(self.br.urlopen(self._serverDate).read().strip()) open(location_manager.VERSION, 'w').write(str(servDate))
def connection(): try: br = URLopener() response = br.open(configuration.WEBSITE_TO_CHECK_CONNECTION).read() if configuration.KEYWORD in response: return 1 else: return 0 except: return 0
def utGrabFromUrl(p_url): """ Takes a file from a remote server """ from urllib import URLopener try: l_opener = URLopener() l_file = l_opener.open(p_url) ctype = l_file.headers['Content-Type'] l_opener.close() return (l_file.read(), ctype) except: return (None, 'text/x-unknown-content-type')
class Updater: """ takes a server location and an info file as parameters in the constructor it will use this server to fetch the new information there should be a /hash and /info.json dir on this server """ def __init__(self,server,infoFile): self._server = server self._infoFile = infoFile self.br = URLopener() """ hasNewInfo :: Boolean compare the local info file hash with the one found on the server and returns true if they are different """ def hasNewInfo(self): f = open(self._infoFile,'r').read() m = md5.new(f).hexdigest() response = self.br.open(self._server+'/hash').read() response = response.replace("\n","") return (m!=response) """ generateTimeStamp :: String returns a string that is used to timestamp old config backup files """ def generateTimeStamp(self): return str(time.gmtime().tm_year)+"_"+str(time.gmtime().tm_mday)+"_"+str(time.gmtime().tm_hour)+"_"+str(time.gmtime().tm_min) """ fetchNewInfo :: Void it will download the info file from the server use the timestamp to back it up and overwrite it """ def fetchNewInfo(self): response = self.br.open(self._server+'/info.json').read() oldInfo = open(self._infoFile,'r').read() open(self._infoFile+"."+self.generateTimeStamp(),'w').write(oldInfo) open(self._infoFile,'w').write(response)
class Updater: def __init__(self,server,infoFile): self._server = server self._infoFile = infoFile self.br = URLopener() def hasNewInfo(self): f = open(self._infoFile,'r').read() m = md5.new(f).hexdigest() response = self.br.open(self._server+'/hash').read() response = response.replace("\n","") return (m!=response) def generateTimeStamp(self): return str(time.gmtime().tm_year)+"_"+str(time.gmtime().tm_mday)+"_"+str(time.gmtime().tm_hour)+"_"+str(time.gmtime().tm_min) def fetchNewInfo(self): response = self.br.open(self._server+'/info.json').read() oldInfo = open(self._infoFile,'r').read() open(self._infoFile+"."+self.generateTimeStamp(),'w').write(oldInfo) open(self._infoFile,'w').write(response)
def call_remote(self, category, params): ''' The meetup api is set up such that the root url does not change much other than the'name' of the thing you call into. In other words, I can just use category to sprintf my way to a valid url, then tack on the rest of the query string specified in params. ''' url = self.root_url url = url % (category) # Every call has to include key url = url + "?" + params + "&key=" + self.key client = URLopener() request = client.open(url) raw_str = request.read() results = json.loads(raw_str) # Let the caller interpret the results of the call. Both the # meta info and the results are passed back return results
def call_remote(self,category,params): ''' The meetup api is set up such that the root url does not change much other than the'name' of the thing you call into. In other words, I can just use category to sprintf my way to a valid url, then tack on the rest of the query string specified in params. ''' url = self.root_url url = url % (category) # Every call has to include key url = url + "?" + params + "&key=" + self.key client = URLopener() request = client.open(url) raw_str = request.read() results = json.loads(raw_str) # Let the caller interpret the results of the call. Both the # meta info and the results are passed back return results
memoHeadings = {} posCount = 0 for elem in courtElems: if not elem.text: continue country = normalStr(elem.text) if country == "Congo RDC": continue strIO = cStringIO.StringIO() urlStub = elem.attrib['href'] if urlStub == '/wlg/courts/nofr/usstates/lxctusa.htm': continue if urlStub == '/wlg/courts/nofr/oeur/lxctjap.htm': continue print country countryHtml = urlh.open(siteRoot + urlStub).read() options = dict(output_xhtml=1, add_xml_decl=1, indent=1, tidy_mark=0) countryHtml = tidy.parseString(countryHtml,**options) countryHtml.write(strIO) strIO.seek(0) countryHtml = strIO.read() strIO.close() countryHtml = re.sub('xmlns="[^"]+"',"",countryHtml) countryDoc = etree.fromstring(countryHtml) courtHeadingElems = countryDoc.xpath("//font[@color='#009944']") for e in courtHeadingElems: heading = normalStr(e.text) #if not allHeadings.has_key(heading): # posCount += 1 # continue
class mangareader_downloader(object): def __init__(self,manga_name,chapter,end_chapter,manga_location,dl_manager): self.manga_location = manga_location self.manga_name = manga_name self.chapter = chapter self.end_chapter = end_chapter self.flag = False self.current_image = "000" self.img = "" self.next_link = "" self.current_page = "http://www.mangareader.net/"+self.manga_name+"/"+self.chapter+"/" self.next_regex = "<span class=\"next\"><a href=\"([^\"]*)\">Next</a></span>" self.nb_of_pages = 0 self.page_counter = 2 self.br = URLopener() self.response = "" self.response_lines = "" self.dl_manager = dl_manager self.resolved = { 'http://i0':'188.132.173.122', 'http://i1':'188.132.173.3', 'http://i2':'188.132.173.6', 'http://i3':'188.132.173.9', 'http://i4':'188.132.173.12', 'http://i5':'188.132.173.15', 'http://i6':'188.132.173.18', 'http://i7':'188.132.173.21', 'http://i8':'188.132.173.24', 'http://i9':'188.132.173.27', 'http://i10':'188.132.173.30', 'http://i11':'188.132.173.33', 'http://i12':'188.132.173.36', 'http://i13':'188.132.173.39', 'http://i14':'188.132.173.42', 'http://i15':'188.132.173.45', 'http://i16':'188.132.173.48', 'http://i17':'188.132.173.51', 'http://i18':'188.132.173.54', 'http://i19':'188.132.173.57', 'http://i20':'188.132.173.60', 'http://i21':'188.132.173.63', 'http://i22':'188.132.173.66', 'http://i23':'188.132.173.69', 'http://i24':'188.132.173.72', 'http://i25':'188.132.173.75', 'http://i26':'188.132.173.78', 'http://i27':'188.132.173.81', 'http://i28':'188.132.173.84', 'http://i29':'188.132.173.87', 'http://i30':'188.132.173.90', 'http://i31':'188.132.173.93', 'http://i32':'188.132.173.96', 'http://i33':'188.132.173.99', 'http://i34':'188.132.173.126', 'http://i35':'188.132.173.129', 'http://i36':'188.132.173.132', 'http://i37':'188.132.173.135', 'http://i38':'188.132.173.138', 'http://i39':'188.132.173.141', 'http://i40':'188.132.173.144', 'http://i41':'188.132.173.200', 'http://i1000':'188.132.173.200', 'http://i999':'188.132.173.12', 'http://i998':'188.132.173.48', 'http://i997':'188.132.173.72', 'http://i996':'188.132.173.96', 'http://i995':'188.132.173.144', 'http://i994':'188.132.173.200' } def increase_current(self): self.current_image = str(int(self.current_image)+1) if len(self.current_image) == 1: self.current_image = "00"+self.current_image elif len(self.current_image) == 2: self.current_image = "0"+self.current_image self.page_counter+=1 def increase_chapter(self): self.nb_of_pages = 0 self.page_counter = 1 self.chapter = str(int(self.chapter)+1) self.current_image = "000" self.next_link = "http://www.mangareader.net/"+self.manga_name+"/"+self.chapter+"/"+str(self.page_counter) self.page_counter +=1 def check_chapter_end(self): if self.page_counter-1 == self.nb_of_pages: return True else : return False def not_published(self): if "is not published yet. Once" in self.response or self.chapter == str(int(self.end_chapter)+1): return True return False def go_to_next_page(self): if not self.check_chapter_end(): self.increase_current() else: self.increase_chapter() self.current_page = self.next_link def scrap_page(self): if self.nb_of_pages == 0: for a in self.response_lines: if "</select> of " in a: self.nb_of_pages = int(re.findall("</select> of (\d+)",a)[0]) break for a in self.response_lines: if '"><img id=\"img\"' in a: self.img = re.findall("src=\"([^\"]*)\" alt",a)[0] break self.next_link = "http://www.mangareader.net/"+self.manga_name+"/"+self.chapter+"/"+str(self.page_counter) def manage_chapters(self): if not os.path.exists(self.manga_location): os.mkdir(self.manga_location) os.chdir(self.manga_location) if not os.path.exists(self.manga_name): os.mkdir(self.manga_name) os.chdir(self.manga_name) if not os.path.exists(self.manga_name+"-"+self.chapter): os.mkdir(self.manga_name+"-"+self.chapter) os.chdir(self.manga_name+"-"+self.chapter) def download_image(self): self.manage_chapters() caching = self.img.split('.')[0] if caching in self.resolved: self.img = self.img.replace( caching+".mangareader.net", "http://"+self.resolved[caching]) if self.dl_manager == 'default': urlretrieve(self.img, self.current_image+'.jpg' ) else: status = 1 while int(status) != 0: status = os.system(self.dl_manager +" "+self.img+ " -o "+self.current_image+".jpg") print "[*] Image saved to "+ os.getcwd() + "/"+self.current_image+".jpg" def start_downloading(self): try: self.response = self.br.open(self.current_page).read() self.response_lines = self.response.split("\n") if not self.not_published(): self.scrap_page() self.manage_chapters() self.download_image() self.go_to_next_page() else : self.flag = True except Exception,e: print e time.sleep(2) self.start_downloading()
We need to liaise with Shima-san about contributing to the data set. ''' import os,sys,re from urllib import URLopener from lxml import etree import json obj = {} urlh = URLopener() html = urlh.open("http://en.wikipedia.org/wiki/List_of_supreme_courts_by_country").read() doc = etree.fromstring(html) entries = doc.xpath("//table[@class='wikitable']//tr") stops = ["the","of","a"] def makeID(court): courtID = court.lower() courtID = re.split("\s+",courtID) for i in range(len(courtID)-1,-1,-1): word = courtID[i] if word in stops: courtID = courtID[0:i] + courtID[i+1:] return ".".join(courtID)
word. We need to liaise with Shima-san about contributing to the data set. ''' import os, sys, re from urllib import URLopener from lxml import etree import json obj = {} urlh = URLopener() html = urlh.open( "http://en.wikipedia.org/wiki/List_of_supreme_courts_by_country").read() doc = etree.fromstring(html) entries = doc.xpath("//table[@class='wikitable']//tr") stops = ["the", "of", "a"] def makeID(court): courtID = court.lower() courtID = re.split("\s+", courtID) for i in range(len(courtID) - 1, -1, -1): word = courtID[i] if word in stops: courtID = courtID[0:i] + courtID[i + 1:]
class mangareader_downloader(object): def __init__(self, manga_name, chapter, end_chapter, manga_location, dl_manager): self.manga_location = manga_location self.manga_name = manga_name self.chapter = chapter self.end_chapter = end_chapter self.flag = False self.current_image = "000" self.img = "" self.next_link = "" self.current_page = "http://www.mangareader.net/" + self.manga_name + "/" + self.chapter + "/" self.next_regex = "<span class=\"next\"><a href=\"([^\"]*)\">Next</a></span>" self.nb_of_pages = 0 self.page_counter = 2 self.br = URLopener() self.response = "" self.response_lines = "" self.dl_manager = dl_manager self.resolved = { 'http://i0': '188.132.173.122', 'http://i1': '188.132.173.3', 'http://i2': '188.132.173.6', 'http://i3': '188.132.173.9', 'http://i4': '188.132.173.12', 'http://i5': '188.132.173.15', 'http://i6': '188.132.173.18', 'http://i7': '188.132.173.21', 'http://i8': '188.132.173.24', 'http://i9': '188.132.173.27', 'http://i10': '188.132.173.30', 'http://i11': '188.132.173.33', 'http://i12': '188.132.173.36', 'http://i13': '188.132.173.39', 'http://i14': '188.132.173.42', 'http://i15': '188.132.173.45', 'http://i16': '188.132.173.48', 'http://i17': '188.132.173.51', 'http://i18': '188.132.173.54', 'http://i19': '188.132.173.57', 'http://i20': '188.132.173.60', 'http://i21': '188.132.173.63', 'http://i22': '188.132.173.66', 'http://i23': '188.132.173.69', 'http://i24': '188.132.173.72', 'http://i25': '188.132.173.75', 'http://i26': '188.132.173.78', 'http://i27': '188.132.173.81', 'http://i28': '188.132.173.84', 'http://i29': '188.132.173.87', 'http://i30': '188.132.173.90', 'http://i31': '188.132.173.93', 'http://i32': '188.132.173.96', 'http://i33': '188.132.173.99', 'http://i34': '188.132.173.126', 'http://i35': '188.132.173.129', 'http://i36': '188.132.173.132', 'http://i37': '188.132.173.135', 'http://i38': '188.132.173.138', 'http://i39': '188.132.173.141', 'http://i40': '188.132.173.144', 'http://i41': '188.132.173.200', 'http://i1000': '188.132.173.200', 'http://i999': '188.132.173.12', 'http://i998': '188.132.173.48', 'http://i997': '188.132.173.72', 'http://i996': '188.132.173.96', 'http://i995': '188.132.173.144', 'http://i994': '188.132.173.200' } def increase_current(self): self.current_image = str(int(self.current_image) + 1) if len(self.current_image) == 1: self.current_image = "00" + self.current_image elif len(self.current_image) == 2: self.current_image = "0" + self.current_image self.page_counter += 1 def increase_chapter(self): self.nb_of_pages = 0 self.page_counter = 1 self.chapter = str(int(self.chapter) + 1) self.current_image = "000" self.next_link = "http://www.mangareader.net/" + self.manga_name + "/" + self.chapter + "/" + str( self.page_counter) self.page_counter += 1 def check_chapter_end(self): if self.page_counter - 1 == self.nb_of_pages: return True else: return False def not_published(self): if "is not published yet. Once" in self.response or self.chapter == str( int(self.end_chapter) + 1): return True return False def go_to_next_page(self): if not self.check_chapter_end(): self.increase_current() else: self.increase_chapter() self.current_page = self.next_link def scrap_page(self): if self.nb_of_pages == 0: for a in self.response_lines: if "</select> of " in a: self.nb_of_pages = int( re.findall("</select> of (\d+)", a)[0]) break for a in self.response_lines: if '"><img id=\"img\"' in a: self.img = re.findall("src=\"([^\"]*)\" alt", a)[0] break self.next_link = "http://www.mangareader.net/" + self.manga_name + "/" + self.chapter + "/" + str( self.page_counter) def manage_chapters(self): if not os.path.exists(self.manga_location): os.mkdir(self.manga_location) os.chdir(self.manga_location) if not os.path.exists(self.manga_name): os.mkdir(self.manga_name) os.chdir(self.manga_name) if not os.path.exists(self.manga_name + "-" + self.chapter): os.mkdir(self.manga_name + "-" + self.chapter) os.chdir(self.manga_name + "-" + self.chapter) def download_image(self): self.manage_chapters() caching = self.img.split('.')[0] if caching in self.resolved: self.img = self.img.replace(caching + ".mangareader.net", "http://" + self.resolved[caching]) if self.dl_manager == 'default': urlretrieve(self.img, self.current_image + '.jpg') else: status = 1 while int(status) != 0: status = os.system(self.dl_manager + " " + self.img + " -o " + self.current_image + ".jpg") print "[*] Image saved to " + os.getcwd( ) + "/" + self.current_image + ".jpg" def start_downloading(self): try: self.response = self.br.open(self.current_page).read() self.response_lines = self.response.split("\n") if not self.not_published(): self.scrap_page() self.manage_chapters() self.download_image() self.go_to_next_page() else: self.flag = True except Exception, e: print e time.sleep(2) self.start_downloading()
#!/usr/bin/env python from re import sub from BeautifulSoup import BeautifulSoup from urllib import URLopener opener = URLopener() html = opener.open('http://www.dailyzen.com/').read() html = html[html.index('<!--Add Quote for correct day-->'):] html1 = html[:html.index('<br>')] html2 = html[html.index('<A class="artist">'):] html2 = html2[:html2.index('</a></i>')] html2 = sub('<A class="artist">','',html2).strip() zen = BeautifulSoup(html1) zen = zen.prettify().strip() for x in ['<!--Add Quote for correct day-->','<br />','<p>','</p>','^\n','\n$']: zen = sub(x,'',zen).strip() zen = sub('\n \n \n','\n \n',zen).strip() print print zen print print '\t\t',html2
memoHeadings = {} posCount = 0 for elem in courtElems: if not elem.text: continue country = normalStr(elem.text) if country == "Congo RDC": continue strIO = cStringIO.StringIO() urlStub = elem.attrib['href'] if urlStub == '/wlg/courts/nofr/usstates/lxctusa.htm': continue if urlStub == '/wlg/courts/nofr/oeur/lxctjap.htm': continue print country countryHtml = urlh.open(siteRoot + urlStub).read() options = dict(output_xhtml=1, add_xml_decl=1, indent=1, tidy_mark=0) countryHtml = tidy.parseString(countryHtml, **options) countryHtml.write(strIO) strIO.seek(0) countryHtml = strIO.read() strIO.close() countryHtml = re.sub('xmlns="[^"]+"', "", countryHtml) countryDoc = etree.fromstring(countryHtml) courtHeadingElems = countryDoc.xpath("//font[@color='#009944']") for e in courtHeadingElems: heading = normalStr(e.text) #if not allHeadings.has_key(heading): # posCount += 1 # continue
def test_ping_play1(): from urllib import URLopener u = URLopener() text = "<title>pypy.js various demos</title>" assert u.open("http://play1.pypy.org/").read().find(text) != -1
def test_ping_play1(): from urllib import URLopener u = URLopener() text = "<title>pypy.js various demos</title>" assert u.open("http://play1.pypy.org/").read().find(text) != -1
def open(self, *args): f = URLopener.open(self, *args) return XML(f)