class GetDelivRecords: def __init__(self, verbose=False, debug=False): self.__dbg__ = debug self.__verbos = verbose self._crawler = Crawler() self._crawler.set_headers(( ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.19) Gecko/2010040116 Ubuntu/9.04 (jaunty) Firefox/3.0.19'), \ ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') )) self._wraper = HTMLSequenceWrapper(childcoef=5.0, headercoef=3.0, mintextlen=20) self._unwanted_titles = ['Download here', 'PDF format'] self._records = [] ################################ #manual processing self.agent = GetHTMLAndParse() # to get region where to search for records self.regionHandler = GetDeliverableRegion() # init text formatter (encoding, erasing white chars etc.) self.formatter = TextFormatUtils() self._omitted_tags = ('br', 'img', 'html', 'body') # tag tolerance self.tagtol = 1 def __debug(self, msg): _err = "cannot decode debug info." if self.__dbg__ == True: try: print("Debug message: " + str(msg)) except UnicodeError: print(_err) def __verbose(self, msg): _err = "cannot decode debug info." if self.__verbose == True: try: print("Verbose: " + str(msg)) except UnicodeError: print(_err) ########################Processing sequencewrapper output###################### """function gets an entry from output of sequence wrapper it tries to create deliv record and retruns true if succed. """ def _make_deliv_record(self, entry): text = [] links = [] #harvest links and text form entry for e in entry.iter(): if e.text != None: text.append(e.text) if e.attrib.get("link") != None: if self.agent.is_wanted_mime(e.attrib.get( "link")) and e.attrib.get("link") not in links: links.append(e.attrib.get("link")) res = self._deliv_in_text(text, links) if type(res) == RRSPublication: self._entriesFoundInText.append(res) self.__debug("Record found cause of text") return True elif type(res) == list: res = self._more_entry_in_record(entry) if (res == True): self.__debug("") return True else: return False res = self._deliv_in_link(text, links, entry) if type(res) == RRSPublication: self._entriesFoundInLinks.append(res) self.__debug("Record found cause of link") return True return False """look for keyword in text""" def _deliv_in_text(self, text, links): #print text #print links #print "*"*40 _title = False _description = "" pattern = re.compile("(DELIVERABLES?)|(D[0-9][0-9]*(.[0-9][0-9]*)?)", re.I) #loop through text in entry looking for title and description for t in text: if _title == False: if pattern.search(t): _title = t #set the longest string as description of deliverable if len(_description) < len(t): _description = t if _title == _description: _description = "" _link = False if type(links) == str: if self.agent.is_wanted_mime(links): _link = links elif type(links) == list: for l in links: if self.agent.is_wanted_mime(l): if _link == False: _link = l else: #if there was already found link if _link[:s.rfind(_link, '.')] == l[:s.rfind(l, '.')]: break else: return ['-3', 'Probably more records in one entry'] #create object if _title: #print "TITLE:"+_title pub = RRSPublication(title=_title, abstract=_description) _typ = RRSPublication_type(type='techreport') pub['type'] = _typ self.__debug("*" * 40) self.__debug("Title: " + _title) self.__debug("Description: " + _description) if _link: #print "LINK:"+_link self.__debug("Link: " + _link) l = RRSUrl(link=_link) pl_rel = RRSRelationshipPublicationUrl() pl_rel.set_entity(l) pub['url'] = pl_rel return pub else: #this entry is not probably deliverable return False """look for a key word in link""" def _deliv_in_link(self, text, links, entry=False): ##print text ##print links #print "*"*40 _title = False _description = "" pattern = re.compile("(DELIVERABLES?)|(D[0-9][0-9]*(.[0-9][0-9]*)?)", re.I) _link = False for l in links: if pattern.search(l): if _link == False: _link = l else: return ['-3', 'Probably more records in one entry'] #loop through text in entry looking for title and description for t in text: if _title == False: if len(t) > 10: _title = t #set the longest string as description of deliverable if len(_description) < len(t): _description = t if _title == _description: _description = "" #if chosen title is not valid try to find better in parent entry if _title and not self._check_title(_title) and entry != False: _title = self._repair_title(entry) #create object if _link: pub = RRSPublication(title=_title, abstract=_description) typ = RRSPublication_type(type='techreport') pub['type'] = typ self.__debug("*" * 40) self.__debug("Title: " + _title) self.__debug("Description: " + _description) self.__debug("Link: " + _link) l = RRSUrl(link=_link) pl_rel = RRSRelationshipPublicationUrl() pl_rel.set_entity(l) pub['url'] = pl_rel return pub else: #this entry is not probably deliverable return False """Check if title contents only unwanted string with some tolerance return true if title is ok """ def _check_title(self, title, tolerance=10): for t in self._unwanted_titles: if (s.find(s.lower(title), s.lower(t))) != -1: if (len(t) + tolerance) > len(title): return False return True "looks for an element with highest visibility rank in parent elemet" def _repair_title(self, entry): parent = entry.getparent() visibility = 0 title = "" for i in parent.iter(): try: if i.attrib.get('visibility') > visibility: visibility = i.attrib.get('visibility') title = i.text except AttributeError: pass if title != "": return title else: return False "Function try to create array of deliverables from one entry in xml tree" def _more_entry_in_record(self, entry): for ch in entry.iter('chunk'): if ch.text != None and ch.attrib.get("link") != None: if self.agent.is_wanted_mime(ch.attrib.get("link")): _pub = RRSPublication(title=ch.text) typ = RRSPublication_type(type='techreport') _pub['type'] = typ _l = RRSUrl(link=ch.attrib.get("link")) _rel = RRSRelationshipPublicationUrl() _rel.set_entity(_l) _pub['url'] = _rel self._entriesFoundInLinks.append(_pub) "Process pages definied by urls" def process_pages(self, pages): self._entriesFoundInText = [] self._entriesFoundInLinks = [] self._urls = pages self._pages = self._crawler.start(pages) #creates RRSPublication objects with information about deliverables for u in self._urls: self._wraper.wrap(self._pages[u], u) self._tree = self._wraper.get_etree() #print self._wraper.get_xml() for entry in self._tree.iter("entry"): self._make_deliv_record(entry) if len(self._entriesFoundInText) > 3: self.__debug("Deliverbles descriptions content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInText)) + " deliv records") self._records = self._entriesFoundInText elif len(self._entriesFoundInLinks) > 3: self.__debug("Deliverbles links content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInLinks)) + " deliv records") self._records = self._entriesFoundInLinks else: self._manual_processing() "This method is called when ther was no records found in output of sequencewrapper" def _manual_processing(self): self._entriesFoundInLinks = [] self._entriesFoundInText = [] self._manual_process_page(self._urls, urlsplit(self._urls[0])[1]) if len(self._entriesFoundInText) > 0: self.__debug("Deliverbles descriptions content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInText)) + " deliv records") self._records = self._entriesFoundInText elif len(self._entriesFoundInLinks) > 0: self.__debug("Deliverbles links content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInLinks)) + " deliv records") self._records = self._entriesFoundInLinks ########################### TABLE HANDLING METHODS ############################ """ Get texts from element and his descendants. If string isset, returns texts as one string with spaces. # elem: lxml element """ def _get_descendats_texts(self, elem, string=True): texts = [] for child in elem.iter(): if child.text and isinstance(child.tag, basestring): if re.search("[a-z0-9]", child.text, re.I): texts.append(self.formatter.format(child.text)) if string: return " ".join(texts) return texts """ Get link from row of table - go through columns and the only href leading to deliverable is returned. """ def _get_row_link(self, row): # find all anchors where parent is row linkanch = row.findall('.//a[@href]') if len(linkanch) == 0: return None for link in linkanch: anchor_link = link.get('href') if self.agent.is_wanted_mime( anchor_link): # check if it is file we want return anchor_link return None """ Handle region as a table. Work with region as it's a table. Try to get table semantic (table order) and get all records out of it. """ def _handle_table(self): for row in self.parentetree: if not row.tag == 'tr': continue row_list = [] _thislink = self._get_row_link(row) if _thislink == None: continue for column in row: text = self._get_descendats_texts(column) if not text: continue row_list.append(text) res = self._deliv_in_text(row_list, [_thislink]) if type(res) == RRSPublication: self._entriesFoundInText.append(res) self.__debug("Record found cause of text") else: res = self._deliv_in_link(row_list, [_thislink]) if type(res) == RRSPublication: self._entriesFoundInLinks.append(res) self.__debug("Record found cause of link") del (row_list) return ######################## TAG SEQUENCE RECOGNIZING METHODS #################### """ Tag check. If it is anchor with href leading to deliverable, returns True """ def _is_deliv_anch(self, tag): if tag.tag == 'a': href = tag.get('href') if self.agent.is_wanted_mime(href): return True return False """ Filters useless and messy tags. Return false if useless, true if normal tag """ def _tagfilter(self, tag): if tag.tag in self._omitted_tags: return False #if tag.text: # if not re.search("[a-z0-9\[\]]", tag.text, re.I): # return False return True """ Gets difference between first two anchors. """ def _getdiff(self, reg, tol): # etree reg = element tree region # int tol: accepted tolerance of tags d = {} index = 0 # fill the dictionary with differences and their occurences for tag in reg.iter(): if not self._tagfilter(tag): continue if self._is_deliv_anch(tag) and not index == 0: try: d[index] += 1 except: d[index] = 1 index = 0 index += 1 # check differencies if the variety isn't higher then $tol tolerance difflist = d.keys() self.__debug("difflist: " + str(difflist)) if len(difflist) == 0: return -1 _max = max(difflist) _min = min(difflist) dlen = len(d.keys()) if dlen == 1: return d.keys()[0] if dlen > ((2 * tol) + 1): # tolerance to both sides return -1 if (_max - _min) > 2 * tol: # some acceptable tolerance return -1 # get the most frequent difference most_freq = max(d.values()) for key in d: if d[key] == most_freq: return key return -1 """ Only anchors found. No optional information. """ def _get_anch_only(self): anchlist = self.agent.find_anchor_elem(self.baseUrl, self.parentetree) # We have to make list of list because XMLOutput return [[anch] for anch in anchlist] """ Main method handling tag sequences and recognizing records. Returns list of records. """ def _get_tag_sequences(self, tag_tol=1): records = [] self._rec = [] if len(self.parentetree) == 0: return [[self.parentetree]] # get interval between anchors, use tolerance tag_tol self.difference = self._getdiff(self.parentetree, self.tagtol) while self.difference == -1: if self.tagtol > 5: self.__verbose("Variety of intervals between anchors is too huge. "+\ "Getting data out of anchors only") return self._get_anch_only() self.tagtol += 1 self.difference = self._getdiff(self.parentetree, self.tagtol) # get sequence of first n tags, where n is average interval between anchors # this could be tag-sequence describing all records in region. self.record_seq = [] i = 0 for tag in self.parentetree.iter(): if not self._tagfilter(tag): continue if i >= self.difference: if not 'a' in self.record_seq: del self.record_seq[0] else: break self.record_seq.append(tag.tag) i += 1 # counter indicates on which position in tag sequence we actually are counter = 0 # make sequence of tags as they go regionlist = filter(self._tagfilter, [tag for tag in self.parentetree.iter()]) recseqlen = len(self.record_seq) reglistlen = len(regionlist) # flag indicating begin of records - in region on the beginning can be some garbage self.begin = False # indicating unpredictable separator between deliverable records self.separator = 0 for i, tag in enumerate(regionlist): # skip and save the sequence at the end if counter > self.difference - 1: records.append(self._rec) # save self._rec = [] # erase the list counter = 0 # reset counter if not self.begin: if tag.tag != self.record_seq[0]: continue else: try: if regionlist[i + 1].tag != self.record_seq[1]: continue except: pass self.begin = True # handle tolerances, try to compare sibling tags self.match = False # match flag # tolerance algorithm. Goes through html and tries to pass irregular tags in sequence. for tol in range(self.tagtol + 1): if tag.tag == self.record_seq[(counter + tol) % recseqlen] or \ regionlist[(i + tol) % reglistlen].tag == self.record_seq[counter % recseqlen]: self.match = True self._rec.append(tag) counter += tol + 1 break elif tag.tag == self.record_seq[(counter - tol) % recseqlen] or \ regionlist[(i - tol) % reglistlen].tag == self.record_seq[counter % recseqlen]: self.match = True self._rec.append(tag) counter -= tol counter += 1 break # if nothing matched, its probably out of tolerance if not self.match: self.separator += 1 # tolerance 10 separators (tags between boxes or tables of deliverables) if self.separator > 10: self.__verbose("Tag sequence doesnt match, probably out of "+\ "tolerance, getting data out of anchors only") # maybe here could be tolerance++ # we didnt catch the sequence with tolerance... return self._get_anch_only() records.append(self._rec) return filter(self._validseq, records) """ Helper method - check if sequence of tags rec contains deliv anchor """ def _validseq(self, rec): for _atr in rec: # if we have anchor containing link to document, return true if self._is_deliv_anch(_atr): return True return False """ Get element texts only, dont look for descendants texts """ def _get_tag_content(self, tag): links = [] texts = [] if tag.tag == 'a': href = tag.get('href') # if link leading to document found, add string to list if href is not None and self.agent.is_wanted_mime(href): links.append(self.formatter.format(href)) title = tag.get('title') # if title found in tag, add string to list if title: texts.append(self.formatter.format(title)) # if not anchor, search text in tag.text if tag.text: if re.search("[a-z0-9]", tag.text, re.I): texts.append(self.formatter.format(tag.text)) return [links, texts] """ Harvest texts out of tags and return list of lists (record) """ def _harvest_text(self, record_tag_list): self._records = [] self._rec = [] _links = [] _texts = [] # loop over records and search all possible useful texts for rec_list in record_tag_list: for tag in rec_list: harvested = (self._get_tag_content(tag)) _links.extend(harvested[0]) _texts.extend(harvested[1]) #self._records.append(self._rec) res = self._deliv_in_text(_texts, _links) if type(res) == RRSPublication: self._entriesFoundInText.append(res) self.__debug("Record found cause of text") else: res = self._deliv_in_link(_texts, _links) if type(res) == RRSPublication: self._entriesFoundInLinks.append(res) self.__debug("Record found cause of link") _links = [] _texts = [] self._rec = [] return self._records """ Text harvesting for sequences. """ def _handle_sequence(self): seq = self._get_tag_sequences() return self._harvest_text(seq) """ Get records from region according document links this method is used when there was no records found in output of sequencewrapper""" def _manual_process_page(self, links, baseurl): _err = None self.baseUrl = baseurl for link in links: # find region with tolerance self.parentetree = self.regionHandler.get_region(link, baseurl, 1) if type(self.parentetree) == tuple: # error _err = self.parentetree self.__debug(_err) continue #make all links absolute in parent tree hrefs = self.parentetree.findall('.//a[@href]') for href in hrefs: href.make_links_absolute('http://' + urlsplit(link)[1] + '/') # get the charset. We dont have etree in htmlHandler, # so we have to use the one from regionHandler self.formatter.set_charset( self.regionHandler.formatter.get_charset()) self.__debug("*" * 100 + '\n' + "*" * 40 + " DATA REGION " + "*" * 40) self.__debug( lxml.etree.tostring(self.parentetree, pretty_print=True)) # get root tag try: self.parentetree = self.parentetree.getroot() except: pass # Parent tag is table # call _handle_table if self.parentetree.tag in ('table', 'tbody'): self.__verbose("Handling table") self._handle_table() else: self.__verbose("Handling sequences") self._handle_sequence() #############PUBLIC METHODS TO GET RESULTS def get_deliverables_XML(self): """return infromations about deliverables stored in objects as xml""" if len(self.get_deliverables()) == 0: return derrno.__err__(derrno.ENOREC) output = StringIO.StringIO() converter = Model2XMLConverter(stream=output) converter.convert(self.get_deliverables()) result = output.getvalue() output.close() return result def get_deliverables(self): """return objects containing infromations""" if len(self._records) == 0: return derrno.__err__(derrno.ENOREC) else: return self._records
class GetDelivRecords: def __init__(self,verbose=False,debug=False): self.__dbg__ = debug self.__verbos = verbose self._crawler = Crawler() self._crawler.set_headers(( ('User-Agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.19) Gecko/2010040116 Ubuntu/9.04 (jaunty) Firefox/3.0.19'), \ ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8') )) self._wraper = HTMLSequenceWrapper(childcoef=5.0, headercoef=3.0, mintextlen=20) self._unwanted_titles = ['Download here','PDF format'] self._records = [] ################################ #manual processing self.agent = GetHTMLAndParse() # to get region where to search for records self.regionHandler = GetDeliverableRegion() # init text formatter (encoding, erasing white chars etc.) self.formatter = TextFormatUtils() self._omitted_tags = ('br', 'img', 'html', 'body') # tag tolerance self.tagtol = 1 def __debug(self, msg): _err = "cannot decode debug info." if self.__dbg__ == True: try: print("Debug message: "+str(msg)) except UnicodeError: print(_err) def __verbose(self,msg): _err = "cannot decode debug info." if self.__verbose == True: try: print("Verbose: "+str(msg)) except UnicodeError: print(_err) ########################Processing sequencewrapper output###################### """function gets an entry from output of sequence wrapper it tries to create deliv record and retruns true if succed. """ def _make_deliv_record(self,entry): text = [] links = [] #harvest links and text form entry for e in entry.iter(): if e.text != None: text.append(e.text) if e.attrib.get("link")!=None: if self.agent.is_wanted_mime(e.attrib.get("link")) and e.attrib.get("link") not in links: links.append(e.attrib.get("link")) res = self._deliv_in_text(text,links) if type(res) == RRSPublication: self._entriesFoundInText.append(res) self.__debug("Record found cause of text") return True elif type(res)==list: res=self._more_entry_in_record(entry) if(res==True): self.__debug("") return True else: return False res = self._deliv_in_link(text,links,entry) if type(res) == RRSPublication: self._entriesFoundInLinks.append(res) self.__debug("Record found cause of link") return True return False """look for keyword in text""" def _deliv_in_text(self,text,links): #print text #print links #print "*"*40 _title = False _description = "" pattern = re.compile("(DELIVERABLES?)|(D[0-9][0-9]*(.[0-9][0-9]*)?)",re.I) #loop through text in entry looking for title and description for t in text: if _title == False: if pattern.search(t): _title = t #set the longest string as description of deliverable if len(_description)<len(t): _description = t if _title == _description: _description = "" _link = False if type(links) == str: if self.agent.is_wanted_mime(links): _link = links elif type(links) ==list: for l in links: if self.agent.is_wanted_mime(l): if _link == False: _link = l else: #if there was already found link if _link[:s.rfind(_link,'.')] == l[:s.rfind(l,'.')]: break else: return ['-3','Probably more records in one entry'] #create object if _title: #print "TITLE:"+_title pub = RRSPublication(title=_title,abstract=_description) _typ = RRSPublication_type(type='techreport') pub['type'] = _typ self.__debug("*"*40) self.__debug("Title: "+_title) self.__debug("Description: "+_description) if _link: #print "LINK:"+_link self.__debug("Link: "+_link) l = RRSUrl(link=_link) pl_rel = RRSRelationshipPublicationUrl() pl_rel.set_entity(l) pub['url'] = pl_rel return pub else: #this entry is not probably deliverable return False """look for a key word in link""" def _deliv_in_link(self,text,links,entry = False): ##print text ##print links #print "*"*40 _title = False _description = "" pattern = re.compile("(DELIVERABLES?)|(D[0-9][0-9]*(.[0-9][0-9]*)?)",re.I) _link = False for l in links: if pattern.search(l): if _link == False: _link =l else: return ['-3','Probably more records in one entry'] #loop through text in entry looking for title and description for t in text: if _title == False: if len(t)>10 : _title = t #set the longest string as description of deliverable if len(_description)<len(t): _description = t if _title == _description: _description = "" #if chosen title is not valid try to find better in parent entry if _title and not self._check_title(_title) and entry != False: _title = self._repair_title(entry) #create object if _link: pub = RRSPublication(title=_title,abstract=_description) typ = RRSPublication_type(type='techreport') pub['type'] = typ self.__debug("*"*40) self.__debug("Title: "+_title) self.__debug("Description: "+_description) self.__debug("Link: "+_link) l = RRSUrl(link=_link) pl_rel = RRSRelationshipPublicationUrl() pl_rel.set_entity(l) pub['url'] = pl_rel return pub else: #this entry is not probably deliverable return False """Check if title contents only unwanted string with some tolerance return true if title is ok """ def _check_title(self,title,tolerance=10): for t in self._unwanted_titles: if (s.find(s.lower(title),s.lower(t))) != -1: if (len(t)+tolerance) > len(title): return False return True "looks for an element with highest visibility rank in parent elemet" def _repair_title(self,entry): parent = entry.getparent() visibility = 0 title = "" for i in parent.iter(): try: if i.attrib.get('visibility') > visibility: visibility = i.attrib.get('visibility') title = i.text except AttributeError: pass if title != "": return title else: return False "Function try to create array of deliverables from one entry in xml tree" def _more_entry_in_record(self,entry): for ch in entry.iter('chunk'): if ch.text != None and ch.attrib.get("link")!=None: if self.agent.is_wanted_mime(ch.attrib.get("link")): _pub= RRSPublication(title=ch.text) typ = RRSPublication_type(type='techreport') _pub['type'] = typ _l = RRSUrl(link=ch.attrib.get("link")) _rel = RRSRelationshipPublicationUrl() _rel.set_entity(_l) _pub['url'] = _rel self._entriesFoundInLinks.append(_pub) "Process pages definied by urls" def process_pages(self,pages): self._entriesFoundInText = [] self._entriesFoundInLinks = [] self._urls = pages self._pages = self._crawler.start(pages) #creates RRSPublication objects with information about deliverables for u in self._urls: self._wraper.wrap(self._pages[u],u) self._tree = self._wraper.get_etree() #print self._wraper.get_xml() for entry in self._tree.iter("entry"): self._make_deliv_record(entry) if len(self._entriesFoundInText)>3: self.__debug("Deliverbles descriptions content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInText)) + " deliv records") self._records = self._entriesFoundInText elif len(self._entriesFoundInLinks)>3: self.__debug("Deliverbles links content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInLinks)) + " deliv records") self._records = self._entriesFoundInLinks else: self._manual_processing() "This method is called when ther was no records found in output of sequencewrapper" def _manual_processing(self): self._entriesFoundInLinks = [] self._entriesFoundInText = [] self._manual_process_page(self._urls, urlsplit(self._urls[0])[1]) if len(self._entriesFoundInText)>0: self.__debug("Deliverbles descriptions content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInText)) + " deliv records") self._records = self._entriesFoundInText elif len(self._entriesFoundInLinks)>0: self.__debug("Deliverbles links content keywords") self.__debug("Found " + "{0}".format(len(self._entriesFoundInLinks)) + " deliv records") self._records = self._entriesFoundInLinks ########################### TABLE HANDLING METHODS ############################ """ Get texts from element and his descendants. If string isset, returns texts as one string with spaces. # elem: lxml element """ def _get_descendats_texts(self, elem, string=True): texts = [] for child in elem.iter(): if child.text and isinstance(child.tag, basestring): if re.search("[a-z0-9]", child.text, re.I): texts.append(self.formatter.format(child.text)) if string: return " ".join(texts) return texts """ Get link from row of table - go through columns and the only href leading to deliverable is returned. """ def _get_row_link(self, row): # find all anchors where parent is row linkanch = row.findall('.//a[@href]') if len(linkanch) == 0: return None for link in linkanch: anchor_link = link.get('href') if self.agent.is_wanted_mime(anchor_link): # check if it is file we want return anchor_link return None """ Handle region as a table. Work with region as it's a table. Try to get table semantic (table order) and get all records out of it. """ def _handle_table(self): for row in self.parentetree: if not row.tag == 'tr': continue row_list = [] _thislink = self._get_row_link(row) if _thislink == None: continue for column in row: text = self._get_descendats_texts(column) if not text: continue row_list.append(text) res = self._deliv_in_text(row_list, [_thislink]) if type(res) == RRSPublication: self._entriesFoundInText.append(res) self.__debug("Record found cause of text") else: res = self._deliv_in_link(row_list, [_thislink]) if type(res) == RRSPublication: self._entriesFoundInLinks.append(res) self.__debug("Record found cause of link") del(row_list) return ######################## TAG SEQUENCE RECOGNIZING METHODS #################### """ Tag check. If it is anchor with href leading to deliverable, returns True """ def _is_deliv_anch(self, tag): if tag.tag == 'a': href = tag.get('href') if self.agent.is_wanted_mime(href): return True return False """ Filters useless and messy tags. Return false if useless, true if normal tag """ def _tagfilter(self, tag): if tag.tag in self._omitted_tags: return False #if tag.text: # if not re.search("[a-z0-9\[\]]", tag.text, re.I): # return False return True """ Gets difference between first two anchors. """ def _getdiff(self, reg, tol): # etree reg = element tree region # int tol: accepted tolerance of tags d = {} index = 0 # fill the dictionary with differences and their occurences for tag in reg.iter(): if not self._tagfilter(tag): continue if self._is_deliv_anch(tag) and not index == 0: try: d[index] += 1 except: d[index] = 1 index = 0 index += 1 # check differencies if the variety isn't higher then $tol tolerance difflist = d.keys() self.__debug("difflist: "+str(difflist)) if len(difflist) == 0: return -1 _max = max(difflist) _min = min(difflist) dlen = len(d.keys()) if dlen == 1: return d.keys()[0] if dlen > ((2*tol)+1): # tolerance to both sides return -1 if (_max - _min) > 2*tol: # some acceptable tolerance return -1 # get the most frequent difference most_freq = max(d.values()) for key in d: if d[key] == most_freq: return key return -1 """ Only anchors found. No optional information. """ def _get_anch_only(self): anchlist = self.agent.find_anchor_elem(self.baseUrl, self.parentetree) # We have to make list of list because XMLOutput return [[anch] for anch in anchlist] """ Main method handling tag sequences and recognizing records. Returns list of records. """ def _get_tag_sequences(self, tag_tol=1): records = [] self._rec = [] if len(self.parentetree) == 0: return [[self.parentetree]] # get interval between anchors, use tolerance tag_tol self.difference = self._getdiff(self.parentetree, self.tagtol) while self.difference == -1: if self.tagtol > 5: self.__verbose("Variety of intervals between anchors is too huge. "+\ "Getting data out of anchors only") return self._get_anch_only() self.tagtol += 1 self.difference = self._getdiff(self.parentetree, self.tagtol) # get sequence of first n tags, where n is average interval between anchors # this could be tag-sequence describing all records in region. self.record_seq = [] i = 0 for tag in self.parentetree.iter(): if not self._tagfilter(tag): continue if i >= self.difference: if not 'a' in self.record_seq: del self.record_seq[0] else: break self.record_seq.append(tag.tag) i += 1 # counter indicates on which position in tag sequence we actually are counter = 0 # make sequence of tags as they go regionlist = filter(self._tagfilter, [tag for tag in self.parentetree.iter()]) recseqlen = len(self.record_seq) reglistlen = len(regionlist) # flag indicating begin of records - in region on the beginning can be some garbage self.begin = False # indicating unpredictable separator between deliverable records self.separator = 0 for i, tag in enumerate(regionlist): # skip and save the sequence at the end if counter > self.difference-1: records.append(self._rec) # save self._rec = [] # erase the list counter = 0 # reset counter if not self.begin: if tag.tag != self.record_seq[0]: continue else: try: if regionlist[i+1].tag != self.record_seq[1]: continue except: pass self.begin = True # handle tolerances, try to compare sibling tags self.match = False # match flag # tolerance algorithm. Goes through html and tries to pass irregular tags in sequence. for tol in range(self.tagtol+1): if tag.tag == self.record_seq[(counter + tol) % recseqlen] or \ regionlist[(i + tol) % reglistlen].tag == self.record_seq[counter % recseqlen]: self.match = True self._rec.append(tag) counter += tol+1 break elif tag.tag == self.record_seq[(counter - tol) % recseqlen] or \ regionlist[(i - tol) % reglistlen].tag == self.record_seq[counter % recseqlen]: self.match = True self._rec.append(tag) counter -= tol counter += 1 break # if nothing matched, its probably out of tolerance if not self.match: self.separator += 1 # tolerance 10 separators (tags between boxes or tables of deliverables) if self.separator > 10: self.__verbose("Tag sequence doesnt match, probably out of "+\ "tolerance, getting data out of anchors only") # maybe here could be tolerance++ # we didnt catch the sequence with tolerance... return self._get_anch_only() records.append(self._rec) return filter(self._validseq, records) """ Helper method - check if sequence of tags rec contains deliv anchor """ def _validseq(self, rec): for _atr in rec: # if we have anchor containing link to document, return true if self._is_deliv_anch(_atr): return True return False """ Get element texts only, dont look for descendants texts """ def _get_tag_content(self, tag): links = [] texts = [] if tag.tag == 'a': href = tag.get('href') # if link leading to document found, add string to list if href is not None and self.agent.is_wanted_mime(href): links.append(self.formatter.format(href)) title = tag.get('title') # if title found in tag, add string to list if title: texts.append(self.formatter.format(title)) # if not anchor, search text in tag.text if tag.text: if re.search("[a-z0-9]", tag.text, re.I): texts.append(self.formatter.format(tag.text)) return [links,texts] """ Harvest texts out of tags and return list of lists (record) """ def _harvest_text(self, record_tag_list): self._records = [] self._rec = [] _links = [] _texts = [] # loop over records and search all possible useful texts for rec_list in record_tag_list: for tag in rec_list: harvested = (self._get_tag_content(tag)) _links.extend(harvested[0]) _texts.extend(harvested[1]) #self._records.append(self._rec) res = self._deliv_in_text(_texts, _links) if type(res) == RRSPublication: self._entriesFoundInText.append(res) self.__debug("Record found cause of text") else: res = self._deliv_in_link(_texts, _links) if type(res) == RRSPublication: self._entriesFoundInLinks.append(res) self.__debug("Record found cause of link") _links = [] _texts = [] self._rec = [] return self._records """ Text harvesting for sequences. """ def _handle_sequence(self): seq = self._get_tag_sequences() return self._harvest_text(seq) """ Get records from region according document links this method is used when there was no records found in output of sequencewrapper""" def _manual_process_page(self, links, baseurl): _err = None self.baseUrl = baseurl for link in links: # find region with tolerance self.parentetree = self.regionHandler.get_region(link, baseurl, 1) if type(self.parentetree) == tuple: # error _err = self.parentetree self.__debug(_err) continue #make all links absolute in parent tree hrefs = self.parentetree.findall('.//a[@href]') for href in hrefs: href.make_links_absolute('http://'+urlsplit(link)[1]+'/') # get the charset. We dont have etree in htmlHandler, # so we have to use the one from regionHandler self.formatter.set_charset(self.regionHandler.formatter.get_charset()) self.__debug("*"*100+'\n'+"*"*40+" DATA REGION "+"*"*40) self.__debug(lxml.etree.tostring(self.parentetree, pretty_print=True)) # get root tag try: self.parentetree = self.parentetree.getroot() except: pass # Parent tag is table # call _handle_table if self.parentetree.tag in ('table','tbody'): self.__verbose("Handling table") self._handle_table() else: self.__verbose("Handling sequences") self._handle_sequence() #############PUBLIC METHODS TO GET RESULTS def get_deliverables_XML(self): """return infromations about deliverables stored in objects as xml""" if len(self.get_deliverables())==0: return derrno.__err__(derrno.ENOREC) output = StringIO.StringIO() converter = Model2XMLConverter(stream=output) converter.convert(self.get_deliverables()) result = output.getvalue() output.close() return result def get_deliverables(self): """return objects containing infromations""" if len(self._records) == 0: return derrno.__err__(derrno.ENOREC) else: return self._records
class GetDelivPage: def __init__(self, url, verbose=False, debug=False, addkeyw=None): # keywords used for document page search self._sigwords = ["d((eliverables?)|[0-9])", "documents?", "reports?", "public(ation)?s?", "results?", "presentations?", "library", #"projects?", "outocomes?", "downloads?", "outputs?"] if addkeyw != None: self._sigwords.append(addkeyw) """ Associative array containing links with their flags { url : [Index/NoIndex/Frame, Visit/Visited, Rank] } index = 0, noindex = 1, frame = 2, unvisited = 0, visited = 1 """ self._link_stack = { url : [0,0,0] } self.base_url = url # save base (input) url # Open an parsing agent to get needed data from page self.agent = GetHTMLAndParse() self._current_url = url # a constant used to set rank in order of importance of the expression # being tested (self._sigwords) self.rank_const = len(self._sigwords) # few a constants for dictionary - just for good-looking source code self.IND_FR = 0 # index/noindex/frame/special self.VISIT = 1 # unvisited/visited self.RANK = 2 # value of rank # set verbose flag self.__verbose__ = verbose #set debug flag self.__dbg__ = debug # checking data types if not type(self.__verbose__) == bool: raise ValueError("Verbose flag has to be boolean.") def __verbose(self, msg): _err = "cannot decode verbose message." if self.__verbose__ == True: try: print(str(msg)) except UnicodeError: print(_err) def __debug(self, msg): _err = "cannot decode debug info." if self.__dbg__ == True: try: print("Debug message: "+str(msg)) except UnicodeError: print(_err) ################################################################################ """ Initialize item in dictionary to noindex/unvisited/rank=0 """ def _link_item_init__(self, link, index=1, visit=0, rank=0): # default setting: noindex,unvisited,norank if not self._link_stack.has_key(link): self._link_stack[link] = [index,visit,rank] return """ Edits item in dictionary self._link_stack """ def _link_item_edit(self, link, index=None, visit=None, rank=None): if index is not None: self._link_stack[link][self.IND_FR] = index if visit is not None: self._link_stack[link][self.VISIT] = visit if rank is not None: # null rank if zero is argument if rank == 0: self._link_stack[link][self.RANK] = 0 # add rank else: self._link_stack[link][self.RANK] += rank return """ Method representing one level of cascade. Do almost any job to search one word in dictionary """ def _level_job(self, index=None): # get list of links from anchors containing one of expression # from self_sigwords result = 0 if index is not None: # searching with one link_list = self.agent.get_all_links( regul = re.compile(self._sigwords[index], re.I), base = self._current_url) else: link_list = self.agent.get_all_links(base = self._current_url) index = self.rank_const if link_list: # # RANK giving & filter # if index is None: rank = 0 elif index == 0: rank = self.rank_const * 2 else: rank = self.rank_const - index for link in link_list: # GTFO javascript if not link or "javascript:" in link or "mailto:" in link: continue if "#" in link: # if pointer delete it link = re.sub('#.*$', '', link) if len(link) > 200: continue if self._link_stack.get(link): # RANK if you see those links for first if self._link_stack[link][self.VISIT] == 0: self._link_item_edit(self._current_url, rank=rank) continue if not self.agent.compare_domains(self.base_url, link): continue split_link = re.sub("https?://.+?/", "", link) # check whether it is file or not if self.agent.is_wanted_mime(link): # # Some PDF or DOC found # # RANK self._link_item_edit(self._current_url, rank=10) self.__debug("Added rank 10 to "+self._current_url) # if re.search("de?l?(iverable)?[0-9]+([\._-][0-9])?", split_link, re.I): self.__debug("Type D on "+self._current_url) # debug print # RANK self._link_item_edit(self._current_url, rank=100) continue elif not self.agent.is_page(link): continue self.__debug("UNWATED") # # Add link # # RANK # initialization of link item in dict self._link_item_init__(link) self._link_item_edit(self._current_url, rank=rank) result += 1 # debug print self.__debug("ADD "+link[7:60]) self.__debug("Rank "+str(rank)+" "+self._current_url) return result """ Cascade search. May improve the speed of script """ def _cascade_search(self): result = 0 # first cascade - look for links cont. deliverables result += self._level_job(0) if not result == 0: return # second cascade - look for links cont. documents and publications result += self._level_job(1) result += self._level_job(2) if not result == 0: return # last cascade - all the rest for i in range(3,self.rank_const): result += self._level_job(i) # check Intro page (all links) only on index if result == 0 and self._link_stack[self._current_url][0] == 0: result += self._level_job() """if result == 0: # RANK DOWN self._link_item_edit(self._current_url, rank=0) print "No anchors on the page""" return """ TRY TO repair link. But for now only append / in base """ def _repair_links(self, base=None): if base is None: base = self.base_url if re.match(".*[^/]$", base): base += "/" if self.agent.get_etree() == -1: return -1 links = self.agent.get_all_links(base = base) # compare link with base url for link in links: if not self.agent.compare_domains(self.base_url, link): continue link = re.sub("https?://.+?/", base, link) # if match, save it as special case self._link_item_init__(link, index=3) """ Checking intro page. It is page without content, only with Enter label """ def _check_intro(self): links = self.agent.get_all_links(base = self._current_url) self.__debug("We've found intro links: "+str(links)) for link in links: if not self.agent.compare_domains(self.base_url, link): continue # save new link as normal page self._link_item_init__(link, index=1) """ Looks for frames on the page """ def _check_frames(self): frames = self.agent.look_for_frame(base = self._current_url) if not frames: return None fcount = len(frames) # debug print self.__debug("We've found frames ("+str(fcount)+") on "+self._current_url) # save new link as frame page for link in frames: if self.agent.compare_domains(self._current_url, link): self._link_item_init__(link, index=2) return fcount """ Checks for titles and gives rank according the result """ def _check_titles(self): for i in range(self.rank_const): hcount = self.agent.count_all_headers( re.compile( self._sigwords[i], re.I )) if not hcount == 0: if i == 0: # # "deliverable" match, the highest rank # # RANK constant is multiplied by 4 self.__debug("deliverable match"+str(self.rank_const * 4)+" "+self._current_url) self._link_item_edit(self._current_url, rank = self.rank_const * 4) else: # # other word match # # do not multiplied rank constant self.__debug("Rank "+str(self.rank_const - i)+" "+self._current_url) self._link_item_edit(self._current_url, rank = self.rank_const - i) """ Get information about current link """ def _check_anchor(self): # tt is Text and Title tt = self.agent.get_anchor_from_link(self._current_url) # return 0 if no anchor match if tt == 0: return tt; # match for deliverables if re.search(self._sigwords[0], tt, re.I): self.__debug("Anchor matched "+self._current_url) # debug print return 1 """ Returns list of unvisited links. Useful in cycle. """ def _check_unvisited_links(self): unvisitedLinks = [] for link in self._link_stack: if self._link_stack[link][self.VISIT] == 0: # if unvisited unvisitedLinks.append(link) return unvisitedLinks # list of unvisited page links """ Aplying all methods to unvisited links - next level of searching. It is main private method. Only this method can decide end of searching """ def _handle_unvis_links(self): unvisLinks = self._check_unvisited_links() if not unvisLinks: return None # end of searching for link in unvisLinks: # cycle in unvisited links # visit and parse page self._link_item_edit(link, visit = 1) (res, err) = self.agent.ghap(link) if res == -1: self.__debug(str(err)+" "+str(link)) # debug print # if link is broken (IND_FR == 3) if self._link_stack[link][self.IND_FR] != 3: self._repair_links() continue # little hack with error message, there is no error but URL! if res == 2: self.base_url = err # URL of the new base self.__debug("Getting url in ghap(): "+str(link)) # debug print self.__verbose("Searching... URL: "+str(link)) # verbose print self._current_url = link if self._link_stack[link][self.IND_FR] == 2: dname = self.agent.get_domain_name(link) if dname is not None: self.base_url = dname ############### # frame check # self._check_frames() ################ # titles check # self._check_titles() # rank giving here ################ # anchor check # if self._check_anchor(): self._link_item_edit(link, rank = 10) # rank giving here too self._cascade_search() # search for next links on this page # when no unvisited links in list, return return 1 """ Returns link of the highest value of rank in self._link_stack. It is called in the end of process.""" def _get_highest_ranks_link(self): hRank = 0 hLink = "" # check all links and choose link with the highest rank for link in self._link_stack: if self._link_stack[link][self.RANK] > hRank: hLink = link hRank = self._link_stack[link][self.RANK] return hLink # WINNER """ Returns list of all links leading to deliverables. Try to find more sites with deliverables.. i.e. like www.awissenet.com has. Maybe test for name of link - anchor: i.e. next, prev, [0-9]+ and so one... Page usualy looks like: next pages: 1 2 3 4 ... """ def _get_deliv_link_list(self,first_link): # agent gets first_link final_list = [] nonvisited = [first_link] current = nonvisited.pop() while current: if not current or "javascript:" in current or "mailto:" in current: try: current = nonvisited.pop() except: break continue if self.agent.ghap(current)[0] == -1: # CACHE ??? maybe try: current = nonvisited.pop() except: break continue nonvisited.extend(self.agent.get_pager_links(base=current)) final_list.append(current) # append only one link try: current = nonvisited.pop() except: break return final_list # returning all pages with deliverables """ Returns list of links on pages with deliverable-documents. If found returns list, if not found, return -1. Only public method in module. """ def get_deliverable_page(self): # the main searching loop # while we have some unvisited links, search while self._handle_unvis_links(): # security case if len(self._link_stack) > 10: break self.__debug("Stack content: "+str(self._link_stack)) if len(self._link_stack) == 1 : return derrno.__err__(derrno.ELNOTFOUND) final_link = self._get_highest_ranks_link() if not final_link or self._link_stack[final_link][2] == 0: return derrno.__err__(derrno.ELNOTFOUND) self.__debug('#'*79) self.__debug("DELIVERABLE PAGE: "+final_link) return [final_link] ####### not in use ############# result = self._get_deliv_link_list(final_link) if len(result) == 0: return derrno.__err__(derrno.ELNOTFOUND) else: return result
class GetDelivPage: def __init__(self, url, verbose=False, debug=False, addkeyw=None): # keywords used for document page search self._sigwords = ["d((eliverables?)|[0-9])", "documents?", "reports?", "public(ation)?s?", "results?", "presentations?", "library", #"projects?", "outocomes?", "downloads?", "outputs?"] if addkeyw != None: self._sigwords.append(addkeyw) """ Associative array containing links with their flags { url : [Index/NoIndex/Frame, Visit/Visited, Rank] } index = 0, noindex = 1, frame = 2, unvisited = 0, visited = 1 """ self._link_stack = { url : [0,0,0] } self.base_url = url # save base (input) url # Open an parsing agent to get needed data from page self.agent = GetHTMLAndParse() self._current_url = url # a constant used to set rank in order of importance of the expression # being tested (self._sigwords) self.rank_const = len(self._sigwords) # few a constants for dictionary - just for good-looking source code self.IND_FR = 0 # index/noindex/frame/special self.VISIT = 1 # unvisited/visited self.RANK = 2 # value of rank # set verbose flag self.__verbose__ = verbose #set debug flag self.__dbg__ = debug # checking data types if not type(self.__verbose__) == bool: raise ValueError("Verbose flag has to be boolean.") def __verbose(self, msg): _err = "cannot decode verbose message." if self.__verbose__ == True: try: print(str(msg)) except UnicodeError: print(_err) def __debug(self, msg): _err = "cannot decode debug info." if self.__dbg__ == True: try: print("Debug message: "+str(msg)) except UnicodeError: print(_err) ################################################################################ """ Initialize item in dictionary to noindex/unvisited/rank=0 """ def _link_item_init__(self, link, index=1, visit=0, rank=0): # default setting: noindex,unvisited,norank if not self._link_stack.has_key(link): self._link_stack[link] = [index,visit,rank] return """ Edits item in dictionary self._link_stack """ def _link_item_edit(self, link, index=None, visit=None, rank=None): if index is not None: self._link_stack[link][self.IND_FR] = index if visit is not None: self._link_stack[link][self.VISIT] = visit if rank is not None: # null rank if zero is argument if rank == 0: self._link_stack[link][self.RANK] = 0 # add rank else: self._link_stack[link][self.RANK] += rank return """ Method representing one level of cascade. Do almost any job to search one word in dictionary """ def _level_job(self, index=None): # get list of links from anchors containing one of expression # from self_sigwords result = 0 if index is not None: # searching with one link_list = self.agent.get_all_links( regul = re.compile(self._sigwords[index], re.I), base = self._current_url) else: link_list = self.agent.get_all_links(base = self._current_url) index = self.rank_const if link_list: # # RANK giving & filter # if index is None: rank = 0 elif index == 0: rank = self.rank_const * 2 else: rank = self.rank_const - index for link in link_list: # GTFO javascript if not link or "javascript:" in link or "mailto:" in link: continue if "#" in link: # if pointer delete it link = re.sub('#.*$', '', link) if len(link) > 200: continue if self._link_stack.get(link): # RANK if you see those links for first if self._link_stack[link][self.VISIT] == 0: self._link_item_edit(self._current_url, rank=rank) continue if not self.agent.compare_domains(self.base_url, link): continue split_link = re.sub("https?://.+?/", "", link) # check whether it is file or not if self.agent.is_wanted_mime(link): # # Some PDF or DOC found # # RANK self._link_item_edit(self._current_url, rank=10) self.__debug("Added rank 10 to "+self._current_url) # if re.search("de?l?(iverable)?[0-9]+([\._-][0-9])?", split_link, re.I): self.__debug("Type D on "+self._current_url) # debug print # RANK self._link_item_edit(self._current_url, rank=100) continue elif not self.agent.is_page(link): continue self.__debug("UNWATED") # # Add link # # RANK # initialization of link item in dict self._link_item_init__(link) self._link_item_edit(self._current_url, rank=rank) result += 1 # debug print self.__debug("ADD "+link[7:60]) self.__debug("Rank "+str(rank)+" "+self._current_url) return result """ Cascade search. May improve the speed of script """ def _cascade_search(self): result = 0 # first cascade - look for links cont. deliverables result += self._level_job(0) if not result == 0: return # second cascade - look for links cont. documents and publications result += self._level_job(1) result += self._level_job(2) if not result == 0: return # last cascade - all the rest for i in range(3,self.rank_const): result += self._level_job(i) # check Intro page (all links) only on index if result == 0 and self._link_stack[self._current_url][0] == 0: result += self._level_job() """if result == 0: # RANK DOWN self._link_item_edit(self._current_url, rank=0) print "No anchors on the page""" return """ TRY TO repair link. But for now only append / in base """ def _repair_links(self, base=None): if base is None: base = self.base_url if re.match(".*[^/]$", base): base += "/" if self.agent.get_etree() == -1: return -1 links = self.agent.get_all_links(base = base) # compare link with base url for link in links: if not self.agent.compare_domains(self.base_url, link): continue link = re.sub("https?://.+?/", base, link) # if match, save it as special case self._link_item_init__(link, index=3) """ Checking intro page. It is page without content, only with Enter label """ def _check_intro(self): links = self.agent.get_all_links(base = self._current_url) self.__debug("We've found intro links: "+str(links)) for link in links: if not self.agent.compare_domains(self.base_url, link): continue # save new link as normal page self._link_item_init__(link, index=1) """ Looks for frames on the page """ def _check_frames(self): frames = self.agent.look_for_frame(base = self._current_url) if not frames: return None fcount = len(frames) # debug print self.__debug("We've found frames ("+str(fcount)+") on "+self._current_url) # save new link as frame page for link in frames: if self.agent.compare_domains(self._current_url, link): self._link_item_init__(link, index=2) return fcount """ Checks for titles and gives rank according the result """ def _check_titles(self): for i in range(self.rank_const): hcount = self.agent.count_all_headers( re.compile( self._sigwords[i], re.I )) if not hcount == 0: if i == 0: # # "deliverable" match, the highest rank # # RANK constant is multiplied by 4 self.__debug("deliverable match"+str(self.rank_const * 4)+" "+self._current_url) self._link_item_edit(self._current_url, rank = self.rank_const * 4) else: # # other word match # # do not multiplied rank constant self.__debug("Rank "+str(self.rank_const - i)+" "+self._current_url) self._link_item_edit(self._current_url, rank = self.rank_const - i) """ Get information about current link """ def _check_anchor(self): # tt is Text and Title tt = self.agent.get_anchor_from_link(self._current_url) # return 0 if no anchor match if tt == 0: return tt; # match for deliverables if re.search(self._sigwords[0], tt, re.I): self.__debug("Anchor matched "+self._current_url) # debug print return 1 """ Returns list of unvisited links. Useful in cycle. """ def _check_unvisited_links(self): unvisitedLinks = [] for link in self._link_stack: if self._link_stack[link][self.VISIT] == 0: # if unvisited unvisitedLinks.append(link) return unvisitedLinks # list of unvisited page links """ Aplying all methods to unvisited links - next level of searching. It is main private method. Only this method can decide end of searching """ def _handle_unvis_links(self): unvisLinks = self._check_unvisited_links() if not unvisLinks: return None # end of searching for link in unvisLinks: # cycle in unvisited links # visit and parse page self._link_item_edit(link, visit = 1) (res, err) = self.agent.ghap(link) if res == -1: self.__debug(str(err)+" "+str(link)) # debug print # if link is broken (IND_FR == 3) if self._link_stack[link][self.IND_FR] != 3: self._repair_links() continue # little hack with error message, there is no error but URL! if res == 2: self.base_url = err # URL of the new base self.__debug("Getting url in ghap(): "+str(link)) # debug print self.__verbose("Searching... URL: "+str(link)) # verbose print self._current_url = link if self._link_stack[link][self.IND_FR] == 2: dname = self.agent.get_domain_name(link) if dname is not None: self.base_url = dname ############### # frame check # self._check_frames() ################ # titles check # self._check_titles() # rank giving here ################ # anchor check # if self._check_anchor(): self._link_item_edit(link, rank = 10) # rank giving here too self._cascade_search() # search for next links on this page # when no unvisited links in list, return return 1 """ Returns link of the highest value of rank in self._link_stack. It is called in the end of process.""" def _get_highest_ranks_link(self): hRank = 0 hLink = "" # check all links and choose link with the highest rank for link in self._link_stack: if self._link_stack[link][self.RANK] > hRank: hLink = link hRank = self._link_stack[link][self.RANK] return hLink # WINNER """ Returns list of all links leading to deliverables. Try to find more sites with deliverables.. i.e. like www.awissenet.com has. Maybe test for name of link - anchor: i.e. next, prev, [0-9]+ and so one... Page usualy looks like: next pages: 1 2 3 4 ... """ def _get_deliv_link_list(self,first_link): # agent gets first_link final_list = [] nonvisited = [first_link] current = nonvisited.pop() while current: if not current or "javascript:" in current or "mailto:" in current: try: current = nonvisited.pop() except: break continue if self.agent.ghap(current)[0] == -1: # CACHE ??? maybe try: current = nonvisited.pop() except: break continue nonvisited.extend(self.agent.get_pager_links(base=current)) final_list.append(current) # append only one link try: current = nonvisited.pop() except: break return final_list # returning all pages with deliverables """ Returns list of links on pages with deliverable-documents. If found returns list, if not found, return -1. Only public method in module. """ def get_deliverable_page(self): # the main searching loop # while we have some unvisited links, search while self._handle_unvis_links(): # security case if len(self._link_stack) > 10: break self.__debug("Stack content: "+str(self._link_stack)) if len(self._link_stack) == 1 : return derrno.__err__(derrno.ELNOTFOUND) final_link = self._get_highest_ranks_link() if not final_link or self._link_stack[final_link][2] == 0: return derrno.__err__(derrno.ELNOTFOUND) self.__debug('#'*79) self.__debug("DELIVERABLE PAGE: "+final_link) return [final_link] ####### not in use ############# result = self._get_deliv_link_list(final_link) if len(result) == 0: return derrno.__err__(derrno.ELNOTFOUND) else: return result