Beispiel #1
0
    def integrate_test_pubs(self, pub_candidates):
        """
			For Debug Errors
		"""
        print "- INTEGRATE TEST -:", self.integrate_test_pubs.__doc__.strip()

        extractor = Extractor.getInstance()
        matcher = PubMatcher.getInstance()

        # print queries
        query, used_pubs = Extractor.pinMaxQuery(pub_candidates)
        print "Test %s pub, query: \n\t%s" % (len(used_pubs), query)
        url = self.settings.urltemplate_by_pubs % URLCleaner.encodeUrlForDownload(query)
        # url = URLCleaner.encodeUrlForDownload(url)
        print "\t", url

        # do
        all_models = extractor.getNodesByPubs(used_pubs)
        (pubs_found, pubs_notfound) = matcher.matchPub(used_pubs, all_models, debug_output=True)

        # print out
        print "-" * 100
        for pub in pubs_found:
            print "[%s] %s" % (pub.ncitation, pub)
        print "-" * 100
        for pub in pubs_notfound:
            print "[%s] %s" % ("-", pub)
        print "-" * 100
        print "- test done -"
Beispiel #2
0
    def integrate_test_pubs(self, pub_candidates):
        '''
			For Debug Errors
		'''
        print '- INTEGRATE TEST -:', self.integrate_test_pubs.__doc__.strip()

        extractor = Extractor.getInstance()
        matcher = PubMatcher.getInstance()

        # print queries
        query, used_pubs = Extractor.pinMaxQuery(pub_candidates)
        print 'Test %s pub, query: \n\t%s' % (len(used_pubs), query)
        url = self.settings.urltemplate_by_pubs % URLCleaner.encodeUrlForDownload(
            query)
        # url = URLCleaner.encodeUrlForDownload(url)
        print "\t", url

        # do
        all_models = extractor.getNodesByPubs(used_pubs)
        (pubs_found, pubs_notfound) = matcher.matchPub(used_pubs,
                                                       all_models,
                                                       debug_output=True)

        # print out
        print '-' * 100
        for pub in pubs_found:
            print '[%s] %s' % (pub.ncitation, pub)
        print '-' * 100
        for pub in pubs_notfound:
            print '[%s] %s' % ('-', pub)
        print '-' * 100
        print '- test done -'
Beispiel #3
0
    def getNodesByPubs(self, pubs):
        '''Get by pubs.
		Return: 
			all_models, {key_title:[ExtractedModel,...]}, can be None, or []
		Param: 
			pubs, [models.Publication], query generated by this pubs must less than 256.
		'''
        query, used_pubs, nouse_pubs = Extractor.pinMaxQuery(pubs)
        url = self.settings.urltemplate_by_pubs % URLCleaner.encodeUrlForDownload(
            query)
        message = "Search Pub[%s] by url[%s]\n" % (used_pubs[0], url)
        # url = URLCleaner.encodeUrlForDownload(url)

        html = self.htmlRetriever.getHtmlRetry(url)
        if html is None:
            print "Download Page failed."
            return None

        models = self.extract_from_source(html)
        if models is None or len(models) == 0:
            return None
        # save models
        all_models = self.__merge_into_extractedmap(
            None, models)  # {key_title:[ExtractedModel,...]}
        #		for k, v in all_models.items():
        #			message += "models find: [%s]\n" % v
        #		print message
        return all_models
Beispiel #4
0
	def getNodesByPubs(self, pubs):
		'''Get by pubs.
		Return: 
			all_models, {key_title:[ExtractedModel,...]}, can be None, or []
		Param: 
			pubs, [models.Publication], query generated by this pubs must less than 256.
		'''
		query, used_pubs, nouse_pubs = Extractor.pinMaxQuery(pubs)
		url = self.settings.urltemplate_by_pubs % URLCleaner.encodeUrlForDownload(query)
		message =  "Search Pub[%s] by url[%s]\n" % (used_pubs[0], url)
		# url = URLCleaner.encodeUrlForDownload(url)
		
		html = self.htmlRetriever.getHtmlRetry(url)
		if html is None:
			print "Download Page failed."
			return None
		
		models = self.extract_from_source(html)
		if models is None or len(models) == 0: 
			return None
		# save models
		all_models = self.__merge_into_extractedmap(None, models)  # {key_title:[ExtractedModel,...]}
#		for k, v in all_models.items():
#			message += "models find: [%s]\n" % v
#		print message
		return all_models
Beispiel #5
0
    def test_retrieve_html(self):
        print 'Test1: test_retrieve_html()'
        url = '''allintitle:"Augmenting Branching Temporal Logics with Existential Quantification over Atomic Propositions" OR "Branching-Depth Hierarchies" OR "On the Relative Succinctness of Nondeterministic Buchi and co-Buchi Word Automata"'''
        url2 = "http://scholar.google.com/scholar?hl=en&num=100&q=%s" % url
        url2 = URLCleaner.encodeUrlForDownload(url2)

        url2 = '''http://scholar.google.com/scholar?hl=en&num=100&as_subj=eng&q=%22Finding%20the%20Number%20of%20Factors%20of%20a%20Polynomial%22OR%22Probabilistic%20Models%20of%20Database%20Locking:%20Solutions,%20Computational%20Algorithms,%20and%20Asymptotics%22OR%22The%20AWK%20Programming%20Language%22OR%22Factoring%20Polynomials%20Over%20Algebraic%20Number%20Fields%22'''
        getter = HtmlRetriever(use_proxy=False)
        print getter.getHtmlRetry(url2, 1)
Beispiel #6
0
	def test_retrieve_html(self):
		print 'Test1: test_retrieve_html()'
		url = '''allintitle:"Augmenting Branching Temporal Logics with Existential Quantification over Atomic Propositions" OR "Branching-Depth Hierarchies" OR "On the Relative Succinctness of Nondeterministic Buchi and co-Buchi Word Automata"'''
		url2 = "http://scholar.google.com/scholar?hl=en&num=100&q=%s" % url
		url2 = URLCleaner.encodeUrlForDownload(url2)
		
		url2 = '''http://scholar.google.com/scholar?hl=en&num=100&as_subj=eng&q=%22Finding%20the%20Number%20of%20Factors%20of%20a%20Polynomial%22OR%22Probabilistic%20Models%20of%20Database%20Locking:%20Solutions,%20Computational%20Algorithms,%20and%20Asymptotics%22OR%22The%20AWK%20Programming%20Language%22OR%22Factoring%20Polynomials%20Over%20Algebraic%20Number%20Fields%22'''
		getter = HtmlRetriever(use_proxy=False)
		print getter.getHtmlRetry(url2, 1)
Beispiel #7
0
 def test_retrieve_html2(self):
     url = '''allintitle:"Augmenting Branching Temporal Logics with Existential Quantification over Atomic Propositions" OR "Branching-Depth Hierarchies" OR "On the Relative Succinctness of Nondeterministic Buchi and co-Buchi Word Automata"'''
     url2 = "http://scholar.google.com/scholar?hl=en&num=100&q=%s" % url
     url2 = URLCleaner.encodeUrlForDownload(url2)
     getter = HtmlRetriever(use_proxy=True)
     print getter.getHtmlRetry(url2, 1)
Beispiel #8
0
	def test_retrieve_html2(self):
		url = '''allintitle:"Augmenting Branching Temporal Logics with Existential Quantification over Atomic Propositions" OR "Branching-Depth Hierarchies" OR "On the Relative Succinctness of Nondeterministic Buchi and co-Buchi Word Automata"'''
		url2 = "http://scholar.google.com/scholar?hl=en&num=100&q=%s" % url
		url2 = URLCleaner.encodeUrlForDownload(url2)
		getter = HtmlRetriever(use_proxy=True)
		print getter.getHtmlRetry(url2, 1)