def startElementHandler(self, name, attrs): # preprocessing the xml schema if self.is_schema: if len(attrs) == 1: schema = list(attrs.values())[0] handle = self.open_xsd_file(os.path.basename(schema)) # if there is no local xsd file grab the url and parse the file if not handle: handle = _urlopen(schema) text = handle.read() self.save_xsd_file(os.path.basename(schema), text) handle.close() self.parse_xsd(ET.fromstring(text)) else: self.parse_xsd(ET.fromstring(handle.read())) handle.close() self.content = "" if name in self.lists: object = ListElement() elif name in self.dictionaries: object = DictionaryElement() elif name in self.structures: object = StructureElement(self.structures[name]) elif name in self.items: # Only appears in ESummary name = str(attrs["Name"]) # convert from Unicode del attrs["Name"] itemtype = str(attrs["Type"]) # convert from Unicode del attrs["Type"] if itemtype == "Structure": object = DictionaryElement() elif name in ("ArticleIds", "History"): object = StructureElement(["pubmed", "medline"]) elif itemtype == "List": object = ListElement() else: object = StringElement() object.itemname = name object.itemtype = itemtype elif name in self.strings + self.errors + self.integers: self.attributes = attrs return else: # Element not found in DTD if self.validating: raise ValidationError(name) else: # this will not be stored in the record object = "" if object != "": object.tag = name if attrs: object.attributes = dict(attrs) if len(self.stack) != 0: current = self.stack[-1] try: current.append(object) except AttributeError: current[name] = object self.stack.append(object)
def _open(cgi, params=None, post=None, ecitmatch=False): """Build the URL and open a handle to it (PRIVATE). Open a handle to Entrez. cgi is the URL for the cgi script to access. params is a dictionary with the options to pass to it. Does some simple error checking, and will raise an IOError if it encounters one. The arugment post should be a boolean to explicitly control if an HTTP POST should be used rather an HTTP GET based on the query length. By default (post=None), POST is used if the URL encoded paramters would be over 1000 characters long. This function also enforces the "up to three queries per second rule" to avoid abusing the NCBI servers. """ # NCBI requirement: At most three queries per second if no API key is provided. # Equivalently, at least a third of second between queries params = _construct_params(params) options = _encode_options(ecitmatch, params) delay = 0.1 if api_key else 0.333333334 current = time.time() wait = _open.previous + delay - current if wait > 0: time.sleep(wait) _open.previous = current + wait else: _open.previous = current # By default, post is None. Set to a boolean to over-ride length choice: if post is None and len(options) > 1000: post = True cgi = _construct_cgi(cgi, post, options) try: if post: handle = _urlopen(cgi, data=_as_bytes(options)) else: handle = _urlopen(cgi) except _HTTPError as exception: raise exception return _binary_to_string_handle(handle)
def externalEntityRefHandler(self, context, base, systemId, publicId): """Handle external entiry reference in order to cache DTD locally. The purpose of this function is to load the DTD locally, instead of downloading it from the URL specified in the XML. Using the local DTD results in much faster parsing. If the DTD is not found locally, we try to download it. If new DTDs become available from NCBI, putting them in Bio/Entrez/DTDs will allow the parser to see them. """ urlinfo = _urlparse(systemId) # Following attribute requires Python 2.5+ # if urlinfo.scheme=='http': if urlinfo[0] in ['http', 'https', 'ftp']: # Then this is an absolute path to the DTD. url = systemId elif urlinfo[0] == '': # Then this is a relative path to the DTD. # Look at the parent URL to find the full path. try: source = self.dtd_urls[-1] except IndexError: # Assume the default URL for DTDs if the top parent # does not contain an absolute path source = "http://www.ncbi.nlm.nih.gov/dtd/" else: source = os.path.dirname(source) # urls always have a forward slash, don't use os.path.join url = source.rstrip("/") + "/" + systemId else: raise ValueError("Unexpected URL scheme %r" % (urlinfo[0])) self.dtd_urls.append(url) # First, try to load the local version of the DTD file location, filename = os.path.split(systemId) handle = self.open_dtd_file(filename) if not handle: # DTD is not available as a local file. Try accessing it through # the internet instead. try: handle = _urlopen(url) except IOError: raise RuntimeError("Failed to access %s at %s" % (filename, url)) text = handle.read() handle.close() self.save_dtd_file(filename, text) handle = BytesIO(text) parser = self.parser.ExternalEntityParserCreate(context) parser.ElementDeclHandler = self.elementDecl parser.ParseFile(handle) handle.close() self.dtd_urls.pop() return 1
def startElementHandler(self, name, attrs): # First, check if the current consumer can use the tag if self.consumer is not None: consumed = self.consumer.startElementHandler(name, attrs) if consumed: return # preprocessing the xml schema if self.is_schema: if len(attrs) == 1: schema = list(attrs.values())[0] handle = self.open_xsd_file(os.path.basename(schema)) # if there is no local xsd file grab the url and parse the file if not handle: handle = _urlopen(schema) text = handle.read() self.save_xsd_file(os.path.basename(schema), text) handle.close() self.parse_xsd(ET.fromstring(text)) else: self.parse_xsd(ET.fromstring(handle.read())) handle.close() cls = self.classes.get(name) if cls is None: # Element not found in DTD if self.validating: raise ValidationError(name) else: # this will not be stored in the record consumer = Consumer(name, attrs) else: consumer = cls(name, attrs) consumer.parent = self.consumer if self.consumer is None: # This is relevant only for Entrez.parse, not for Entrez.read. # If self.consumer is None, then this is the first start tag we # encounter, and it should refer to a list. Store this list in # the record attribute, so that Entrez.parse can iterate over it. # The record attribute will be set again at the last end tag; # However, it doesn't hurt to set it twice. value = consumer.value if value is not None: self.record = value self.consumer = consumer
def _open(cgi, params=None, post=None, ecitmatch=False): """Helper function to build the URL and open a handle to it (PRIVATE). Open a handle to Entrez. cgi is the URL for the cgi script to access. params is a dictionary with the options to pass to it. Does some simple error checking, and will raise an IOError if it encounters one. The arugment post should be a boolean to explicitly control if an HTTP POST should be used rather an HTTP GET based on the query length. By default (post=None), POST is used if the query URL would be over 1000 characters long. The arugment post should be a boolean to explicitly control if an HTTP POST should be used rather an HTTP GET based on the query length. This function also enforces the "up to three queries per second rule" to avoid abusing the NCBI servers. """ if params is None: params = {} # NCBI requirement: At most three queries per second. # Equivalently, at least a third of second between queries delay = 0.333333334 current = time.time() wait = _open.previous + delay - current if wait > 0: time.sleep(wait) _open.previous = current + wait else: _open.previous = current # Remove None values from the parameters for key, value in list(params.items()): if value is None: del params[key] # Tell Entrez that we are using Biopython (or whatever the user has # specified explicitly in the parameters or by changing the default) if "tool" not in params: params["tool"] = tool # Tell Entrez who we are if "email" not in params: if email is not None: params["email"] = email else: warnings.warn(""" Email address is not specified. To make use of NCBI's E-utilities, NCBI requires you to specify your email address with each request. As an example, if your email address is [email protected], you can specify it as follows: from Bio import Entrez Entrez.email = '*****@*****.**' In case of excessive usage of the E-utilities, NCBI will attempt to contact a user at the email address provided before blocking access to the E-utilities.""", UserWarning) # Open a handle to Entrez. options = _urlencode(params, doseq=True) # _urlencode encodes pipes, which NCBI expects in ECitMatch if ecitmatch: options = options.replace('%7C', '|') # print cgi + "?" + options # By default, post is None. Set to a boolean to over-ride length choice: if post is None and len(options) > 1000: post = True try: if post: # HTTP POST handle = _urlopen(cgi, data=_as_bytes(options)) else: # HTTP GET cgi += "?" + options handle = _urlopen(cgi) except _HTTPError as exception: raise exception return _binary_to_string_handle(handle)
def _open(cgi, params=None, post=None, ecitmatch=False): """Helper function to build the URL and open a handle to it (PRIVATE). Open a handle to Entrez. cgi is the URL for the cgi script to access. params is a dictionary with the options to pass to it. Does some simple error checking, and will raise an IOError if it encounters one. The arugment post should be a boolean to explicitly control if an HTTP POST should be used rather an HTTP GET based on the query length. By default (post=None), POST is used if the query URL would be over 1000 characters long. The arugment post should be a boolean to explicitly control if an HTTP POST should be used rather an HTTP GET based on the query length. This function also enforces the "up to three queries per second rule" to avoid abusing the NCBI servers. """ if params is None: params = {} # NCBI requirement: At most three queries per second. # Equivalently, at least a third of second between queries delay = 0.333333334 current = time.time() wait = _open.previous + delay - current if wait > 0: time.sleep(wait) _open.previous = current + wait else: _open.previous = current # Remove None values from the parameters for key, value in list(params.items()): if value is None: del params[key] # Tell Entrez that we are using Biopython (or whatever the user has # specified explicitly in the parameters or by changing the default) if "tool" not in params: params["tool"] = tool # Tell Entrez who we are if "email" not in params: if email is not None: params["email"] = email else: warnings.warn( """ Email address is not specified. To make use of NCBI's E-utilities, NCBI requires you to specify your email address with each request. As an example, if your email address is [email protected], you can specify it as follows: from Bio import Entrez Entrez.email = '*****@*****.**' In case of excessive usage of the E-utilities, NCBI will attempt to contact a user at the email address provided before blocking access to the E-utilities.""", UserWarning) # Open a handle to Entrez. options = _urlencode(params, doseq=True) # _urlencode encodes pipes, which NCBI expects in ECitMatch if ecitmatch: options = options.replace('%7C', '|') # print cgi + "?" + options # By default, post is None. Set to a boolean to over-ride length choice: if post is None and len(options) > 1000: post = True try: if post: # HTTP POST handle = _urlopen(cgi, data=_as_bytes(options)) else: # HTTP GET cgi += "?" + options handle = _urlopen(cgi) except _HTTPError as exception: raise exception return _binary_to_string_handle(handle)