コード例 #1
0
def audit(osmfile, audit_value = ''):
    osm_file = open(osmfile, "r")
    if audit_value == 'postal':
        zip_fixed= set()
        for event, elem in ET.iterparse(osm_file, events=("start",)):
    
            if elem.tag == "node" or elem.tag == "way":
                for tag in elem.iter("tag"):
                    if is_postal(tag):
                        zip_fixed.add(audit_postalcode(tag.attrib['v']))
        return zip_fixed

    elif audit_value == 'city':
        city_fixed = set()
        for event, elem in ET.iterparse(osm_file, events=("start",)):
            if elem.tag == "node" or elem.tag == "way":
                for tag in elem.iter("tag"):
                    if is_city(tag):
                        city_fixed.add(audit_city(tag.attrib['v']))
        return city_fixed                        
    
    else:
        street_types = defaultdict(set)
        for event, elem in ET.iterparse(osm_file, events=("start",)):

            if elem.tag == "node" or elem.tag == "way":
                for tag in elem.iter("tag"):
                    if is_street_name(tag):
                        audit_street_type(street_types, tag.attrib['v'])

        return street_types
コード例 #2
0
    def test_getAllNestedElementInformation(self):
        expectedResultPeaks = {'fullName': 'Proteomics Standards Initiative Mass Spectrometry Ontology', 'id': 'MS', 'tagName': '{http://psi.hupo.org/ms/mzml}cv', 
                               'URI': 'http://psidev.cvs.sourceforge.net/*checkout*/psidev/psi/psi-ms/mzML/controlledVocabulary/psi-ms.obo'}
        expectedResultMzml = {'fullName': 'Proteomics Standards Initiative Mass Spectrometry Ontology', 'id': 'MS', 'tagName': '{http://psi.hupo.org/ms/mzml}cv', 
                              'URI': 'http://psidev.cvs.sourceforge.net/*checkout*/psidev/psi/psi-ms/mzML/controlledVocabulary/psi-ms.obo', 'version':'2.26.0'}
        expectedResultFeatureXML = {'name': 'FeatureFinder', 'tagName': 'software', 'version': '1.8.0'}

        actualResultPeaks = {}
        elementFile = open(testFolder+'peaksMzmlTestfile.peaks.mzML')
        for event, element in cElementTree.iterparse(elementFile):
            actualResultPeaks = elementFunctions.getAllNestedElementInformation(element)
            # only doing one to test, break
            break
        
        actualResultMzml = {}
        elementFile = open(testFolder+'mzml_test_file_1.mzML')
        for event, element in cElementTree.iterparse(elementFile):
            actualResultMzml = elementFunctions.getAllNestedElementInformation(element)
            # only doing one to test, break
            break

        actualResultFeatureXML = {}
        elementFile = open(testFolder+'featurexmlTestFile_1.featureXML')
        for event, element in cElementTree.iterparse(elementFile):
            actualResultFeatureXML = elementFunctions.getAllNestedElementInformation(element)
            # only doing one to test, break
            break
            
        
        self.assertDictEqual(expectedResultPeaks, actualResultPeaks)
        self.assertDictEqual(expectedResultMzml, actualResultMzml)
        self.assertDictEqual(expectedResultFeatureXML, actualResultFeatureXML)
コード例 #3
0
ファイル: blastin.py プロジェクト: arendsee/blastdbm
def parse_blast_xml(args, cur):
    if args.input:
        for f in args.input:
            con = et.iterparse(f, events=('end', 'start'))
            _parse_blast_xml(args, cur, con)
    else:
        con = et.iterparse(sys.stdin, events=('end', 'start'))
        _parse_blast_xml(args, cur, con)
コード例 #4
0
 def __init__(self):
     self.ATC_dict = json.loads('all_level.json')
     self.result = []
     for event, drug = ET.iterparse('drugbank.xml'):
         if drug.tag != '{http://www.drugbank.ca}drug':
             continue
         for first_level in drug:
             if first_level.tag == '{http://www.drugbank.ca}calculated-properties':
                 self.result += self.calculated_properties(first_level)
コード例 #5
0
ファイル: xml.py プロジェクト: ssato/python-anyconfig
def _iterparse(xmlfile):
    """
    Avoid bug in python 3.{2,3}. See http://bugs.python.org/issue9257.

    :param xmlfile: XML file or file-like object
    """
    try:
        return ET.iterparse(xmlfile, events=("start-ns", ))
    except TypeError:
        return ET.iterparse(xmlfile, events=(b"start-ns", ))
コード例 #6
0
def main():
  global args
  options = argparse.ArgumentParser(epilog="Example: \
%(prog)s dmarc-xml-file 1> outfile.log")
  options.add_argument("dmarcfile", help="dmarc file in XML format")
  args = options.parse_args()

  # get an iterable and turn it into an iterator
  meta_fields = get_meta(iter(etree.iterparse(args.dmarcfile, events=("start", "end"))));
  if not meta_fields:
    print >> sys.stderr, "Error: No valid 'policy_published' and 'report_metadata' xml tags found; File: " + args.dmarcfile 
    sys.exit(1)

  print_record(iter(etree.iterparse(args.dmarcfile, events=("start", "end"))), meta_fields, args)
コード例 #7
0
ファイル: weather.py プロジェクト: tehmaze/x84
def do_search(search):
    """
    Given any arbitrary string, return list of possible matching locations.
    """
    import StringIO
    from x84.bbs import echo, getch
    disp_msg(u'SEARChiNG')
    resp = requests.get(u'http://apple.accuweather.com'
                        + u'/adcbin/apple/Apple_find_city.asp',
                        params=(('location', search),))
    locations = list()
    if resp is None:
        disp_notfound()
    elif resp.status_code != 200:
        # todo: logger.error
        echo(u'\r\n' + u'Status Code: %s\r\n\r\n' % (resp.status_code,))
        echo(repr(resp.content))
        echo(u'\r\n\r\n' + 'Press any key')
        getch()
    else:
        # print resp.content
        xml_stream = StringIO.StringIO(resp.content)
        locations = list([dict(elem.attrib.items())
                          for _event, elem in ET.iterparse(xml_stream)
                          if elem.tag == 'location'])
        if 0 == len(locations):
            disp_notfound()
        else:
            disp_found(len(locations))
    return locations
コード例 #8
0
ファイル: users.py プロジェクト: knopthakorn/Data-Analyst
def process_map(filename):
    users = set()
    for _, element in ET.iterparse(filename, events = ("start",)):
        if 'uid' in element.attrib:
            users.add(get_user(element))

    return users
コード例 #9
0
ファイル: repodb.py プロジェクト: pombredanne/cvs-pyrpm
 def readPrimary(self):
     # If we have either a local cache of the primary.xml.gz file or if
     # it is already local (nfs or local file system) we calculate it's
     # checksum and compare it with the one from repomd. If they are
     # the same we don't need to cache it again and can directly use it.
     if self.repomd.has_key("primary"):
         if not self.repomd["primary"].has_key("location"):
             return 0
         primary = self.repomd["primary"]["location"]
         (csum, destfile) = self.nc.checksum(primary, "sha")
         if self.repomd["primary"].has_key("checksum") and \
                csum == self.repomd["primary"]["checksum"]:
             filename = destfile
         else:
             filename = self.nc.cache(primary, 1)
         if not filename:
             return 0
         try:
             fd = PyGZIP(filename)
             ip = iterparse(fd, events=("start","end"))
             ip = iter(ip)
         except IOError:
             log.error("Couldn't parse primary.xml")
             return 0
         self._parse(ip)
     return 1
コード例 #10
0
ファイル: xml_parse.py プロジェクト: chenjz/online_lda_python
def parse_and_write(xml_file, outfile, fields, tag, n, interval=1):
    # get an iterable
    context = ET.iterparse(xml_file, events=("start", "end"))
    # turn it into an iterator
    context = iter(context)
    # get the root element
    event, root = context.next()
    i = 0
    with open(outfile, 'w') as f:
        for event, row in context:
            if event == "end" and row.tag == tag:
                if i % 100000 == 0:
                    pct = round((i * 1.0 / n) * 100, 1)
                    Printer("Processed {0} records. ~ {1}\% complete.".format(i, pct))
                if interval == 1 or i % interval == 0:
                    if all(map(lambda x: x in row.attrib, fields)):
                        field_data = []
                        for fd in fields:
                            if fd == 'Tags':
                                field_data.extend(parse_tags(row.attrib[fd].encode('ascii', 'ignore')))
                            else:
                                field_data.append(clean(row.attrib[fd].encode('ascii', 'ignore')))
                        text = " ".join(field_data) + "\n"
                        f.write(text)
                i += 1
                root.clear()
                if i >= n:
                    break
コード例 #11
0
ファイル: scanner.py プロジェクト: legoktm/dumpscanner
    def new_parse(self):
        """Generator using cElementTree iterparse function"""
        if self.filename.endswith('.bz2'):
            import bz2
            source = bz2.BZ2File(self.filename)
        elif self.filename.endswith('.gz'):
            import gzip
            source = gzip.open(self.filename)
        elif self.filename.endswith('.7z'):
            import subprocess
            source = subprocess.Popen('7za e -bd -so %s 2>/dev/null'
                                      % self.filename,
                                      shell=True,
                                      stdout=subprocess.PIPE,
                                      bufsize=65535).stdout
        else:
            # assume it's an uncompressed XML file
            source = open(self.filename)
        context = iterparse(source, events=("start", "end", "start-ns"))
        self.root = None

        for event, elem in context:
            if event == "start-ns" and elem[0] == "":
                self.uri = elem[1]
                continue
            if event == "start" and self.root is None:
                self.root = elem
                continue
            for rev in self._parse(event, elem):
                yield rev
コード例 #12
0
def parseOsm(source, handler):
  for event, elem in ElementTree.iterparse(source, events=('start', 'end')):
    if event == 'start':
      handler.startElement(elem.tag, elem.attrib)
    elif event == 'end':
      handler.endElement(elem.tag)
    elem.clear()
コード例 #13
0
def process_map(filename):
    keys = {"lower": 0, "lower_colon": 0, "problemchars": 0, "other": 0}
    for _, element in ET.iterparse(filename):
#        print element.tag
        keys = key_type(element, keys)

    return keys
コード例 #14
0
ファイル: users.py プロジェクト: j-bennet/udacity-nano-da
def process_map(filename):
    users = set()
    for _, el in ET.iterparse(filename):
        if 'user' in el.attrib:
            users.add(el.attrib['user'])

    return users
def list_country(filename):
    governorate_set=set()
    for _,element in ET.iterparse(filename):
        for tag in element.iter('tag'):
            if 'governorate' in tag.attrib['k'] and tag.attrib['v'] not in governorate_set:
                governorate_set.add(tag.attrib['v'])
    return governorate_set
コード例 #16
0
ファイル: audit_street_names.py プロジェクト: rbmayer/Udacity
def audit(osmfile):
    osm_file = open(osmfile, "r")
    street_types = defaultdict(set)
    for event, elem in ET.iterparse(osm_file, events=("start",)):
        # audit street names in "nodes" containing "addr:street"
        if elem.tag == "node":
            for tag in elem.iter("tag"):
                if is_street_name(tag) and tag.attrib['v'] != "":
                    street_type, street_name = audit_street_type(
                        tag.attrib['v'])
                    add_street_type(street_types, street_type, street_name)
        # audit street names in "ways" containing "highway"
        elif elem.tag == "way":
            highway = 0
            # check if way matches an included highway type
            for tag in elem.iter("tag"):
                if (tag.attrib['k'] == "highway") and (tag.attrib['v'] 
                in highway_types):
                    highway = 1
            if highway == 1:
                for tag in elem.iter("tag"):
                    if is_name(tag) and tag.attrib['v'] != "":
                        street_type, street_name = audit_street_type(
                            tag.attrib['v'])
                        if street_type != None:
                            add_street_type(street_types, street_type, 
                                            street_name)
                            
    return street_types
コード例 #17
0
    def _parse_results(self, stream):
        """Parse results and messages out of *stream*."""
        result = None
        values = None
        try:
            for event, elem in et.iterparse(stream, events=('start', 'end')):
                if elem.tag == 'results' and event == 'start':
                    # The wrapper element is a <results preview="0|1">. We
                    # don't care about it except to tell is whether these
                    # are preview results, or the final results from the
                    # search.
                    is_preview = elem.attrib['preview'] == '1'
                    self.is_preview = is_preview
                if elem.tag == 'result':
                    if event == 'start':
                        result = OrderedDict()
                    elif event == 'end':
                        yield result
                        result = None
                        elem.clear()

                elif elem.tag == 'field' and result is not None:
                    # We need the 'result is not None' check because
                    # 'field' is also the element name in the <meta>
                    # header that gives field order, which is not what we
                    # want at all.
                    if event == 'start':
                        values = []
                    elif event == 'end':
                        field_name = elem.attrib['k'].encode('utf8')
                        if len(values) == 1:
                            result[field_name] = values[0]
                        else:
                            result[field_name] = values
                        # Calling .clear() is necessary to let the
                        # element be garbage collected. Otherwise
                        # arbitrarily large results sets will use
                        # arbitrarily large memory intead of
                        # streaming.
                        elem.clear()

                elif elem.tag in ('text', 'v') and event == 'end':
                    text = "".join(elem.itertext())
                    values.append(text.encode('utf8'))
                    elem.clear()

                elif elem.tag == 'msg':
                    if event == 'start':
                        msg_type = elem.attrib['type']
                    elif event == 'end':
                        text = elem.text if elem.text is not None else ""
                        yield Message(msg_type, text.encode('utf8'))
                        elem.clear()
        except SyntaxError as pe:
            # This is here to handle the same incorrect return from
            # splunk that is described in __init__.
            if 'no element found' in pe.msg:
                return
            else:
                raise
コード例 #18
0
def process_osm(file_in):
	with open(file_in) as file:
		for _, element in ET.iterparse(file):
			el = shape_data(element)
			if el:
				#pprint.pprint(el)
				way_node_collection.insert(el)
コード例 #19
0
    def parseFile(self, f):
        """ Parses a single Doxygen XML file
            :param f: XML file path
        """
        documentable_members = 0
        documented_members = 0

        # Wrap everything in a try, as sometimes Doxygen XML is malformed
        try:
            for event, elem in ET.iterparse(f):
                if event == 'end' and elem.tag == 'compounddef':
                    if self.elemIsPublicClass(elem):
                        # store documentation status
                        members, documented, undocumented, bindable, has_brief_description, found_version_added = self.parseClassElem(elem)
                        documentable_members += members
                        documented_members += documented
                        class_name = elem.find('compoundname').text
                        acceptable_missing = self.acceptable_missing.get(class_name, [])

                        if not self.hasGroup(class_name) and not class_name in self.acceptable_missing_group:
                            self.classes_missing_group.append(class_name)
                        if not class_name in self.acceptable_missing_brief and not has_brief_description:
                            self.classes_missing_brief.append(class_name)
                        if not class_name in self.acceptable_missing_added_note and not found_version_added:
                            self.classes_missing_version_added.append(class_name)

                        # GEN LIST
                        # if len(undocumented) > 0:
                        #     print('"%s": [%s],' % (class_name, ", ".join(['"%s"' % e.replace('"', '\\"') for e in undocumented])))

                        unacceptable_undocumented = undocumented - set(acceptable_missing)

                        # do a case insensitive check too
                        unacceptable_undocumented_insensitive = set([u.lower() for u in undocumented]) - set([u.lower() for u in acceptable_missing])

                        if len(unacceptable_undocumented_insensitive) > 0:
                            self.undocumented_members[class_name] = {}
                            self.undocumented_members[class_name]['documented'] = documented
                            self.undocumented_members[class_name]['members'] = members
                            self.undocumented_members[class_name]['missing_members'] = unacceptable_undocumented

                        # store bindable members
                        if self.classElemIsBindable(elem):
                            for m in bindable:
                                self.bindable_members.append(m)

                    elem.clear()
        except ET.ParseError as e:
            # sometimes Doxygen generates malformed xml (e.g., for < and > operators)
            line_num, col = e.position
            with open(f, 'r') as xml_file:
                for i, l in enumerate(xml_file):
                    if i == line_num - 1:
                        line = l
                        break
            caret = '{:=>{}}'.format('^', col)
            print(('ParseError in {}\n{}\n{}\n{}'.format(f, e, line, caret)))

        self.documentable_members += documentable_members
        self.documented_members += documented_members
コード例 #20
0
ファイル: utils.py プロジェクト: ad-m/django-teryt
def parse(stream):
    for event, element in et.iterparse(stream):
        if element.tag != 'row':
            continue
        yield {
            x.get('name'): x.text.strip() if x.text else None for x in element
        }
コード例 #21
0
def count_tags(filename):
        # YOUR CODE HERE
        d = defaultdict(int)
        for events, elem in ET.iterparse(filename):
            d[elem.tag] += 1
                
        return d
コード例 #22
0
def parse_mediawiki_xml(xml_fo, dump_func):
    """
    Process mediawiki xml dump of wiktionary.
    """
    # create tag names in advance (setting default xmlns doesn't help)
    page_tag = tag("page")
    text_tag = tag("text")
    title_tag = tag("title")

    # using event based eltree parser
    itree = ET.iterparse(xml_fo, events=("start","end"))

    word_name = None
    etymology_list = None

    for event, elem in itree:
        # reset data for new word entry
        if event == "start":
            if elem.tag == page_tag:
                word_name = None
            continue
        # get data for current word entry (event == end)
        if elem.tag == title_tag:
            if not elem.text.startswith("Wiktionary:"):
                word_name = elem.text
        elif word_name is not None:
            if elem.tag == text_tag:
                etymology_list = get_etymology(elem.text)
            elif elem.tag == page_tag:
                # all data for current word entry has beed processed
                dump_func(word_name, etymology_list)
コード例 #23
0
def examine_tags(osmfile, tag_range, item_limit):
    assert len(tag_range) == 2
    # use pre-loaded tag_keys list of tuples, if exists
    if TAG_KEYS:
        tag_keys = TAG_KEYS
    # else call mapparser count_tags method to pull sorted list of top tags
    else:
        _, tag_keys = count_tags(osmfile)
    # list comprehension for producing a list of tag_keys in string format
    tag_keys = [tag_key[0] for tag_key in tag_keys][tag_range[0]:tag_range[1]]
    print "Examining tag keys: {}".format(tag_keys)

    # open osm file
    osm_file = open(osmfile, "r")

    # initialize data with default set data structure
    data = defaultdict(set)

    # iterate through elements
    for _, elem in ET.iterparse(osm_file, events=("start",)):
        # check if the element is a node or way
        if elem.tag == "node" or elem.tag == "way":
            # iterate through children matching `tag`
            for tag in elem.iter("tag"):
                # skip if does not contain key-value pair
                if 'k' not in tag.attrib or 'v' not in tag.attrib:
                    continue
                key = tag.get('k')
                val = tag.get('v')
                # add to set if in tag keys of interest and is below the item limit
                if key in tag_keys and len(data[key]) < item_limit:
                    data[key].add(val)
    return data
コード例 #24
0
ファイル: __init__.py プロジェクト: cbrafter/sumo
def parse(xmlfile, element_names, element_attrs={}, attr_conversions={}):
    """
    Parses the given element_names from xmlfile and yield compound objects for
    their xml subtrees (no extra objects are returned if element_names appear in
    the subtree) The compound objects provide all element attributes of
    the root of the subtree as attributes unless attr_names are supplied. In this
    case attr_names maps element names to a list of attributes which are
    supplied. If attr_conversions is not empty it must map attribute names to
    callables which will be called upon the attribute value before storing under
    the attribute name. 
    The compound objects gives dictionary style access to list of compound
    objects o for any children with the given element name 
    o['child_element_name'] = [osub0, osub1, ...]
    As a shorthand, attribute style access to the list of child elements is
    provided unless an attribute with the same name as the child elements
    exists (i.e. o.child_element_name = [osub0, osub1, ...])
    @Note: All elements with the same name must have the same type regardless of
    the subtree in which they occur
    @Note: Attribute names may be modified to avoid name clashes
    with python keywords.
    @Note: The element_names may be either a single string or a list of strings.
    @Example: parse('plain.edg.xml', ['edge'])
    """
    if isinstance(element_names, str):
        element_names = [element_names]
    elementTypes = {}
    for event, parsenode in ET.iterparse(xmlfile):
        if parsenode.tag in element_names:
            yield _get_compound_object(parsenode, elementTypes,
                                       parsenode.tag, element_attrs, attr_conversions)
            parsenode.clear()
コード例 #25
0
ファイル: stream.py プロジェクト: bradjasper/concurrence
    def elements(self):
        
        if not self.parser:
            reader = self.stream.reader
            class f(object):
                def read(self, n):
                    if reader.buffer.remaining == 0:
                        #read more data into buffer
                        reader._read_more()
                    return reader.buffer.read_bytes(min(n, reader.buffer.remaining))

            self.parser = iter(iterparse(f(), events=("start", "end")))
            event, self.root = self.parser.next()
            level = 0
        
        for event, element in self.parser:
            if event == 'start':
                level += 1
            elif event == 'end':
                level -= 1
                if level == 0:
                    yield element
                #TODO clear root
            else:
                assert False, "unexpected event"
コード例 #26
0
ファイル: z_xml_iterparser.py プロジェクト: chenziliang/src
def iterparse(text, interested_path_handlers):
    '''
    interested_path_handlers => {'start': ((interested_path, handler), (interested_path, handler), ...),
                                 'end':((interested_path, handler), (interested_path, handler), ...)}
    interested_path => (tag1, tag2, tag3, ...)
    An incremental XML parser. ElementTree.findall has too high CPU/Memory footprint when data set is big
    '''
    strf = StringIO()
    strf.write(text)
    strf.seek(0)

    context = ElementTree.iterparse(strf, events=('start', 'end'))
    context = iter(context)
    all_start_handlers = interested_path_handlers.get('start', ())
    all_end_handlers = interested_path_handlers.get('end', ())
    current_path = []

    for ev, elem in context:
        tag, value = elem.tag, elem.text
        if ev == 'start':
            current_path.append(tag)
            if all_start_handlers:
                _do_handlers(ev, elem, current_path, all_start_handlers)
        elif ev == 'end':
            if all_end_handlers:
                _do_handlers(ev, elem, current_path, all_end_handlers)
            current_path.pop()
            elem.clear()
コード例 #27
0
def count_tags(filename, limit=-1, verbose=False):
    """
    Parses the OSM file and counts the tags by type.
    """
    # initialize dict objects and counter
    tag_count = {}
    tag_keys = {}
    counter = 0

    # iterate through elements
    for _, element in ET.iterparse(filename, events=("start",)):
        # add to tag count
        add_tag(element.tag, tag_count)

        # if tag and has key, add the tag key to tag_keys dict
        if element.tag == 'tag' and 'k' in element.attrib:
            add_tag(element.get('k'), tag_keys)

        # print if verbose output enabled
        if verbose:
            print "{0}: {1}".format(counter, element.tag)

        # break if exceed limit
        if limit > 0 and counter >= limit:
            break
        counter += 1

    # produces a sorted-by-decreasing list of tag key-count pairs
    tag_keys = sorted(tag_keys.items(), key=operator.itemgetter(1))[::-1]

    # return values
    return tag_count, tag_keys
コード例 #28
0
ファイル: __init__.py プロジェクト: pombredanne/django-rdf
 def _xml_namespaces(self):
     for _, e in iterparse(self._path, events=('start-ns',)):
         lcode, uri = e
         if 1 > Namespace.objects.filter(resource__name=uri).count():
             r = Resource(name=uri)
             yield r
             yield Namespace(code=lcode, resource=r)
コード例 #29
0
def iterArticles(f):
    pmid = None
    title = None
    abstract = None
    journal= None
    mesh_list = []
    for event,elem in ET.iterparse(f, events=("start","end")):
        if event == 'start':
            if elem.tag == 'PubmedArticle':
                pmid,title,abstract,journal,mesh_list= None,None,None, None,[]
        elif event == 'end':            
            if elem.tag == 'PubmedArticle':
                yield pmid, title, abstract,journal,mesh_list
            elif elem.tag == 'PMID':
                pmid = elem.text
            elif elem.tag == 'ArticleTitle':
                title = elem.text               
            elif elem.tag == 'AbstractText':
                abstract = elem.text
            elif elem.tag == 'Title':
                journal = elem.text    
            elif elem.tag == 'KeywordList':
                keyword_list=elem.findall("Keyword")
                for aa in keyword_list:
                    mesh_list.append(aa.text)
            elif elem.tag == 'MeshHeadingList':
                mhlist = elem.findall("MeshHeading")  
                for child in mhlist:
                    if child.findtext('DescriptorName'):
                         mesh_list.append(child.findtext('DescriptorName'))
                    if child.findtext('QualifierName'):     
                         mesh_list.append(child.findtext('QualifierName'))
コード例 #30
0
ファイル: readers.py プロジェクト: filannim/ManTIME
    def _get_annotations(self, source, start_offset=0):
        """It returns the annotations found in the document.

        It follows the following format:
           [
            ('TAG', {ATTRIBUTES}, (start_offset, end_offset)),
            ('TAG', {ATTRIBUTES}, (start_offset, end_offset)),
            ...
            ('TAG', {ATTRIBUTES}, (start_offset, end_offset))
           ]

        """
        annotations = []
        for event, element in etree.iterparse(
                StringIO(source), events=('start', 'end')):
            if event == 'start':
                if element.tag in self.tags_to_spot:
                    try:
                        end_offset = start_offset + len(element.text)
                    except TypeError:
                        continue
                    annotations.append((element.tag, element.attrib,
                                        (start_offset, end_offset)))
                start_offset += len(element.text)
            elif event == 'end':
                if element.text is not None and element.tail is not None:
                    start_offset += len(element.tail)
        return annotations
コード例 #31
0
def convert(blastxml_filename, output_handle):
    blast_program = None
    # get an iterable
    try:
        context = ElementTree.iterparse(blastxml_filename, events=("start", "end"))
    except Exception:
        sys.exit("Invalid data format.")
    # turn it into an iterator
    context = iter(context)
    # get the root element
    try:
        event, root = context.next()
    except Exception:
        sys.exit("Invalid data format.")
    for event, elem in context:
        if event == "end" and elem.tag == "BlastOutput_program":
            blast_program = elem.text
        # for every <Iteration> tag
        if event == "end" and elem.tag == "Iteration":
            # Expecting either this, from BLAST 2.2.25+ using FASTA vs FASTA
            # <Iteration_query-ID>sp|Q9BS26|ERP44_HUMAN</Iteration_query-ID>
            # <Iteration_query-def>Endoplasmic reticulum resident protein 44
            # OS=H**o sapiens GN=ERP44 PE=1 SV=1</Iteration_query-def>
            # <Iteration_query-len>406</Iteration_query-len>
            # <Iteration_hits></Iteration_hits>
            #
            # Or, from BLAST 2.2.24+ run online
            # <Iteration_query-ID>Query_1</Iteration_query-ID>
            # <Iteration_query-def>Sample</Iteration_query-def>
            # <Iteration_query-len>516</Iteration_query-len>
            # <Iteration_hits>...
            qseqid = elem.findtext("Iteration_query-ID")
            if re_default_query_id.match(qseqid):
                # Place holder ID, take the first word of the query definition
                qseqid = elem.findtext("Iteration_query-def").split(None, 1)[0]
            qlen = int(elem.findtext("Iteration_query-len"))

            # for every <Hit> within <Iteration>
            for hit in elem.findall("Iteration_hits/Hit"):
                # Expecting either this,
                # <Hit_id>gi|3024260|sp|P56514.1|OPSD_BUFBU</Hit_id>
                # <Hit_def>RecName: Full=Rhodopsin</Hit_def>
                # <Hit_accession>P56514</Hit_accession>
                # or,
                # <Hit_id>Subject_1</Hit_id>
                # <Hit_def>gi|57163783|ref|NP_001009242.1| rhodopsin [Felis catus]</Hit_def>
                # <Hit_accession>Subject_1</Hit_accession>
                #
                # apparently depending on the parse_deflines switch
                #
                # Or, with a local database not using -parse_seqids can get this,
                # <Hit_id>gnl|BL_ORD_ID|2</Hit_id>
                # <Hit_def>chrIII gi|240255695|ref|NC_003074.8| Arabidopsis
                # thaliana chromosome 3, complete sequence</Hit_def>
                # <Hit_accession>2</Hit_accession>
                sseqid = hit.findtext("Hit_id").split(None, 1)[0]
                hit_def = sseqid + " " + hit.findtext("Hit_def")
                if re_default_subject_id.match(sseqid) and sseqid == hit.findtext("Hit_accession"):
                    # Place holder ID, take the first word of the subject definition
                    hit_def = hit.findtext("Hit_def")
                    sseqid = hit_def.split(None, 1)[0]
                if sseqid.startswith("gnl|BL_ORD_ID|") and sseqid == "gnl|BL_ORD_ID|" + hit.findtext("Hit_accession"):
                    # Alternative place holder ID, again take the first word of hit_def
                    hit_def = hit.findtext("Hit_def")
                    sseqid = hit_def.split(None, 1)[0]
                # for every <Hsp> within <Hit>
                for hsp in hit.findall("Hit_hsps/Hsp"):
                    nident = hsp.findtext("Hsp_identity")
                    length = hsp.findtext("Hsp_align-len")
                    # As of NCBI BLAST+ 2.4.0 this is given to 3dp (not 2dp)
                    pident = "%0.3f" % (100 * float(nident) / float(length))

                    q_seq = hsp.findtext("Hsp_qseq")
                    h_seq = hsp.findtext("Hsp_hseq")
                    m_seq = hsp.findtext("Hsp_midline")
                    assert len(q_seq) == len(h_seq) == len(m_seq) == int(length)
                    gapopen = str(len(q_seq.replace('-', ' ').split()) - 1 +
                                  len(h_seq.replace('-', ' ').split()) - 1)

                    mismatch = m_seq.count(' ') + m_seq.count('+') - q_seq.count('-') - h_seq.count('-')
                    # TODO - Remove this alternative mismatch calculation and test
                    # once satisifed there are no problems
                    expected_mismatch = len(q_seq) - sum(1 for q, h in zip(q_seq, h_seq)
                                                         if q == h or q == "-" or h == "-")
                    xx = sum(1 for q, h in zip(q_seq, h_seq) if q == "X" and h == "X")
                    if not (expected_mismatch - q_seq.count("X") <= int(mismatch) <= expected_mismatch + xx):
                        sys.exit("%s vs %s mismatches, expected %i <= %i <= %i"
                                 % (qseqid, sseqid, expected_mismatch - q_seq.count("X"),
                                    int(mismatch), expected_mismatch))

                    # TODO - Remove this alternative identity calculation and test
                    # once satisifed there are no problems
                    expected_identity = sum(1 for q, h in zip(q_seq, h_seq) if q == h)
                    if not (expected_identity - xx <= int(nident) <= expected_identity + q_seq.count("X")):
                        sys.exit("%s vs %s identities, expected %i <= %i <= %i"
                                 % (qseqid, sseqid, expected_identity, int(nident),
                                    expected_identity + q_seq.count("X")))

                    evalue = hsp.findtext("Hsp_evalue")
                    if evalue == "0":
                        evalue = "0.0"
                    else:
                        evalue = "%0.0e" % float(evalue)

                    bitscore = float(hsp.findtext("Hsp_bit-score"))
                    if bitscore < 100:
                        # Seems to show one decimal place for lower scores
                        bitscore = "%0.1f" % bitscore
                    else:
                        # Note BLAST does not round to nearest int, it truncates
                        bitscore = "%i" % bitscore

                    values = [qseqid,
                              sseqid,
                              pident,
                              length,  # hsp.findtext("Hsp_align-len")
                              str(mismatch),
                              gapopen,
                              hsp.findtext("Hsp_query-from"),  # qstart,
                              hsp.findtext("Hsp_query-to"),  # qend,
                              hsp.findtext("Hsp_hit-from"),  # sstart,
                              hsp.findtext("Hsp_hit-to"),  # send,
                              evalue,  # hsp.findtext("Hsp_evalue") in scientific notation
                              bitscore,  # hsp.findtext("Hsp_bit-score") rounded
                              ]

                    if extended:
                        try:
                            sallseqid = ";".join(name.split(None, 1)[0] for name in hit_def.split(" >"))
                            salltitles = "<>".join(name.split(None, 1)[1] for name in hit_def.split(" >"))
                        except IndexError as e:
                            sys.exit("Problem splitting multuple hits?\n%r\n--> %s" % (hit_def, e))
                        # print hit_def, "-->", sallseqid
                        positive = hsp.findtext("Hsp_positive")
                        ppos = "%0.2f" % (100 * float(positive) / float(length))
                        qframe = hsp.findtext("Hsp_query-frame")
                        sframe = hsp.findtext("Hsp_hit-frame")
                        if blast_program == "blastp":
                            # Probably a bug in BLASTP that they use 0 or 1 depending on format
                            if qframe == "0":
                                qframe = "1"
                            if sframe == "0":
                                sframe = "1"
                        slen = int(hit.findtext("Hit_len"))
                        values.extend([sallseqid,
                                       hsp.findtext("Hsp_score"),  # score,
                                       nident,
                                       positive,
                                       hsp.findtext("Hsp_gaps"),  # gaps,
                                       ppos,
                                       qframe,
                                       sframe,
                                       # NOTE - for blastp, XML shows original seq, tabular uses XXX masking
                                       q_seq,
                                       h_seq,
                                       str(qlen),
                                       str(slen),
                                       salltitles,
                                       ])
                    if cols:
                        # Only a subset of the columns are needed
                        values = [values[colnames.index(c)] for c in cols]
                    # print "\t".join(values)
                    output_handle.write("\t".join(values) + "\n")
            # prevents ElementTree from growing large datastructure
            root.clear()
            elem.clear()
コード例 #32
0

#        elif ep[0].text == "Monoisotopic Weight":
#            pass
#        else:
#            pass

    return result

with open("json_dict.txt") as f:
    atc_dict = f.read()
d = json.loads(atc_dict)
count = 0
flag = 0
output_file = open('output', 'ab')
for event, drug in ET.iterparse('drugbank.xml'):
    if drug.tag != '{http://www.drugbank.ca}drug':
        continue
    if not drug.get('type'):
        continue

    result = [0] * 5
    result.append([0.11, 0.01, 0, 0.99, 0.5])
    if drug.get('type') == 'small molecule':
        result[0] = [1]
    else:
        result[0] = [0]
    items = 0
    for first_level in drug:
        flag = 0
コード例 #33
0
# -*- coding: utf-8 -*-
"""
Created on Wed Jul 19 20:51:11 2017

@author: Burky
"""

import xml.etree.cElementTree as ET
import pprint
from collections import defaultdict

filename = 'cleveland_ohio.osm'

zip_codes = defaultdict(int)
for event, elem in ET.iterparse(filename):
    if elem.tag == 'way':
        for tag in elem.findall('tag'):
            if tag.attrib['k'] == 'addr:postcode':
                zip_codes[tag.attrib['v']] += 1
pprint.pprint(zip_codes)
コード例 #34
0
import xml.etree.cElementTree as cElementTree
import csv

folder = 'D:/Will/GIS Data/Raw OSM Data/EP/'
raw_osm_file = folder + 'arizona_8-12-2016.osm'
outfile = folder + 'parking_lots.csv'

writer = csv.writer(open(outfile, 'wb'))
writer.writerow(['OSMID'])

context = cElementTree.iterparse(raw_osm_file, events=("start", "end"))
context = iter(context)
event, root = next(context)

for event, elem in context:
    if event == 'end' and elem.tag == 'way':
        is_parking = False
        iterator = iter(elem)
        for child in iterator:
            if child.get('k') == 'service' and (
                    child.get('v') == 'parking_aisle'
                    or child.get('v') == 'parking'):
                is_parking = True
            if child.get('k') == 'amenity' and child.get('v') == 'parking':
                is_parking = True

        if is_parking:
            writer.writerow([elem.get('id')])
    root.clear()
コード例 #35
0
def audit():
    for event, elem in ET.iterparse(filename):
        if is_street_name(elem):
            audit_street_type(street_types, elem.attrib['v'])
    print_sorted_dict(street_types)
コード例 #36
0
    def parse(self, file_obj):
        nodes = []
        ways = []
        print('----------------')
        context = iter(iterparse(file_obj, events=('start', 'end')))
        event, root = next(context)
        
        for (event, elem) in context:
            name = elem.tag
            attrs = elem.attrib
            
            if 'start' == event:
                """Parse the XML element at the start"""
                if name == 'node':
                    record = self.fillDefault(attrs)
                    loc = [float(attrs['lon']),
                           float(attrs['lat'])]
                    record['loc'] = loc
                    record['geometry'] = {'type':'Point', 'coordinates': loc}
                elif name == 'tag':
                    k = attrs['k']
                    v = attrs['v']
                    # MongoDB doesn't let us have dots in the key names.
                    #k = k.replace('.', ',,')
                    record['tg'].append((k, v))
                    record['ky'].append(k)
                elif name == 'way':
                    # Insert remaining nodes
                    if len(nodes) > 0:
                        self.client.osm2.nodes.insert_many(nodes)
                        nodes = []

                    record = self.fillDefault(attrs)
                    record['nd'] = []
                elif name == 'relation':
                    # Insert remaining ways
                    if len(ways) > 0:
                        self.client.osm2.ways.insert_many(ways)
                        ways = []

                    record = self.fillDefault(attrs)
                    record['mm'] = []
                elif name == 'nd':
                    ref = int(attrs['ref'])
                    record['nd'].append(ref)
                elif name == 'member':
                    record['mm'].append(dict(type=attrs['type'],
                                                  ref=int(attrs['ref']),
                                                  role=attrs['role']))
                    
                    if attrs['type'] == 'way':
                        ways2relations = self.client.osm2.ways.find_one({ '_id' : ref})
                        if ways2relations:
                            if 'relations' not in ways2relations:
                                ways2relations['relations'] = []
                            ways2relations['relations'].append(record['_id'])
                            self.client.osm2.ways.save(ways2relations)
                    elif attrs['type'] == 'node':
                        nodes2relations = self.client.osm2.nodes.find_one({ '_id' : ref})
                        if nodes2relations:
                            if 'relations' not in nodes2relations:
                                nodes2relations['relations'] = []
                            nodes2relations['relations'].append(record['_id'])
                            self.client.osm2.nodes.save(nodes2relations)
            elif 'end' == event:
                """Finish parsing an element
                (only really used with nodes, ways and relations)"""
                if name == 'node':
                    if len(record['tg']) == 0:
                        del record['tg']
                    if len(record['ky']) == 0:
                        del record['ky']
                    nodes.append(record)
                    if len(nodes) > 2500:
                        self.client.osm2.nodes.insert_many(nodes)
                        nodes = []
                        self.writeStatsToScreen()

                    record = {}
                    self.stat_nodes = self.stat_nodes + 1
                elif name == 'way':
                    if len(record['tg']) == 0:
                        del record['tg']
                    if len(record['ky']) == 0:
                        del record['ky']
                    nds = dict((rec['_id'], rec) for rec in self.client.osm2.nodes.find({ '_id': { '$in': record['nd'] } }, { 'loc': 1, '_id': 1 }))
                    record['loc'] = []
                    record['geometry'] = dict()
                    locs = []
                    for node in record['nd']:
                        if node in nds:
                            record['loc'].append(nds[node]['loc'])
                            locs.append(nds[node]['loc'])
                        else:
                            print('node not found: '+ str(node))
                    record['geometry'] = {'type':'LineString', 'coordinates': locs}

                    ways.append(record)
                    if len(ways) > 2000:
                        self.client.osm2.ways.insert_many(ways)
                        ways = []

                    record = {}
                    self.statsCount = self.statsCount + 1
                    if self.statsCount > 1000:
                        self.writeStatsToScreen()
                        self.statsCount = 0
                    self.stat_ways = self.stat_ways + 1
                elif name == 'relation':
                    if len(record['tg']) == 0:
                        del record['tg']
                    if len(record['ky']) == 0:
                        del record['ky']
                    self.client.osm2.relations.save(record)
                    record = {}
                    self.statsCount = self.statsCount + 1
                    if self.statsCount > 10:
                        self.writeStatsToScreen()
                        self.statsCount = 0
                    self.stat_relations = self.stat_relations + 1
            elem.clear()
            root.clear()
コード例 #37
0
#llamado con 7za e -so eowiki-20091128-pages-meta-history.xml.7z | python stubmetahistory-fetch-celementtree.py eo

lang = 'es'  #idioma que será analizado
if len(sys.argv) >= 2:
    lang = sys.argv[1]

rawtotalrevisions = 0.0
site = wikipedia.Site(lang, 'wikipedia')
data = site.getUrl("/wiki/Special:Statistics?action=raw")
rawtotalrevisions += float(data.split("edits=")[1].split(";")[0])

source = sys.stdin
outputfile = "/mnt/user-store/dump/%swiki-fetched.txt.bz" % (lang)
g = bz2.BZ2File(outputfile, "w")

context = iterparse(source, events=("start", "end"))
context = iter(context)

r_newlines = re.compile(ur"(?im)[\n\r\t\s]")
r_redirect = re.compile(
    ur"(?i)^\s*#\s*(REDIRECCIÓN|REDIRECT)\s*\[\[[^\]]+?\]\]")
r_disambig = re.compile(
    ur"(?i)\{\{\s*(d[ei]sambig|desambiguaci[oó]n|des|desamb)\s*[\|\}]")
r_links = re.compile(ur"\[\[\s*[^\]]+?\s*[\]\|]")
r_categories = re.compile(
    ur"(?i)\[\[\s*(category|categoría)\s*\:\s*[^\]\|]+\s*[\]\|]")
r_sections = re.compile(ur"(?im)^(\=+)[^\=]+?\1")
r_templates = ""
r_interwikis = re.compile(ur"(?i)\[\[\s*[a-z]{2,3}(-[a-z]{2,3})?\s*\:")
r_externallinks = re.compile(ur"://")
r_bold = ""
コード例 #38
0
def audit(osmfile, options=None):
    '''
    Audits the OSM file using the different audit functions defined herein.
    
    osm_file: str. Filepath to the OSM file being audited
    options: list of str. Dictates what types of audits are run. Allowed options values:
                        'counting'
                        'zips'
                        'county/state counting'
                        'county/state reporting'
                        'lat/long'
                        'amenities'
                        'property types'
                        'property type counts'
    
    '''

    with open(osmfile, "rb") as fileIn:
        if options:
            #Setting up the necessary beginning parameters for each function
            if 'counting' in options:
                tag_counts = {}
            if 'zips' in options:
                zipLength = 5
                zipLengthDict = {zipLength: 0, "Non-number": 0}
                known_zips = set()
                knownZipTags = set()
                zip_tags_ignored = []
            if 'county/state counting' in options:
                county_tags = {}
                state_tags = {}
                state_tags_ignored = [
                    'state_capital', 'source:hgv:state_network',
                    'hgv:state_network'
                ]
            if 'county/state reporting' in options:
                counties_found = set()
                states_found = set()
                countyKeys = [
                    'gnis:County', 'gnis:County_num', 'gnis:county_id',
                    'gnis:county_name', 'is_in:county', 'tiger:county'
                ]
                stateKeys = [
                    'addr:state', 'gnis:ST_alpha', 'gnis:state_id',
                    'nist:state_fips', 'ST_num'
                ]
            if 'lat/long' in options:
                badNodes = defaultdict(
                    list
                )  #ensures that each new key will automatically have an empty list value
            if 'amenities' in options:
                known_amenities = defaultdict(set)
            if 'property types' in options:
                propTypes = defaultdict(set)
            if 'property type counts' in options:
                propRecords = defaultdict(int)
                allowed_propTypes = {
                    'landuse': [
                        'residential', 'village_green', 'recreation_ground',
                        'allotments', 'commercial', 'depot', 'industrial',
                        'landfill', 'orchard', 'plant_nursery', 'port',
                        'quarry', 'retail'
                    ],
                    'building': [
                        'apartments', 'farm', 'house', 'detached',
                        'residential', 'dormitory', 'houseboat', 'bungalow',
                        'static_caravan', 'cabin', 'hotel', 'commercial',
                        'industrial', 'retail', 'warehouse', 'kiosk',
                        'hospital', 'stadium'
                    ]
                }

    #----------------------------------------------------------------------
    #Iterating through the XML file
            for _, elem in ET.iterparse(fileIn):
                if 'counting' in options:
                    tag_counts = count_tags(elem, tag_counts)

                if 'zips' in options:
                    zipLengthDict, known_zips, knownZipTags = zipCheck(elem, zipLengthDict, known_zips, \
                                                                       knownZipTags, zip_tags_ignored, digits=zipLength)

                if 'county/state counting' in options:
                    county_tags, state_tags = countyStateTypeCounter(
                        elem, county_tags, state_tags, state_tags_ignored)

                if 'county/state reporting' in options:
                    counties_found, states_found = countyStateReporter(
                        elem, countyKeys, stateKeys, counties_found,
                        states_found)

                if 'lat/long' in options:
                    badNodes = lat_long_checker(elem, badNodes)

                if 'amenities' in options:
                    known_amenities = amenityFinder(elem, known_amenities)

                if 'property types' in options:
                    propTypes = propertyType(elem, propTypes)

                if 'property type counts' in options:
                    propRecords = propertyCounter(elem, allowed_propTypes,
                                                  propRecords)

    #----------------------------------------------------------------------
    #printing everything once done iterating
            if 'counting' in options:
                print("Tags Found")
                pprint.pprint(tag_counts)
            if 'zips' in options:
                print("\nZip Lengths")
                pprint.pprint(zipLengthDict)
                print("\nUnique Zip Codes")
                pprint.pprint(known_zips)
                print("\nZip Code Tag Keys Found")
                pprint.pprint(knownZipTags)
            if 'county/state counting' in options:
                print("\nTypes of County Tags")
                pprint.pprint(county_tags)
                print("\nTypes of State Tags")
                pprint.pprint(state_tags)
            if 'county/state reporting' in options:
                print("\nStates Identified")
                pprint.pprint(states_found)
                print("\nCounties Identified")
                pprint.pprint(counties_found)
            if 'lat/long' in options:
                print("\nNodes with Incorrect Latitudes and/or Longitudes")
                pprint.pprint(badNodes)
            if 'amenities' in options:
                print("\nUnique Amenity and Shop Types Identified")
                pprint.pprint(known_amenities)
            if 'property types' in options:
                print("\nUnique Landuse Types")
                pprint.pprint(propTypes)
            if 'property type counts' in options:
                print("\nCounts of Relevant Landuse Types")
                pprint.pprint(propRecords)
コード例 #39
0
def test_saving_network_with_bonkers_attributes_with_geometry(tmpdir):
    # attributes are assumed to be a nested dictionary of very specific format. Due to the fact that user can
    # do virtually anything to edge attributes, or due to calculation error, this may not be the case. If it's not
    # of correct format, we don't expect it to get saved to the matsim network.xml
    network = Network('epsg:27700')
    network.add_node('0',
                     attribs={
                         'id': '0',
                         'x': 1,
                         'y': 2,
                         'lat': 1,
                         'lon': 2
                     })
    network.add_node('1',
                     attribs={
                         'id': '1',
                         'x': 2,
                         'y': 2,
                         'lat': 2,
                         'lon': 2
                     })

    link_attribs = {
        'id': '0',
        'from': '0',
        'to': '1',
        'length': 1,
        'freespeed': 1,
        'capacity': 20,
        'permlanes': 1,
        'oneway': '1',
        'modes': ['car'],
        'geometry': LineString([(1, 2), (2, 3), (3, 4)]),
        'attributes': float('nan')
    }

    network.add_link('0', '0', '1', attribs=link_attribs)
    network.write_to_matsim(tmpdir)

    assert_semantically_equal(dict(network.links()), {'0': link_attribs})

    assert_semantically_equal(
        matsim_xml_writer.check_link_attributes(link_attribs), {
            'id': '0',
            'from': '0',
            'to': '1',
            'length': 1,
            'freespeed': 1,
            'capacity': 20,
            'permlanes': 1,
            'oneway': '1',
            'modes': ['car'],
            'geometry': LineString([(1, 2), (2, 3), (3, 4)])
        })

    found_geometry_attrib = False
    for event, elem in ET.iterparse(os.path.join(tmpdir, 'network.xml'),
                                    events=('start', 'end')):
        if event == 'start':
            if elem.tag == 'attribute':
                if elem.attrib['name'] == 'geometry':
                    assert elem.text == '_ibE_seK_ibE_ibE_ibE_ibE'
                    found_geometry_attrib = True
    assert found_geometry_attrib
コード例 #40
0
    def _parse_wikipedia(self, maximum_number_of_documents=None):
        assert self.__path.exists(), "Wikipedia data does not exist"

        # Determine size of file
        compressed_size = self.__path.stat().st_size

        # Initialise container for documents
        documents = []

        with open(self.__path, mode="rb") as compressed_file:
            with bz2.BZ2File(compressed_file, mode="rb") as uncompressed_file:

                total_compressed_bytes_read_at_last_batch = 0
                tag_prefix = ""
                namespaces = []
                article_namespace_key = None
                in_page = False

                with tqdm(desc="",
                          total=compressed_size,
                          unit="B",
                          unit_scale=True) as progress_bar:

                    for event_number, (event, element) in enumerate(
                            ElementTree.iterparse(
                                uncompressed_file,
                                events=["start", "end", "start-ns",
                                        "end-ns"])):

                        if event == "start-ns":
                            namespaces.append(element)
                            namespace_id, namespace_uri = element
                            if namespace_id == "":
                                tag_prefix = f"{{{namespace_uri}}}"

                        elif event == "end-ns":
                            namespace = namespaces.pop()
                            namespace_id, namespace_uri = namespace
                            if namespace_id == "":
                                tag_prefix = ""

                        elif event == "start":
                            if element.tag == f"{tag_prefix}page":
                                in_page = True
                                title = None
                                text = None
                                page_namespace_keys = []
                                page_redirect = False

                        elif event == "end":

                            tag = element.tag

                            if tag.startswith(tag_prefix):
                                tag = tag.replace(tag_prefix, "", 1)

                            if tag == "namespace":
                                if element.text is None:
                                    article_namespace_key = element.attrib[
                                        "key"]

                            elif in_page and tag == "title":
                                if not title:
                                    title = element.text
                                else:
                                    progress_bar.write(
                                        "Multiple titles found for article "
                                        f"\"{title}\". First one used.")

                            elif in_page and tag == "text":
                                if not text:
                                    text = element.text
                                else:
                                    progress_bar.write(
                                        "Multiple text sections found for article "
                                        f"\"{title}\". First one used.")

                            elif in_page and tag == "ns":
                                page_namespace_keys.append(element.text)

                            elif in_page and tag == "redirect":
                                page_redirect = True

                            elif in_page and tag == "page":

                                in_page = False

                                if article_namespace_key not in page_namespace_keys \
                                    or page_redirect:
                                    continue

                                url = self.__page_base_url \
                                    + title.replace(" ", "_")

                                abstract = self._parse_wikipedia_article(
                                    article_text=text,
                                    sections="first paragraph",
                                    include_header_image_captions=False,
                                    include_header_infoboxes=False)

                                fulltext = self._parse_wikipedia_article(
                                    article_text=text,
                                    sections="all",
                                    include_header_image_captions=False,
                                    include_header_infoboxes=False)

                                document = {
                                    "title": title,
                                    "url": url,
                                    "abstract": abstract,
                                    "text": fulltext
                                }

                                documents.append(document)

                            element.clear()

                        if maximum_number_of_documents and \
                            len(documents) >= maximum_number_of_documents:
                            break

                        if event_number % 1000 == 0:
                            total_compressed_bytes_read = \
                                compressed_file.tell()
                            compressed_bytes_read_for_batch = \
                                total_compressed_bytes_read \
                                - total_compressed_bytes_read_at_last_batch
                            total_compressed_bytes_read_at_last_batch = \
                                total_compressed_bytes_read
                            progress_bar.update(
                                compressed_bytes_read_for_batch)

        return documents
コード例 #41
0
ファイル: 10_data_import.py プロジェクト: JasonCarter80/DAND
def process_to_csv(file_in):
    """ Main process to clean data and save to CSV"""
    LOGGER.info("Processing elements in %s", file_in)
    nodes = []
    nodes_tags = []
    addresses = []
    ways = []
    ways_nodes = []
    ways_tags = []

    for _, elem in ET.iterparse(file_in):
        # Process the nodes
        if elem.tag == 'node':
            node = {}
            node_id = 0
            if elem.keys():
                for name, value in elem.items():
                    if name == 'id':
                        node_id = value
                    node[name] = value

                # Process any tags
                if len(elem):
                    address = {'id': node_id}
                    for tag in elem.iter('tag'):
                        # Build a seperate table for real addresses
                        if 'addr' in tag.attrib['k']:
                            address = add_address(tag, address)
                        else:
                            newtag = {'id': node_id}
                            newtag['key'] = tag.attrib['k'].lower()
                            newtag['value'] = tag.attrib['v']
                            nodes_tags.append(newtag)

                    if len(address) > 1:
                        address = audit_address(address)
                        addresses.append(address)

                nodes.append(node)

        # Process ways
        elif elem.tag == 'way':
            position = 0
            way = {}
            way_id = 0
            if elem.keys():
                for name, value in elem.items():
                    if name == 'id':
                        way_id = value
                    way[name] = value

                # Process any Children Found
                if len(elem):
                    # Process Tags
                    for tag in elem.iter('tag'):
                        way_tag = {'id': way_id}
                        way_tag['key'] = tag.attrib['k'].lower()
                        way_tag['value'] = tag.attrib['v']
                        ways_tags.append(way_tag)
                    # Process Node Relations
                    for ndr in elem.iter('nd'):
                        position += 1
                        way_node = {'id': way_id}
                        way_node['node_id'] = ndr.attrib['ref']
                        way_node['position'] = position
                        ways_nodes.append(way_node)

            ways.append(way)

    write_csv(nodes, 'output/nodes.csv', NODES_FIELDS)
    write_csv(nodes_tags, 'output/nodes_tags.csv', TAGS_FIELDS)
    write_csv(addresses, 'output/node_addresses.csv', ADDRESS_FIELDS)
    write_csv(ways, 'output/ways.csv', WAYS_FIELDS)
    write_csv(ways_tags, 'output/ways_tags.csv', TAGS_FIELDS)
    write_csv(ways_nodes, 'output/ways_nodes.csv', WAYS_NODES_FIELDS)
    return
コード例 #42
0
ファイル: pmcxml.py プロジェクト: ocbier/biotext
def processPMCFile(source):
    # Skip to the article element in the file
    for event, elem in etree.iterparse(source,
                                       events=('start', 'end', 'start-ns',
                                               'end-ns')):
        if (event == 'end' and elem.tag == 'article'):
            pmidText, pmcidText, doiText, pubYear, pubMonth, pubDay, journal, journalISO = getMetaInfoForPMCArticle(
                elem)

            # We're going to process the main article along with any subarticles
            # And if any of the subarticles have distinguishing IDs (e.g. PMID), then
            # that'll be used, otherwise the parent article IDs will be used
            subarticles = [elem] + elem.findall('./sub-article')

            for articleElem in subarticles:
                if articleElem == elem:
                    # This is the main parent article. Just use its IDs
                    subPmidText, subPmcidText, subDoiText, subPubYear, subPubMonth, subPubDay, subJournal, subJournalISO = pmidText, pmcidText, doiText, pubYear, pubMonth, pubDay, journal, journalISO
                else:
                    # Check if this subarticle has any distinguishing IDs and use them instead
                    subPmidText, subPmcidText, subDoiText, subPubYear, subPubMonth, subPubDay, subJournal, subJournalISO = getMetaInfoForPMCArticle(
                        articleElem)
                    if subPmidText == '' and subPmcidText == '' and subDoiText == '':
                        subPmidText, subPmcidText, subDoiText = pmidText, pmcidText, doiText
                    if subPubYear == None:
                        subPubYear = pubYear
                        subPubMonth = pubMonth
                        subPubDay = pubDay
                    if subJournal == None:
                        subJournal = journal
                        subJournalISO = journalISO

                # Extract the title of paper
                title = articleElem.findall(
                    './front/article-meta/title-group/article-title'
                ) + articleElem.findall(
                    './front-stub/title-group/article-title')
                assert len(title) <= 1
                titleText = extractTextFromElemList(title)
                titleText = [
                    removeWeirdBracketsFromOldTitles(t) for t in titleText
                ]

                # Get the subtitle (if it's there)
                subtitle = articleElem.findall(
                    './front/article-meta/title-group/subtitle'
                ) + articleElem.findall('./front-stub/title-group/subtitle')
                subtitleText = extractTextFromElemList(subtitle)
                subtitleText = [
                    removeWeirdBracketsFromOldTitles(t) for t in subtitleText
                ]

                # Extract the abstract from the paper
                abstract = articleElem.findall(
                    './front/article-meta/abstract') + articleElem.findall(
                        './front-stub/abstract')
                abstractText = extractTextFromElemList(abstract)

                # Extract the full text from the paper as well as supplementaries and floating blocks of text
                articleText = extractTextFromElemList(
                    articleElem.findall('./body'))
                backText = extractTextFromElemList(
                    articleElem.findall('./back'))
                floatingText = extractTextFromElemList(
                    articleElem.findall('./floats-group'))

                document = {
                    'pmid': subPmidText,
                    'pmcid': subPmcidText,
                    'doi': subDoiText,
                    'pubYear': subPubYear,
                    'pubMonth': subPubMonth,
                    'pubDay': subPubDay,
                    'journal': subJournal,
                    'journalISO': subJournalISO
                }

                textSources = {}
                textSources['title'] = titleText
                textSources['subtitle'] = subtitleText
                textSources['abstract'] = abstractText
                textSources['article'] = articleText
                textSources['back'] = backText
                textSources['floating'] = floatingText

                for k in textSources.keys():
                    tmp = textSources[k]
                    tmp = [t for t in tmp if len(t) > 0]
                    tmp = [html.unescape(t) for t in tmp]
                    tmp = [removeBracketsWithoutWords(t) for t in tmp]
                    textSources[k] = tmp

                document['textSources'] = textSources
                yield document

            # Less important here (compared to abstracts) as each article file is not too big
            elem.clear()
コード例 #43
0
def process_map(filename):
    users = set()
    for _, element in ET.iterparse(filename):
        if 'user' in element.attrib:
            users.add(get_user(element))
    return users
コード例 #44
0
    def parseFile(self, f):
        """ Parses a single Doxygen XML file
            :param f: XML file path
        """
        documentable_members = 0
        documented_members = 0

        # Wrap everything in a try, as sometimes Doxygen XML is malformed
        try:
            for event, elem in ET.iterparse(f):
                if event == 'end' and elem.tag == 'compounddef':
                    if self.elemIsPublicClass(elem):
                        # store documentation status
                        members, documented, undocumented, bindable, has_brief_description, found_version_added = self.parseClassElem(
                            elem)
                        documentable_members += members
                        documented_members += documented
                        class_name = elem.find('compoundname').text
                        acceptable_missing = self.acceptable_missing.get(
                            class_name, [])

                        if not self.hasGroup(
                                class_name
                        ) and not class_name in self.acceptable_missing_group:
                            self.classes_missing_group.append(class_name)
                        if not class_name in self.acceptable_missing_brief and not has_brief_description:
                            self.classes_missing_brief.append(class_name)
                        if not class_name in self.acceptable_missing_added_note and not found_version_added:
                            self.classes_missing_version_added.append(
                                class_name)

                        # GEN LIST
                        # if len(undocumented) > 0:
                        #     print('"%s": [%s],' % (class_name, ", ".join(['"%s"' % e.replace('"', '\\"') for e in undocumented])))

                        unacceptable_undocumented = undocumented - set(
                            acceptable_missing)

                        # do a case insensitive check too
                        unacceptable_undocumented_insensitive = set([
                            u.lower() for u in undocumented
                        ]) - set([u.lower() for u in acceptable_missing])

                        if len(unacceptable_undocumented_insensitive) > 0:
                            self.undocumented_members[class_name] = {}
                            self.undocumented_members[class_name][
                                'documented'] = documented
                            self.undocumented_members[class_name][
                                'members'] = members
                            self.undocumented_members[class_name][
                                'missing_members'] = unacceptable_undocumented

                        # store bindable members
                        if self.classElemIsBindable(elem):
                            for m in bindable:
                                self.bindable_members.append(m)

                    elem.clear()
        except ET.ParseError as e:
            # sometimes Doxygen generates malformed xml (eg for < and > operators)
            line_num, col = e.position
            with open(f, 'r') as xml_file:
                for i, l in enumerate(xml_file):
                    if i == line_num - 1:
                        line = l
                        break
            caret = '{:=>{}}'.format('^', col)
            print(('ParseError in {}\n{}\n{}\n{}'.format(f, e, line, caret)))

        self.documentable_members += documentable_members
        self.documented_members += documented_members
コード例 #45
0
# first data import
import xml.etree.cElementTree as ET
from consts import *


file_name = r'C:\Users\Meital\Desktop\Posts.xml'
questions_cnt = 0
answers_cnt = 0

if __name__ == "__main__":
    DB = client.sof_new
    POSTS_DB = DB.posts
    #POSTS_DB.drop()
    POSTS_DB.ensure_index("Id", unique=True)

    for event, elem in ET.iterparse(file_name, events=("start", "end")):
        if event == 'start':
            post_type = elem.get('PostTypeId')
            if post_type == '1':
                title = elem.get('Title')
                tags = elem.get('Tags')
                post_id = elem.get('Id')
                if POSTS_DB.find({"Id": post_id}).count() == 0:
                    rep_id = POSTS_DB.insert(elem.attrib, w=0)
                    questions_cnt += 1
                else:
                    print 'id ' + id + ' already exists'
            elem.clear()

    print 'inserted:' + str(questions_cnt)
    for event, elem in ET.iterparse(file_name, events=("start", "end")):
コード例 #46
0
                end_tag = str[index:].index('>')
                yield str[(index + 1):(index + end_tag)]
                index += end_tag + 1
            except ValueError:
                raise Exception("Tag parsing error in \"%s\"" % str)
        else:
            raise Exception("Tag parsing error in \"%s\"" % str)


if len(sys.argv) != 2:
    raise Exception("Usage: %s so-files-directory" % sys.argv[0])

os.chdir(sys.argv[1])

filename = "Posts.xml"
posts = ElementTree.iterparse(filename)
tags = {}
tag_id = 1
print "COPY posts (id, type, creation, score, viewcount, title, body, userid, lastactivity, tags, answercount, commentcount) FROM stdin;"

for event, post in posts:
    if event == "end" and post.tag == "row":
        id = int(post.attrib["Id"])

        if post.attrib.has_key("PostTypeId"):
            type = int(post.attrib["PostTypeId"])
        else:
            type = "\N"

        creation = post.attrib["CreationDate"]
コード例 #47
0
def processArticleFiles(filelist, outFile, processFunction):
    if not isinstance(filelist, list):
        filelist = [filelist]

    # Go through the list of filenames and open each one
    for filename in filelist:
        with open(filename, 'r') as openfile:

            # Skip to the article element in the file
            for event, elem in etree.iterparse(openfile,
                                               events=('start', 'end',
                                                       'start-ns', 'end-ns')):
                if (event == 'end' and elem.tag == 'article'):

                    pmidText, pmcidText, doiText, pubYear = getMetaInfoForPMCArticle(
                        elem)

                    # We're going to process the main article along with any subarticles
                    # And if any of the subarticles have distinguishing IDs (e.g. PMID), then
                    # that'll be used, otherwise the parent article IDs will be used
                    subarticles = [elem] + elem.findall('./sub-article')

                    for articleElem in subarticles:
                        if articleElem == elem:
                            # This is the main parent article. Just use its IDs
                            subPmidText, subPmcidText, subDoiText, subPubYear = pmidText, pmcidText, doiText, pubYear
                        else:
                            # Check if this subarticle has any distinguishing IDs and use them instead
                            subPmidText, subPmcidText, subDoiText, subPubYear = getMetaInfoForPMCArticle(
                                articleElem)
                            if subPmidText == '' and subPmcidText == '' and subDoiText == '':
                                subPmidText, subPmcidText, subDoiText = pmidText, pmcidText, doiText
                            if subPubYear == '':
                                subPubYear = pubYear

                        # Information about the source of this text
                        textSourceInfo = {
                            'pmid': subPmidText,
                            'pmcid': subPmcidText,
                            'doi': subDoiText,
                            'pubYear': subPubYear
                        }

                        # Extract the title of paper
                        title = articleElem.findall(
                            './front/article-meta/title-group/article-title'
                        ) + articleElem.findall(
                            './front-stub/title-group/article-title')
                        assert len(title) <= 1
                        titleText = extractTextFromElemList(title)
                        titleText = [
                            removeWeirdBracketsFromOldTitles(t)
                            for t in titleText
                        ]

                        # Get the subtitle (if it's there)
                        subtitle = articleElem.findall(
                            './front/article-meta/title-group/subtitle'
                        ) + articleElem.findall(
                            './front-stub/title-group/subtitle')
                        subtitleText = extractTextFromElemList(subtitle)
                        subtitleText = [
                            removeWeirdBracketsFromOldTitles(t)
                            for t in subtitleText
                        ]

                        # Extract the abstract from the paper
                        abstract = articleElem.findall(
                            './front/article-meta/abstract'
                        ) + articleElem.findall('./front-stub/abstract')
                        abstractText = extractTextFromElemList(abstract)

                        # Extract the full text from the paper as well as supplementaries and floating blocks of text
                        articleText = extractTextFromElemList(
                            articleElem.findall('./body'))
                        backText = extractTextFromElemList(
                            articleElem.findall('./back'))
                        floatingText = extractTextFromElemList(
                            articleElem.findall('./floats-group'))

                        # Combine all the text we want to process
                        allText = titleText + subtitleText + abstractText + articleText + backText + floatingText
                        allText = [t for t in allText if len(t) > 0]
                        allText = [htmlUnescape(t) for t in allText]
                        allText = [
                            removeBracketsWithoutWords(t) for t in allText
                        ]

                        # Get the co-occurrences using a single list
                        processFunction(outFile, allText, textSourceInfo)

                    # Less important here (compared to abstracts) as each article file is not too big
                    elem.clear()
コード例 #48
0
def audit(osm_file):
    audit_results = {}
    field_types = {'node': {}, 'node_tags': {}, 'way': {}, 'way_tags': {}, 'way_nodes': {}}
    field_validity = {'node': {}, 'node_tags': {}, 'way': {}, 'way_tags': {}, 'way_nodes': {}}

    name_en_re = re.compile(r'\b\S+\.?$', re.IGNORECASE)
    expected = ['Road', 'Street', 'Expressway', 'Bridge', 'Highway', 'River', 'Lake', "Hutong"
                'Park', 'Zone', 'Area', 'Alley', 'Market', 'Campus', 'Gate', 'Hall', 'Engineering',
                'China', 'Elegance', 'Avenue', 'Mansion', 'Square', 'Palace', 'Hotel', 'Rail',
                'Quarter', "Building", "Line", "Apartment", "Airport", "Institute", "College"]

    # for a given tag, update the set of its field types based on current element
    def update_field_types(e, tag):
        for field in FIELDS[tag]:
            if field not in field_types[tag]:
                field_types[tag][field] = set()
            field_types[tag][field].add(detect_type(e.attrib[field]))

    # 验证时间标记
    def validate_timestamp(e, tag):
        timestamp = parse_datetime(e.attrib['timestamp'])
        if timestamp is not None:
            if timestamp < field_validity[tag]['timestamp'][0]:
                field_validity[tag]['timestamp'][0] = timestamp
            if timestamp > field_validity[tag]['timestamp'][1]:
                field_validity[tag]['timestamp'][1] = timestamp

    # 验证经纬度和时间信息是否有效
    def validate_node(e, tag):
        if field_validity[tag] == {}:
            field_validity[tag] = {
                'lat': [90, 0],
                'lon': [180, -180],
                'timestamp': [datetime.now(), datetime(1970, 1, 1, 0, 0, 0)]
            }

        lat = parse_float(e.attrib['lat'])
        if lat is not None:
            if lat < field_validity[tag]['lat'][0]:
                field_validity[tag]['lat'][0] = lat
            if lat > field_validity[tag]['lat'][1]:
                field_validity[tag]['lat'][1] = lat

        lon = parse_float(e.attrib['lon'])
        if lon is not None:
            if lon < field_validity[tag]['lon'][0]:
                field_validity[tag]['lon'][0] = lon
            if lon > field_validity[tag]['lon'][1]:
                field_validity[tag]['lon'][1] = lon
        validate_timestamp(e, tag)

    # 验证 way 的 timestamp 是否有效
    def validate_way(e, tag):
        if field_validity[tag] == {}:
            field_validity[tag] = {
                'timestamp': [datetime.now(), datetime(1970, 1, 1, 0, 0, 0)]
            }
        validate_timestamp(e, tag)

    # 验证 postcode 格式是否正确
    def validate_postcode(e, tag):
        if e.attrib['k'] == 'addr:postcode':
            postcode = e.attrib['v']
            if not check_postcode(postcode):
                field_validity[tag]['postcode'].add(postcode)


    def validate_node_tags(e, tag):
        if field_validity[tag] == {}:
            field_validity[tag] = {
                'postcode': set()
            }
        validate_postcode(e, tag)

    # 验证 name:en
    def validate_way_name_en(e, tag):
        if e.attrib['k'] == 'name:en':
            name_en = e.attrib['v']
            m = name_en_re.search(name_en)
            if m:
                way_type = m.group()
                if way_type not in expected:
                    field_validity[tag]['name_en'][way_type].add(name_en)
    
    # 验证 way 的 'name_en'、'postcode'
    def validate_way_tags(e, tag):
        if field_validity[tag] == {}:
            field_validity[tag] = {
                'name_en': defaultdict(set),
                'postcode': set()
            }
        validate_way_name_en(e, tag)
        validate_postcode(e, tag)


    for _, ele in ET.iterparse(osm_file):
        if ele.tag == 'node':
            update_field_types(ele, 'node')
            validate_node(ele, 'node')

            for e_tag in ele.iter('tag'):
                update_field_types(e_tag, 'node_tags')
                validate_node_tags(e_tag, 'node_tags')

        if ele.tag == 'way':
            update_field_types(ele, 'way')
            validate_way(ele, 'way')

            for e_tag in ele.iter('tag'):
                update_field_types(e_tag, 'way_tags')
                validate_way_tags(e_tag, 'way_tags')

            for e_nd in ele.iter('nd'):
                update_field_types(e_nd, 'way_nodes')


    audit_results['field_types'] = field_types
    audit_results['field_validity'] = field_validity

    return audit_results
コード例 #49
0
def process_medline_file(
    source: Union[str, TextIO],
    tag_handlers: Dict[str,
                       TagHandlerFunction] = {}) -> Iterable[MedlineArticle]:
    """
    Args:
        source: path to the MEDLINE xml file
    """
    for event, elem in etree.iterparse(source,
                                       events=("start", "end", "start-ns",
                                               "end-ns")):
        if event == "end" and elem.tag == "PubmedArticle":  # MedlineCitation'):
            # Try to extract the pmid_id
            pmid_field = elem.find("./MedlineCitation/PMID")
            assert pmid_field is not None
            pmid = pmid_field.text

            journal_year, journal_month, journal_day = get_journal_date_for_medline_file(
                elem, pmid)
            entry_year, entry_month, entry_day = get_pubmed_entry_date(
                elem, pmid)

            jComparison = tuple(
                9999 if d is None else d
                for d in [journal_year, journal_month, journal_day])
            eComparison = tuple(9999 if d is None else d
                                for d in [entry_year, entry_month, entry_day])
            if (
                    jComparison < eComparison
            ):  # The PubMed entry has been delayed for some reason so let's try the journal data
                pub_year, pub_month, pub_day = journal_year, journal_month, journal_day
            else:
                pub_year, pub_month, pub_day = entry_year, entry_month, entry_day

            # Extract the authors
            author_elems = elem.findall(
                "./MedlineCitation/Article/AuthorList/Author")
            authors = []
            for author_elem in author_elems:
                forename = author_elem.find("./ForeName")
                lastname = author_elem.find("./LastName")
                collectivename = author_elem.find("./CollectiveName")

                name = None
                if (forename is not None and lastname is not None
                        and forename.text is not None
                        and lastname.text is not None):
                    name = "%s %s" % (forename.text, lastname.text)
                elif lastname is not None and lastname.text is not None:
                    name = lastname.text
                elif forename is not None and forename.text is not None:
                    name = forename.text
                elif collectivename is not None and collectivename.text is not None:
                    name = collectivename.text
                else:
                    raise RuntimeError(
                        "Unable to find authors in Pubmed citation (PMID=%s)" %
                        pmid)
                authors.append(name)

            chemicals = []
            chemical_elems = elem.findall(
                "./MedlineCitation/ChemicalList/Chemical/NameOfSubstance")
            for chemical_elem in chemical_elems:
                chem_id = chemical_elem.attrib["UI"]
                name = chemical_elem.text
                # chemicals.append((chem_id,name))
                chemicals.append("%s|%s" % (chem_id, name))
            chemicals_txt = "\t".join(chemicals)

            mesh_headings = []
            mesh_elems = elem.findall(
                "./MedlineCitation/MeshHeadingList/MeshHeading")
            for mesh_elem in mesh_elems:
                descriptor_elem = mesh_elem.find("./DescriptorName")
                mesh_id = descriptor_elem.attrib["UI"]
                major_topic_yn = descriptor_elem.attrib["MajorTopicYN"]
                name = descriptor_elem.text

                assert "|" not in mesh_id and "~" not in mesh_id, "Found delimiter in %s" % mesh_id
                assert "|" not in major_topic_yn and "~" not in major_topic_yn, (
                    "Found delimiter in %s" % major_topic_yn)
                assert "|" not in name and "~" not in name, "Found delimiter in %s" % name

                mesh_heading = "Descriptor|%s|%s|%s" % (mesh_id,
                                                        major_topic_yn, name)

                qualifier_elems = mesh_elem.findall("./QualifierName")
                for qualifier_elem in qualifier_elems:
                    mesh_id = qualifier_elem.attrib["UI"]
                    major_topic_yn = qualifier_elem.attrib["MajorTopicYN"]
                    name = qualifier_elem.text

                    assert "|" not in mesh_id and "~" not in mesh_id, (
                        "Found delimiter in %s" % mesh_id)
                    assert "|" not in major_topic_yn and "~" not in major_topic_yn, (
                        "Found delimiter in %s" % major_topic_yn)
                    assert "|" not in name and "~" not in name, "Found delimiter in %s" % name

                    mesh_heading += "~Qualifier|%s|%s|%s" % (
                        mesh_id, major_topic_yn, name)

                mesh_headings.append(mesh_heading)
            mesh_headings_txt = "\t".join(mesh_headings)

            supplementary_concepts = []
            concept_elems = elem.findall(
                "./MedlineCitation/SupplMeshList/SupplMeshName")
            for concept_elem in concept_elems:
                concept_id = concept_elem.attrib["UI"]
                concept_type = concept_elem.attrib["Type"]
                concept_name = concept_elem.text
                # supplementary_concepts.append((concept_id,concept_type,concept_name))
                supplementary_concepts.append(
                    "%s|%s|%s" % (concept_id, concept_type, concept_name))
            supplementary_concepts_txt = "\t".join(supplementary_concepts)

            doi_elems = elem.findall(
                "./PubmedData/ArticleIdList/ArticleId[@IdType='doi']")
            dois = [
                doi_elem.text for doi_elem in doi_elems
                if doi_elem.text and doi_regex.match(doi_elem.text)
            ]

            doi = None
            if dois:
                doi = dois[0]  # We'll just use DOI the first one provided

            pmc_elems = elem.findall(
                "./PubmedData/ArticleIdList/ArticleId[@IdType='pmc']")
            assert len(pmc_elems
                       ) <= 1, "Foud more than one PMCID with PMID: %s" % pmid
            pmcid = None
            if len(pmc_elems) == 1:
                pmcid = pmc_elems[0].text

            pub_type_elems = elem.findall(
                "./MedlineCitation/Article/PublicationTypeList/PublicationType"
            )
            pub_type = [
                e.text for e in pub_type_elems if e.text not in pub_type_skips
            ]
            pub_type_txt = "|".join(pub_type)

            # Extract the title of paper
            title = elem.findall("./MedlineCitation/Article/ArticleTitle")
            title_text = extract_text_chunks(title, tag_handlers=tag_handlers)
            title_text = [
                remove_weird_brackets_from_old_titles(chunk.text)
                for chunk in title_text if chunk.text
            ]
            title_text = [t for t in title_text if len(t) > 0]
            title_text = [html.unescape(t) for t in title_text]
            title_text = [remove_brackets_without_words(t) for t in title_text]

            # Extract the abstract from the paper
            abstract = elem.findall(
                "./MedlineCitation/Article/Abstract/AbstractText")
            abstract_text = extract_text_chunks(abstract,
                                                tag_handlers=tag_handlers)
            abstract_text = [
                chunk.text for chunk in abstract_text if len(chunk.text) > 0
            ]
            abstract_text = [html.unescape(t) for t in abstract_text]
            abstract_text = [
                remove_brackets_without_words(t) for t in abstract_text
            ]

            journal_title_fields = elem.findall(
                "./MedlineCitation/Article/Journal/Title")
            journal_title_iso_fields = elem.findall(
                "./MedlineCitation/Article/Journal/ISOAbbreviation")

            journal_title, journal_iso_title = "", ""
            assert len(journal_title_fields) <= 1, "Error with pmid=%s" % pmid
            assert len(
                journal_title_iso_fields) <= 1, "Error with pmid=%s" % pmid
            if journal_title_fields:
                journal_title = journal_title_fields[0].text
            if journal_title_iso_fields:
                journal_iso_title = journal_title_iso_fields[0].text

            document = {}
            document["pmid"] = pmid
            document["pmcid"] = pmcid
            document["doi"] = doi
            document["pubYear"] = pub_year
            document["pubMonth"] = pub_month
            document["pubDay"] = pub_day
            document["title"] = title_text
            document["abstract"] = abstract_text
            document["journal"] = journal_title
            document["journalISO"] = journal_iso_title
            document["authors"] = authors
            document["chemicals"] = chemicals_txt
            document["meshHeadings"] = mesh_headings_txt
            document["supplementaryMesh"] = supplementary_concepts_txt
            document["publicationTypes"] = pub_type_txt

            yield MedlineArticle(document)

            # Important: clear the current element from memory to keep memory usage low
            elem.clear()
コード例 #50
0
def audit():
	for event, elem in ET.iterparse(osm_file):
		if is_street_name(elem):
		audit_street_type(street_types, elem.attrib['v'])    
	print_sorted_dict(street_types)    

if __name__ == '__main__':
	audit()



#mapparser.py

import xml.etree.cElementTree as ET #导入库函数 ET
import pprint

def count_tags(filename):
	elem_dict = {}.fromkeys(('bounds','member','nd','node','osm','relation','tag','way'),0) 
	#创建空白字典,包含key值 且value值等于0
	for _, elem in ET.iterparse(filename, events=("start",)): #遍历数据文件
		if elem.tag in elem_dict: #通过筛选是否存在空字典中key值对应的标签
			elem_dict[elem.tag] += 1 #改变字典的value值
		else:
			elem_dict[elem.tag] = 1 #此处的[]使用 elem.tag 而没有使用 '' 符号    ????
	return elem_dict

def test():

	tags = count_tags('example.osm')
	pprint.pprint(tags)
	assert tags =={'bounds': 1,
	'member': 3,
'nd': 4,
'node': 20,
'osm': 1,
'relation': 1,
'tag': 7,
'way': 1}

if __name__ =="__main__":
	test()

# Iterating through Ways Tags

import xml.etree.cElementTree as ET
from collections import defaultdict
import re
import pprint


#练习: 标签类型

import xml.etree.cElementTree as ET
import pprint
import re
"""
Your task is to explore the data a bit more.
Before you process the data and add it into your database, you should check the
"k" value for each "<tag>" and see if there are any potential problems.

We have provided you with 3 regular expressions to check for certain patterns
in the tags. As we saw in the quiz earlier, we would like to change the data
model and expand the "addr:street" type of keys to a dictionary like this:
{"address": {"street": "Some value"}}
So, we have to see if we have such tags, and if we have any tags with
problematic characters.

Please complete the function 'key_type', such that we have a count of each of
four tag categories in a dictionary:
  "lower", for tags that contain only lowercase letters and are valid,
  "lower_colon", for otherwise valid tags with a colon in their names,
  "problemchars", for tags with problematic characters, and
  "other", for other tags that do not fall into the other three categories.
See the 'process_map' and 'test' functions for examples of the expected format.
"""


lower = re.compile(r'^([a-z]|_)*$')
lower_colon = re.compile(r'^([a-z]|_)*:([a-z]|_)*$')
problemchars = re.compile(r'[=\+/&<>;\'"\?%#$@\,\. \t\r\n]')


def key_type(element, keys):
    
    if element.tag == "tag":
        if lower.match(element.attrib['k']):
            keys["lower"] +=1
        elif lower_colon.match(element.attrib['k']):
            keys["lower_colon"] +=1
        elif problemchars.search(element.attrib['k']):
            keys["problemchars"] +=1
        else:
            keys["other"] +=1
    return keys



def process_map(filename):
    keys = {"lower": 0, "lower_colon": 0, "problemchars": 0, "other": 0}
    for _, element in ET.iterparse(filename):
        keys = key_type(element, keys)

    return keys

#练习:探索用户

import xml.etree.cElementTree as ET
import pprint
import re
"""
Your task is to explore the data a bit more.
The first task is a fun one - find out how many unique users
have contributed to the map in this particular area!

The function process_map should return a set of unique user IDs ("uid")
"""

def get_user(element):
    if 'uid' in element.attrib:
        return element.attrib['uid']


def process_map(filename):
    users = set()
    for _, element in ET.iterparse(filename):
        if get_user(element):
            users.add(get_user(element))

    return users


def test():

    users = process_map('example.osm')
    pprint.pprint(users)
    assert len(users) == 6



if __name__ == "__main__":
    test()



def test():
    # You can use another testfile 'map.osm' to look at your solution
    # Note that the assertion below will be incorrect then.
    # Note as well that the test function here is only used in the Test Run;
    # when you submit, your code will be checked against a different dataset.
    keys = process_map('example.osm')
    pprint.pprint(keys)
    assert keys == {'lower': 5, 'lower_colon': 0, 'other': 1, 'problemchars': 1}


if __name__ == "__main__":
    test()
コード例 #51
0
#helps to identify important tag keys in the sample data.
#Here we will ctreat a dictionary holding key as key name and value as number of times it has appeard.
import xml.etree.cElementTree as ET
import pprint
import re

filename = 'sample20.osm'
node_tag_keys = dict()
for _, element in ET.iterparse(filename):
    if element.tag == 'node':
        for e in element:
            if e.attrib['k'] in node_tag_keys:
                node_tag_keys[e.attrib['k']] += 1
            else:
                node_tag_keys[e.attrib['k']] = 1
for nodes in node_tag_keys:
    #if the count of repetition is greater than 50 then it will print those names and their values
    if node_tag_keys[nodes] >= 50:
        print('{}->{}'.format(nodes, node_tag_keys[nodes]))
コード例 #52
0
def process_map(filename):
    check = {"lower": 0, "lower_colon": 0, "problemchars": 0, "other": 0}
    for event, element in ET.iterparse(filename, events=("start", )):
        check = key_check(element, check)
    return check
コード例 #53
0
ファイル: drpm.py プロジェクト: wenjiedeng/yum
                kwargs['failfunc'] = failfunc
                kwargs['async'] = True
            try:
                mdpath[repo] = repo._retrieveMD(name, **kwargs)
            except RepoError, e:
                failfunc(e)
        if async:
            grabber.parallel_wait()

        # parse metadata, create DeltaPackage instances
        for repo, cpath in mdpath.items():
            pinfo_repo = pinfo[repo]
            path = repo_gen_decompress(cpath,
                                       'prestodelta.xml',
                                       cached=repo.cache)
            for ev, el in iterparse(path):
                if el.tag != 'newpackage': continue
                name = el.get('name')
                arch = el.get('arch')
                new = name, arch, el.get('epoch'), el.get('version'), el.get(
                    'release')
                index = pinfo_repo.get(new)
                if index is not None:
                    po = pkgs[index]
                    perc = repo.deltarpm_percentage
                    if perc is None:
                        perc = ayum.conf.deltarpm_percentage
                    best = po.size * (perc / 100.0)
                    have = oldrpms.get(repo, {}).get((name, arch), {})
                    for el in el.findall('delta'):
                        size = int(el.find('size').text)
コード例 #54
0
ファイル: ieval.py プロジェクト: seongminp/GEC-Metrics
# Do we have what we need?
if not file_ref or not file_hyp:
    print help_str
    exit(0)

# Totals
# System
t_counts_sys  = Counter({ D:Counter({TP:0,TN:0,FP:0,FN:0,FPN:0}), 
                          C:Counter({TP:0,TN:0,FP:0,FN:0,FPN:0}) })
# Baseline
t_counts_base = Counter({ D:Counter({TP:0,TN:0,FP:0,FN:0,FPN:0}),
                          C:Counter({TP:0,TN:0,FP:0,FN:0,FPN:0}) })

cg = CG.CandidateGenerator()
f_hyp = open(file_hyp,"r")
context = ET.iterparse(file_ref, events=("start", "end"))
context = iter(context)
event, root = context.next()

print_info(file_ref, file_hyp, max_a, max_m, optimise, b, w)

# Show results per sentence?
if per_sent: 
    print "\nSENTENCE RESULTS"
    print_header(per_sent)

# Read gold standard and process each sentence
for event, elem in context:
    if event == "end":
        if elem.tag == "sentence":
            sid = elem.get("id") # Sentence ID
コード例 #55
0
ファイル: parser.py プロジェクト: zhangfeifei02/python
def xml_parse(xm_file, ifilter, tfilter, nfilter, list):
    """
    Function for parsing XML files created by DNSRecon and apply filters.
    """
    iplist = []
    for event, elem in cElementTree.iterparse(xm_file):
        # Check if it is a record
        if elem.tag == "record":
            # Check that it is a RR Type that has an IP Address
            if "address" in elem.attrib:
                # Check if the IP is in the filter list of IPs to ignore
                if (len(ifilter) == 0 or IPAddress(elem.attrib['address']) in ifilter) and (elem.attrib['address'] != "no_ip"):
                    # Check if the RR Type against the types
                    if re.match(tfilter, elem.attrib['type'], re.I):
                        # Process A, AAAA and PTR Records
                        if re.search(r'PTR|^[A]$|AAAA', elem.attrib['type']) \
                        and re.search(nfilter, elem.attrib['name'], re.I):
                            if list:
                                if elem.attrib['address'] not in iplist:
                                    print elem.attrib['address']
                            else:
                                print_good("{0} {1} {2}".format(elem.attrib['type'], elem.attrib['name'], elem.attrib['address']))

                        # Process NS Records
                        elif re.search(r'NS', elem.attrib['type']) and \
                        re.search(nfilter, elem.attrib['target'], re.I):
                            if list:
                                if elem.attrib['address'] not in iplist:
                                    iplist.append(elem.attrib['address'])
                            else:
                                print_good("{0} {1} {2}".format(elem.attrib['type'], elem.attrib['target'], elem.attrib['address']))

                        # Process SOA Records
                        elif re.search(r'SOA', elem.attrib['type']) and \
                        re.search(nfilter, elem.attrib['mname'], re.I):
                            if list:
                                if elem.attrib['address'] not in iplist:
                                    iplist.append(elem.attrib['address'])
                            else:
                                print_good("{0} {1} {2}".format(elem.attrib['type'], elem.attrib['mname'], elem.attrib['address']))

                        # Process MS Records
                        elif re.search(r'MX', elem.attrib['type']) and \
                        re.search(nfilter, elem.attrib['exchange'], re.I):
                            if list:
                                if elem.attrib['address'] not in iplist:
                                    iplist.append(elem.attrib['address'])
                            else:
                                print_good("{0} {1} {2}".format(elem.attrib['type'], elem.attrib['exchange'], elem.attrib['address']))

                        # Process SRV Records
                        elif re.search(r'SRV', elem.attrib['type']) and \
                        re.search(nfilter, elem.attrib['target'], re.I):
                            if list:
                                if elem.attrib['address'] not in iplist:
                                    iplist.append(elem.attrib['address'])
                            else:
                                print_good("{0} {1} {2} {3}".format(elem.attrib['type'], elem.attrib['name'], elem.attrib['address'], elem.attrib['target'], elem.attrib['port']))
            else:
                if re.match(tfilter, elem.attrib['type'], re.I):
                    # Process TXT and SPF Records
                    if re.search(r'TXT|SPF', elem.attrib['type']):
                        if not list:
                            print_good("{0} {1}".format(elem.attrib['type'], elem.attrib['strings']))
    # Process IPs in list
    if len(iplist) > 0:
        try:
            for ip in filter(None, iplist):
                print_line(ip)
        except IOError:
            sys.exit(0)
コード例 #56
0
 def parse(self):
     for event, elem in ElementTree.iterparse(self.dump):
         if elem.tag == "artist":
             artist = self.proc_artist(elem)
             self.download_artist(artist)
コード例 #57
0
def process_map(filename):
    keys = {"lower": 0, "lower_colon": 0, "problemchars": 0, "other": 0}
    for _, element in ET.iterparse(filename):
        keys = key_type(element, keys)

    return keys
コード例 #58
0
def process_users(filename):
    users = []
    for _, element in ET.iterparse(filename):
        if "user" in element.attrib:
            users.append(element.attrib["user"])
    return users
コード例 #59
0
def iterative_files_changelog_parser(file_extension, filelists_xml_path,
                                     other_xml_path):
    """
    Iteratively parse filelists.xml and other.xml, to avoid over-use of memory.

    createrepo_c parses everything in bulk, into memory. For large repositories such as
    RHEL 7 or OL 7, this can require more than 5gb of memory. That isn't acceptable, especially
    when many repositories are being synced at once. The main offenders are other.xml (changelogs)
    and filelists.xml (list of owned files). These also happen to be relatively easy to parse.

    This function, ported from Pulp 2, takes a path to filelists.xml and other.xml, creates
    a streaming parser for each, and then yields one package worth of data from each file.
    """
    # it's basically always gzip, but we'll cover our bases w/ all the possibilites
    if file_extension == "gz":
        open_func = gzip.open
    elif file_extension == "xz":
        open_func = lzma.open
    elif file_extension == "bz2":
        open_func = bz2.open
    elif file_extension == "xml":
        open_func = open
    else:
        raise TypeError("Unknown metadata compression type")
    # TODO: zstd

    with open_func(filelists_xml_path) as filelists_xml, open_func(
            other_xml_path) as other_xml:
        filelists_parser = iterparse(filelists_xml, events=("start", "end"))
        filelists_xml_iterator = iter(filelists_parser)

        other_parser = iterparse(other_xml, events=("start", "end"))
        other_xml_iterator = iter(other_parser)

        # get a hold of the root element so we can clear it
        # this prevents the entire parsed document from building up in memory
        try:
            filelists_root_element = next(filelists_xml_iterator)[1]
            other_root_element = next(other_xml_iterator)[1]
        # I know. This is a terrible misuse of SyntaxError. Don't blame the messenger.
        except SyntaxError:
            log.error("failed to parse XML metadata file")
            raise

        while True:
            for event, filelists_element in filelists_xml_iterator:
                # if we're not at a fully parsed package element, keep going
                if event != "end":
                    continue
                # make this work whether the file has namespace as part of the tag or not
                if not (filelists_element.tag == "package" or re.sub(
                        NS_STRIP_RE, "", filelists_element.tag) == "package"):
                    continue

                break

            for event, other_element in other_xml_iterator:
                # if we're not at a fully parsed package element, keep going
                if event != "end":
                    continue
                # make this work whether the file has namespace as part of the tag or not
                if not (other_element.tag == "package" or re.sub(
                        NS_STRIP_RE, "", other_element.tag) == "package"):
                    continue

                break

            (filelists_pkgid,
             files) = process_filelists_package_element(filelists_element)
            (other_pkgid,
             changelogs) = process_other_package_element(other_element)

            filelists_root_element.clear(
            )  # clear all previously parsed ancestors of the root
            other_root_element.clear()

            assert (
                filelists_pkgid == other_pkgid
            ), "Package id for filelists.xml ({}) and other.xml ({}) do not match".format(
                filelists_pkgid, other_pkgid)

            yield filelists_pkgid, files, changelogs
def audit():
    osm_file = open(OSMFILE, encoding='utf8')
    for event, elem in ET.iterparse(osm_file):
        if is_phone_number(elem):
            audit_phone_numbers(elem.attrib["v"])
    osm_file.close()