Example #1
0
    def upload_multipart(self,
                         key,
                         content_type='application/octet-stream',
                         MultipartUpload=MultipartUpload,
                         encryption_key=None):
        """Upload file to S3 by uploading multiple chunks"""

        if isinstance(key, Key):
            key = key.key
        result = yield from self._request(
            Request("POST",
                    '/' + key, {'uploads': ''}, {
                        'HOST': self._host,
                        'CONTENT-TYPE': content_type,
                    },
                    payload=b'',
                    encryption_key=encryption_key))
        try:
            if result.status != 200:
                xml = yield from result.read()
                raise errors.AWSException.from_bytes(result.status, xml)
            xml = yield from result.read()
            upload_id = parse_xml(xml).find('s3:UploadId', namespaces=NS).text
            assert upload_id, xml
            return MultipartUpload(self, key, upload_id)
        finally:
            result.close()
Example #2
0
 def read_next():
     nonlocal final, marker
     result = yield from self._request(
         Request(
             "GET",
             "/",
             {
                 'prefix': prefix,
                 'max-keys': str(max_keys),
                 'marker': marker
             },
             {'HOST': self._host},
             b'',
         ))
     data = (yield from result.read())
     if result.status != 200:
         raise errors.AWSException.from_bytes(result.status, data)
     x = parse_xml(data)
     result = list(
         map(Key.from_xml, x.findall('s3:Contents', namespaces=NS)))
     if (x.find('s3:IsTruncated', namespaces=NS).text == 'false'
             or len(result) == 0):
         final = True
     else:
         marker = result[-1].key
     return result
Example #3
0
def check_upstream(api_proxy_base, api_key):
    ''' Check connectivity and consistency of NameCheap-hosted records.
    
        Throw exceptions if a problem is found, otherwise return nothing.
    '''
    query = dict(
        ApiKey=api_key,
        Command='namecheap.domains.dns.getHosts',
        ClientIp=get_proxy_ipaddr(api_proxy_base)
        )
        
    query.update(api_defaults)
    
    got = get(api_proxy_base + '?' + urlencode(query))
    tree = parse_xml(got.content)
    
    for el in tree.iter('{http://api.namecheap.com/xml.response}Error'):
        raise ValueError('Upstream API error: {}'.format(el.text))
    
    hosts, expected_hash = [], None
    
    for el in tree.iter('{http://api.namecheap.com/xml.response}host'):
        if (el.attrib['Type'], el.attrib['Name']) == ('TXT', 'hosts-hash'):
            expected_hash = el.attrib['Address']
        else:
            hosts.append(format_xml_element(el))

    found_hash = hash_host_records(hosts)
    
    if expected_hash != found_hash:
        raise ValueError('Calculated hash {} but expected {}'.format(found_hash, expected_hash))
    
    print >> stderr, 'Remote host checks out with hash "{}"'.format(found_hash)
Example #4
0
    def target_outdated(self):
        outdated = False
        error = False

        if not self.source.exists():
            log.error('File {} does not exist.'.format(self.source))
            raise ValueError('Fatal error.')

        if self.target.exists():
            if self_mtime > self.target.lstat().st_mtime:
                outdated = True

            target_mtime = self.target.lstat().st_mtime
            if self.source.lstat().st_mtime > target_mtime:
                outdated = True
        else:
            target_mtime = -1
            outdated = True

        files = [Path(file.text) for file in parse_xml(self.source.open('rb')).findall('./qresource/file')]
        for file in files:
            real_file = self.source.parent / file
            if not real_file.exists():
                log.error('File {} does not exist. (defined in {})'.format(file, self.source))
                error = True
            if real_file.lstat().st_mtime > target_mtime:
                outdated = True

        if error:
            raise ValueError('Fatal error.')

        return outdated
Example #5
0
def load_data(url, format):
    raw = urlopen(url).read()
    if format == "json":
        data = json.loads(raw)
    elif format == "xml":
        data = parse_xml(raw)
    return data
Example #6
0
def try_parse_single_tag_from_xml_document(path: Path, tag_name: str) -> Optional[str]:
    assert tag_name.startswith("{"), "Should start with schema"

    root = parse_xml(str(path)).getroot()
    tags = tuple(_iter_tag_recursive(root, tag_name))
    if is_empty(tags):
        return None
    else:
        assert len(tags) == 1  # Should only be specified once
        tag = tags[0]
        return tag.text
Example #7
0
def qtc():
    data = urllib.urlopen(url).read()
    xml = parse_xml(data)
    date = xml.get("date")
    index = {}
    for group in xml.findall("index"):
        this = {}
        timeperiod = int(group.get("timeperiod"))
        for item in group.getchildren():
            name = fixname(item.tag)
            price = float(item.find("price").text)
            volume = float(item.find("volume").text)
            this[name] = (price, volume)
        index[timeperiod] = this
    return (date, index)
Example #8
0
 def _load_hadoop_config(self):
   if not self._hadoop_configs:
     self._hadoop_configs = dict()
     for file_name in os.listdir(self.local_hadoop_conf_dir):
       if not file_name.lower().endswith(".xml"):
         continue
       xml_doc = parse_xml(os.path.join(self.local_hadoop_conf_dir, file_name))
       for property in xml_doc.getiterator("property"):
         name = property.find("name")
         if name is None or name.text is None:
           continue
         value = property.find("value")
         if value is None or value.text is None:
           continue
         self._hadoop_configs[name.text] = value.text
Example #9
0
 def _load_hadoop_config(self):
     if not self._hadoop_configs:
         self._hadoop_configs = dict()
         for file_name in os.listdir(self.local_hadoop_conf_dir):
             if not file_name.lower().endswith(".xml"):
                 continue
             xml_doc = parse_xml(
                 os.path.join(self.local_hadoop_conf_dir, file_name))
             for property in xml_doc.getiterator("property"):
                 name = property.find("name")
                 if name is None or name.text is None:
                     continue
                 value = property.find("value")
                 if value is None or value.text is None:
                     continue
                 self._hadoop_configs[name.text] = value.text
Example #10
0
 async def _parse_legends_file(self,
                               file_path: str,
                               parse_only: typing.List[str] = None):
     if not paths.exists(file_path):
         raise InternalError(f'{file_path} does not exist')
     self.context.logger.debug(f'Parsing {file_path}')
     tree = parse_xml(file_path)
     root = tree.getroot()
     for elem in root:  # type: Element
         if elem.tag in self._parsers and (parse_only is None
                                           or elem.tag in parse_only):
             self.context.logger.debug(f'Parsing {elem.tag}')
             await self._parsers[elem.tag].parse(elem)
         else:
             self.context.logger.debug(f'Not parsing {elem.tag}')
     self.context.logger.debug(f'Done parsing {file_path}')
Example #11
0
 def exists(self, prefix=''):
     result = yield from self._request(Request(
         "GET",
         "/",
         {'prefix': prefix,
          'separator': '/',
          'max-keys': '1'},
         {'HOST': self._host},
         b'',
         ))
     data = (yield from result.read())
     if result.status != 200:
         raise errors.AWSException.from_bytes(result.status, data)
     x = parse_xml(data)
     return any(map(Key.from_xml,
                     x.findall('s3:Contents', namespaces=NS)))
Example #12
0
File: wthr.py Project: lavelle/wthr
    def getWeather(self, place, unit):
        if place == 'here':
            place = self._getLocation()

        WOEID = self._getWOEID(place)
        url = 'http://weather.yahooapis.com/forecastrss?w=' + WOEID + '&u=' + unit

        response = parse_xml(urlopen(url).read())
        title = response.find('channel/description').text[7:] + ' on ' + response.find('channel/item/pubDate').text + ':\n'
        raw = response.find('channel/item/description').text

        start = 'Forecast:</b><BR />'
        end = '<a href'

        weather = raw[raw.find(start) + len(start): raw.find(end) - 1].replace('<br />', '')[:-1]

        return title + weather
Example #13
0
 def from_bytes(status, body):
     if not body:
         raise RuntimeError("HTTP Error {}".format(status))
     try:
         xml = parse_xml(body)
     except ParseError:
         raise RuntimeError(body)
     code_el = xml.find("Code")
     if code_el is None or not code_el.text:
         raise RuntimeError(body)
     class_name = code_el.text
     try:
         cls = globals()[class_name]
     except KeyError:
         raise RuntimeError("Error {} is unknown".format(class_name))
     msg = xml.find("Message")
     return cls(class_name if msg is None else msg.text)
Example #14
0
 def list(self, prefix='', max_keys=1000):
     result = yield from self._request(Request(
         "GET",
         "/",
         {'prefix': prefix,
          'max-keys': str(max_keys)},
         {'HOST': self._host},
         b'',
         ))
     data = (yield from result.read())
     if result.status != 200:
         raise errors.AWSException.from_bytes(result.status, data)
     x = parse_xml(data)
     if x.find('s3:IsTruncated', namespaces=NS).text != 'false':
         raise AssertionError(
             "File list is truncated, use bigger max_keys")
     return list(map(Key.from_xml,
                     x.findall('s3:Contents', namespaces=NS)))
Example #15
0
    def parse_file(file: IO) -> Dict[int, instructions.InstructionBase]:
        """ Nacteni XML dat z datoveho proudu a zpracovani. 

        Perameters
        ----------
        file: IO
            Vstupni datovy proud.

        Returns
        -------
        Dict[int, instructions.InstructionBase]
            Serazeny slovnik, kde klicem bude obsah XML atributu order.
        """
        try:
            xml_data = parse_xml(file).getroot()
            return InstructionsParser.parse(xml_data)
        except ParseError:
            exit_app(exitCodes.INVALID_XML_FORMAT, 'Invalid XML format.', True)
Example #16
0
 def exists(self, prefix=''):
     result = yield from self._request(
         Request(
             "GET",
             "/",
             {
                 'prefix': prefix,
                 'separator': '/',
                 'max-keys': '1'
             },
             {'HOST': self._host},
             b'',
         ))
     data = (yield from result.read())
     if result.status != 200:
         raise errors.AWSException.from_bytes(result.status, data)
     x = parse_xml(data)
     return any(map(Key.from_xml, x.findall('s3:Contents', namespaces=NS)))
Example #17
0
 def list(self, prefix='', max_keys=1000):
     result = yield from self._request(
         Request(
             "GET",
             "/",
             {
                 'prefix': prefix,
                 'max-keys': str(max_keys)
             },
             {'HOST': self._host},
             b'',
         ))
     data = (yield from result.read())
     if result.status != 200:
         raise errors.AWSException.from_bytes(result.status, data)
     x = parse_xml(data)
     if x.find('s3:IsTruncated', namespaces=NS).text != 'false':
         raise AssertionError("File list is truncated, use bigger max_keys")
     return list(map(Key.from_xml, x.findall('s3:Contents', namespaces=NS)))
Example #18
0
 def from_bytes(status, body):
     if not body:
         # sometimes Riak CS doesn't have response body :(
         # TODO(tailhook) maybe use status to create specific error?
         raise RuntimeError("HTTP Error {}".format(status))
     try:
         xml = parse_xml(body)
     except ParseError:
         raise RuntimeError(body)
     code_el = xml.find("Code")
     if code_el is None or not code_el.text:
         raise RuntimeError(body)
     class_name = code_el.text
     try:
         cls = globals()[class_name]
     except KeyError:
         raise RuntimeError("Error {} is unknown".format(class_name))
     msg = xml.find("Message")
     return cls(class_name if msg is None else msg.text)
Example #19
0
 def from_bytes(status, body):
     if not body:
         # sometimes Riak CS doesn't have response body :(
         # TODO(tailhook) maybe use status to create specific error?
         raise RuntimeError("HTTP Error {}".format(status))
     try:
         xml = parse_xml(body)
     except ParseError:
         raise RuntimeError(body)
     code_el = xml.find("Code")
     if code_el is None or not code_el.text:
         raise RuntimeError(body)
     class_name = code_el.text
     try:
         cls = globals()[class_name]
     except KeyError:
         raise RuntimeError("Error {} is unknown".format(class_name))
     msg = xml.find("Message")
     return cls(class_name if msg is None else msg.text)
Example #20
0
 def commit(self):
     if self._done:
         raise RuntimeError("Can't commit twice or after close")
     self._done = True
     data = xml_tostring(self.xml)
     result = yield from self.bucket._request(Request("POST",
         '/' + self.key, {
             'uploadId': self.upload_id,
         }, headers={
             'CONTENT-LENGTH': str(len(data)),
             'HOST': self.bucket._host,
             'CONTENT-TYPE': 'application/xml',
         }, payload=data))
     try:
         xml = yield from result.read()
         if result.status != 200:
             raise errors.AWSException.from_bytes(result.status, xml)
         xml = parse_xml(xml)
         return xml.find('s3:ETag', namespaces=NS)
     finally:
         result.close()
Example #21
0
    def get_weather(self, place, unit):
        """Get Weather information.
        Use the argument 'place' to say which city you want to know about. if
        the place argument is "here", it'll fetch your place from your IP.
        """
        if place == 'here':
            place = self._get_location()

        woeid = self._get_woeid(place)
        url = 'http://weather.yahooapis.com/forecastrss?w=%s&u=%s' % (woeid, unit)

        response = parse_xml(urlopen(url).read())
        title = response.find('channel/description').text[7:] + \
            ' on ' + response.find('channel/item/pubDate').text + ':\n'
        raw = response.find('channel/item/description').text

        start = 'Forecast:</b><BR />'
        end = '<a href'

        weather = raw[raw.find(start) + len(start): raw.find(end) - 1].replace('<br />', '')[:-1]

        return title + weather
Example #22
0
def update_version_file(versions_file_name, new_version_str):
	new_version = Version.parse(new_version_str)
	
	doc = parse_xml(versions_file_name)
	branch, = (branch for branch in doc.findall('branch') if int(branch.get('id')) == new_version.major)
	
	old_stable_version = Version.parse(branch.get('stable'))
	old_unstable_version = Version.parse(branch.get('unstable'))
	
	assert_upgrade(old_stable_version, new_version)
		
	if new_version.is_stable:
		branch.attrib['stable'] = str(new_version)
		if old_unstable_version is not None and old_unstable_version < new_version:
			del branch.attrib['unstable']
	else:
		if old_unstable_version is not None:
			assert_upgrade(old_unstable_version, new_version)
				
		branch.attrib['unstable'] = str(new_version)
	
	doc.write(open(versions_file_name, "w"))
Example #23
0
def push_upstream(api_proxy_base, api_key, host_records):
    ''' Post replacement host records to NameCheap.
    
        Throw exceptions if a problem is found, otherwise return nothing.
    '''
    hash = hash_host_records(map(format_csv_row, host_records))
    
    form = dict(
        ApiKey=api_key,
        Command='namecheap.domains.dns.setHosts',
        ClientIp=get_proxy_ipaddr(api_proxy_base),

        # Hash record is the first record.
        HostName1='hosts-hash',
        RecordType1='TXT',
        Address1=hash,
        MXPref1=0,
        TTL1=300
        )
    
    form.update(api_defaults)
    
    for (record, number) in zip(host_records, count(2)):
        form.update({
            'HostName{:d}'.format(number): record['Host'],
            'RecordType{:d}'.format(number): record['Type'],
            'Address{:d}'.format(number): record['Value'],
            'MXPref{:d}'.format(number): record['MXPref'] or '0',
            'TTL{:d}'.format(number): record['TTL']
            })
    
    posted = post(api_proxy_base, data=form)
    tree = parse_xml(posted.content)
    
    for el in tree.iter('{http://api.namecheap.com/xml.response}Error'):
        raise ValueError('Upstream API error: {}'.format(el.text))
    
    if posted.status_code not in range(200, 299):
        raise Exception('Bad response status {}'.format(posted.status_code))
Example #24
0
    def upload_multipart(self, key,
            content_type='application/octed-stream',
            MultipartUpload=MultipartUpload):
        """Upload file to S3 by uploading multiple chunks"""

        if isinstance(key, Key):
            key = key.key
        result = yield from self._request(Request("POST",
            '/' + key, {'uploads': ''}, {
            'HOST': self._host,
            'CONTENT-TYPE': content_type,
            }, payload=b''))
        try:
            if result.status != 200:
                xml = yield from result.read()
                raise errors.AWSException.from_bytes(result.status, xml)
            xml = yield from result.read()
            upload_id = parse_xml(xml).find('s3:UploadId',
                                            namespaces=NS).text
            assert upload_id, xml
            return MultipartUpload(self, key, upload_id)
        finally:
            result.close()
Example #25
0
def format_style_sheet(xml_doc, layer_name, element_name, element_attrs,
                       element_text, target_xml):
    # formats template XML stylesheet during runtime.
    et = parse_xml(xml_doc)  # parsing XML document
    for i in et.getroot().findall('Layer'):
        # iterates over all available layers in XML and selects those which is mentioned in layer_name parameter
        if (not layer_name):
            break  # if layer_name is exhausted, just break loop
        if (i.attrib.get('name') in layer_name):
            # remove that layer which is found and to be processed
            layer_name.remove(i.attrib.get('name'))
            target_elem = i.find('Datasource')  # this is our target element
            if (not target_elem):
                # if something goes wrong with target Layer Name, denote failure
                return False
            # creating a new element, which is child of our target `Datasource` element
            subelement = SubElement(target_elem, element_name)
            subelement.text = element_text  # setting text for newly added element
            subelement.set(*element_attrs)  # sets attribute of new element
    if (exists(target_xml)):
        unlink(target_xml)
    et.write(target_xml)  # writes to another XML stylesheet
    return True
Example #26
0
 def read_next():
     nonlocal final, marker
     result = yield from self._request(Request(
         "GET",
         "/",
         {'prefix': prefix,
          'max-keys': str(max_keys),
          'marker': marker},
         {'HOST': self._host},
         b'',
         ))
     data = (yield from result.read())
     if result.status != 200:
         raise errors.AWSException.from_bytes(result.status, data)
     x = parse_xml(data)
     result = list(map(Key.from_xml,
                     x.findall('s3:Contents', namespaces=NS)))
     if(x.find('s3:IsTruncated', namespaces=NS).text == 'false' or
             len(result) == 0):
         final = True
     else:
         marker = result[-1].key
     return result
Example #27
0
def update_version_file(versions_file_name, new_version_str):
    new_version = Version.parse(new_version_str)

    doc = parse_xml(versions_file_name)
    branch, = (branch for branch in doc.findall('branch')
               if int(branch.get('id')) == new_version.major)

    old_stable_version = Version.parse(branch.get('stable'))
    old_unstable_version = Version.parse(branch.get('unstable'))

    assert_upgrade(old_stable_version, new_version)

    if new_version.is_stable:
        branch.attrib['stable'] = str(new_version)
        if old_unstable_version is not None and old_unstable_version < new_version:
            del branch.attrib['unstable']
    else:
        if old_unstable_version is not None:
            assert_upgrade(old_unstable_version, new_version)

        branch.attrib['unstable'] = str(new_version)

    doc.write(open(versions_file_name, "w"))
Example #28
0
 def commit(self):
     if self._done:
         raise RuntimeError("Can't commit twice or after close")
     self._done = True
     data = xml_tostring(self.xml)
     result = yield from self.bucket._request(
         Request("POST",
                 '/' + self.key, {
                     'uploadId': self.upload_id,
                 },
                 headers={
                     'CONTENT-LENGTH': str(len(data)),
                     'HOST': self.bucket._host,
                     'CONTENT-TYPE': 'application/xml',
                 },
                 payload=data))
     try:
         xml = yield from result.read()
         if result.status != 200:
             raise errors.AWSException.from_bytes(result.status, xml)
         xml = parse_xml(xml)
         return xml.find('s3:ETag', namespaces=NS)
     finally:
         result.close()
def main(argv):

    try:

        (sFile) = smr.loadArgs(argv, 'calls')

        sFilePrefix = sFile

        aColumns = [
            'year', 'mon', 'day', 'time', 'dur(m)', 'contact', 'phone', 'type',
            'timestamp'
        ]

        out_fname = 'z_{}.tsv'.format(sFilePrefix)

        tree = parse_xml(sFile)

        calls = tree.getroot()

        pc("""Backup info: 
  File: {}
  count={}
  backup_set={}
  backup_date={}""".format(sFile, calls.get('count'), calls.get('backup_set'),
                           calls.get('backup_date')))

        rows = []
        iC = 0
        for m in calls.findall('call'):
            iC = iC + 1
            # pc('type(m) = {}', type(m))
            # pc('m = {}', smr.dump(m, True))

            iType = smr.aget("m", m, 'type', req=True, toint=True)
            if iType == 1:
                sType = "1"
            elif iType == 2:
                sType = "2"
            elif iType == 3:
                sType = "3"
            elif iType == 5:
                sType = "5"
            else:
                raise Exception("Unexpected type {} in xml: {}".format(
                    iType, smr.dump(m)))

            sPhone = smr.fixPhone(smr.aget("m", m, 'number', req=True))

            d = smr.dateFromTimestamp(smr.aget("m", m, 'date', req=True))

            rows.append([
                d.year, d.month, d.day,
                d.strftime('%H:%M:%S'),
                round(
                    smr.aget("m", m, 'duration', req=True, toint=True) / 60,
                    1),
                smr.aget("m", m, 'contact_name', req=True), sPhone, sType,
                d.strftime('%y%m%d-%H%M%S')
            ])

        pc("calls found: {}", iC)

        # Sort
        rows = sorted(rows, key=lambda r: (r[0], r[1], r[2], r[3]))

        fTmp = open(out_fname, 'w', newline='')
        bNewFile = True
        wri = csv.writer(fTmp, delimiter='\t')
        wri.writerow(aColumns)

        for r in rows:
            wri.writerow(r)

        fTmp.close()

        pc("Data written to: {}".format(out_fname))

        pc('.: done :.')
        pc('.')

    except Exception as e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        sTB = '\n'.join(traceback.format_tb(exc_traceback))
        pc("Fatal exception: {}\n - msg: {}\n stack: {}".format(
            exc_type, exc_value, sTB))
Example #30
0
def parse_specdata(source, doc_name, doc_type, doc_version):
    """
	
	Reads a schema specification from a file (e.g., specdata.xml) or file-like object, and returns a tuple containing:
	
		* a mapping of class names to Element subclasses
		* a Document subclass
	
	:arg source: the file or file-like object
	:type source: str or file-like object
	:arg schema_name: the name of the schema
	:type schema_name: str
	:returns: tuple
	
	"""

    tree = parse_xml(source)
    elements = {}
    globals = []

    def child_elements(parent_level, element_list, upper_recursive=None):
        children = []
        while element_list:
            raw_element = element_list[0]
            raw_attrs = raw_element.attrib

            element_level = int(raw_attrs['level'])

            is_global = False
            if element_level == -1:
                is_global = True
            elif parent_level is not None and not element_level > parent_level:
                break
            element_list = element_list[1:]

            element_name = '%sElement' % raw_attrs.get(
                'cppname', raw_attrs.get('name')).translate(None, '-')
            element_attrs = {
                '__module__':
                None,
                'id':
                int(raw_attrs['id'], 0),
                'name':
                raw_attrs['name'],
                'type':
                SPECDATA_TYPES[raw_attrs['type']],
                'mandatory':
                True if raw_attrs.get('mandatory', False) == '1' else False,
                'multiple':
                True if raw_attrs.get('multiple', False) == '1' else False
            }
            try:
                element_attrs['default'] = {
                    INT: lambda default: int(default),
                    UINT: lambda default: int(default),
                    FLOAT: lambda default: float(default),
                    STRING: lambda default: str(default),
                    UNICODE: lambda default: unicode(default)
                }.get(element_attrs['type'],
                      lambda default: default)(raw_attrs['default'])
            except (KeyError, ValueError):
                element_attrs['default'] = None

            element = type(element_name, (Element, ), element_attrs)
            elements[element_name] = element

            recursive = []
            if upper_recursive:
                recursive.extend(upper_recursive)
            if raw_attrs.get('recursive', False) == '1':
                recursive.append(element)

            element_children, element_list = child_elements(
                element_level if not is_global else 0, element_list, recursive)
            element_children += tuple(recursive)
            element.children = element_children

            if is_global:
                globals.append(element)
            else:
                children.append(element)
        return tuple(children), element_list

    children = child_elements(None, tree.getroot().getchildren())[0]

    document_attrs = {
        '__module__': None,
        'type': doc_type,
        'version': doc_version,
        'children': children,
        'globals': tuple(globals)
    }
    document = type(doc_name, (Document, ), document_attrs)

    return elements, document
Example #31
0
def get(baseurl, args):
    url = "%s?%s" % (baseurl, urllib.urlencode(args))
    data = urllib.urlopen(url).read()
    return parse_xml(data)
Example #32
0
# In[9]:


def _voc_cls_name_to_idx(name):
    idx = dict(list(zip(VOC_CLASSES, list(range(len(VOC_CLASSES))))))[name]
    return idx


sample_id = '000330'

# Read the image file.
img = cv2.imread(osp.join(DATASET_DIR, 'JPEGImages/{}.jpg'.format(sample_id)))

# Read and parse the corresponding annotation file.
anno = parse_xml(osp.join(DATASET_DIR,
                          'Annotations/{}.xml'.format(sample_id))).getroot()
gt_boxes = []
for obj in anno.findall('object'):
    bndbox = obj.find('bndbox')
    bbox = [
        float(bndbox.find('xmin').text) - 1,
        float(bndbox.find('ymin').text) - 1,
        float(bndbox.find('xmax').text) - 1,
        float(bndbox.find('ymax').text) - 1,
        _voc_cls_name_to_idx(obj.find('name').text)
    ]
    gt_boxes.append(bbox)
gt_boxes = np.array(gt_boxes, dtype=np.float32)

# Resize the image and rescale the GT boxes accordingly.
img_size_min = np.min(img.shape[:2])
Example #33
0
def parse_specdata(source, doc_name, doc_type, doc_version):
	"""
	
	Reads a schema specification from a file (e.g., specdata.xml) or file-like object, and returns a tuple containing:
	
		* a mapping of class names to Element subclasses
		* a Document subclass
	
	:arg source: the file or file-like object
	:type source: str or file-like object
	:arg schema_name: the name of the schema
	:type schema_name: str
	:returns: tuple
	
	"""
	
	tree = parse_xml(source)
	elements = {}
	globals = []
	
	def child_elements(parent_level, element_list, upper_recursive=None):
		children = []
		while element_list:
			raw_element = element_list[0]
			raw_attrs = raw_element.attrib
			
			element_level = int(raw_attrs['level'])
			
			is_global = False
			if element_level == -1:
				is_global = True
			elif parent_level is not None and not element_level > parent_level:
				break
			element_list = element_list[1:]

			element_name = '%sElement' % raw_attrs.get('cppname', raw_attrs.get('name')).translate(None, '-')
			element_attrs = {
				'__module__': None,
				'id': int(raw_attrs['id'], 0),
				'name': raw_attrs['name'],
				'type': SPECDATA_TYPES[raw_attrs['type']],
				'mandatory': True if raw_attrs.get('mandatory', False) == '1' else False,
				'multiple': True if raw_attrs.get('multiple', False) == '1' else False
			}
			try:
				element_attrs['default'] = {
					INT: lambda default: int(default),
					UINT: lambda default: int(default),
					FLOAT: lambda default: float(default),
					STRING: lambda default: str(default),
					UNICODE: lambda default: unicode(default)
				}.get(element_attrs['type'], lambda default: default)(raw_attrs['default'])
			except (KeyError, ValueError):
				element_attrs['default'] = None
			
			element = type(element_name, (Element,), element_attrs)
			elements[element_name] = element
			
			recursive = []
			if upper_recursive:
				recursive.extend(upper_recursive)
			if raw_attrs.get('recursive', False) == '1':
				recursive.append(element)
			
			element_children, element_list = child_elements(element_level if not is_global else 0, element_list, recursive)
			element_children += tuple(recursive)
			element.children = element_children
			
			if is_global:
				globals.append(element)
			else:
				children.append(element)
		return tuple(children), element_list
	
	children = child_elements(None, tree.getroot().getchildren())[0]
	
	document_attrs = {
		'__module__': None,
		'type': doc_type,
		'version': doc_version,
		'children': children,
		'globals': tuple(globals)
	}
	document = type(doc_name, (Document,), document_attrs)
	
	return elements, document
Example #34
0
def main(argv):

    try:

        (sFile) = smr.loadArgs(argv, "sms")

        sFilePrefix = sFile

        # First lets write SMS
        aColumns = ['type', 'year', 'mon', 'day', 'time', 'contact', 'phone', 'dir', 'body', 'timestamp']

        out_fname = 'z_{}.tsv'.format(sFilePrefix)
            
        tree = parse_xml(sFile)

        smses = tree.getroot()

        rows = []; iC = 0
        for m in smses.findall('sms'):
            iC = iC + 1
            # pc('type(m) = {}', type(m))
            # pc('m = {}', smr.dump(m, True))

            iType = smr.aget("m", m, 'type', req=True, toint=True)
            if iType == 1:
                sType = "in"
            elif iType == 2:
                sType = "out"
            else:
                raise Exception("Unexpected type {} in xml: {}".format(iType, smr.dump(m)))

            sPhone = smr.fixPhone(smr.aget("m", m, 'address', req=True))

            d = smr.dateFromTimestamp(smr.aget("m", m, 'date', req=True))
            
            rows.append([
                 'sms'
                ,d.year, d.month, d.day, d.strftime('%H:%M:%S')
                ,smr.aget("m", m, 'contact_name', req=True)
                ,sPhone
                ,sType
                ,smr.aget("m", m, 'body', req=True)
                ,d.strftime('%y%m%d-%H%M%S')
            ])

        pc("sms records found: {}", iC)

        iC = 0
        for m in smses.findall('mms'):
            iC = iC + 1
            # pc('type(m) = {}', type(m))
            # pc('m = {}', smr.dump(m, True))
            iType = smr.aget("m", m, 'type', req=True, toint=True)
            if iType == -1:
                sType = "?"
            else:
                raise Exception("Unexpected type {} in xml: {}".format(iType, smr.dump(m)))

            sPhone = smr.fixPhone(smr.aget("m", m, 'address', req=True))

            sMsg = ""
            for part in m.findall('part'):
            # for part in m.find('parts').getchildren():
                iSeq = smr.aget("part", part, 'seq', req=True, toint=True)
                if iSeq == 0:
                    sMsg = smr.aget("part", part, 'text', req=True)

            d = smr.dateFromTimestamp(smr.aget("m", m, 'date', req=True))

            rows.append([
                 'mms'
                ,d.year, d.month, d.day, d.strftime('%H:%M:%S')
                ,smr.aget("m", m, 'contact_name', req=True)
                ,sPhone
                ,sType
                ,sMsg
                ,d.strftime('%y%m%d-%H%M%S')
            ])

        pc("mms records found: {}", iC)

        # Sort
        rows = sorted(rows, key=lambda r: (r[1], r[2], r[3], r[4]) )

        fTmp = open(out_fname,'w', newline='')
        bNewFile=True
        wri = csv.writer(fTmp, delimiter='\t')
        wri.writerow(aColumns)

        for r in rows:
            wri.writerow(r)

        fTmp.close()
        

        pc("starting argv: {}".format(argv))

        pc("Data written to: {}".format(out_fname))

        pc('.: done :.')
        pc('.')

    except Exception as e:
        exc_type, exc_value, exc_traceback = sys.exc_info()
        sTB = '\n'.join(traceback.format_tb(exc_traceback))
        pc("Fatal exception: {}\n - msg: {}\n stack: {}".format(exc_type, exc_value, sTB))
def dlcs_parse_xml(data, split_tags=False):
    """Parse any del.icio.us XML document and return Python data structure.

    Recognizes all XML document formats as returned by the version 1 API and
    translates to a JSON-like data structure (dicts 'n lists).

    Returned instance is always a dictionary. Examples::

     {'posts': [{'url':'...','hash':'...',},],}
     {'tags':['tag1', 'tag2',]}
     {'dates': [{'count':'...','date':'...'},], 'tag':'', 'user':'******'}
     {'result':(True, "done")}
     # etcetera.
    """
    # TODO: split_tags is not implemented

    if DEBUG > 3:
        print("dlcs_parse_xml: parsing from ", data, sys.stderr)

    if not hasattr(data, 'read'):
        data = StringIO(data)

    doc = parse_xml(data)
    root = doc.getroot()
    fmt = root.tag

    # Split up into three cases: Data, Result or Update
    if fmt in ('tags', 'posts', 'dates', 'bundles'):
        # Data: expect a list of data elements, 'resources'.
        # Use `fmt` (without last 's') to find data elements, elements
        # don't have contents, attributes contain all the data we need:
        # append to list
        elist = [el.attrib for el in doc.findall(fmt[:-1])]

        # Return list in dict, use tagname of rootnode as keyname.
        data = {fmt: elist}

        # Root element might have attributes too, append dict.
        data.update(root.attrib)

        return data

    elif fmt == 'result':
        # Result: answer to operations
        if 'code' in root.attrib:
            msg = root.attrib['code']
        else:
            msg = root.text

        # XXX: Return {'result':(True, msg)} for /known/ O.K. messages,
        # use (False, msg) otherwise. Move this to DeliciousAPI?
        v = msg in DLCS_OK_MESSAGES
        return {fmt: (v, msg)}

    elif fmt == 'update':
        # Update: "time"
        return {fmt: {
            'time': time.strptime(root.attrib['time'], ISO_8601_DATETIME)}}

    else:
        raise PyDeliciousException("Unknown XML document format '%s'" % fmt)
Example #36
0
import os
import datetime

from tomlkit import comment
from tomlkit import document
from tomlkit import nl
from tomlkit import table
from tomlkit import integer as toml_int
from tomlkit.toml_file import TOMLFile
from xml.etree.ElementTree import parse as parse_xml

from lib import toml_file_header

xml_file = os.path.join(os.path.dirname(__file__), "..", "srcdata", "stm32_db",
                        "mcu", "families.xml")
xml_tree = parse_xml(xml_file)
xml_root = xml_tree.getroot()

toml_file = os.path.join(os.path.dirname(__file__), "..", "data",
                         "mcu_families", "stm.toml")
toml = TOMLFile(toml_file)

doc = document()

toml_file_header(doc)

for xml_family in xml_root.findall("./Family"):
    family = table()
    family.add("name", xml_family.get("Name"))
    family.add("object_type", "family")
    for xml_subfamily in xml_family.findall("./SubFamily"):