Exemplo n.º 1
0
    def get(self):
        # Loading current weather data…
        current_url = CURRENT_URL.format(self.config.location)
        with closing(urllib.urlopen(current_url)) as file_handle:
            current_data = parse_xml(file_handle)

        # Loading forecast data…
        forecast_url = FORECAST_URL.format(self.config.location)
        with closing(urllib.urlopen(forecast_url)) as file_handle:
            forecast_data = parse_xml(file_handle)

        forecast = []

        for day in forecast_data.find('simpleforecast').getchildren()[1:4]:
            forecast.append(day.find('icon').text)

        return {
            'current': {
                'weather': current_data.find('weather').text,
                'temperature': current_data.find('temp_c').text,
                'humidity': current_data.find('relative_humidity').text[:-1], # strip the '%'
                'wind_direction': current_data.find('wind_dir').text,
                'wind_speed': str(round(float(current_data.find('wind_mph').text) * KILOMETER_MILES_RATIO, 1)),
                'pressure': current_data.find('pressure_mb').text,
                'visibility': current_data.find('visibility_km').text,
                'icon': current_data.find('icon').text
            },
            'forecast': forecast
        }
Exemplo n.º 2
0
    def read_scheme(self):

        if not os.path.isfile(self.scheme_path):
            from . import MissingConfigurationDefinitionFile
            raise MissingConfigurationDefinitionFile("Could not find %r." % self.scheme_path)

        tree = parse_xml(self.scheme_path)
        root = tree.getroot()
        scheme = ordereddict()

        for child in root.getchildren():
            option_name = child.tag
            attributes = dict(child.attrib)
            option_type = attributes.pop('type')
            if option_type.startswith('multioption'):
                # TODO: Hrm
                attributes['default'] = child.attrib.pop('default', None)
                attributes['options'] = unserialize_atomic(child, FIELD_TYPE_MAP)
            else:
                if not (
                    FIELD_TYPE_MAP.get(option_type) in ('list', 'tuple', 'dict')
                    and not child.getchildren()
                ):
                    attributes['default'] = unserialize_atomic(child, FIELD_TYPE_MAP)
            scheme[option_name] = get_field(option_type)(**attributes)

        return scheme
def get_info(superevent):
    superevent_id = superevent['superevent_id']
    result = {'superevent_id': superevent_id}
    api = client.superevents[superevent_id]

    # Scan VOEvents in reverse order (newest to oldest).
    voevents = api.voevents.get()[::-1]
    for voevent in voevents:
        try:
            voevent_xml = parse_xml(api.files[voevent['filename']].get())
        except requests.exceptions.HTTPError as e:
            # Some VOEvents cannot be found because the files in GraceDB were
            # not exposed to the public. Skip them.
            if e.response.status_code == 404:
                warnings.warn(f'HTTP Error 404 for f{voevent["filename"]}')
                continue
            else:
                raise
    
        # Get source classification and source properties.
        for group in ['Classification', 'Properties']:
            for key, value in get_params_for_group(voevent_xml, group).items():
                result.setdefault(key, value)

        # Look for a BAYESTAR sky map.
        elem = voevent_xml.find(".//Param[@name='skymap_fits']")
        if elem is not None and 'bayestar' in elem.attrib['value'].lower():
            skymap = get_skymap(elem.attrib['value'])
            for key, value in get_skymap_stats(skymap).items():
                result.setdefault(key, value)

        if all(key in result for key in ['Terrestrial', 'area(90)']):
            return result
    else:
        raise RuntimeError(f'Missing some information for {superevent_id}')
Exemplo n.º 4
0
    def __init__(self, path):
        self.root_namespace = Namespace()
        self.functions = []
        self.classes = []
        self.structs = []
        self.enums = []

        # Begin temporary, the variables only exist during import,
        # meaning while __init__ runs

        # A dict of {namespace: {function: [signatures]}}
        self.function_signatures = defaultdict(partial(defaultdict, list))
        self.qualtypes = []
        self.nodes = {}

        # End temporary variables

        doc = parse_xml(path)
        for xml_node in doc.xpath('/HeaderExport/Class'):
            self.parse_class(xml_node)
        for xml_node in doc.xpath('/HeaderExport/Struct'):
            self.parse_struct(xml_node)
        for xml_node in doc.xpath('/HeaderExport/Function'):
            self.parse_function(xml_node)
        for xml_node in doc.xpath('/HeaderExport/Enum'):
            self.parse_enumeration(xml_node)
        self.process_functions()
        self.process_contexts()
        self.process_types()
        self.process_bases()

        del self.nodes
Exemplo n.º 5
0
def main():
    engine = create_engine('sqlite:///test.sqlite')

    session.bind = engine
    session.autoflush = False

    filename = sys.argv[1]

    xml = parse_xml(filename)

    ctx = {}
    for game in xml.xpath('/wild/game'):
        ctx['version'] = get_version(game.get('version'))
        # XXX region should be set based on the location
        ctx['region'] = ctx['version'].version_group.generation.main_region
        for loc in game.xpath('location'):
            ctx['location'] = get_or_create_location(loc, ctx)
            for area in loc.xpath('area'):
                ctx['area'] = create_area(area, ctx)

                if area.get('name', False):
                    print loc.get('name') + "/" + area.get('name')
                else:
                    print loc.get('name')

                encounters = list(reduce_encounters(area))
                insert_encounters(encounters, ctx)
                #for e in sorted(encounters,
                #                key=itemgetter('method', 'terrain')):
                #    print e
        session.flush()
    session.commit()
Exemplo n.º 6
0
    def from_xml(self, content):
        """
        Given some XML data, returns a Python dictionary of the decoded data.
        """
        if lxml is None:
            raise ImproperlyConfigured("Usage of the XML aspects requires lxml.")

        return self.from_etree(parse_xml(StringIO(content)).getroot())
Exemplo n.º 7
0
    def from_xml(self, content):
        """
        Given some XML data, returns a Python dictionary of the decoded data.
        """
        if lxml is None:
            raise UnsupportedDeserializationFormat("Usage of the XML aspects requires lxml.")

        return self.from_etree(parse_xml(StringIO(content)).getroot())
Exemplo n.º 8
0
 def from_xml(self, content):
     """
     Given some XML data, returns a Python dictionary of the decoded data.
     """
     if lxml is None:
         raise ImproperlyConfigured("Usage of the XML aspects requires lxml.")
     
     return self.from_etree(parse_xml(StringIO(content)).getroot())
Exemplo n.º 9
0
async def _preprocess_soap(req):
    from lxml.etree import fromstring as parse_xml

    body = await req.read()
    root = parse_xml(body)

    action = _find_element(root, 'Body/*[1]')

    return action
Exemplo n.º 10
0
def htmlXPath(xpathExpr, body):
    try:
        xmlNode = parse_xml(StringIO(body), parser=HTMLParser()).getroot()
    except XMLSyntaxError:
        print(body)
        raise

    for result in xmlNode.xpath(xpathExpr):
        yield result
Exemplo n.º 11
0
def htmlXPath(xpathExpr, body):
    try:
        xmlNode = parse_xml(BytesIO(body), parser=HTMLParser()).getroot()
    except XMLSyntaxError:
        print(body)
        raise

    for result in xmlNode.xpath(xpathExpr):
        yield result
def main():
    global base_url, ajax_url, page_param, page_num_of_article, page_start, page_step, page_encoding, page_sleep, count

    count = 0

    last_page      = scraperwiki.sqlite.get_var('last_page', -1)
    latest_article = None
    start_over     = False
    if last_page == -1:
        last_page      = page_start
        latest_article = scraperwiki.sqlite.get_var('latest_article', None)
        start_over     = True

    opener = urllib2.build_opener()
    opener.addheaders = [
        ('User-agent', 'Mozilla/5.0'),
        ('Referer', base_url),
        ('X-Requested-With', 'XMLHttpRequest')
    ]
    urllib2.install_opener(opener)

    error_count    = 0
    num_of_article = page_num_of_article
    while num_of_article == page_num_of_article:
        params[page_param] = last_page

        try:
            xml = scraperwiki.scrape(ajax_url, params)
        except urllib2.URLError, e:
            print 'Cannot reach the server:',
            if hasattr(e, 'reason'): print e.reason
            elif hasattr(e, 'code'): print e.code
            error_count += 1
            if error_count < 3: continue
            else: break

        # try:
        #     xml = xml.decode(page_encoding)
        # except UnicodeDecodeError:
        #     encoded = ''
        #     for word in xml.split(' '):
        #         try:
        #             encoded += word.decode(page_encoding) + ' '
        #         except UnicodeDecodeError:
        #             pass
        #     xml = encoded.rstrip()

        num_of_article = scrape(parse_xml(xml), latest_article, start_over)

        page = last_page / page_step
        if (page_start == 0): page += 1

        scraperwiki.sqlite.save_var('last_page', last_page)
        print 'Page', page, ',', num_of_article, 'article(s)'

        last_page += page_step
        time.sleep(page_sleep)
Exemplo n.º 13
0
    def get_temperature(self, location):

        print
        print location
        print

        current_url = CURRENT_URL.format(location)
        with closing(urllib.urlopen(current_url)) as file_handle:
            current_data = parse_xml(file_handle)

        return {'temperature': current_data.find('temp_c').text, 'location': location}
Exemplo n.º 14
0
def parseHtmlAsXml(body):
    def forceXml(body):
        newBody = body
        for entity, replacement in _entities.items():
            newBody = newBody.replace(entity, replacement)
        newBody = _scriptTagRegex.sub('', newBody)
        return newBody
    try: 
        return parse_xml(StringIO(forceXml(body)))
    except XMLSyntaxError:
        print body 
        raise
Exemplo n.º 15
0
def main(args=None):
    if args is None:
        args = sys.argv[1:]

    try:
        filename = args[0]
    except IndexError:
        filename = "-"

    xml = parse_xml(filename if filename != "-" else sys.stdin)
    collapse(xml)
    xml.write(sys.stdout.buffer, pretty_print=True)
Exemplo n.º 16
0
def createReturnValue(header, body, parse):
    if parse and body.strip() != '':
        try:
            body = parse_xml(StringIO(body))
        except:
            try:
                body = HTML(body, HTMLParser(recover=True))
            except:
                print "Exception parsing:"
                print body
                raise
    return header, body
Exemplo n.º 17
0
 def _subtitle_file_as_mpl2(self, xmlfile, language):
     # see http://lists.mplayerhq.hu/pipermail/mplayer-users/2003-February/030222.html
     tempfile = NamedTempfile(self.video_id+'-subtitle-'+language+'.mpl2')
     if not tempfile.isempty():
         return tempfile.name
     with tempfile:
         xmltree = parse_xml(xmlfile).getroot()
         for element in xmltree:
            start = int(float(element.attrib['start'])*1000)
            end = start + int(float(element.attrib['dur'])*1000)
            text = element.text.replace('\n', '|')
            tempfile.file.write('[{0}][{1}]{2}\n'.format(start, end, text))
     return tempfile.name
Exemplo n.º 18
0
 async def read_rrd(remote,fn):
     """
     Dump a RRD fiel to XML and parse it.
     """
     if remote:
         # This is a hack
         fn = "/var/lib/rrdcached/db/collectd/"+str(fn)[7:]
         # ssh is broken WRT quoting
         p = await trio.run_process(["ssh","-n",remote,"rrdtool","dump",shlex.quote(fn)], capture_stdout=True)
     else:
         p = await trio.run_process(["rrdtool","dump",str(fn)], capture_stdout=True)
     # Moving the parser to a background task doesn't pseed things up
     return parse_xml(p.stdout)
Exemplo n.º 19
0
    def __init__(self, path):
        self.functions = []
        self.classes = []
        self.structs = []
        self.enums = []

        # Begin temporaries
        self.nodes = {}
        self.namespaces = []

        # A dict of {context: {function: [signatures]}}
        self.signatures = defaultdict(partial(defaultdict, list))
        # End temporaries

        parsers = {
            'Struct': self.parse_class, # Not all Struct tags are POD
            'Class': self.parse_class,
            'Function': self.parse_function,
            'Method': self.parse_method,
            'Constructor': self.parse_method,
            'Destructor': self.parse_method,
            'Field': self.parse_field,
            'Enumeration': self.parse_enum,
            'ArrayType': self.parse_array,
            'PointerType': self.parse_pointer,
            'ReferenceType': self.parse_reference,
            'CvQualifiedType': self.parse_qualified_type,
            'FundamentalType': self.parse_builtin,
            'Typedef': self.parse_typedef,
            'File': self.parse_file,
            'Namespace': self.parse_namespace,
        }

        doc = parse_xml(path)
        for xml_node in doc.getroot().iterchildren():
            parser = parsers.get(xml_node.tag)
            if parser:
                parser(xml_node)

        self.process_locations()
        self.process_contexts()
        self.process_namespaces()
        self.process_signatures()
        self.process_classes()
        self.process_types()

        del self.nodes
        del self.namespaces
Exemplo n.º 20
0
Arquivo: config.py Projeto: cream/bob
    def __init__(self, path):
        
        self.pools = {}
        self.repositories = {}
        self.options = {}
        
        self.path = path
        self.tree = parse_xml(path)
        
        root = self.tree.getroot()

        for elm in root.xpath('//configuration/pool'):
            self.pools[elm.get('id')] = {
                'path': elm.get('path')
            }

        for elm in root.xpath('//configuration/repository'):
            self.repositories[elm.get('id')] = {
                'remote': elm.get('remote'),
                'pool': elm.get('pool'),
            }

        for elm in root.xpath('//configuration/option'):
            self.options[elm.get('key')] = elm.get('value')
Exemplo n.º 21
0
def createReturnValue(header, body, parse):
    if parse and body.strip() != '':
        body = parse_xml(StringIO(body))
    return header, body
Exemplo n.º 22
0
 def load(cls, file_path):
     return cls.from_xml(parse_xml(file_path))
Exemplo n.º 23
0
 def parse_response(self, response):
     """Parse the raw XML content."""
     # Requesting the text content of the response doesn't remove the BOM so
     # we request the binary content and decode it ourselves to remove it.
     f = io.StringIO(response.content.decode('utf-8-sig'))
     return parse_xml(f)
Exemplo n.º 24
0
 def parse(page, filename):
     estimator = Estimator()
     document = parse_xml(filename)  # type: ElementTree
     estimator.estimate(page, document)
Exemplo n.º 25
0
 def from_xml(self, content):
     if lxml is None:
         raise ImproperlyConfigured("Usage of the XML aspects requires lxml.")
     return self.from_etree(parse_xml(StringIO(content)).getroot())