def main():
    start = time()

    yplib.setUp()
    yplib.set_debugging(False)

    geosite = Geosite.objects.get(code='OCDE')

    countries = GeoCountry.objects.all()
    countries = countries.values_list('iso', flat=True)

    sql = """
    SELECT `value`
    FROM variables
    WHERE `name`='last_ocde_updated'
    """
    lastdate = sql2val(sql);
    if not lastdate:
        lastdate = '20000101000000'
    statuses = []
    types = []
    oc_count = 0
    gc_count = 0
    nc_count = 0

    k = 0
    uc = 0
    nc = 0

    for country in countries:
        url = 'http://opencaching.de/xml/ocxml11.php?modifiedsince=%s&cache=1&country=%s' % \
            (lastdate, country)
        response = urllib2.urlopen(url)
        xml = response.read()
        try:
            root = ET.XML(xml)
        except Exception as e:
            print 'PARSING ERROR', country, e
            continue

        # session id
        current_session = root[0]
        session_id = current_session.text

        # count
        records = root[1]
        caches_count = int(records.get("cache") or 0)
        if caches_count:
            page_count = int(round(caches_count * 1.0 / CACHES_PER_PAGE, 0)) + 1
            for p in range(page_count):
                page_url = 'http://www.opencaching.de/xml/ocxml11.php?sessionid=%s&file=%s' % \
                    (session_id, p + 1)
                page_response = urllib2.urlopen(page_url).read()
                from StringIO import StringIO
                zipdata = StringIO()
                zipdata.write(page_response)
                try:
                    zf = zipfile.ZipFile(zipdata)
                except:
                    continue
                for name in zf.namelist():
                    uncompressed = zf.read(name)
                    cache_root = ET.XML(uncompressed)
                    latitude = None
                    longitude = None
                    status = None
                    created_date_str = ''
                    for cache in  cache_root.getchildren():
                        k += 1
                        if cache.tag == 'cache':
                            the_geothing = TheGeothing()
                            the_location = TheLocation()
                            for param in cache:
                                if param.tag == 'id':
                                    the_geothing.pid = param.get('id')
                                if param.tag == 'userid':
                                    the_geothing.author = param.text
                                if param.tag == 'name':
                                    the_geothing.name = param.text
                                if param.tag == 'longitude':
                                    longitude = param.text
                                if param.tag == 'latitude':
                                    latitude = param.text
                                if param.tag == 'type':
                                    cache_type = param.get('short')
                                    the_geothing.type_code = OCDE_TYPES.get(cache_type)
                                    type_ = (param.get('id'), param.get('short'))
                                    if not type_ in types:
                                        types.append(type_)
                                if param.tag == 'status':
                                    status = int(param.get('id') or 0)
                                    status_ = (status, param.text)
                                    if not status_ in statuses:
                                        statuses.append(status_)
                                if param.tag == 'waypoints':
                                    the_geothing.code = param.get('oc')
                                    if the_geothing.code:
                                        oc_count += 1
                                    gccode = param.get('gccom')
                                    if gccode:
                                        gc_count += 1
                                    nccode = param.get('nccom')
                                    if nccode:
                                        nc_count += 1
                                if param.tag == 'datecreated':
                                    created_date_str = param.text
                                    parts = strptime(created_date_str, '%Y-%m-%d %H:%M:%S')
                                    dt = datetime(parts[0], parts[1], parts[2],
                                                  parts[3], parts[4], parts[5])
                                    the_geothing.created_date = dt

                                if latitude and longitude and status == 1:

                                    the_location.NS_degree = float(latitude)
                                    the_location.EW_degree = float(longitude)
                                    if the_geothing.code and the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
                                        geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite)
                                        if geothing is not None:
                                            uc += update_geothing(geothing, the_geothing, the_location) or 0
                                        else:
                                            create_new_geothing(the_geothing, the_location, geosite)
                                            nc += 1

    message = 'OK. updated %s, new %s' % (uc, nc)
    log('map_ocde_caches', message)
    print message
    sql = """
    UPDATE `variables`
    SET `value`='%s'
    WHERE `name`='last_ocde_updated'
    """ % ocde_timestamp()
    execute_query(sql)

    elapsed = time() - start
    print "Elapsed time -->", elapsed
def process(rectangle, geosite, params):
    #print 'process'
    #print rectangle, geosite, params
    print geosite
    bbox = [str(x) for x in rectangle]
    url = params.get('url_pattern') % (params['MY_CONSUMER_KEY'],
                                       '|'.join(bbox), params['FIELDS'])
    #print geosite, rectangle
    #print 'URL'
    #print url
    try:
        response = urllib2.urlopen(url)
        print 'GOT'
    except Exception as e:
        print 'exception', e
        return
    data = response.read()
    caches_data = json.loads(data)
    caches = caches_data.get('results')
    #print 'count', len(caches)
    more_data = caches_data.get('more')
    if more_data:
        BlockNeedBeDivided.objects.create(geosite=geosite,
                                          bb='|'.join(bbox),
                                          added=datetime.now())
    if not len(caches):
        return
    k = 0
    uc = 0
    nc = 0
    #print caches
    for code, cache in caches.iteritems():
        #print cache
        k += 1
        the_geothing = TheGeothing()
        the_location = TheLocation()
        locations = cache.get('location').split('|')
        lat_degree = float(locations[0])
        the_location.NS_degree = lat_degree
        lon_degree = float(locations[1])
        the_location.EW_degree = lon_degree
        the_geothing.code = cache.get('code')
        the_geothing.name = cache.get('name')

        if cache.get('status') != 'Available':
            continue

        the_geothing.type_code = OCPL_TYPES.get(cache.get('type'))
        cache_url = cache.get('url')
        if not cache_url:
            continue
        p = re.compile(params['code_re'])
        dgs = p.findall(cache_url)
        if not dgs:
            continue
        the_geothing.pid = int(dgs[0], 16)
        if cache.get('owner'):
            owner_name = cache.get('owner').get('username')
        the_geothing.author = owner_name
        date_created = cache.get('date_created')
        if date_created:
            date_created = date_created[:10]
            parts = date_created.split('-')
            if parts and len(parts) == 3:
                dt = datetime(int(parts[0]), int(parts[1]), int(parts[2]))
                the_geothing.created_date = dt

        if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
            geothing = get_object_or_none(Geothing,
                                          pid=the_geothing.pid,
                                          geosite=geosite)
            #print
            #print geothing
            if geothing is not None:
                #print 'update'
                uc += update_geothing(geothing, the_geothing,
                                      the_location) or 0
            else:
                #print 'new'
                create_new_geothing(the_geothing, the_location, geosite)
                nc += 1
    message = 'OK. updated %s, new %s' % (uc, nc)
    log(params.get('log_key'), message)
Exemple #3
0
def main():
    start = time()

    yplib.setUp()
    yplib.set_debugging(False)

    geosite = Geosite.objects.get(code='OCCZ')

    statuses = []
    types = []
    oc_count = 0

    k = 0
    uc = 0
    nc = 0

    url = 'http://www.opencaching.cz/search.php?searchto=searchbydistance&showresult=1&output=XML&sort=byname&latNS=N&lat_h=50&lat_min=5.123&lonEW=E&lon_h=14&lon_min=20.123&distance=1500&unit=km&count=500&startat=0'

    response = urllib2.urlopen(url).read()

    cache_root = ET.XML(response)

    docinfo = cache_root.getchildren()[0]
    result_count = 0
    for tag in docinfo.getchildren():
        if tag.tag == 'results':
            result_count = int(tag.text or 0)
    if result_count:
        for cache in cache_root.getchildren()[1:]:
            latitude = None
            longitude = None
            status = None
            created_date_str = ''
            k += 1
            if cache.tag == 'cache':
                the_geothing = TheGeothing()
                the_location = TheLocation()

                for param in cache:
                    if param.tag == 'id':
                        the_geothing.pid = param.text
                    if param.tag == 'owner':
                        the_geothing.author = param.text
                    if param.tag == 'name':
                        the_geothing.name = param.text
                    if param.tag == 'lon':
                        longitude = param.text
                    if param.tag == 'lat':
                        latitude = param.text
                    if param.tag == 'type':
                        cache_type = param.text
                        the_geothing.type_code = OCCZ_TYPES.get(cache_type)
                        if not cache_type in types:
                            types.append(cache_type)
                    if param.tag == 'status':
                        status = param.text
                        if not status in statuses:
                            statuses.append(status)
                    if param.tag == 'waypoint':
                        the_geothing.code = param.text
                        if the_geothing.code:
                            oc_count += 1
                    if param.tag == 'hidden':
                        created_date_str = param.text
                        parts = strptime(created_date_str, '%d.%m.%Y')
                        dt = datetime(parts[0], parts[1], parts[2], parts[3],
                                      parts[4], parts[5])
                        the_geothing.created_date = dt

                if latitude and longitude:

                    the_location.NS_degree = get_degree(latitude)
                    the_location.EW_degree = get_degree(longitude)
                    if the_geothing.code and the_geothing.pid and \
                       the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
                        geothing = get_object_or_none(Geothing,
                                                      pid=the_geothing.pid,
                                                      geosite=geosite)
                        if geothing is not None:
                            uc += update_geothing(geothing, the_geothing,
                                                  the_location) or 0
                        else:
                            create_new_geothing(the_geothing, the_location,
                                                geosite)
                            nc += 1

    message = 'OK. updated %s, new %s' % (uc, nc)
    log('map_occz_caches', message)
    print message

    print
    print types
    print statuses

    elapsed = time() - start
    print "Elapsed time -->", elapsed
Exemple #4
0
def main():
    LOAD_CACHES = True
    
    start = time() 
    
    yplib.setUp()
    yplib.set_debugging(False)
    
    url = 'http://www.geocaching.su/rss/geokrety/api.php?interval=1y&ctypes=1,2,3,7&changed=1'
    
    f = urllib2.urlopen(url)
    xml = f.read()
    xml = xml
    
    try:
        sxml = ET.XML(xml)
    except Exception as e:
        print type(e)
        print e
        return
    
    cnt_new = 0
    cnt_upd = 0
    caches = sxml.getchildren()     
    
    geosite = Geosite.objects.get(code='GC_SU')
    
    for cache in caches:
        if cache.tag == 'cache':
            the_geothing = TheGeothing()
            the_location = TheLocation()
            for tag_ in cache.getchildren():
                if tag_.tag == 'code':
                    the_geothing.code = tag_.text
                if tag_.tag == 'autor':
                    the_geothing.author = tag_.text
                if tag_.tag == 'name':
                    the_geothing.name = tag_.text
                if tag_.tag == 'position':
                    lat_degree = float(tag_.get('lat'))
                    the_location.NS_degree = lat_degree
                    lon_degree = float(tag_.get('lon'))
                    the_location.EW_degree = lon_degree                    
                if tag_.tag == 'cdate':
                    date_str = tag_.text
                    date_ = date_str.split('-')
                    if len(date_) == 3:
                        the_geothing.created_date = datetime(int(date_[0]), int(date_[1]), int(date_[2]))
            if  the_geothing.code:
                p = re.compile('(\D+)(\d+)') 
                dgs = p.findall(the_geothing.code)
                if dgs:
                    code_data = dgs[0]
                    the_geothing.pid = int(code_data[1])
                    the_geothing.type_code = code_data[0]

            if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
                geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite)
                if geothing is not None:
                    cnt_upd += update_geothing(geothing, the_geothing, the_location) or 0

                else:
                    create_new_geothing(the_geothing, the_location, geosite)
                    cnt_new += 1

    message = 'OK %s/%s'%(cnt_new, cnt_upd)
    log('map_gcsu_caches', message)
    print message
    elapsed = time() - start
    print "Elapsed time -->", elapsed
    def handle(self, *args, **options):
        url = 'http://www.geocaching.su/rss/geokrety/api.php?interval=1y&ctypes=1,2,3,7&changed=1'
        with requests.Session() as session:
            xml = session.get(url, data=LOGIN_DATA).content

        # print(xml)

        try:
            sxml = ET.XML(xml)
        except Exception as e:
            print(type(e))
            print(e)
            return

        cnt_new = 0
        cnt_upd = 0

        caches = sxml.getchildren()

        geosite = Geosite.objects.get(code='GC_SU')

        for cache in caches:
            if cache.tag == 'cache':
                the_geothing = TheGeothing()
                the_location = TheLocation()
                for tag_ in cache.getchildren():
                    if tag_.tag == 'code':
                        the_geothing.code = tag_.text
                    if tag_.tag == 'autor':
                        the_geothing.author = tag_.text
                    if tag_.tag == 'name':
                        the_geothing.name = tag_.text
                    if tag_.tag == 'position':
                        lat_degree = float(tag_.get('lat'))
                        the_location.NS_degree = lat_degree
                        lon_degree = float(tag_.get('lon'))
                        the_location.EW_degree = lon_degree
                    if tag_.tag == 'cdate':
                        date_str = tag_.text
                        date_ = date_str.split('-')
                        date_[2] = date_[2].split()[0]
                        if len(date_) == 3:
                            the_geothing.created_date = datetime(
                                int(date_[0]), int(date_[1]), int(date_[2]))
                if the_geothing.code:
                    p = re.compile('(\D+)(\d+)')
                    dgs = p.findall(the_geothing.code)
                    if dgs:
                        code_data = dgs[0]
                        the_geothing.pid = int(code_data[1])
                        the_geothing.type_code = code_data[0]

                if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
                    geothing = get_object_or_none(Geothing,
                                                  pid=the_geothing.pid,
                                                  geosite=geosite)
                if geothing is not None:
                    cnt_upd += update_geothing(geothing, the_geothing,
                                               the_location) or 0

                else:
                    create_new_geothing(the_geothing, the_location, geosite)
                    cnt_new += 1

        message = 'OK %s/%s' % (cnt_new, cnt_upd)
        log('map_gcsu_caches', message)
        print(message)

        return 'List of caches from geocaching.su has updated'