def main():
start = time()
yplib.setUp()
yplib.set_debugging(False)
geosite = Geosite.objects.get(code='OCDE')
countries = GeoCountry.objects.all()
countries = countries.values_list('iso', flat=True)
sql = """
SELECT `value`
FROM variables
WHERE `name`='last_ocde_updated'
"""
lastdate = sql2val(sql);
if not lastdate:
lastdate = '20000101000000'
statuses = []
types = []
oc_count = 0
gc_count = 0
nc_count = 0
k = 0
uc = 0
nc = 0
for country in countries:
url = 'http://opencaching.de/xml/ocxml11.php?modifiedsince=%s&cache=1&country=%s' % \
(lastdate, country)
response = urllib2.urlopen(url)
xml = response.read()
try:
root = ET.XML(xml)
except Exception as e:
print 'PARSING ERROR', country, e
continue
# session id
current_session = root[0]
session_id = current_session.text
# count
records = root[1]
caches_count = int(records.get("cache") or 0)
if caches_count:
page_count = int(round(caches_count * 1.0 / CACHES_PER_PAGE, 0)) + 1
for p in range(page_count):
page_url = 'http://www.opencaching.de/xml/ocxml11.php?sessionid=%s&file=%s' % \
(session_id, p + 1)
page_response = urllib2.urlopen(page_url).read()
from StringIO import StringIO
zipdata = StringIO()
zipdata.write(page_response)
try:
zf = zipfile.ZipFile(zipdata)
except:
continue
for name in zf.namelist():
uncompressed = zf.read(name)
cache_root = ET.XML(uncompressed)
latitude = None
longitude = None
status = None
created_date_str = ''
for cache in cache_root.getchildren():
k += 1
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for param in cache:
if param.tag == 'id':
the_geothing.pid = param.get('id')
if param.tag == 'userid':
the_geothing.author = param.text
if param.tag == 'name':
the_geothing.name = param.text
if param.tag == 'longitude':
longitude = param.text
if param.tag == 'latitude':
latitude = param.text
if param.tag == 'type':
cache_type = param.get('short')
the_geothing.type_code = OCDE_TYPES.get(cache_type)
type_ = (param.get('id'), param.get('short'))
if not type_ in types:
types.append(type_)
if param.tag == 'status':
status = int(param.get('id') or 0)
status_ = (status, param.text)
if not status_ in statuses:
statuses.append(status_)
if param.tag == 'waypoints':
the_geothing.code = param.get('oc')
if the_geothing.code:
oc_count += 1
gccode = param.get('gccom')
if gccode:
gc_count += 1
nccode = param.get('nccom')
if nccode:
nc_count += 1
if param.tag == 'datecreated':
created_date_str = param.text
parts = strptime(created_date_str, '%Y-%m-%d %H:%M:%S')
dt = datetime(parts[0], parts[1], parts[2],
parts[3], parts[4], parts[5])
the_geothing.created_date = dt
if latitude and longitude and status == 1:
the_location.NS_degree = float(latitude)
the_location.EW_degree = float(longitude)
if the_geothing.code and the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite)
if geothing is not None:
uc += update_geothing(geothing, the_geothing, the_location) or 0
else:
create_new_geothing(the_geothing, the_location, geosite)
nc += 1
message = 'OK. updated %s, new %s' % (uc, nc)
log('map_ocde_caches', message)
print message
sql = """
UPDATE `variables`
SET `value`='%s'
WHERE `name`='last_ocde_updated'
""" % ocde_timestamp()
execute_query(sql)
elapsed = time() - start
print "Elapsed time -->", elapsed
def process(rectangle, geosite, params):
#print 'process'
#print rectangle, geosite, params
print geosite
bbox = [str(x) for x in rectangle]
url = params.get('url_pattern') % (params['MY_CONSUMER_KEY'],
'|'.join(bbox), params['FIELDS'])
#print geosite, rectangle
#print 'URL'
#print url
try:
response = urllib2.urlopen(url)
print 'GOT'
except Exception as e:
print 'exception', e
return
data = response.read()
caches_data = json.loads(data)
caches = caches_data.get('results')
#print 'count', len(caches)
more_data = caches_data.get('more')
if more_data:
BlockNeedBeDivided.objects.create(geosite=geosite,
bb='|'.join(bbox),
added=datetime.now())
if not len(caches):
return
k = 0
uc = 0
nc = 0
#print caches
for code, cache in caches.iteritems():
#print cache
k += 1
the_geothing = TheGeothing()
the_location = TheLocation()
locations = cache.get('location').split('|')
lat_degree = float(locations[0])
the_location.NS_degree = lat_degree
lon_degree = float(locations[1])
the_location.EW_degree = lon_degree
the_geothing.code = cache.get('code')
the_geothing.name = cache.get('name')
if cache.get('status') != 'Available':
continue
the_geothing.type_code = OCPL_TYPES.get(cache.get('type'))
cache_url = cache.get('url')
if not cache_url:
continue
p = re.compile(params['code_re'])
dgs = p.findall(cache_url)
if not dgs:
continue
the_geothing.pid = int(dgs[0], 16)
if cache.get('owner'):
owner_name = cache.get('owner').get('username')
the_geothing.author = owner_name
date_created = cache.get('date_created')
if date_created:
date_created = date_created[:10]
parts = date_created.split('-')
if parts and len(parts) == 3:
dt = datetime(int(parts[0]), int(parts[1]), int(parts[2]))
the_geothing.created_date = dt
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
#print
#print geothing
if geothing is not None:
#print 'update'
uc += update_geothing(geothing, the_geothing,
the_location) or 0
else:
#print 'new'
create_new_geothing(the_geothing, the_location, geosite)
nc += 1
message = 'OK. updated %s, new %s' % (uc, nc)
log(params.get('log_key'), message)
def main():
start = time()
yplib.setUp()
yplib.set_debugging(False)
geosite = Geosite.objects.get(code='OCCZ')
statuses = []
types = []
oc_count = 0
k = 0
uc = 0
nc = 0
url = 'http://www.opencaching.cz/search.php?searchto=searchbydistance&showresult=1&output=XML&sort=byname&latNS=N&lat_h=50&lat_min=5.123&lonEW=E&lon_h=14&lon_min=20.123&distance=1500&unit=km&count=500&startat=0'
response = urllib2.urlopen(url).read()
cache_root = ET.XML(response)
docinfo = cache_root.getchildren()[0]
result_count = 0
for tag in docinfo.getchildren():
if tag.tag == 'results':
result_count = int(tag.text or 0)
if result_count:
for cache in cache_root.getchildren()[1:]:
latitude = None
longitude = None
status = None
created_date_str = ''
k += 1
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for param in cache:
if param.tag == 'id':
the_geothing.pid = param.text
if param.tag == 'owner':
the_geothing.author = param.text
if param.tag == 'name':
the_geothing.name = param.text
if param.tag == 'lon':
longitude = param.text
if param.tag == 'lat':
latitude = param.text
if param.tag == 'type':
cache_type = param.text
the_geothing.type_code = OCCZ_TYPES.get(cache_type)
if not cache_type in types:
types.append(cache_type)
if param.tag == 'status':
status = param.text
if not status in statuses:
statuses.append(status)
if param.tag == 'waypoint':
the_geothing.code = param.text
if the_geothing.code:
oc_count += 1
if param.tag == 'hidden':
created_date_str = param.text
parts = strptime(created_date_str, '%d.%m.%Y')
dt = datetime(parts[0], parts[1], parts[2], parts[3],
parts[4], parts[5])
the_geothing.created_date = dt
if latitude and longitude:
the_location.NS_degree = get_degree(latitude)
the_location.EW_degree = get_degree(longitude)
if the_geothing.code and the_geothing.pid and \
the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
if geothing is not None:
uc += update_geothing(geothing, the_geothing,
the_location) or 0
else:
create_new_geothing(the_geothing, the_location,
geosite)
nc += 1
message = 'OK. updated %s, new %s' % (uc, nc)
log('map_occz_caches', message)
print message
print
print types
print statuses
elapsed = time() - start
print "Elapsed time -->", elapsed
def main():
LOAD_CACHES = True
start = time()
yplib.setUp()
yplib.set_debugging(False)
url = 'http://www.geocaching.su/rss/geokrety/api.php?interval=1y&ctypes=1,2,3,7&changed=1'
f = urllib2.urlopen(url)
xml = f.read()
xml = xml
try:
sxml = ET.XML(xml)
except Exception as e:
print type(e)
print e
return
cnt_new = 0
cnt_upd = 0
caches = sxml.getchildren()
geosite = Geosite.objects.get(code='GC_SU')
for cache in caches:
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for tag_ in cache.getchildren():
if tag_.tag == 'code':
the_geothing.code = tag_.text
if tag_.tag == 'autor':
the_geothing.author = tag_.text
if tag_.tag == 'name':
the_geothing.name = tag_.text
if tag_.tag == 'position':
lat_degree = float(tag_.get('lat'))
the_location.NS_degree = lat_degree
lon_degree = float(tag_.get('lon'))
the_location.EW_degree = lon_degree
if tag_.tag == 'cdate':
date_str = tag_.text
date_ = date_str.split('-')
if len(date_) == 3:
the_geothing.created_date = datetime(int(date_[0]), int(date_[1]), int(date_[2]))
if the_geothing.code:
p = re.compile('(\D+)(\d+)')
dgs = p.findall(the_geothing.code)
if dgs:
code_data = dgs[0]
the_geothing.pid = int(code_data[1])
the_geothing.type_code = code_data[0]
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite)
if geothing is not None:
cnt_upd += update_geothing(geothing, the_geothing, the_location) or 0
else:
create_new_geothing(the_geothing, the_location, geosite)
cnt_new += 1
message = 'OK %s/%s'%(cnt_new, cnt_upd)
log('map_gcsu_caches', message)
print message
elapsed = time() - start
print "Elapsed time -->", elapsed
def handle(self, *args, **options):
url = 'http://www.geocaching.su/rss/geokrety/api.php?interval=1y&ctypes=1,2,3,7&changed=1'
with requests.Session() as session:
xml = session.get(url, data=LOGIN_DATA).content
# print(xml)
try:
sxml = ET.XML(xml)
except Exception as e:
print(type(e))
print(e)
return
cnt_new = 0
cnt_upd = 0
caches = sxml.getchildren()
geosite = Geosite.objects.get(code='GC_SU')
for cache in caches:
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for tag_ in cache.getchildren():
if tag_.tag == 'code':
the_geothing.code = tag_.text
if tag_.tag == 'autor':
the_geothing.author = tag_.text
if tag_.tag == 'name':
the_geothing.name = tag_.text
if tag_.tag == 'position':
lat_degree = float(tag_.get('lat'))
the_location.NS_degree = lat_degree
lon_degree = float(tag_.get('lon'))
the_location.EW_degree = lon_degree
if tag_.tag == 'cdate':
date_str = tag_.text
date_ = date_str.split('-')
date_[2] = date_[2].split()[0]
if len(date_) == 3:
the_geothing.created_date = datetime(
int(date_[0]), int(date_[1]), int(date_[2]))
if the_geothing.code:
p = re.compile('(\D+)(\d+)')
dgs = p.findall(the_geothing.code)
if dgs:
code_data = dgs[0]
the_geothing.pid = int(code_data[1])
the_geothing.type_code = code_data[0]
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
if geothing is not None:
cnt_upd += update_geothing(geothing, the_geothing,
the_location) or 0
else:
create_new_geothing(the_geothing, the_location, geosite)
cnt_new += 1
message = 'OK %s/%s' % (cnt_new, cnt_upd)
log('map_gcsu_caches', message)
print(message)
return 'List of caches from geocaching.su has updated'