def restore(self, id=None):
state = None
if id is None and self.last_profile_key in self.request.session:
id = self.request.session[self.last_profile_key]
elif id:
self.request.session[self.last_profile_key] = id
profile_qs = TableViewProfile.objects.filter(
tableview_name=self.table.id)
if self.table.global_profile:
profile_qs = profile_qs.filter(user__isnull=True)
else:
profile_qs = profile_qs.filter(user=self.request.user)
if (id is None and self.session_key
not in self.request.session) or id == 'default':
# load default state
self.profile = get_object_or_none(profile_qs, is_default=True)
elif id and id.isdigit():
self.profile = get_object_or_none(profile_qs,
is_default=False,
pk=id)
if self.session_key in self.request.session:
state = self.request.session[self.session_key]
if self.profile:
state = self.profile.state
if state:
self.apply_state(state)
def main():
is_updating = get_object_or_none(Variable, name='updating')
if is_updating is None:
return
is_updating.value = '1'
is_updating.save()
updated = get_object_or_none(Variable, name='updated')
if updated is None:
return
updated.value = 'successful'
updated.save()
def main():
updated = get_object_or_none(Variable, name='updated')
if not updated:
return
if updated.value != 'successful':
return
is_updating = get_object_or_none(Variable, name='updating')
if not is_updating:
return
is_updating.value = '0'
is_updating.save()
log('gcsu', 'succsess')
def main():
updated = get_object_or_none(Variable, name='map_updated')
if not updated:
return
updated.value = 'successful'
updated.save()
log('map', 'success')
def process(rectangle, geosite, uc, nc):
print 'process'
print rectangle
print geosite
print uc, nc
bbox = [str(x) for x in rectangle]
#print bbox
url = url_pattern % (MY_KEY, ','.join(bbox))
print url
response = urllib2.urlopen(url)
data = response.read()
#print data
caches = json.loads(data)
for cache in caches:
name = cache.get('name', '')
if name:
name = name.replace('\n', '').strip()
cache['name'] = name
the_location = TheLocation()
l = cache.get('location')
if l:
lat_degree = l.get('lat')
lon_degree = l.get('lon')
the_location.NS_degree = lat_degree
the_location.EW_degree = lon_degree
p = re.compile('OX([\dA-Z]+)')
code = cache.get('oxcode').upper()
dgs = p.findall(code)
if dgs:
code_data = dgs[0]
cache['pid'] = dig32(code_data)
cache['type_code'] = type_code_occom(cache.get('type'))
cache['created_date'] = get_occom_date(cache.get('hidden'))
a = cache.get('hidden_by')
if a:
cache['author'] = a.get('name')
region = cache.get('region')
if region:
cache['country_name'] = region.get('country')
cache['country_code'] = iso_by_country_name(cache['country_name'])
if not code or not cache.get('pid'):
print 'ERROR: NO CODE'
print cache
continue
geothing = get_object_or_none(Geothing,
pid=cache.get('pid'),
geosite=geosite)
if geothing is not None:
uc += update_geothing(geothing, cache, the_location) or 0
else:
create_new_geothing(cache, the_location, geosite)
nc += 1
return uc, nc
def main():
LOAD_CACHES = True
start = time()
file = '/tmp/export_oc-full.xml'
# sanity checking, only work on wpt files
if file.endswith('.xml') == 0: sys.exit(-1)
#sql = """ UPDATE geokret gk SET gk.state = NULL """
#r = execute_query(sql)
print "Reading file: "+file
fh = open(file,'r')
xml = fh.read()
fh.close()
sxml = ET.XML(xml)
all_krety = sxml.getchildren()
for kret in all_krety:
gkid = int(kret.get('id') or 0)
if not gkid:
continue
geokret = get_object_or_none(GeoKret, gkid=gkid)
if geokret:
for node in kret.getchildren():
if node.tag == 'state':
geokret.state = int(node.text) if node.text else None
geokret.save()
elapsed = time() - start
print "Elapsed time -->", elapsed
def geocacher_exists(pid):
return get_object_or_none(Geocacher, pid=pid)
def main():
start = time()
yplib.setUp()
yplib.set_debugging(False)
geosite = Geosite.objects.get(code='OCDE')
countries = GeoCountry.objects.all()
countries = countries.values_list('iso', flat=True)
sql = """
SELECT `value`
FROM variables
WHERE `name`='last_ocde_updated'
"""
lastdate = sql2val(sql);
if not lastdate:
lastdate = '20000101000000'
statuses = []
types = []
oc_count = 0
gc_count = 0
nc_count = 0
k = 0
uc = 0
nc = 0
for country in countries:
url = 'http://opencaching.de/xml/ocxml11.php?modifiedsince=%s&cache=1&country=%s' % \
(lastdate, country)
response = urllib2.urlopen(url)
xml = response.read()
try:
root = ET.XML(xml)
except Exception as e:
print 'PARSING ERROR', country, e
continue
# session id
current_session = root[0]
session_id = current_session.text
# count
records = root[1]
caches_count = int(records.get("cache") or 0)
if caches_count:
page_count = int(round(caches_count * 1.0 / CACHES_PER_PAGE, 0)) + 1
for p in range(page_count):
page_url = 'http://www.opencaching.de/xml/ocxml11.php?sessionid=%s&file=%s' % \
(session_id, p + 1)
page_response = urllib2.urlopen(page_url).read()
from StringIO import StringIO
zipdata = StringIO()
zipdata.write(page_response)
try:
zf = zipfile.ZipFile(zipdata)
except:
continue
for name in zf.namelist():
uncompressed = zf.read(name)
cache_root = ET.XML(uncompressed)
latitude = None
longitude = None
status = None
created_date_str = ''
for cache in cache_root.getchildren():
k += 1
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for param in cache:
if param.tag == 'id':
the_geothing.pid = param.get('id')
if param.tag == 'userid':
the_geothing.author = param.text
if param.tag == 'name':
the_geothing.name = param.text
if param.tag == 'longitude':
longitude = param.text
if param.tag == 'latitude':
latitude = param.text
if param.tag == 'type':
cache_type = param.get('short')
the_geothing.type_code = OCDE_TYPES.get(cache_type)
type_ = (param.get('id'), param.get('short'))
if not type_ in types:
types.append(type_)
if param.tag == 'status':
status = int(param.get('id') or 0)
status_ = (status, param.text)
if not status_ in statuses:
statuses.append(status_)
if param.tag == 'waypoints':
the_geothing.code = param.get('oc')
if the_geothing.code:
oc_count += 1
gccode = param.get('gccom')
if gccode:
gc_count += 1
nccode = param.get('nccom')
if nccode:
nc_count += 1
if param.tag == 'datecreated':
created_date_str = param.text
parts = strptime(created_date_str, '%Y-%m-%d %H:%M:%S')
dt = datetime(parts[0], parts[1], parts[2],
parts[3], parts[4], parts[5])
the_geothing.created_date = dt
if latitude and longitude and status == 1:
the_location.NS_degree = float(latitude)
the_location.EW_degree = float(longitude)
if the_geothing.code and the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite)
if geothing is not None:
uc += update_geothing(geothing, the_geothing, the_location) or 0
else:
create_new_geothing(the_geothing, the_location, geosite)
nc += 1
message = 'OK. updated %s, new %s' % (uc, nc)
log('map_ocde_caches', message)
print message
sql = """
UPDATE `variables`
SET `value`='%s'
WHERE `name`='last_ocde_updated'
""" % ocde_timestamp()
execute_query(sql)
elapsed = time() - start
print "Elapsed time -->", elapsed
def check_cach(cach_pid):
def get_coordinates(cell):
coordinates = cell.text
parts = t2.findall(coordinates)[0]
if len(parts) == 4:
ns_degree, ns_minute, ew_degree, ew_minute = parts
parts = t3.findall(coordinates)
NS = parts[0]
parts = t4.findall(coordinates)
EW = parts[0]
return ns_degree, ns_minute, ew_degree, ew_minute, NS, EW
def get_type(cell):
return cell.text
def get_class(cell):
class_ = None
if cell:
parts = cell.contents
items = []
for p in parts:
txt = p.string
if txt and nottag(txt):
items.append(txt)
class_ = ';'.join(items)
return class_
def get_mestnost(cell):
oblast = country = None
parts = cell.contents
if len(parts):
country = parts[0]
if len(parts) > 2:
oblast = parts[2]
return country, oblast
def get_dostupnost(cell):
parts = cell.contents
dostupnost = parts[0].split(':')[1].strip()
mestnost = parts[2].split(':')[1].strip()
return dostupnost, mestnost
def get_town(cell):
return cell.text
def get_grade(cell):
grade = None
if cell.img:
grade = cell.img.get('title')
return grade
def get_attributes(element):
attr = None
items = []
imgs = element.findAll('img')
for img in imgs:
if 'images/attrib/' in img.get('src'):
items.append(img.get('title'))
attr = ';'.join(items)
return attr
url = 'http://www.geocaching.su/?pn=101&cid=%d'%int(cach_pid)
try:
yplib.get(url)
except:
print 'exception'
return False
soup=yplib.soup()
h = soup.find('h1', {'class':'hdr'})
t = re.compile('([^\[]+)\[.+\]')
t1 = re.compile('[^\[]+\[([^\[\]]+\/[^\[\]]+)\]')
t2 = re.compile('[N,S]\s(\d+)\&\#176\;\s([\d\.]+).+[E,W]\s(\d+)\&\#176\;\s([\d\.]+)')
t3 = re.compile('([N,S]\s\d+\&\#176\;\s[\d\.]+.)')
t4 = re.compile('([E,W]\s\d+\&\#176\;\s[\d\.]+.)')
t5 = re.compile('WinPopup\(\'profile\.php\?pid\=(\d+)')
name = None
items = t.findall(h.text)
if items:
name = items[0]
full_code = None
items = t1.findall(h.text)
if items:
full_code = items[0]
type_code, pid = full_code.split('/')
tbl = soup.find('table', attrs={'cellpadding':3, 'width':160})
rows = tbl.findAll('tr')
ns_degree = ns_minute = ew_degree = ew_minute = NS = EW = None
country = oblast = town = None
dostupnost = mestnost = None
cach_type = cach_class = None
grade = attr = None
act = None
for row in rows:
tds = row.findAll('td')
ths = row.findAll('th')
td = None
if tds:
td = tds[0]
cell = None
if act:
if ths:
cell = ths[0]
elif tds:
cell = tds[1]
if act == 'coord':
ns_degree, ns_minute, ew_degree, ew_minute, NS, EW = get_coordinates(cell)
if act == 'mestnost':
country, oblast = get_mestnost(cell)
if act == 'dostupnost':
dostupnost, mestnost = get_dostupnost(cell)
if act == 'town':
town = get_town(cell)
if act == 'grade':
grade = get_grade(cell)
act = None
if td and td.text.startswith(u'Тип:'):
cach_type = get_type(tds[1])
act = None
if td and td.text.startswith(u'Класс:'):
cach_class = get_class(tds[1])
act = None
if td and td.text.startswith(u'КООРДИНАТЫ'):
act = 'coord'
if td and td.text.startswith(u'МЕСТНОСТЬ'):
act = 'mestnost'
if td and td.text.startswith(u'БЛИЖАЙШИЙ'):
act = 'town'
if td and td.text.startswith(u'ОЦЕНКИ'):
act = 'dostupnost'
if td and td.text.startswith(u'РЕЙТИНГ'):
act = 'grade'
if td and td.text.startswith(u'АТРИБУТЫ'):
attr = get_attributes(tbl)
act = None
created_by = created_date = changed_date = coauthors = None
div = soup.findAll('div', attrs={'style':'padding: 5px; font-family: Verdana; font-weight: bold;'})[0]
a = div.a
if a:
onclick = a.get('onclick')
if onclick:
pid = t5.findall(onclick)
if pid:
created_by = int(pid[0])
parts = div.contents
for p in parts:
txt = p.string
#if txt:
#print txt.encode('utf8'), type(txt)
if txt and nottag(txt):
txt = txt.string.strip()
if txt.startswith(u'Создан:'):
items = txt.split()
if len(items) == 2:
created_date = items[1]
if created_date:
day, month, year = [int(s) for s in created_date.split('.')]
created_date = date(year, month, day)
if txt.startswith(u'(отредактирован'):
txt = txt[1:-1]
items = txt.split()
if len(items) == 2:
changed_date = items[1]
if changed_date:
day, month, year = [int(s) for s in changed_date.split('.')]
changed_date = date(year, month, day)
if txt.startswith(u'Компаньоны:'):
coauthors = 'yes'
the_cach = TheCach()
the_cach.pid = cach_pid
the_cach.code = '%s%s' % (type_code, the_cach.pid)
the_cach.type_code = type_code
#print
#print cach.pid
#print '|%s|'%the_cach.code.encode('utf8')
the_cach.name = text_or_none(name)
the_cach.cach_type = text_or_none(cach_type)
the_cach.cach_class = text_or_none(cach_class)
the_cach.loc_NS = char_or_none(NS)
the_cach.loc_EW = char_or_none(EW)
the_cach.loc_NS_degree = int_or_none(ns_degree)
the_cach.loc_EW_degree = int_or_none(ew_degree)
the_cach.loc_NS_minute = float_or_none(ns_minute)
the_cach.loc_EW_minute = float_or_none(ew_minute)
the_cach.country = text_or_none(country)
the_cach.oblast = text_or_none(oblast)
the_cach.town = text_or_none(town)
the_cach.dostupnost = int_or_none(dostupnost)
the_cach.mestnost = int_or_none(mestnost)
the_cach.grade = float_or_none(grade)
the_cach.cach_attr = text_or_none(attr)
the_cach.created_by = created_by
the_cach.created_date = created_date
the_cach.changed_date = changed_date
the_cach.coauthors = coauthors
print the_cach.name.encode('utf8')
geocache = get_object_or_none(Cach, pid=cach_pid)
if geocache is not None:
update_geocache(geocache, the_cach)
else:
cach = Cach.objects.create(pid=cach_pid)
cach.__dict__.update(the_cach.__dict__)
print 'save', cach.pid
cach.save()
#nc += 1
#if True:
#cach.__dict__.update(the_cach.__dict__)
#print 'save', cach.pid
#cach.save()
return True
def process(rectangle, geosite, params):
#print 'process'
#print rectangle, geosite, params
print geosite
bbox = [str(x) for x in rectangle]
url = params.get('url_pattern') % (params['MY_CONSUMER_KEY'],
'|'.join(bbox), params['FIELDS'])
#print geosite, rectangle
#print 'URL'
#print url
try:
response = urllib2.urlopen(url)
print 'GOT'
except Exception as e:
print 'exception', e
return
data = response.read()
caches_data = json.loads(data)
caches = caches_data.get('results')
#print 'count', len(caches)
more_data = caches_data.get('more')
if more_data:
BlockNeedBeDivided.objects.create(geosite=geosite,
bb='|'.join(bbox),
added=datetime.now())
if not len(caches):
return
k = 0
uc = 0
nc = 0
#print caches
for code, cache in caches.iteritems():
#print cache
k += 1
the_geothing = TheGeothing()
the_location = TheLocation()
locations = cache.get('location').split('|')
lat_degree = float(locations[0])
the_location.NS_degree = lat_degree
lon_degree = float(locations[1])
the_location.EW_degree = lon_degree
the_geothing.code = cache.get('code')
the_geothing.name = cache.get('name')
if cache.get('status') != 'Available':
continue
the_geothing.type_code = OCPL_TYPES.get(cache.get('type'))
cache_url = cache.get('url')
if not cache_url:
continue
p = re.compile(params['code_re'])
dgs = p.findall(cache_url)
if not dgs:
continue
the_geothing.pid = int(dgs[0], 16)
if cache.get('owner'):
owner_name = cache.get('owner').get('username')
the_geothing.author = owner_name
date_created = cache.get('date_created')
if date_created:
date_created = date_created[:10]
parts = date_created.split('-')
if parts and len(parts) == 3:
dt = datetime(int(parts[0]), int(parts[1]), int(parts[2]))
the_geothing.created_date = dt
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
#print
#print geothing
if geothing is not None:
#print 'update'
uc += update_geothing(geothing, the_geothing,
the_location) or 0
else:
#print 'new'
create_new_geothing(the_geothing, the_location, geosite)
nc += 1
message = 'OK. updated %s, new %s' % (uc, nc)
log(params.get('log_key'), message)
def main():
LOAD_CACHES = True
start = time()
yplib.setUp()
yplib.set_debugging(False)
url = 'http://www.geocaching.su/rss/geokrety/api.php?interval=1y&ctypes=1,2,3,7&changed=1'
f = urllib2.urlopen(url)
xml = f.read()
xml = xml
try:
sxml = ET.XML(xml)
except Exception as e:
print type(e)
print e
return
cnt_new = 0
cnt_upd = 0
caches = sxml.getchildren()
geosite = Geosite.objects.get(code='GC_SU')
for cache in caches:
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for tag_ in cache.getchildren():
if tag_.tag == 'code':
the_geothing.code = tag_.text
if tag_.tag == 'autor':
the_geothing.author = tag_.text
if tag_.tag == 'name':
the_geothing.name = tag_.text
if tag_.tag == 'position':
lat_degree = float(tag_.get('lat'))
the_location.NS_degree = lat_degree
lon_degree = float(tag_.get('lon'))
the_location.EW_degree = lon_degree
if tag_.tag == 'cdate':
date_str = tag_.text
date_ = date_str.split('-')
if len(date_) == 3:
the_geothing.created_date = datetime(int(date_[0]), int(date_[1]), int(date_[2]))
if the_geothing.code:
p = re.compile('(\D+)(\d+)')
dgs = p.findall(the_geothing.code)
if dgs:
code_data = dgs[0]
the_geothing.pid = int(code_data[1])
the_geothing.type_code = code_data[0]
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite)
if geothing is not None:
cnt_upd += update_geothing(geothing, the_geothing, the_location) or 0
else:
create_new_geothing(the_geothing, the_location, geosite)
cnt_new += 1
message = 'OK %s/%s'%(cnt_new, cnt_upd)
log('map_gcsu_caches', message)
print message
elapsed = time() - start
print "Elapsed time -->", elapsed
def main():
LOAD_CACHES = True
start = time()
file = '../shukach_all.wpt'
# sanity checking, only work on wpt files
if file.endswith('.wpt') == 0: sys.exit(-1)
print "Reading file: " + file
#fh = codecs.open(file,'r',"utf-8")
fh = codecs.open(file, 'r', "cp1251")
wpt = fh.readlines()
fh.close()
WPT_CODE = 1
WPT_LAT = 2
WPT_LON = 3
WPT_TITLE = 10
WPT_DATE = 4
geosite = Geosite.objects.get(code='SHUKACH')
print geosite
print len(wpt), 'points'
k = 0
for point in wpt:
#print
#print point
fields = point.split(',')
if fields[0].isdigit():
the_geothing = TheGeothing()
the_location = TheLocation()
#geothing = Geothing(geosite=geosite)
#for field in fields:
#print field
#l = Location()
lat_degree = float(fields[WPT_LAT])
the_location.NS_degree = round(lat_degree, 4)
#the_location.NS_minute = round((abs(lat_degree) - abs(the_location.NS_degree)) * 60, 2)
lon_degree = float(fields[WPT_LON])
the_location.EW_degree = round(lon_degree, 4)
#the_location.EW_minute = round((abs(lon_degree) - abs(the_location.EW_degree)) * 60, 2)
#l.save()
#geothing.location = l
p = re.compile('(\D+)(\d+)')
dgs = p.findall(fields[WPT_CODE])
if dgs:
code_data = dgs[0]
the_geothing.code = fields[WPT_CODE]
the_geothing.pid = int(code_data[1])
the_geothing.type_code = code_data[0]
p = re.compile(u'(.+)от(.+)')
dgs = p.findall(fields[WPT_TITLE])
if dgs:
title = dgs[0]
the_geothing.name = title[0]
the_geothing.author = title[1]
d = float(fields[WPT_DATE])
#print Dephi_date_to_python_date(d)
the_geothing.created_date = Dephi_date_to_python_date(d)
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
#import pdb; pdb.set_trace()
if geothing is not None:
update_geothing(geothing, the_geothing, the_location)
else:
create_new_geothing(the_geothing, the_location, geosite)
elapsed = time() - start
print "Elapsed time -->", elapsed
def cache_exists(cach_pid):
return get_object_or_none(Cach, pid=cach_pid)
def main():
start = time()
yplib.setUp()
yplib.set_debugging(False)
geosite = Geosite.objects.get(code='OCCZ')
statuses = []
types = []
oc_count = 0
k = 0
uc = 0
nc = 0
url = 'http://www.opencaching.cz/search.php?searchto=searchbydistance&showresult=1&output=XML&sort=byname&latNS=N&lat_h=50&lat_min=5.123&lonEW=E&lon_h=14&lon_min=20.123&distance=1500&unit=km&count=500&startat=0'
response = urllib2.urlopen(url).read()
cache_root = ET.XML(response)
docinfo = cache_root.getchildren()[0]
result_count = 0
for tag in docinfo.getchildren():
if tag.tag == 'results':
result_count = int(tag.text or 0)
if result_count:
for cache in cache_root.getchildren()[1:]:
latitude = None
longitude = None
status = None
created_date_str = ''
k += 1
if cache.tag == 'cache':
the_geothing = TheGeothing()
the_location = TheLocation()
for param in cache:
if param.tag == 'id':
the_geothing.pid = param.text
if param.tag == 'owner':
the_geothing.author = param.text
if param.tag == 'name':
the_geothing.name = param.text
if param.tag == 'lon':
longitude = param.text
if param.tag == 'lat':
latitude = param.text
if param.tag == 'type':
cache_type = param.text
the_geothing.type_code = OCCZ_TYPES.get(cache_type)
if not cache_type in types:
types.append(cache_type)
if param.tag == 'status':
status = param.text
if not status in statuses:
statuses.append(status)
if param.tag == 'waypoint':
the_geothing.code = param.text
if the_geothing.code:
oc_count += 1
if param.tag == 'hidden':
created_date_str = param.text
parts = strptime(created_date_str, '%d.%m.%Y')
dt = datetime(parts[0], parts[1], parts[2], parts[3],
parts[4], parts[5])
the_geothing.created_date = dt
if latitude and longitude:
the_location.NS_degree = get_degree(latitude)
the_location.EW_degree = get_degree(longitude)
if the_geothing.code and the_geothing.pid and \
the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
if geothing is not None:
uc += update_geothing(geothing, the_geothing,
the_location) or 0
else:
create_new_geothing(the_geothing, the_location,
geosite)
nc += 1
message = 'OK. updated %s, new %s' % (uc, nc)
log('map_occz_caches', message)
print message
print
print types
print statuses
elapsed = time() - start
print "Elapsed time -->", elapsed
def iso_by_country_name(country_name):
r = None
country = get_object_or_none(GeoCountry, name=country_name)
if country:
r = country.iso
return r
def main():
LOAD_CACHES = True
start = time()
yplib.setUp()
yplib.set_debugging(False)
r = yplib.post2('http://www.geocaching.su/?pn=108',
(('Log_In', 'Log_In'), ('email', '*****@*****.**'),
('passwd', 'zaebalixakeryvas'), ('longterm', '1')))
soup = yplib.soup()
a = soup.find('a', attrs={'class': "profilelink"}, text='galdor')
if not a:
print 'Authorization failed'
return False
r = yplib.get('http://www.geocaching.su/site/popup/selex.php')
soup = yplib.soup()
chbox_list = soup.findAll('input', type='checkbox')
regions = []
for chbox in chbox_list:
v = chbox.get('value')
if v and chbox.get('name', '') == 'point[]':
regions.append(v)
data = [
('translit', '0'),
('fmt', 'wpt'),
('code_to_name', '1'),
('finded', '2'),
]
for r in regions:
data.append(('point[]', r))
r = yplib.post2('http://www.geocaching.su/site/popup/export.php', data)
soup = yplib.soup()
wpt = soup.text.split('\n')
WPT_CODE = 1
WPT_LAT = 2
WPT_LON = 3
WPT_TITLE = 10
WPT_DATE = 4
geosite = Geosite.objects.get(code='GC_SU')
print len(wpt), 'points'
k = 0
for point in wpt:
k += 1
fields = point.split(',')
if fields[0].isdigit():
the_geothing = TheGeothing()
the_location = TheLocation()
lat_degree = float(fields[WPT_LAT])
the_location.NS_degree = lat_degree
#the_location.NS_minute = (abs(lat_degree) - abs(the_location.NS_degree)) * 60
lon_degree = float(fields[WPT_LON])
the_location.EW_degree = lon_degree
#the_location.EW_minute = (abs(lon_degree) - abs(the_location.EW_degree)) * 60
p = re.compile('(\D+)(\d+)')
dgs = p.findall(fields[WPT_CODE])
if dgs:
code_data = dgs[0]
the_geothing.code = fields[WPT_CODE]
the_geothing.pid = int(code_data[1])
the_geothing.type_code = code_data[0]
p = re.compile(u'(.+)от(.+)')
dgs = p.findall(fields[WPT_TITLE])
if dgs:
title = dgs[0]
the_geothing.name = title[0]
the_geothing.author = title[1]
d = float(fields[WPT_DATE])
the_geothing.created_date = Dephi_date_to_python_date(d)
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
if geothing is not None:
update_geothing(geothing, the_geothing, the_location)
else:
create_new_geothing(the_geothing, the_location, geosite)
log('map_gcsu_caches', 'OK')
elapsed = time() - start
print "Elapsed time -->", elapsed
def main():
LOAD_CACHES = True
start = time()
yplib.setUp()
yplib.set_debugging(False)
# log in
r = yplib.post2('http://opencaching.pl/login.php',
(('LogMeIn', 'zaloguj'), ('email', 'kurianin'),
('password', 'gjhjkjy'), ('action', 'login'),
('target', 'index.php')))
soup = yplib.soup()
a = soup.find('a', text='kurianin')
if not a:
print 'Authorization failed'
return False
print 'OK'
## search page
#r = yplib.get('http://opencaching.pl/search.php')
#soup = yplib.soup()
# get wpt file
r = yplib.get(
'http://opencaching.pl/search.php?searchto=searchbyname&showresult=1&expert=0&output=HTML&sort=bycreated&f_inactive=1&f_ignored=1&f_userfound=1&f_userowner=1&f_watched=0&f_geokret=0&country=PL®ion=&cachetype=1111111110&cache_attribs=&cache_attribs_not=&cachesize_1=1&cachesize_2=1&cachesize_3=1&cachesize_4=1&cachesize_5=1&cachesize_6=1&cachesize_7=1&cachevote_1=-3&cachevote_2=3.000&cachenovote=1&cachedifficulty_1=1&cachedifficulty_2=5&cacheterrain_1=1&cacheterrain_2=5&cacherating=0&cachename=%25&cachename='
)
soup = yplib.soup(cp='utf8')
link_to_wpt = ''
#the_div = soup.find('div', {'class':"content2-pagetitle"})
wpt_link = re.compile('ocpl\d+\.wpt\?.+count\=max.*')
a_list = soup.findAll('a', {'class': "links", 'title': "Oziexplorer .wpt"})
if a_list:
for a in a_list:
if a.get('href') and wpt_link.match(a.get('href')):
link_to_wpt = a.get('href')
break
print link_to_wpt
if link_to_wpt:
r = yplib.get(link_to_wpt)
soup = yplib.soup(cp='utf8')
wpt = soup.text.split('\n')
else:
print 'oblom'
return
WPT_CODE = 10
WPT_LAT = 2
WPT_LON = 3
WPT_TITLE = 1
WPT_DATE = 4
MY_CONSUMER_KEY = 'fky3LF9xvWz9y7Gs3tZ6'
FIELDS = 'code|name|location|type|status|url|owner|date_created'
geocach_api_request = 'http://opencaching.pl/okapi/services/caches/geocache?cache_code=%s&consumer_key=%s&fields=%s'
geosite = Geosite.objects.get(code='OCPL')
print geosite
print len(wpt), 'points'
k = 0
uc = 0
nc = 0
for point in wpt:
k += 1
fields = point.split(',')
if fields[0] == '-1':
the_geothing = TheGeothing()
the_geothing.pid = 1
the_location = TheLocation()
lat_degree = float(fields[WPT_LAT])
the_location.NS_degree = lat_degree
#the_location.NS_minute = (abs(lat_degree) - abs(the_location.NS_degree)) * 60
lon_degree = float(fields[WPT_LON])
the_location.EW_degree = lon_degree
#the_location.EW_minute = (abs(lon_degree) - abs(the_location.EW_degree)) * 60
code_str = fields[WPT_CODE]
parts = code_str.split('/')
if len(parts) == 4:
cache_code = parts[0]
the_geothing.code = cache_code
the_geothing.name = fields[WPT_TITLE]
geothing_items = Geothing.objects.filter(
code=the_geothing.code, geosite=geosite)
if geothing_items.count() > 0:
geothing = geothing_items[0]
if the_geothing.name == geothing.name and not location_was_changed(
geothing.location, the_location):
continue
url = geocach_api_request % (cache_code, MY_CONSUMER_KEY,
FIELDS)
try:
response = urllib2.urlopen(url)
json_str = response.read()
cache_data = json.loads(json_str)
if cache_data.get('status') != 'Available':
continue
#print cache_data.get('type')
the_geothing.type_code = OCPL_TYPES.get(
cache_data.get('type'))
#print the_geothing.type_code
cache_url = cache_data.get('url')
if not cache_url:
continue
p = re.compile(u'OP([\dA-F]+)$')
dgs = p.findall(cache_url)
the_geothing.pid = int(dgs[0], 16)
owner_name = ''
if cache_data.get('owner'):
owner_name = cache_data.get('owner').get('username')
the_geothing.author = owner_name
date_created = cache_data.get('date_created')
if date_created:
date_created = date_created[:10]
parts = date_created.split('-')
if parts and len(parts) == 3:
dt = datetime(int(parts[0]), int(parts[1]),
int(parts[2]))
the_geothing.created_date = dt
except:
print
print 'exception.'
print url
print cache_data
#break
continue
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
if geothing is not None:
update_geothing(geothing, the_geothing, the_location)
uc += 1
else:
create_new_geothing(the_geothing, the_location, geosite)
nc += 1
#break
sql = """
select COUNT(*)
FROM
(
select g.code as code, count(id) as cnt
from geothing g
group by g.code
having cnt > 1
) as tbl
"""
dc = sql2val(sql)
message = 'OK. updated %s, new %s, doubles %s' % (uc, nc, dc)
log('map_ocpl_caches', message)
elapsed = time() - start
print "Elapsed time -->", elapsed
def main():
LOAD_CACHES = True
start = time()
file = '/tmp/ocpl.wpt'
# sanity checking, only work on wpt files
if file.endswith('.wpt') == 0: sys.exit(-1)
print "Reading file: " + file
fh = codecs.open(file, 'r', "utf-8")
wpt = fh.readlines()
fh.close()
WPT_CODE = 10
WPT_LAT = 2
WPT_LON = 3
WPT_TITLE = 1
WPT_DATE = 4
MY_CONSUMER_KEY = 'fky3LF9xvWz9y7Gs3tZ6'
FIELDS = 'code|name|location|type|status|url|owner|date_created'
geocach_api_request = 'http://opencaching.pl/okapi/services/caches/geocache?cache_code=%s&consumer_key=%s&fields=%s'
geosite = Geosite.objects.get(code='OCPL')
print geosite
print len(wpt), 'points'
k = 0
for point in wpt:
k += 1
fields = point.split(',')
if fields[0] == '-1':
the_geothing = TheGeothing()
the_location = TheLocation()
lat_degree = float(fields[WPT_LAT])
the_location.NS_degree = lat_degree
#the_location.NS_minute = (abs(lat_degree) - abs(the_location.NS_degree)) * 60
lon_degree = float(fields[WPT_LON])
the_location.EW_degree = lon_degree
#the_location.EW_minute = (abs(lon_degree) - abs(the_location.EW_degree)) * 60
code_str = fields[WPT_CODE]
parts = code_str.split('/')
if len(parts) == 4:
cache_code = parts[0]
the_geothing.code = cache_code
the_geothing.name = fields[WPT_TITLE]
geothing = get_object_or_none(Geothing,
code=the_geothing.code,
geosite=geosite)
if geothing:
if the_geothing.name == geothing.name and not location_was_changed(
geothing.location, the_location):
continue
print the_geothing.code, 'changed'
url = geocach_api_request % (cache_code, MY_CONSUMER_KEY,
FIELDS)
try:
response = urllib2.urlopen(url)
json_str = response.read()
cache_data = json.loads(json_str)
if cache_data.get('status') != 'Available':
continue
the_geothing.type_code = OCPL_TYPES.get(
cache_data.get('type'))
cache_url = cache_data.get('url')
if not cache_url:
continue
p = re.compile(u'(\d+)$')
dgs = p.findall(cache_url)
the_geothing.pid = int(dgs[0])
owner_name = ''
if cache_data.get('owner'):
owner_name = cache_data.get('owner').get('username')
the_geothing.author = owner_name
date_created = cache_data.get('date_created')
if date_created:
date_created = date_created[:10]
parts = date_created.split('-')
if parts and len(parts) == 3:
dt = datetime(int(parts[0]), int(parts[1]),
int(parts[2]))
the_geothing.created_date = dt
except:
print 'exception'
continue
if the_geothing.type_code in GEOCACHING_ONMAP_TYPES:
geothing = get_object_or_none(Geothing,
pid=the_geothing.pid,
geosite=geosite)
if geothing is not None:
update_geothing(geothing, the_geothing, the_location)
else:
create_new_geothing(the_geothing, the_location, geosite)
#break
elapsed = time() - start
print "Elapsed time -->", elapsed
