def restore(self, id=None): state = None if id is None and self.last_profile_key in self.request.session: id = self.request.session[self.last_profile_key] elif id: self.request.session[self.last_profile_key] = id profile_qs = TableViewProfile.objects.filter( tableview_name=self.table.id) if self.table.global_profile: profile_qs = profile_qs.filter(user__isnull=True) else: profile_qs = profile_qs.filter(user=self.request.user) if (id is None and self.session_key not in self.request.session) or id == 'default': # load default state self.profile = get_object_or_none(profile_qs, is_default=True) elif id and id.isdigit(): self.profile = get_object_or_none(profile_qs, is_default=False, pk=id) if self.session_key in self.request.session: state = self.request.session[self.session_key] if self.profile: state = self.profile.state if state: self.apply_state(state)
def main(): is_updating = get_object_or_none(Variable, name='updating') if is_updating is None: return is_updating.value = '1' is_updating.save() updated = get_object_or_none(Variable, name='updated') if updated is None: return updated.value = 'successful' updated.save()
def main(): updated = get_object_or_none(Variable, name='updated') if not updated: return if updated.value != 'successful': return is_updating = get_object_or_none(Variable, name='updating') if not is_updating: return is_updating.value = '0' is_updating.save() log('gcsu', 'succsess')
def main(): updated = get_object_or_none(Variable, name='map_updated') if not updated: return updated.value = 'successful' updated.save() log('map', 'success')
def process(rectangle, geosite, uc, nc): print 'process' print rectangle print geosite print uc, nc bbox = [str(x) for x in rectangle] #print bbox url = url_pattern % (MY_KEY, ','.join(bbox)) print url response = urllib2.urlopen(url) data = response.read() #print data caches = json.loads(data) for cache in caches: name = cache.get('name', '') if name: name = name.replace('\n', '').strip() cache['name'] = name the_location = TheLocation() l = cache.get('location') if l: lat_degree = l.get('lat') lon_degree = l.get('lon') the_location.NS_degree = lat_degree the_location.EW_degree = lon_degree p = re.compile('OX([\dA-Z]+)') code = cache.get('oxcode').upper() dgs = p.findall(code) if dgs: code_data = dgs[0] cache['pid'] = dig32(code_data) cache['type_code'] = type_code_occom(cache.get('type')) cache['created_date'] = get_occom_date(cache.get('hidden')) a = cache.get('hidden_by') if a: cache['author'] = a.get('name') region = cache.get('region') if region: cache['country_name'] = region.get('country') cache['country_code'] = iso_by_country_name(cache['country_name']) if not code or not cache.get('pid'): print 'ERROR: NO CODE' print cache continue geothing = get_object_or_none(Geothing, pid=cache.get('pid'), geosite=geosite) if geothing is not None: uc += update_geothing(geothing, cache, the_location) or 0 else: create_new_geothing(cache, the_location, geosite) nc += 1 return uc, nc
def main(): LOAD_CACHES = True start = time() file = '/tmp/export_oc-full.xml' # sanity checking, only work on wpt files if file.endswith('.xml') == 0: sys.exit(-1) #sql = """ UPDATE geokret gk SET gk.state = NULL """ #r = execute_query(sql) print "Reading file: "+file fh = open(file,'r') xml = fh.read() fh.close() sxml = ET.XML(xml) all_krety = sxml.getchildren() for kret in all_krety: gkid = int(kret.get('id') or 0) if not gkid: continue geokret = get_object_or_none(GeoKret, gkid=gkid) if geokret: for node in kret.getchildren(): if node.tag == 'state': geokret.state = int(node.text) if node.text else None geokret.save() elapsed = time() - start print "Elapsed time -->", elapsed
def geocacher_exists(pid): return get_object_or_none(Geocacher, pid=pid)
def main(): start = time() yplib.setUp() yplib.set_debugging(False) geosite = Geosite.objects.get(code='OCDE') countries = GeoCountry.objects.all() countries = countries.values_list('iso', flat=True) sql = """ SELECT `value` FROM variables WHERE `name`='last_ocde_updated' """ lastdate = sql2val(sql); if not lastdate: lastdate = '20000101000000' statuses = [] types = [] oc_count = 0 gc_count = 0 nc_count = 0 k = 0 uc = 0 nc = 0 for country in countries: url = 'http://opencaching.de/xml/ocxml11.php?modifiedsince=%s&cache=1&country=%s' % \ (lastdate, country) response = urllib2.urlopen(url) xml = response.read() try: root = ET.XML(xml) except Exception as e: print 'PARSING ERROR', country, e continue # session id current_session = root[0] session_id = current_session.text # count records = root[1] caches_count = int(records.get("cache") or 0) if caches_count: page_count = int(round(caches_count * 1.0 / CACHES_PER_PAGE, 0)) + 1 for p in range(page_count): page_url = 'http://www.opencaching.de/xml/ocxml11.php?sessionid=%s&file=%s' % \ (session_id, p + 1) page_response = urllib2.urlopen(page_url).read() from StringIO import StringIO zipdata = StringIO() zipdata.write(page_response) try: zf = zipfile.ZipFile(zipdata) except: continue for name in zf.namelist(): uncompressed = zf.read(name) cache_root = ET.XML(uncompressed) latitude = None longitude = None status = None created_date_str = '' for cache in cache_root.getchildren(): k += 1 if cache.tag == 'cache': the_geothing = TheGeothing() the_location = TheLocation() for param in cache: if param.tag == 'id': the_geothing.pid = param.get('id') if param.tag == 'userid': the_geothing.author = param.text if param.tag == 'name': the_geothing.name = param.text if param.tag == 'longitude': longitude = param.text if param.tag == 'latitude': latitude = param.text if param.tag == 'type': cache_type = param.get('short') the_geothing.type_code = OCDE_TYPES.get(cache_type) type_ = (param.get('id'), param.get('short')) if not type_ in types: types.append(type_) if param.tag == 'status': status = int(param.get('id') or 0) status_ = (status, param.text) if not status_ in statuses: statuses.append(status_) if param.tag == 'waypoints': the_geothing.code = param.get('oc') if the_geothing.code: oc_count += 1 gccode = param.get('gccom') if gccode: gc_count += 1 nccode = param.get('nccom') if nccode: nc_count += 1 if param.tag == 'datecreated': created_date_str = param.text parts = strptime(created_date_str, '%Y-%m-%d %H:%M:%S') dt = datetime(parts[0], parts[1], parts[2], parts[3], parts[4], parts[5]) the_geothing.created_date = dt if latitude and longitude and status == 1: the_location.NS_degree = float(latitude) the_location.EW_degree = float(longitude) if the_geothing.code and the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) if geothing is not None: uc += update_geothing(geothing, the_geothing, the_location) or 0 else: create_new_geothing(the_geothing, the_location, geosite) nc += 1 message = 'OK. updated %s, new %s' % (uc, nc) log('map_ocde_caches', message) print message sql = """ UPDATE `variables` SET `value`='%s' WHERE `name`='last_ocde_updated' """ % ocde_timestamp() execute_query(sql) elapsed = time() - start print "Elapsed time -->", elapsed
def check_cach(cach_pid): def get_coordinates(cell): coordinates = cell.text parts = t2.findall(coordinates)[0] if len(parts) == 4: ns_degree, ns_minute, ew_degree, ew_minute = parts parts = t3.findall(coordinates) NS = parts[0] parts = t4.findall(coordinates) EW = parts[0] return ns_degree, ns_minute, ew_degree, ew_minute, NS, EW def get_type(cell): return cell.text def get_class(cell): class_ = None if cell: parts = cell.contents items = [] for p in parts: txt = p.string if txt and nottag(txt): items.append(txt) class_ = ';'.join(items) return class_ def get_mestnost(cell): oblast = country = None parts = cell.contents if len(parts): country = parts[0] if len(parts) > 2: oblast = parts[2] return country, oblast def get_dostupnost(cell): parts = cell.contents dostupnost = parts[0].split(':')[1].strip() mestnost = parts[2].split(':')[1].strip() return dostupnost, mestnost def get_town(cell): return cell.text def get_grade(cell): grade = None if cell.img: grade = cell.img.get('title') return grade def get_attributes(element): attr = None items = [] imgs = element.findAll('img') for img in imgs: if 'images/attrib/' in img.get('src'): items.append(img.get('title')) attr = ';'.join(items) return attr url = 'http://www.geocaching.su/?pn=101&cid=%d'%int(cach_pid) try: yplib.get(url) except: print 'exception' return False soup=yplib.soup() h = soup.find('h1', {'class':'hdr'}) t = re.compile('([^\[]+)\[.+\]') t1 = re.compile('[^\[]+\[([^\[\]]+\/[^\[\]]+)\]') t2 = re.compile('[N,S]\s(\d+)\&\#176\;\s([\d\.]+).+[E,W]\s(\d+)\&\#176\;\s([\d\.]+)') t3 = re.compile('([N,S]\s\d+\&\#176\;\s[\d\.]+.)') t4 = re.compile('([E,W]\s\d+\&\#176\;\s[\d\.]+.)') t5 = re.compile('WinPopup\(\'profile\.php\?pid\=(\d+)') name = None items = t.findall(h.text) if items: name = items[0] full_code = None items = t1.findall(h.text) if items: full_code = items[0] type_code, pid = full_code.split('/') tbl = soup.find('table', attrs={'cellpadding':3, 'width':160}) rows = tbl.findAll('tr') ns_degree = ns_minute = ew_degree = ew_minute = NS = EW = None country = oblast = town = None dostupnost = mestnost = None cach_type = cach_class = None grade = attr = None act = None for row in rows: tds = row.findAll('td') ths = row.findAll('th') td = None if tds: td = tds[0] cell = None if act: if ths: cell = ths[0] elif tds: cell = tds[1] if act == 'coord': ns_degree, ns_minute, ew_degree, ew_minute, NS, EW = get_coordinates(cell) if act == 'mestnost': country, oblast = get_mestnost(cell) if act == 'dostupnost': dostupnost, mestnost = get_dostupnost(cell) if act == 'town': town = get_town(cell) if act == 'grade': grade = get_grade(cell) act = None if td and td.text.startswith(u'Тип:'): cach_type = get_type(tds[1]) act = None if td and td.text.startswith(u'Класс:'): cach_class = get_class(tds[1]) act = None if td and td.text.startswith(u'КООРДИНАТЫ'): act = 'coord' if td and td.text.startswith(u'МЕСТНОСТЬ'): act = 'mestnost' if td and td.text.startswith(u'БЛИЖАЙШИЙ'): act = 'town' if td and td.text.startswith(u'ОЦЕНКИ'): act = 'dostupnost' if td and td.text.startswith(u'РЕЙТИНГ'): act = 'grade' if td and td.text.startswith(u'АТРИБУТЫ'): attr = get_attributes(tbl) act = None created_by = created_date = changed_date = coauthors = None div = soup.findAll('div', attrs={'style':'padding: 5px; font-family: Verdana; font-weight: bold;'})[0] a = div.a if a: onclick = a.get('onclick') if onclick: pid = t5.findall(onclick) if pid: created_by = int(pid[0]) parts = div.contents for p in parts: txt = p.string #if txt: #print txt.encode('utf8'), type(txt) if txt and nottag(txt): txt = txt.string.strip() if txt.startswith(u'Создан:'): items = txt.split() if len(items) == 2: created_date = items[1] if created_date: day, month, year = [int(s) for s in created_date.split('.')] created_date = date(year, month, day) if txt.startswith(u'(отредактирован'): txt = txt[1:-1] items = txt.split() if len(items) == 2: changed_date = items[1] if changed_date: day, month, year = [int(s) for s in changed_date.split('.')] changed_date = date(year, month, day) if txt.startswith(u'Компаньоны:'): coauthors = 'yes' the_cach = TheCach() the_cach.pid = cach_pid the_cach.code = '%s%s' % (type_code, the_cach.pid) the_cach.type_code = type_code #print #print cach.pid #print '|%s|'%the_cach.code.encode('utf8') the_cach.name = text_or_none(name) the_cach.cach_type = text_or_none(cach_type) the_cach.cach_class = text_or_none(cach_class) the_cach.loc_NS = char_or_none(NS) the_cach.loc_EW = char_or_none(EW) the_cach.loc_NS_degree = int_or_none(ns_degree) the_cach.loc_EW_degree = int_or_none(ew_degree) the_cach.loc_NS_minute = float_or_none(ns_minute) the_cach.loc_EW_minute = float_or_none(ew_minute) the_cach.country = text_or_none(country) the_cach.oblast = text_or_none(oblast) the_cach.town = text_or_none(town) the_cach.dostupnost = int_or_none(dostupnost) the_cach.mestnost = int_or_none(mestnost) the_cach.grade = float_or_none(grade) the_cach.cach_attr = text_or_none(attr) the_cach.created_by = created_by the_cach.created_date = created_date the_cach.changed_date = changed_date the_cach.coauthors = coauthors print the_cach.name.encode('utf8') geocache = get_object_or_none(Cach, pid=cach_pid) if geocache is not None: update_geocache(geocache, the_cach) else: cach = Cach.objects.create(pid=cach_pid) cach.__dict__.update(the_cach.__dict__) print 'save', cach.pid cach.save() #nc += 1 #if True: #cach.__dict__.update(the_cach.__dict__) #print 'save', cach.pid #cach.save() return True
def process(rectangle, geosite, params): #print 'process' #print rectangle, geosite, params print geosite bbox = [str(x) for x in rectangle] url = params.get('url_pattern') % (params['MY_CONSUMER_KEY'], '|'.join(bbox), params['FIELDS']) #print geosite, rectangle #print 'URL' #print url try: response = urllib2.urlopen(url) print 'GOT' except Exception as e: print 'exception', e return data = response.read() caches_data = json.loads(data) caches = caches_data.get('results') #print 'count', len(caches) more_data = caches_data.get('more') if more_data: BlockNeedBeDivided.objects.create(geosite=geosite, bb='|'.join(bbox), added=datetime.now()) if not len(caches): return k = 0 uc = 0 nc = 0 #print caches for code, cache in caches.iteritems(): #print cache k += 1 the_geothing = TheGeothing() the_location = TheLocation() locations = cache.get('location').split('|') lat_degree = float(locations[0]) the_location.NS_degree = lat_degree lon_degree = float(locations[1]) the_location.EW_degree = lon_degree the_geothing.code = cache.get('code') the_geothing.name = cache.get('name') if cache.get('status') != 'Available': continue the_geothing.type_code = OCPL_TYPES.get(cache.get('type')) cache_url = cache.get('url') if not cache_url: continue p = re.compile(params['code_re']) dgs = p.findall(cache_url) if not dgs: continue the_geothing.pid = int(dgs[0], 16) if cache.get('owner'): owner_name = cache.get('owner').get('username') the_geothing.author = owner_name date_created = cache.get('date_created') if date_created: date_created = date_created[:10] parts = date_created.split('-') if parts and len(parts) == 3: dt = datetime(int(parts[0]), int(parts[1]), int(parts[2])) the_geothing.created_date = dt if the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) #print #print geothing if geothing is not None: #print 'update' uc += update_geothing(geothing, the_geothing, the_location) or 0 else: #print 'new' create_new_geothing(the_geothing, the_location, geosite) nc += 1 message = 'OK. updated %s, new %s' % (uc, nc) log(params.get('log_key'), message)
def main(): LOAD_CACHES = True start = time() yplib.setUp() yplib.set_debugging(False) url = 'http://www.geocaching.su/rss/geokrety/api.php?interval=1y&ctypes=1,2,3,7&changed=1' f = urllib2.urlopen(url) xml = f.read() xml = xml try: sxml = ET.XML(xml) except Exception as e: print type(e) print e return cnt_new = 0 cnt_upd = 0 caches = sxml.getchildren() geosite = Geosite.objects.get(code='GC_SU') for cache in caches: if cache.tag == 'cache': the_geothing = TheGeothing() the_location = TheLocation() for tag_ in cache.getchildren(): if tag_.tag == 'code': the_geothing.code = tag_.text if tag_.tag == 'autor': the_geothing.author = tag_.text if tag_.tag == 'name': the_geothing.name = tag_.text if tag_.tag == 'position': lat_degree = float(tag_.get('lat')) the_location.NS_degree = lat_degree lon_degree = float(tag_.get('lon')) the_location.EW_degree = lon_degree if tag_.tag == 'cdate': date_str = tag_.text date_ = date_str.split('-') if len(date_) == 3: the_geothing.created_date = datetime(int(date_[0]), int(date_[1]), int(date_[2])) if the_geothing.code: p = re.compile('(\D+)(\d+)') dgs = p.findall(the_geothing.code) if dgs: code_data = dgs[0] the_geothing.pid = int(code_data[1]) the_geothing.type_code = code_data[0] if the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) if geothing is not None: cnt_upd += update_geothing(geothing, the_geothing, the_location) or 0 else: create_new_geothing(the_geothing, the_location, geosite) cnt_new += 1 message = 'OK %s/%s'%(cnt_new, cnt_upd) log('map_gcsu_caches', message) print message elapsed = time() - start print "Elapsed time -->", elapsed
def main(): LOAD_CACHES = True start = time() file = '../shukach_all.wpt' # sanity checking, only work on wpt files if file.endswith('.wpt') == 0: sys.exit(-1) print "Reading file: " + file #fh = codecs.open(file,'r',"utf-8") fh = codecs.open(file, 'r', "cp1251") wpt = fh.readlines() fh.close() WPT_CODE = 1 WPT_LAT = 2 WPT_LON = 3 WPT_TITLE = 10 WPT_DATE = 4 geosite = Geosite.objects.get(code='SHUKACH') print geosite print len(wpt), 'points' k = 0 for point in wpt: #print #print point fields = point.split(',') if fields[0].isdigit(): the_geothing = TheGeothing() the_location = TheLocation() #geothing = Geothing(geosite=geosite) #for field in fields: #print field #l = Location() lat_degree = float(fields[WPT_LAT]) the_location.NS_degree = round(lat_degree, 4) #the_location.NS_minute = round((abs(lat_degree) - abs(the_location.NS_degree)) * 60, 2) lon_degree = float(fields[WPT_LON]) the_location.EW_degree = round(lon_degree, 4) #the_location.EW_minute = round((abs(lon_degree) - abs(the_location.EW_degree)) * 60, 2) #l.save() #geothing.location = l p = re.compile('(\D+)(\d+)') dgs = p.findall(fields[WPT_CODE]) if dgs: code_data = dgs[0] the_geothing.code = fields[WPT_CODE] the_geothing.pid = int(code_data[1]) the_geothing.type_code = code_data[0] p = re.compile(u'(.+)от(.+)') dgs = p.findall(fields[WPT_TITLE]) if dgs: title = dgs[0] the_geothing.name = title[0] the_geothing.author = title[1] d = float(fields[WPT_DATE]) #print Dephi_date_to_python_date(d) the_geothing.created_date = Dephi_date_to_python_date(d) if the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) #import pdb; pdb.set_trace() if geothing is not None: update_geothing(geothing, the_geothing, the_location) else: create_new_geothing(the_geothing, the_location, geosite) elapsed = time() - start print "Elapsed time -->", elapsed
def cache_exists(cach_pid): return get_object_or_none(Cach, pid=cach_pid)
def main(): start = time() yplib.setUp() yplib.set_debugging(False) geosite = Geosite.objects.get(code='OCCZ') statuses = [] types = [] oc_count = 0 k = 0 uc = 0 nc = 0 url = 'http://www.opencaching.cz/search.php?searchto=searchbydistance&showresult=1&output=XML&sort=byname&latNS=N&lat_h=50&lat_min=5.123&lonEW=E&lon_h=14&lon_min=20.123&distance=1500&unit=km&count=500&startat=0' response = urllib2.urlopen(url).read() cache_root = ET.XML(response) docinfo = cache_root.getchildren()[0] result_count = 0 for tag in docinfo.getchildren(): if tag.tag == 'results': result_count = int(tag.text or 0) if result_count: for cache in cache_root.getchildren()[1:]: latitude = None longitude = None status = None created_date_str = '' k += 1 if cache.tag == 'cache': the_geothing = TheGeothing() the_location = TheLocation() for param in cache: if param.tag == 'id': the_geothing.pid = param.text if param.tag == 'owner': the_geothing.author = param.text if param.tag == 'name': the_geothing.name = param.text if param.tag == 'lon': longitude = param.text if param.tag == 'lat': latitude = param.text if param.tag == 'type': cache_type = param.text the_geothing.type_code = OCCZ_TYPES.get(cache_type) if not cache_type in types: types.append(cache_type) if param.tag == 'status': status = param.text if not status in statuses: statuses.append(status) if param.tag == 'waypoint': the_geothing.code = param.text if the_geothing.code: oc_count += 1 if param.tag == 'hidden': created_date_str = param.text parts = strptime(created_date_str, '%d.%m.%Y') dt = datetime(parts[0], parts[1], parts[2], parts[3], parts[4], parts[5]) the_geothing.created_date = dt if latitude and longitude: the_location.NS_degree = get_degree(latitude) the_location.EW_degree = get_degree(longitude) if the_geothing.code and the_geothing.pid and \ the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) if geothing is not None: uc += update_geothing(geothing, the_geothing, the_location) or 0 else: create_new_geothing(the_geothing, the_location, geosite) nc += 1 message = 'OK. updated %s, new %s' % (uc, nc) log('map_occz_caches', message) print message print print types print statuses elapsed = time() - start print "Elapsed time -->", elapsed
def iso_by_country_name(country_name): r = None country = get_object_or_none(GeoCountry, name=country_name) if country: r = country.iso return r
def main(): LOAD_CACHES = True start = time() yplib.setUp() yplib.set_debugging(False) r = yplib.post2('http://www.geocaching.su/?pn=108', (('Log_In', 'Log_In'), ('email', '*****@*****.**'), ('passwd', 'zaebalixakeryvas'), ('longterm', '1'))) soup = yplib.soup() a = soup.find('a', attrs={'class': "profilelink"}, text='galdor') if not a: print 'Authorization failed' return False r = yplib.get('http://www.geocaching.su/site/popup/selex.php') soup = yplib.soup() chbox_list = soup.findAll('input', type='checkbox') regions = [] for chbox in chbox_list: v = chbox.get('value') if v and chbox.get('name', '') == 'point[]': regions.append(v) data = [ ('translit', '0'), ('fmt', 'wpt'), ('code_to_name', '1'), ('finded', '2'), ] for r in regions: data.append(('point[]', r)) r = yplib.post2('http://www.geocaching.su/site/popup/export.php', data) soup = yplib.soup() wpt = soup.text.split('\n') WPT_CODE = 1 WPT_LAT = 2 WPT_LON = 3 WPT_TITLE = 10 WPT_DATE = 4 geosite = Geosite.objects.get(code='GC_SU') print len(wpt), 'points' k = 0 for point in wpt: k += 1 fields = point.split(',') if fields[0].isdigit(): the_geothing = TheGeothing() the_location = TheLocation() lat_degree = float(fields[WPT_LAT]) the_location.NS_degree = lat_degree #the_location.NS_minute = (abs(lat_degree) - abs(the_location.NS_degree)) * 60 lon_degree = float(fields[WPT_LON]) the_location.EW_degree = lon_degree #the_location.EW_minute = (abs(lon_degree) - abs(the_location.EW_degree)) * 60 p = re.compile('(\D+)(\d+)') dgs = p.findall(fields[WPT_CODE]) if dgs: code_data = dgs[0] the_geothing.code = fields[WPT_CODE] the_geothing.pid = int(code_data[1]) the_geothing.type_code = code_data[0] p = re.compile(u'(.+)от(.+)') dgs = p.findall(fields[WPT_TITLE]) if dgs: title = dgs[0] the_geothing.name = title[0] the_geothing.author = title[1] d = float(fields[WPT_DATE]) the_geothing.created_date = Dephi_date_to_python_date(d) if the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) if geothing is not None: update_geothing(geothing, the_geothing, the_location) else: create_new_geothing(the_geothing, the_location, geosite) log('map_gcsu_caches', 'OK') elapsed = time() - start print "Elapsed time -->", elapsed
def main(): LOAD_CACHES = True start = time() yplib.setUp() yplib.set_debugging(False) # log in r = yplib.post2('http://opencaching.pl/login.php', (('LogMeIn', 'zaloguj'), ('email', 'kurianin'), ('password', 'gjhjkjy'), ('action', 'login'), ('target', 'index.php'))) soup = yplib.soup() a = soup.find('a', text='kurianin') if not a: print 'Authorization failed' return False print 'OK' ## search page #r = yplib.get('http://opencaching.pl/search.php') #soup = yplib.soup() # get wpt file r = yplib.get( 'http://opencaching.pl/search.php?searchto=searchbyname&showresult=1&expert=0&output=HTML&sort=bycreated&f_inactive=1&f_ignored=1&f_userfound=1&f_userowner=1&f_watched=0&f_geokret=0&country=PL®ion=&cachetype=1111111110&cache_attribs=&cache_attribs_not=&cachesize_1=1&cachesize_2=1&cachesize_3=1&cachesize_4=1&cachesize_5=1&cachesize_6=1&cachesize_7=1&cachevote_1=-3&cachevote_2=3.000&cachenovote=1&cachedifficulty_1=1&cachedifficulty_2=5&cacheterrain_1=1&cacheterrain_2=5&cacherating=0&cachename=%25&cachename=' ) soup = yplib.soup(cp='utf8') link_to_wpt = '' #the_div = soup.find('div', {'class':"content2-pagetitle"}) wpt_link = re.compile('ocpl\d+\.wpt\?.+count\=max.*') a_list = soup.findAll('a', {'class': "links", 'title': "Oziexplorer .wpt"}) if a_list: for a in a_list: if a.get('href') and wpt_link.match(a.get('href')): link_to_wpt = a.get('href') break print link_to_wpt if link_to_wpt: r = yplib.get(link_to_wpt) soup = yplib.soup(cp='utf8') wpt = soup.text.split('\n') else: print 'oblom' return WPT_CODE = 10 WPT_LAT = 2 WPT_LON = 3 WPT_TITLE = 1 WPT_DATE = 4 MY_CONSUMER_KEY = 'fky3LF9xvWz9y7Gs3tZ6' FIELDS = 'code|name|location|type|status|url|owner|date_created' geocach_api_request = 'http://opencaching.pl/okapi/services/caches/geocache?cache_code=%s&consumer_key=%s&fields=%s' geosite = Geosite.objects.get(code='OCPL') print geosite print len(wpt), 'points' k = 0 uc = 0 nc = 0 for point in wpt: k += 1 fields = point.split(',') if fields[0] == '-1': the_geothing = TheGeothing() the_geothing.pid = 1 the_location = TheLocation() lat_degree = float(fields[WPT_LAT]) the_location.NS_degree = lat_degree #the_location.NS_minute = (abs(lat_degree) - abs(the_location.NS_degree)) * 60 lon_degree = float(fields[WPT_LON]) the_location.EW_degree = lon_degree #the_location.EW_minute = (abs(lon_degree) - abs(the_location.EW_degree)) * 60 code_str = fields[WPT_CODE] parts = code_str.split('/') if len(parts) == 4: cache_code = parts[0] the_geothing.code = cache_code the_geothing.name = fields[WPT_TITLE] geothing_items = Geothing.objects.filter( code=the_geothing.code, geosite=geosite) if geothing_items.count() > 0: geothing = geothing_items[0] if the_geothing.name == geothing.name and not location_was_changed( geothing.location, the_location): continue url = geocach_api_request % (cache_code, MY_CONSUMER_KEY, FIELDS) try: response = urllib2.urlopen(url) json_str = response.read() cache_data = json.loads(json_str) if cache_data.get('status') != 'Available': continue #print cache_data.get('type') the_geothing.type_code = OCPL_TYPES.get( cache_data.get('type')) #print the_geothing.type_code cache_url = cache_data.get('url') if not cache_url: continue p = re.compile(u'OP([\dA-F]+)$') dgs = p.findall(cache_url) the_geothing.pid = int(dgs[0], 16) owner_name = '' if cache_data.get('owner'): owner_name = cache_data.get('owner').get('username') the_geothing.author = owner_name date_created = cache_data.get('date_created') if date_created: date_created = date_created[:10] parts = date_created.split('-') if parts and len(parts) == 3: dt = datetime(int(parts[0]), int(parts[1]), int(parts[2])) the_geothing.created_date = dt except: print print 'exception.' print url print cache_data #break continue if the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) if geothing is not None: update_geothing(geothing, the_geothing, the_location) uc += 1 else: create_new_geothing(the_geothing, the_location, geosite) nc += 1 #break sql = """ select COUNT(*) FROM ( select g.code as code, count(id) as cnt from geothing g group by g.code having cnt > 1 ) as tbl """ dc = sql2val(sql) message = 'OK. updated %s, new %s, doubles %s' % (uc, nc, dc) log('map_ocpl_caches', message) elapsed = time() - start print "Elapsed time -->", elapsed
def main(): LOAD_CACHES = True start = time() file = '/tmp/ocpl.wpt' # sanity checking, only work on wpt files if file.endswith('.wpt') == 0: sys.exit(-1) print "Reading file: " + file fh = codecs.open(file, 'r', "utf-8") wpt = fh.readlines() fh.close() WPT_CODE = 10 WPT_LAT = 2 WPT_LON = 3 WPT_TITLE = 1 WPT_DATE = 4 MY_CONSUMER_KEY = 'fky3LF9xvWz9y7Gs3tZ6' FIELDS = 'code|name|location|type|status|url|owner|date_created' geocach_api_request = 'http://opencaching.pl/okapi/services/caches/geocache?cache_code=%s&consumer_key=%s&fields=%s' geosite = Geosite.objects.get(code='OCPL') print geosite print len(wpt), 'points' k = 0 for point in wpt: k += 1 fields = point.split(',') if fields[0] == '-1': the_geothing = TheGeothing() the_location = TheLocation() lat_degree = float(fields[WPT_LAT]) the_location.NS_degree = lat_degree #the_location.NS_minute = (abs(lat_degree) - abs(the_location.NS_degree)) * 60 lon_degree = float(fields[WPT_LON]) the_location.EW_degree = lon_degree #the_location.EW_minute = (abs(lon_degree) - abs(the_location.EW_degree)) * 60 code_str = fields[WPT_CODE] parts = code_str.split('/') if len(parts) == 4: cache_code = parts[0] the_geothing.code = cache_code the_geothing.name = fields[WPT_TITLE] geothing = get_object_or_none(Geothing, code=the_geothing.code, geosite=geosite) if geothing: if the_geothing.name == geothing.name and not location_was_changed( geothing.location, the_location): continue print the_geothing.code, 'changed' url = geocach_api_request % (cache_code, MY_CONSUMER_KEY, FIELDS) try: response = urllib2.urlopen(url) json_str = response.read() cache_data = json.loads(json_str) if cache_data.get('status') != 'Available': continue the_geothing.type_code = OCPL_TYPES.get( cache_data.get('type')) cache_url = cache_data.get('url') if not cache_url: continue p = re.compile(u'(\d+)$') dgs = p.findall(cache_url) the_geothing.pid = int(dgs[0]) owner_name = '' if cache_data.get('owner'): owner_name = cache_data.get('owner').get('username') the_geothing.author = owner_name date_created = cache_data.get('date_created') if date_created: date_created = date_created[:10] parts = date_created.split('-') if parts and len(parts) == 3: dt = datetime(int(parts[0]), int(parts[1]), int(parts[2])) the_geothing.created_date = dt except: print 'exception' continue if the_geothing.type_code in GEOCACHING_ONMAP_TYPES: geothing = get_object_or_none(Geothing, pid=the_geothing.pid, geosite=geosite) if geothing is not None: update_geothing(geothing, the_geothing, the_location) else: create_new_geothing(the_geothing, the_location, geosite) #break elapsed = time() - start print "Elapsed time -->", elapsed