def login(self, username, password): """Logs the user in. Downloads the relevant cookies to keep the user logged in.""" logging.info("Logging in...") rawPage = Util.urlopen(self._getURL("loginPage")) if not rawPage: logging.error("Cannot load login page.") return False soup = BeautifulSoup(rawPage.read()) logging.debug("Checking for previous login.") logged = self.getLoggedUser(soup) if logged: if logged == username: logging.info("Already logged as %s.", logged) self.loggedIn = True return True else: logging.info("Already logged as %s, but want to log in as %s.", logged, username) self.logout() # continue logging in postValues = dict() logging.debug("Assembling POST data.") # login fields loginElements = soup("input", type=["text", "password", "checkbox"]) for field, val in zip(loginElements, [username, password, 1]): postValues[field["name"]] = val # other nescessary fields for field in soup("input", type=["hidden", "submit"]): postValues[field["name"]] = field["value"] url = self._getURL("loginPage") # login to the site logging.debug("Submiting login form.") rawPage = Util.urlopen(url, postValues) if not rawPage: logging.error("Cannot load response after login.") return False soup = BeautifulSoup(rawPage.read()) logging.debug("Checking the result.") if self.getLoggedUser(soup): logging.info("Logged in successfully as %s.", username) self.loggedIn = True return True else: logging.error("Cannot logon to the site (probably wrong username or password).") return False
def loadCacheQuick(self, wp): """Loads details from map server. Loads just basic cache details, but very quickly""" if not self.loggedIn or not (isinstance(wp, StringTypes) and wp.startswith("GC")): return logging.info("Loading quick details about %s...", wp) # assemble request params = urlencode({"i": wp}) url = self.mapurl + "?" + params # make request rawPage = Util.urlopen(url) if not rawPage: logging.error("Cannot load search page.") return res = json.loads(rawPage.read()) # check for success if res["status"] != "success": return None data = res["data"][0] # prettify some data size = data["container"]["text"].lower() hidden = datetime.strptime(data["hidden"], '%m/%d/%Y').date() # assemble cache object c = Cache(wp, data["name"], data["type"]["text"], None, data["available"], None, size, data["difficulty"]["text"], data["terrain"]["text"], data["owner"]["text"], hidden, None) return c
def getLoggedUser(self, soup=None): """Returns the name of curently logged user. Or None if no user is logged in.""" if not isinstance(soup, BeautifulSoup): logging.debug("No 'soup' passed, loading login page on my own.") page = Util.urlopen(self._getURL("loginPage")) soup = BeautifulSoup(page.read()) logging.debug("Checking for already logged user.") try: return soup.find("div", "LoggedIn").find("strong").text except AttributeError: return None
def loadCache(self, wp): """Loads details from cache page. Loads all cache details and return fully populated cache object.""" if not self.loggedIn or not (isinstance(wp, StringTypes) and wp.startswith("GC")): return logging.info("Loading details about %s...", wp) # assemble request params = urlencode({"wp": wp}) url = self._getURL("cacheDetails") + "?" + params # make request rawPage = Util.urlopen(url) if not rawPage: logging.error("Cannot load search page.") return soup = BeautifulSoup(rawPage) # parse raw data cacheDetails = soup.find(id="cacheDetails") name = cacheDetails.find("h2") cacheType = cacheDetails.find("img").get("alt") author = cacheDetails("a")[1] hidden = cacheDetails.find("div", "minorCacheDetails").findAll("div")[1] location = soup.find(id="uxLatLon") state = soup.find("ul", "OldWarning") found = soup.find("div", "FoundStatus") DandT = soup.find("div", "CacheStarLabels").findAll("img") size = soup.find("div", "CacheSize").find("img") attributesRaw = soup("div", "CacheDetailNavigationWidget")[0].findAll("img") userContent = soup("div", "UserSuppliedContent") hint = soup.find(id="div_hint") favorites = soup.find("span", "favorite-value") # prettify data name = name.text.encode("utf-8", "xmlcharrefreplace") author = author.text.encode("utf-8", "xmlcharrefreplace") hidden = datetime.strptime(hidden.text.split()[2], '%m/%d/%Y').date() try: lat, lon = Util.parseRaw(location.text) location = geo.Point(Util.toDecimal(*lat), Util.toDecimal(*lon)) except ValueError: loggin.debug("Could not parse coordinates") state = state is None found = found and "Found It!" in found.text or False dif, ter = map(lambda e: float(e.get("alt").split()[0]), DandT) size = " ".join(size.get("alt").split()[1:]).lower() attributesRaw = map(lambda e: e.get("src").split('/')[-1].rsplit("-", 1), attributesRaw) attributes = {} # parse attributes by src to know yes/no for attribute_name, appendix in attributesRaw: if appendix.startswith("blank"): continue attributes[attribute_name] = appendix.startswith("yes") summary = userContent[0].text.encode("utf-8", "xmlcharrefreplace") description = userContent[1] hint = Util.rot13(hint.text.strip().encode('utf-8')) favorites = int(favorites.text) # assemble cache object c = Cache(wp, name, cacheType, location, state, found, size, dif, ter, author, hidden, attributes, summary, description, hint, favorites) return c
def _searchGetPage(self, point, page=1): """Returns one page of caches as a list. Searches for a caches around a point and returns N-th page (specifiend by page argument).""" if not self.loggedIn or not isinstance(point, geo.Point) or type(page) is not IntType: return logging.info("Fetching page %d.", page) # assemble request params = urlencode({"lat": point.latitude, "lng": point.longitude}) url = self._getURL("cachesNearest") + "?" + params # we have to add POST for other pages than 1st if page == 1: data = None else: # TODO handle searching on second page without first data = self.paggingHelpers data["__EVENTTARGET"] = self.paggingPostbacks[page] data["__EVENTARGUMENT"] = "" # make request rawPage = Util.urlopen(url, data) if not rawPage: logging.error("Cannot load search page.") return list() soup = BeautifulSoup(rawPage) # root of a few following elements pageBuilders = soup("td", "PageBuilderWidget") # parse pagging widget total, page, pageCount = map(lambda elm: int(elm.text), pageBuilders[0].findAll("b")) logging.debug("Found %d results. Showing page %d of %d.", total, page, pageCount) # save search postbacks for future usage if page == 1: paggingLinks = filter(lambda e: e.get("id"), pageBuilders[1].findAll("a")) paggingPostbacks = {int(link.text): link.get("href").split("'")[1] for link in paggingLinks} self.paggingPostbacks = paggingPostbacks # other nescessary fields self.paggingHelpers = dict() for field in soup("input", type="hidden"): self.paggingHelpers[field["name"]] = field["value"] # parse results table data = soup.find("table", "SearchResultsTable").findAll("tr", "Data") result = [] for cache in data: # parse raw data typeLink, nameLink = cache("a", "lnk") direction, info, DandT, placed, lastFound = cache("span", "small") found = cache.find("img", title="Found It!") is not None size = cache.find("td", "AlignCenter").find("img") author, wp, area = map(lambda t: t.strip(), info.text.split("|")) # prettify data cacheType = typeLink.find("img").get("alt") name = nameLink.span.text.strip().encode("ascii", "xmlcharrefreplace") state = not "Strike" in nameLink.get("class") size = " ".join(size.get("alt").split()[1:]).lower() dif, ter = map(float, DandT.text.split("/")) hidden = datetime.strptime(placed.text, '%m/%d/%Y').date() author = author[3:].encode("ascii", "xmlcharrefreplace") # delete "by " # assemble cache object c = Cache(wp, name, cacheType, None, state, found, size, dif, ter, author, hidden) logging.debug("Parsing cache: %s", c) result.append(c) return result