def oauth_flow(s, oauth_url, username=None, password=None, sandbox=False): """s should be a requests session""" r = s.get(oauth_url) if r.status_code >= 300: raise RuntimeError(r.text) params = urlparse.parse_qs(urlparse.urlparse(r.url).query) data = { "un": username, "width": 2560, "height": 1440, "hasRememberUn": True, "startURL": params['startURL'], "loginURL": "", "loginType": 6, "useSecure": True, "local": "", "lt": "OAUTH", "qs": "r=https%3A%2F%2Flocalhost%3A8443%2Fsalesforce%2F21", "locale": "", "oauth_token": "", "oauth_callback": "", "login": "", "serverid": "", "display": "popup", "username": username, "pw": password, "Login": "", } if sandbox: base = "https://test.salesforce.com" else: base = "https://login.salesforce.com" r2 = s.post(base, data) m = re.search("window.location.href\s*='(.[^']+)'", r2.text) assert m is not None, ("Couldn't find location.href expression in page {} " "(Username or password is wrong)").format(r2.url) u3 = "https://" + urlparse.urlparse(r2.url).hostname + m.group(1) r3 = s.get(u3) m = re.search("window.location.href\s*='(.[^']+)'", r3.text) assert m is not None, ("Couldn't find location.href expression in page {}:" "\n{}").format(r3.url, r3.text) return m.group(1)
def run_firefox_for_android(build_obj, params): """ Launch Firefox for Android on the connected device. Optional 'params' allow parameters to be passed to Firefox. """ device = _get_device(build_obj.substs) try: # # Construct an adb command similar to: # # $ adb shell am start -a android.activity.MAIN \ # -n org.mozilla.fennec_$USER \ # -d <url param> \ # --es args "<params>" # app = build_obj.substs['ANDROID_PACKAGE_NAME'] url = None if params: for p in params: if urlparse.urlparse(p).scheme != "": url = p params.remove(p) break device.launch_fennec(app, extra_args=params, url=url) except Exception: _log_warning("unable to launch Firefox for Android") return 1 return 0
def _discover_auth_versions(self, session, auth_url): # discover the API versions the server is supporting base on the # given URL v2_auth_url = None v3_auth_url = None try: ks_discover = discover.Discover(session=session, auth_url=auth_url) v2_auth_url = ks_discover.url_for('2.0') v3_auth_url = ks_discover.url_for('3.0') except ks_exc.ClientException as e: # Identity service may not support discover API version. # Lets trying to figure out the API version from the original URL. url_parts = urlparse.urlparse(auth_url) (scheme, netloc, path, params, query, fragment) = url_parts path = path.lower() if path.startswith('/v3'): v3_auth_url = auth_url elif path.startswith('/v2'): v2_auth_url = auth_url else: # not enough information to determine the auth version msg = ('Unable to determine the Keystone version ' 'to authenticate with using the given ' 'auth_url. Identity service may not support API ' 'version discovery. Please provide a versioned ' 'auth_url instead. error=%s') % (e) raise exception.GlanceError(msg) return (v2_auth_url, v3_auth_url)
def run_firefox_for_android(build_obj, params, **kwargs): """ Launch Firefox for Android on the connected device. Optional 'params' allow parameters to be passed to Firefox. """ device = _get_device(build_obj.substs) try: # # Construct an adb command similar to: # # $ adb shell am start -a android.activity.MAIN \ # -n org.mozilla.fennec_$USER \ # -d <url param> \ # --es args "<params>" # app = build_obj.substs['ANDROID_PACKAGE_NAME'] msg = "URL specified as '{}'; dropping URL-like parameter '{}'" if params: for p in params: if urlparse.urlparse(p).scheme != "": params.remove(p) if kwargs.get('url'): _log_warning(msg.format(kwargs['url'], p)) else: kwargs['url'] = p device.launch_fennec(app, extra_args=params, **kwargs) except Exception: _log_warning("unable to launch Firefox for Android") return 1 return 0
def is_local_service(name): """ Determine if a service definition describes a service running on the local node. This is true if the service URL is for localhost, matches the machine's name, or ec2 public name """ if name is None: return False if "://" in name: url = urlparse.urlparse(name) if ":" in url.netloc: name = url.netloc.split(":")[0] else: name = url.netloc elif ":" in name: name = name.split(":")[0] if name == "localhost": return True if '.' in name: name = name.split('.')[0] node = socket.getfqdn() if '.' in node: node = node.split('.')[0] if name == node: return True pn = public_name() if pn is not None and pn.split(".")[0] == name: return True return False
def valid_url(url): try: _ = urlparse.urlparse(url) except Exception: raise NotValid("URL") # if part[1] == "localhost" or part[1] == "127.0.0.1": # raise NotValid("URL") return True
def git_fetch(args): parser = argparse.ArgumentParser( prog='git fetch', usage= 'git fetch [http(s)://<remote repo> or remotename] [-u username[:password]]', description="Push to a remote repository") parser.add_argument('url', type=str, nargs='?', help='URL to push to') parser.add_argument('-u', metavar='username[:password]', type=str, required=False, help='username[:password]') result = parser.parse_args(args) repo = _get_repo() origin = 'origin' if not result.url: result.url = repo.remotes.get('origin', '') if result.url in repo.remotes: origin = result.url result.url = repo.remotes.get(origin) if not urlparse.urlparse(result.url).scheme: raise Exception( 'url must match a remote name, or must start with http:// or https://' ) print('Starting fetch, this could take a while') remote_refs = porcelain.fetch(repo.repo.path, result.url) print('Fetch successful. Importing refs') remote_tags = gittle.utils.git.subrefs(remote_refs, 'refs/tags') remote_heads = gittle.utils.git.subrefs(remote_refs, 'refs/heads') # Filter refs clean_remote_tags = gittle.utils.git.clean_refs(remote_tags) clean_remote_heads = gittle.utils.git.clean_refs(remote_heads) # Base of new refs heads_base = 'refs/remotes/' + origin # Import branches repo.import_refs(heads_base, clean_remote_heads) for k, v in clean_remote_heads.items(): print('imported {}/{} {}'.format(heads_base, k, v)) # Import tags repo.import_refs('refs/tags', clean_remote_tags) for k, v in clean_remote_tags.items(): print('imported {}/{} {}'.format('refs/tags', k, v)) print('Checking for deleted remote refs') #delete unused remote refs for k in gittle.utils.git.subrefs(repo.refs, heads_base): if k not in clean_remote_heads: print('Deleting {}'.format('/'.join([heads_base, k]))) del repo.refs['/'.join([heads_base, k])] print('Fetch complete')
def git_fetch(args): parser = argparse.ArgumentParser(prog='git fetch' , usage='git fetch [http(s)://<remote repo> or remotename] [-u username[:password]]' , description="Push to a remote repository") parser.add_argument('url', type=str, nargs='?', help='URL to push to') parser.add_argument('-u', metavar='username[:password]', type=str, required=False, help='username[:password]') result = parser.parse_args(args) repo = _get_repo() origin='origin' if not result.url: result.url = repo.remotes.get('origin','') if result.url in repo.remotes: origin=result.url result.url=repo.remotes.get(origin) if not urlparse.urlparse(result.url).scheme: raise Exception('url must match a remote name, or must start with http:// or https://') print('Starting fetch, this could take a while') remote_refs=porcelain.fetch(repo.repo.path,result.url) print('Fetch successful. Importing refs') remote_tags = gittle.utils.git.subrefs(remote_refs, 'refs/tags') remote_heads = gittle.utils.git.subrefs(remote_refs, 'refs/heads') # Filter refs clean_remote_tags = gittle.utils.git.clean_refs(remote_tags) clean_remote_heads = gittle.utils.git.clean_refs(remote_heads) # Base of new refs heads_base = 'refs/remotes/' + origin # Import branches repo.import_refs( heads_base, clean_remote_heads ) for k,v in clean_remote_heads.items(): print('imported {}/{} {}'.format(heads_base,k,v)) # Import tags repo.import_refs( 'refs/tags', clean_remote_tags ) for k,v in clean_remote_tags.items(): print('imported {}/{} {}'.format('refs/tags',k,v)) print('Checking for deleted remote refs') #delete unused remote refs for k in gittle.utils.git.subrefs(repo.refs,heads_base): if k not in clean_remote_heads: print('Deleting {}'.format('/'.join([heads_base,k]))) del repo.refs['/'.join([heads_base,k])] print('Fetch complete')
def extract_url_arguments(url, urlmap): """ This extracts the URL arguments from a given URL """ parsed_url = urlparse.urlparse(url) map_adapter = urlmap.bind( server_name=parsed_url.netloc, script_name=parsed_url.path, url_scheme=parsed_url.scheme, path_info=parsed_url.path ) return map_adapter.match()[1]
def filter_result(link): try: # Valid results are absolute URLs not pointing to a Google domain # like images.google.com or googleusercontent.com o = urlparse.urlparse(link, 'http') if o.netloc and 'google' not in o.netloc: return link # Decode hidden URLs. if link.startswith('/url?'): link = urlparse.parse_qs(o.query)['q'][0] # Valid results are absolute URLs not pointing to a Google domain # like images.google.com or googleusercontent.com o = urlparse.urlparse(link, 'http') if o.netloc and 'google' not in o.netloc: return link # Otherwise, or on error, return None. except Exception: pass return None
def manage_makeChanges(self, rules_raw, REQUEST=None): '''Perform changes''' request = REQUEST self.rules_raw = request.rules_raw self.rules = [] self.errors = [] tests = [] i = 0 for line in self.rules_raw.split('\n'): i += 1 line = line.strip() if not line or line[0] == '#': continue if line[0] == '=': line = line[1:].strip() parts = re.split(r"(?<!\\)\s+", line) #split on non escaped whitespace if len(parts) == 2: tests.append(parts) parts = re.split(r"(?<!\\)\s+", line) #split on non escaped whitespace if len(parts) == 2: match, url = parts try: self.rules.append((re.compile(match), url)) except Exception as detail: self.errors.append('%d: %s "%s"' % (i, str(detail), line)) else: self.errors.append( "Line should be DOMAIN_REGEX/PATH_REGEX NEW_URL: %s" % line) for oldurl, newurl in tests: scheme, netloc, path, params, query, fragment = urlparse.urlparse( oldurl) res = self.redirect(netloc, path, query) if res != newurl: self.errors.append("TEST: %s %s != %s" % (oldurl, res, newurl)) message = "Saved changes." return self.Redirector_editForm(self, REQUEST, manage_tabs_message=message)
def git_push(args): parser = argparse.ArgumentParser(prog='git push' , usage='git push [http(s)://<remote repo> or remote] [-u username[:password]]' , description="Push to a remote repository") parser.add_argument('url', type=str, nargs='?', help='URL to push to') parser.add_argument('-u', metavar='username[:password]', type=str, required=False, help='username[:password]') result = parser.parse_args(args) user, sep, pw = result.u.partition(':') if result.u else (None,None,None) repo = _get_repo() origin='origin' if not result.url: result.url = repo.remotes.get('origin','') if result.url in repo.remotes: origin=result.url result.url=repo.remotes.get(origin) branch_name = os.path.join('refs','heads', repo.active_branch) #'refs/heads/%s' % repo.active_branch print("Attempting to push to: {0}, branch: {1}".format(result.url, branch_name)) netloc = urlparse.urlparse(result.url).netloc keychainservice = 'stash.git.{0}'.format(netloc) if sep and not user: # -u : clears keychain for this server for service in keychain.get_services(): if service[0]==keychainservice: keychain.delete_password(*service) #Attempt to retrieve user if not user and SAVE_PASSWORDS and result.url.startswith('http'): try: user = dict(keychain.get_services())[keychainservice] except KeyError: user = input('Enter username: '******'Enter password: '******'Enter credentials for {0}'.format(netloc)) outstream = StringIO() if user: if not pw and SAVE_PASSWORDS: pw = keychain.get_password(keychainservice, user) #Check again, did we retrieve a password? if not pw: user, pw = console.login_alert('Enter credentials for {0}'.format(netloc), login=user) host_with_auth='{}:{}@{}'.format(user,pw,netloc) url=urlparse.urlunparse( urlparse.urlparse(result.url)._replace( netloc=host_with_auth)) porcelain.push(repo.repo.path, url, branch_name, errstream=outstream) keychain.set_password(keychainservice, user, pw) else: porcelain.push(repo.repo.path, result.url, branch_name, errstream=outstream) for line in outstream.getvalue().split('\n'): print((line.replace(pw, '*******') if pw else line)) print('success!')
def cfcookie(netloc, ua, timeout): try: headers = {'User-Agent': ua} req = urllib_request.Request(netloc, headers=headers) try: urllib_request.urlopen(req, timeout=int(timeout)) except urllib_request.HTTPError as response: result = response.read(5242880) jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0] init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};', result)[-1] builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0] decryptVal = parseJSString(init) lines = builder.split(';') for line in lines: if len(line) > 0 and '=' in line: sections = line.split('=') line_val = parseJSString(sections[1]) decryptVal = int( eval(str(decryptVal) + sections[0][-1] + str(line_val))) answer = decryptVal + len(urlparse.urlparse(netloc).netloc) query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % ( netloc, jschl, answer) if 'type="hidden" name="pass"' in result: passval = re.findall('name="pass" value="(.*?)"', result)[0] query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % ( netloc, quote_plus(passval), jschl, answer) time.sleep(5) cookies = cookielib.LWPCookieJar() handlers = [ urllib_request.HTTPHandler(), urllib_request.HTTPSHandler(), urllib_request.HTTPCookieProcessor(cookies) ] opener = urllib_request.build_opener(*handlers) urllib_request.install_opener(opener) try: request = urllib_request.Request(query, headers=headers) urllib_request.urlopen(request, timeout=int(timeout)) except: pass cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies]) return cookie except: pass
def git_push(args): parser = argparse.ArgumentParser( prog='git push', usage= 'git push [http(s)://<remote repo> or remote] [-u username[:password]]', description="Push to a remote repository") parser.add_argument('url', type=str, nargs='?', help='URL to push to') parser.add_argument('-u', metavar='username[:password]', type=str, required=False, help='username[:password]') result = parser.parse_args(args) user, sep, pw = result.u.partition(':') if result.u else (None, None, None) repo = _get_repo() origin = 'origin' if not result.url: result.url = repo.remotes.get('origin', '') if result.url in repo.remotes: origin = result.url result.url = repo.remotes.get(origin) branch_name = os.path.join( 'refs', 'heads', repo.active_branch) #'refs/heads/%s' % repo.active_branch print("Attempting to push to: {0}, branch: {1}".format( result.url, branch_name)) netloc = urlparse.urlparse(result.url).netloc keychainservice = 'stash.git.{0}'.format(netloc) if sep and not user: # -u : clears keychain for this server for service in keychain.get_services(): if service[0] == keychainservice: keychain.delete_password(*service) #Attempt to retrieve user if not user and SAVE_PASSWORDS and result.url.startswith('http'): try: user = dict(keychain.get_services())[keychainservice] except KeyError: user = input('Enter username: '******'Enter password: '******'Enter credentials for {0}'.format(netloc)) outstream = StringIO() if user: if not pw and SAVE_PASSWORDS: pw = keychain.get_password(keychainservice, user) #Check again, did we retrieve a password? if not pw: user, pw = console.login_alert( 'Enter credentials for {0}'.format(netloc), login=user) host_with_auth = '{}:{}@{}'.format(user, pw, netloc) url = urlparse.urlunparse( urlparse.urlparse(result.url)._replace(netloc=host_with_auth)) porcelain.push(repo.repo.path, url, branch_name, errstream=outstream) keychain.set_password(keychainservice, user, pw) else: porcelain.push(repo.repo.path, result.url, branch_name, errstream=outstream) for line in outstream.getvalue().split('\n'): print((line.replace(pw, '*******') if pw else line)) print('success!')
def get_endpoint_certificate(endpoint): url = urlparse.urlparse(endpoint) host = url.hostname port = url.port return ssl.get_server_certificate((host, port))
def generic(self, url, pattern=None): if 'youporn.com' in url: u = self.youporn(url) try: r = client.request(url) if pattern: s=re.findall(r'%s' % pattern, r) else: patterns = [ r'''\s*=\s*[\'\"](http.+?)[\'\"]''', \ r'''\s*=\s*['"](http.+?)['"]''', \ r'''['"][0-9_'"]+:\s[\'\"]([^'"]+)''', \ r'''\(\w+\([\'\"]([^\'\"]*)''', \ r'''[\'\"]\w+[\'\"]:['"]([^'"]*)''', \ r'''\s*=\s*[\'\"](http.+?)[\'\"]''', \ r'''\s*:\s*[\'\"](//.+?)[\'\"]''', \ r'''\:[\'\"](\.+?)[\'\"]''', \ r'''\s*\(\s*[\'\"](http.+?)[\'\"]''', \ r'''\s*=\s*[\'\"](//.+?)[\'\"]''', \ r'''\w*:\s*[\'\"](http.+?)[\'\"]''', \ r'''\w*=[\'\"]([^\'\"]*)''', \ r'''\w*\s*=\s*[\'\"]([^\'\"]*)''', \ r'''(?s)<file>([^<]*)''', \ ] s = [] for pattern in patterns: l = re.findall(pattern, r) s += [i for i in l if (urlparse.urlparse(i).path).strip('/').split('/')[-1].split('.')[-1] in ['mp4', 'flv', 'm3u8']] if s: s = [i for i in s if (urlparse.urlparse(i).path).strip('/').split('/')[-1].split('.')[-1] in ['mp4', 'flv', 'm3u8']] else: s = client.parseDOM(r, 'source', ret='src', attrs = {'type': 'video.+?'}) if not s: log_utils.log('Error resolving %s :: Error: %s' % (url,str(e)), log_utils.LOGERROR) return s = ['http:' + i if i.startswith('//') else i for i in s] s = [urlparse.urljoin(url, i) if not i.startswith('http') else i for i in s] s = [x for y,x in enumerate(s) if x not in s[:y]] self.u = [] def request(i): try: i = i.replace(' ','%20') c = client.request(i, output='headers', referer=url) checks = ['video','mpegurl','html'] if any(f for f in checks if f in c['Content-Type']): self.u.append((i, int(c['Content-Length']))) except: pass threads = [] for i in s: threads.append(workers.Thread(request, i)) [i.start() for i in threads] ; [i.join() for i in threads] u = sorted(self.u, key=lambda x: x[1])[::-1] mobile_mode = kodi.get_setting('mobile_mode') if mobile_mode == 'true': u = client.request(u[-1][0], output='geturl', referer=url) else: u = client.request(u[0][0], output='geturl', referer=url) log_utils.log('Returning %s from XXX-O-DUS Resolver' % str(u), log_utils.LOGNOTICE) return u except Exception as e: log_utils.log('Error resolving %s :: Error: %s' % (url,str(e)), log_utils.LOGERROR)
def get_cookie(self, netloc, ua, timeout): try: headers = {'User-Agent': ua} request = urllib2.Request(netloc) _add_request_header(request, headers) try: response = urllib2.urlopen(request, timeout=int(timeout)) except urllib2.HTTPError as response: result = response.read(5242880) try: encoding = response.info().getheader('Content-Encoding') except: encoding = None if encoding == 'gzip': result = gzip.GzipFile( fileobj=StringIO.StringIO(result)).read() jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0] init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};', result)[-1] builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0] decryptVal = self.parseJSString(init) lines = builder.split(';') for line in lines: if len(line) > 0 and '=' in line: sections = line.split('=') line_val = self.parseJSString(sections[1]) decryptVal = int( eval( str(decryptVal) + sections[0][-1] + str(line_val))) answer = decryptVal + len(urlparse.urlparse(netloc).netloc) query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % ( netloc, jschl, answer) if 'type="hidden" name="pass"' in result: passval = re.findall('name="pass" value="(.*?)"', result)[0] query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % ( netloc, urllib.quote_plus(passval), jschl, answer) time.sleep(6) cookies = cookielib.LWPCookieJar() handlers = [ urllib2.HTTPHandler(), urllib2.HTTPSHandler(), urllib2.HTTPCookieProcessor(cookies) ] opener = urllib2.build_opener(*handlers) opener = urllib2.install_opener(opener) try: request = urllib2.Request(query) _add_request_header(request, headers) response = urllib2.urlopen(request, timeout=int(timeout)) except: pass cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies]) if 'cf_clearance' in cookie: self.cookie = cookie except: pass