def oauth_flow(s, oauth_url, username=None, password=None, sandbox=False):
"""s should be a requests session"""
r = s.get(oauth_url)
if r.status_code >= 300:
raise RuntimeError(r.text)
params = urlparse.parse_qs(urlparse.urlparse(r.url).query)
data = {
"un": username,
"width": 2560,
"height": 1440,
"hasRememberUn": True,
"startURL": params['startURL'],
"loginURL": "",
"loginType": 6,
"useSecure": True,
"local": "",
"lt": "OAUTH",
"qs": "r=https%3A%2F%2Flocalhost%3A8443%2Fsalesforce%2F21",
"locale": "",
"oauth_token": "",
"oauth_callback": "",
"login": "",
"serverid": "",
"display": "popup",
"username": username,
"pw": password,
"Login": "",
}
if sandbox:
base = "https://test.salesforce.com"
else:
base = "https://login.salesforce.com"
r2 = s.post(base, data)
m = re.search("window.location.href\s*='(.[^']+)'", r2.text)
assert m is not None, ("Couldn't find location.href expression in page {} "
"(Username or password is wrong)").format(r2.url)
u3 = "https://" + urlparse.urlparse(r2.url).hostname + m.group(1)
r3 = s.get(u3)
m = re.search("window.location.href\s*='(.[^']+)'", r3.text)
assert m is not None, ("Couldn't find location.href expression in page {}:"
"\n{}").format(r3.url, r3.text)
return m.group(1)
def run_firefox_for_android(build_obj, params):
"""
Launch Firefox for Android on the connected device.
Optional 'params' allow parameters to be passed to Firefox.
"""
device = _get_device(build_obj.substs)
try:
#
# Construct an adb command similar to:
#
# $ adb shell am start -a android.activity.MAIN \
# -n org.mozilla.fennec_$USER \
# -d <url param> \
# --es args "<params>"
#
app = build_obj.substs['ANDROID_PACKAGE_NAME']
url = None
if params:
for p in params:
if urlparse.urlparse(p).scheme != "":
url = p
params.remove(p)
break
device.launch_fennec(app, extra_args=params, url=url)
except Exception:
_log_warning("unable to launch Firefox for Android")
return 1
return 0
def _discover_auth_versions(self, session, auth_url):
# discover the API versions the server is supporting base on the
# given URL
v2_auth_url = None
v3_auth_url = None
try:
ks_discover = discover.Discover(session=session, auth_url=auth_url)
v2_auth_url = ks_discover.url_for('2.0')
v3_auth_url = ks_discover.url_for('3.0')
except ks_exc.ClientException as e:
# Identity service may not support discover API version.
# Lets trying to figure out the API version from the original URL.
url_parts = urlparse.urlparse(auth_url)
(scheme, netloc, path, params, query, fragment) = url_parts
path = path.lower()
if path.startswith('/v3'):
v3_auth_url = auth_url
elif path.startswith('/v2'):
v2_auth_url = auth_url
else:
# not enough information to determine the auth version
msg = ('Unable to determine the Keystone version '
'to authenticate with using the given '
'auth_url. Identity service may not support API '
'version discovery. Please provide a versioned '
'auth_url instead. error=%s') % (e)
raise exception.GlanceError(msg)
return (v2_auth_url, v3_auth_url)
def run_firefox_for_android(build_obj, params, **kwargs):
"""
Launch Firefox for Android on the connected device.
Optional 'params' allow parameters to be passed to Firefox.
"""
device = _get_device(build_obj.substs)
try:
#
# Construct an adb command similar to:
#
# $ adb shell am start -a android.activity.MAIN \
# -n org.mozilla.fennec_$USER \
# -d <url param> \
# --es args "<params>"
#
app = build_obj.substs['ANDROID_PACKAGE_NAME']
msg = "URL specified as '{}'; dropping URL-like parameter '{}'"
if params:
for p in params:
if urlparse.urlparse(p).scheme != "":
params.remove(p)
if kwargs.get('url'):
_log_warning(msg.format(kwargs['url'], p))
else:
kwargs['url'] = p
device.launch_fennec(app, extra_args=params, **kwargs)
except Exception:
_log_warning("unable to launch Firefox for Android")
return 1
return 0
def is_local_service(name):
"""
Determine if a service definition describes a service running on
the local node. This is true if the service URL is for localhost,
matches the machine's name, or ec2 public name
"""
if name is None:
return False
if "://" in name:
url = urlparse.urlparse(name)
if ":" in url.netloc:
name = url.netloc.split(":")[0]
else:
name = url.netloc
elif ":" in name:
name = name.split(":")[0]
if name == "localhost":
return True
if '.' in name:
name = name.split('.')[0]
node = socket.getfqdn()
if '.' in node:
node = node.split('.')[0]
if name == node:
return True
pn = public_name()
if pn is not None and pn.split(".")[0] == name:
return True
return False
def valid_url(url):
try:
_ = urlparse.urlparse(url)
except Exception:
raise NotValid("URL")
# if part[1] == "localhost" or part[1] == "127.0.0.1":
# raise NotValid("URL")
return True
def valid_url(url):
try:
_ = urlparse.urlparse(url)
except Exception:
raise NotValid("URL")
# if part[1] == "localhost" or part[1] == "127.0.0.1":
# raise NotValid("URL")
return True
def git_fetch(args):
parser = argparse.ArgumentParser(
prog='git fetch',
usage=
'git fetch [http(s)://<remote repo> or remotename] [-u username[:password]]',
description="Push to a remote repository")
parser.add_argument('url', type=str, nargs='?', help='URL to push to')
parser.add_argument('-u',
metavar='username[:password]',
type=str,
required=False,
help='username[:password]')
result = parser.parse_args(args)
repo = _get_repo()
origin = 'origin'
if not result.url:
result.url = repo.remotes.get('origin', '')
if result.url in repo.remotes:
origin = result.url
result.url = repo.remotes.get(origin)
if not urlparse.urlparse(result.url).scheme:
raise Exception(
'url must match a remote name, or must start with http:// or https://'
)
print('Starting fetch, this could take a while')
remote_refs = porcelain.fetch(repo.repo.path, result.url)
print('Fetch successful. Importing refs')
remote_tags = gittle.utils.git.subrefs(remote_refs, 'refs/tags')
remote_heads = gittle.utils.git.subrefs(remote_refs, 'refs/heads')
# Filter refs
clean_remote_tags = gittle.utils.git.clean_refs(remote_tags)
clean_remote_heads = gittle.utils.git.clean_refs(remote_heads)
# Base of new refs
heads_base = 'refs/remotes/' + origin
# Import branches
repo.import_refs(heads_base, clean_remote_heads)
for k, v in clean_remote_heads.items():
print('imported {}/{} {}'.format(heads_base, k, v))
# Import tags
repo.import_refs('refs/tags', clean_remote_tags)
for k, v in clean_remote_tags.items():
print('imported {}/{} {}'.format('refs/tags', k, v))
print('Checking for deleted remote refs')
#delete unused remote refs
for k in gittle.utils.git.subrefs(repo.refs, heads_base):
if k not in clean_remote_heads:
print('Deleting {}'.format('/'.join([heads_base, k])))
del repo.refs['/'.join([heads_base, k])]
print('Fetch complete')
def git_fetch(args):
parser = argparse.ArgumentParser(prog='git fetch'
, usage='git fetch [http(s)://<remote repo> or remotename] [-u username[:password]]'
, description="Push to a remote repository")
parser.add_argument('url', type=str, nargs='?', help='URL to push to')
parser.add_argument('-u', metavar='username[:password]', type=str, required=False, help='username[:password]')
result = parser.parse_args(args)
repo = _get_repo()
origin='origin'
if not result.url:
result.url = repo.remotes.get('origin','')
if result.url in repo.remotes:
origin=result.url
result.url=repo.remotes.get(origin)
if not urlparse.urlparse(result.url).scheme:
raise Exception('url must match a remote name, or must start with http:// or https://')
print('Starting fetch, this could take a while')
remote_refs=porcelain.fetch(repo.repo.path,result.url)
print('Fetch successful. Importing refs')
remote_tags = gittle.utils.git.subrefs(remote_refs, 'refs/tags')
remote_heads = gittle.utils.git.subrefs(remote_refs, 'refs/heads')
# Filter refs
clean_remote_tags = gittle.utils.git.clean_refs(remote_tags)
clean_remote_heads = gittle.utils.git.clean_refs(remote_heads)
# Base of new refs
heads_base = 'refs/remotes/' + origin
# Import branches
repo.import_refs(
heads_base,
clean_remote_heads
)
for k,v in clean_remote_heads.items():
print('imported {}/{} {}'.format(heads_base,k,v))
# Import tags
repo.import_refs(
'refs/tags',
clean_remote_tags
)
for k,v in clean_remote_tags.items():
print('imported {}/{} {}'.format('refs/tags',k,v))
print('Checking for deleted remote refs')
#delete unused remote refs
for k in gittle.utils.git.subrefs(repo.refs,heads_base):
if k not in clean_remote_heads:
print('Deleting {}'.format('/'.join([heads_base,k])))
del repo.refs['/'.join([heads_base,k])]
print('Fetch complete')
def extract_url_arguments(url, urlmap):
"""
This extracts the URL arguments from a given URL
"""
parsed_url = urlparse.urlparse(url)
map_adapter = urlmap.bind(
server_name=parsed_url.netloc,
script_name=parsed_url.path,
url_scheme=parsed_url.scheme,
path_info=parsed_url.path
)
return map_adapter.match()[1]
def filter_result(link):
try:
# Valid results are absolute URLs not pointing to a Google domain
# like images.google.com or googleusercontent.com
o = urlparse.urlparse(link, 'http')
if o.netloc and 'google' not in o.netloc:
return link
# Decode hidden URLs.
if link.startswith('/url?'):
link = urlparse.parse_qs(o.query)['q'][0]
# Valid results are absolute URLs not pointing to a Google domain
# like images.google.com or googleusercontent.com
o = urlparse.urlparse(link, 'http')
if o.netloc and 'google' not in o.netloc:
return link
# Otherwise, or on error, return None.
except Exception:
pass
return None
def manage_makeChanges(self, rules_raw, REQUEST=None):
'''Perform changes'''
request = REQUEST
self.rules_raw = request.rules_raw
self.rules = []
self.errors = []
tests = []
i = 0
for line in self.rules_raw.split('\n'):
i += 1
line = line.strip()
if not line or line[0] == '#':
continue
if line[0] == '=':
line = line[1:].strip()
parts = re.split(r"(?<!\\)\s+",
line) #split on non escaped whitespace
if len(parts) == 2:
tests.append(parts)
parts = re.split(r"(?<!\\)\s+",
line) #split on non escaped whitespace
if len(parts) == 2:
match, url = parts
try:
self.rules.append((re.compile(match), url))
except Exception as detail:
self.errors.append('%d: %s "%s"' % (i, str(detail), line))
else:
self.errors.append(
"Line should be DOMAIN_REGEX/PATH_REGEX NEW_URL: %s" %
line)
for oldurl, newurl in tests:
scheme, netloc, path, params, query, fragment = urlparse.urlparse(
oldurl)
res = self.redirect(netloc, path, query)
if res != newurl:
self.errors.append("TEST: %s %s != %s" % (oldurl, res, newurl))
message = "Saved changes."
return self.Redirector_editForm(self,
REQUEST,
manage_tabs_message=message)
def git_push(args):
parser = argparse.ArgumentParser(prog='git push'
, usage='git push [http(s)://<remote repo> or remote] [-u username[:password]]'
, description="Push to a remote repository")
parser.add_argument('url', type=str, nargs='?', help='URL to push to')
parser.add_argument('-u', metavar='username[:password]', type=str, required=False, help='username[:password]')
result = parser.parse_args(args)
user, sep, pw = result.u.partition(':') if result.u else (None,None,None)
repo = _get_repo()
origin='origin'
if not result.url:
result.url = repo.remotes.get('origin','')
if result.url in repo.remotes:
origin=result.url
result.url=repo.remotes.get(origin)
branch_name = os.path.join('refs','heads', repo.active_branch) #'refs/heads/%s' % repo.active_branch
print("Attempting to push to: {0}, branch: {1}".format(result.url, branch_name))
netloc = urlparse.urlparse(result.url).netloc
keychainservice = 'stash.git.{0}'.format(netloc)
if sep and not user:
# -u : clears keychain for this server
for service in keychain.get_services():
if service[0]==keychainservice:
keychain.delete_password(*service)
#Attempt to retrieve user
if not user and SAVE_PASSWORDS and result.url.startswith('http'):
try:
user = dict(keychain.get_services())[keychainservice]
except KeyError:
user = input('Enter username: '******'Enter password: '******'Enter credentials for {0}'.format(netloc))
outstream = StringIO()
if user:
if not pw and SAVE_PASSWORDS:
pw = keychain.get_password(keychainservice, user)
#Check again, did we retrieve a password?
if not pw:
user, pw = console.login_alert('Enter credentials for {0}'.format(netloc), login=user)
host_with_auth='{}:{}@{}'.format(user,pw,netloc)
url=urlparse.urlunparse(
urlparse.urlparse(result.url)._replace(
netloc=host_with_auth))
porcelain.push(repo.repo.path, url, branch_name, errstream=outstream)
keychain.set_password(keychainservice, user, pw)
else:
porcelain.push(repo.repo.path, result.url, branch_name, errstream=outstream)
for line in outstream.getvalue().split('\n'):
print((line.replace(pw, '*******') if pw else line))
print('success!')
def cfcookie(netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
req = urllib_request.Request(netloc, headers=headers)
try:
urllib_request.urlopen(req, timeout=int(timeout))
except urllib_request.HTTPError as response:
result = response.read(5242880)
jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0]
init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};',
result)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0]
decryptVal = parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = parseJSString(sections[1])
decryptVal = int(
eval(str(decryptVal) + sections[0][-1] + str(line_val)))
answer = decryptVal + len(urlparse.urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall('name="pass" value="(.*?)"', result)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (
netloc, quote_plus(passval), jschl, answer)
time.sleep(5)
cookies = cookielib.LWPCookieJar()
handlers = [
urllib_request.HTTPHandler(),
urllib_request.HTTPSHandler(),
urllib_request.HTTPCookieProcessor(cookies)
]
opener = urllib_request.build_opener(*handlers)
urllib_request.install_opener(opener)
try:
request = urllib_request.Request(query, headers=headers)
urllib_request.urlopen(request, timeout=int(timeout))
except:
pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
return cookie
except:
pass
def git_push(args):
parser = argparse.ArgumentParser(
prog='git push',
usage=
'git push [http(s)://<remote repo> or remote] [-u username[:password]]',
description="Push to a remote repository")
parser.add_argument('url', type=str, nargs='?', help='URL to push to')
parser.add_argument('-u',
metavar='username[:password]',
type=str,
required=False,
help='username[:password]')
result = parser.parse_args(args)
user, sep, pw = result.u.partition(':') if result.u else (None, None, None)
repo = _get_repo()
origin = 'origin'
if not result.url:
result.url = repo.remotes.get('origin', '')
if result.url in repo.remotes:
origin = result.url
result.url = repo.remotes.get(origin)
branch_name = os.path.join(
'refs', 'heads',
repo.active_branch) #'refs/heads/%s' % repo.active_branch
print("Attempting to push to: {0}, branch: {1}".format(
result.url, branch_name))
netloc = urlparse.urlparse(result.url).netloc
keychainservice = 'stash.git.{0}'.format(netloc)
if sep and not user:
# -u : clears keychain for this server
for service in keychain.get_services():
if service[0] == keychainservice:
keychain.delete_password(*service)
#Attempt to retrieve user
if not user and SAVE_PASSWORDS and result.url.startswith('http'):
try:
user = dict(keychain.get_services())[keychainservice]
except KeyError:
user = input('Enter username: '******'Enter password: '******'Enter credentials for {0}'.format(netloc))
outstream = StringIO()
if user:
if not pw and SAVE_PASSWORDS:
pw = keychain.get_password(keychainservice, user)
#Check again, did we retrieve a password?
if not pw:
user, pw = console.login_alert(
'Enter credentials for {0}'.format(netloc), login=user)
host_with_auth = '{}:{}@{}'.format(user, pw, netloc)
url = urlparse.urlunparse(
urlparse.urlparse(result.url)._replace(netloc=host_with_auth))
porcelain.push(repo.repo.path, url, branch_name, errstream=outstream)
keychain.set_password(keychainservice, user, pw)
else:
porcelain.push(repo.repo.path,
result.url,
branch_name,
errstream=outstream)
for line in outstream.getvalue().split('\n'):
print((line.replace(pw, '*******') if pw else line))
print('success!')
def get_endpoint_certificate(endpoint):
url = urlparse.urlparse(endpoint)
host = url.hostname
port = url.port
return ssl.get_server_certificate((host, port))
def generic(self, url, pattern=None):
if 'youporn.com' in url: u = self.youporn(url)
try:
r = client.request(url)
if pattern: s=re.findall(r'%s' % pattern, r)
else:
patterns = [
r'''\s*=\s*[\'\"](http.+?)[\'\"]''', \
r'''\s*=\s*['"](http.+?)['"]''', \
r'''['"][0-9_'"]+:\s[\'\"]([^'"]+)''', \
r'''\(\w+\([\'\"]([^\'\"]*)''', \
r'''[\'\"]\w+[\'\"]:['"]([^'"]*)''', \
r'''\s*=\s*[\'\"](http.+?)[\'\"]''', \
r'''\s*:\s*[\'\"](//.+?)[\'\"]''', \
r'''\:[\'\"](\.+?)[\'\"]''', \
r'''\s*\(\s*[\'\"](http.+?)[\'\"]''', \
r'''\s*=\s*[\'\"](//.+?)[\'\"]''', \
r'''\w*:\s*[\'\"](http.+?)[\'\"]''', \
r'''\w*=[\'\"]([^\'\"]*)''', \
r'''\w*\s*=\s*[\'\"]([^\'\"]*)''', \
r'''(?s)<file>([^<]*)''', \
]
s = []
for pattern in patterns:
l = re.findall(pattern, r)
s += [i for i in l if (urlparse.urlparse(i).path).strip('/').split('/')[-1].split('.')[-1] in ['mp4', 'flv', 'm3u8']]
if s: s = [i for i in s if (urlparse.urlparse(i).path).strip('/').split('/')[-1].split('.')[-1] in ['mp4', 'flv', 'm3u8']]
else: s = client.parseDOM(r, 'source', ret='src', attrs = {'type': 'video.+?'})
if not s:
log_utils.log('Error resolving %s :: Error: %s' % (url,str(e)), log_utils.LOGERROR)
return
s = ['http:' + i if i.startswith('//') else i for i in s]
s = [urlparse.urljoin(url, i) if not i.startswith('http') else i for i in s]
s = [x for y,x in enumerate(s) if x not in s[:y]]
self.u = []
def request(i):
try:
i = i.replace(' ','%20')
c = client.request(i, output='headers', referer=url)
checks = ['video','mpegurl','html']
if any(f for f in checks if f in c['Content-Type']): self.u.append((i, int(c['Content-Length'])))
except:
pass
threads = []
for i in s: threads.append(workers.Thread(request, i))
[i.start() for i in threads] ; [i.join() for i in threads]
u = sorted(self.u, key=lambda x: x[1])[::-1]
mobile_mode = kodi.get_setting('mobile_mode')
if mobile_mode == 'true': u = client.request(u[-1][0], output='geturl', referer=url)
else: u = client.request(u[0][0], output='geturl', referer=url)
log_utils.log('Returning %s from XXX-O-DUS Resolver' % str(u), log_utils.LOGNOTICE)
return u
except Exception as e:
log_utils.log('Error resolving %s :: Error: %s' % (url,str(e)), log_utils.LOGERROR)
def get_cookie(self, netloc, ua, timeout):
try:
headers = {'User-Agent': ua}
request = urllib2.Request(netloc)
_add_request_header(request, headers)
try:
response = urllib2.urlopen(request, timeout=int(timeout))
except urllib2.HTTPError as response:
result = response.read(5242880)
try:
encoding = response.info().getheader('Content-Encoding')
except:
encoding = None
if encoding == 'gzip':
result = gzip.GzipFile(
fileobj=StringIO.StringIO(result)).read()
jschl = re.findall('name="jschl_vc" value="(.+?)"/>', result)[0]
init = re.findall('setTimeout\(function\(\){\s*.*?.*:(.*?)};',
result)[-1]
builder = re.findall(r"challenge-form\'\);\s*(.*)a.v", result)[0]
decryptVal = self.parseJSString(init)
lines = builder.split(';')
for line in lines:
if len(line) > 0 and '=' in line:
sections = line.split('=')
line_val = self.parseJSString(sections[1])
decryptVal = int(
eval(
str(decryptVal) + sections[0][-1] + str(line_val)))
answer = decryptVal + len(urlparse.urlparse(netloc).netloc)
query = '%s/cdn-cgi/l/chk_jschl?jschl_vc=%s&jschl_answer=%s' % (
netloc, jschl, answer)
if 'type="hidden" name="pass"' in result:
passval = re.findall('name="pass" value="(.*?)"', result)[0]
query = '%s/cdn-cgi/l/chk_jschl?pass=%s&jschl_vc=%s&jschl_answer=%s' % (
netloc, urllib.quote_plus(passval), jschl, answer)
time.sleep(6)
cookies = cookielib.LWPCookieJar()
handlers = [
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(cookies)
]
opener = urllib2.build_opener(*handlers)
opener = urllib2.install_opener(opener)
try:
request = urllib2.Request(query)
_add_request_header(request, headers)
response = urllib2.urlopen(request, timeout=int(timeout))
except:
pass
cookie = '; '.join(['%s=%s' % (i.name, i.value) for i in cookies])
if 'cf_clearance' in cookie: self.cookie = cookie
except:
pass