def __init__(self, *args, **kwargs):
if 'signalwire_space_url' in kwargs:
signalwire_space_url = kwargs.pop('signalwire_space_url',
"api.signalwire.com")
else:
signalwire_space_url = os.environ['SIGNALWIRE_SPACE_URL']
p = urlparse(signalwire_space_url, 'http')
netloc = p.netloc or p.path
path = p.path if p.netloc else ''
p = ParseResult('https', netloc, path, *p[3:])
super(Client, self).__init__(*args, **kwargs)
self._api = TwilioApi(self)
self._api.base_url = p.geturl()
TwilioFax.__init__ = patched_fax_init
TwilioV1.__init__ = patched_fax_v1_init
self._fax = TwilioFax(self)
self._fax.base_url = p.geturl()
TwilioRestException.__str__ = patched_str
AccountInstance.__init__ = patched_accountinstance_init
LocalInstance.__init__ = patched_localinstance_init
TollFreeInstance.__init__ = patched_tollfreeinstance_init
ApplicationInstance.__init__ = patched_applicationinstance_init
IncomingPhoneNumberInstance.__init__ = patched_incomingphonenumberinstance_init
RecordingInstance.__init__ = patched_recordinginstance_init
TranscriptionInstance.__init__ = patched_transcriptioninstance_init
def _import_ts(
parsed_source: ParseResult, # pylint: disable=too-many-locals,too-many-arguments
parsed_destination: ParseResult,
source_grid: Grid,
customer_id: str,
force: bool, # Copy even if the data is identical
merge_ts: bool, # Merge current TS with the new period of TS
envs: Dict[str, str],
use_thread: bool = True):
# Now, it's time to upload the referenced time-series
source_url = parsed_source.geturl()
source_home = source_url[0:source_url.rfind('/') + 1]
destination_url = parsed_destination.geturl()
destination_home = destination_url[0:destination_url.rfind('/') + 1]
requests = []
for row in source_grid:
if "hisURI" in row:
source_time_serie = source_home + row["hisURI"]
destination_time_serie = destination_home + row["hisURI"]
if use_thread:
requests.append((urlparse(source_time_serie),
urlparse(destination_time_serie), customer_id,
True, False, force, True, envs))
else:
_update_grid_on_s3(urlparse(source_time_serie),
urlparse(destination_time_serie),
customer_id=customer_id,
compare_grid=True,
update_time_series=False,
force=force,
merge_ts=True,
envs=envs)
if requests:
with ThreadPool(processes=_POOL_SIZE) as pool:
pool.starmap(_update_grid_on_s3, requests)
def __Download(self, link: ParseResult, dst: str) -> int:
"""
__Download(link: ParseResult, dst: str) -> str
驱动下载本体
:param link: 下载链接
:param dst: 保存路径及文件名
:return: str
"""
size = int(urlopen(link.geturl()).info().get('Content-Length', -1))
bar = tqdm(total=size,
initial=0,
unit='B',
unit_scale=True,
desc=os.path.basename(dst),
ncols=70)
req = requests.get(url=link.geturl(), stream=True)
with open(dst, 'ab') as f:
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
bar.update(len(chunk))
else:
f.close()
bar.close()
return size
def _periodic_refresh_versions(self, parsed_uri: ParseResult,
first_time: bool) -> None:
""" Refresh list of versions """
# Refresh at a rounded period, then all cloud instances refresh data at the same time.
now = datetime.utcnow().replace(tzinfo=pytz.UTC)
next_time = now.replace(minute=0, second=0) + timedelta(
minutes=(now.minute + self._periodic_refresh) //
self._periodic_refresh * self._periodic_refresh)
assert next_time > now
if parsed_uri.scheme == "s3":
assert BOTO3_AVAILABLE, "Use 'pip install boto3'"
start_of_current_period = \
(next_time - timedelta(minutes=self._periodic_refresh)).replace(tzinfo=pytz.UTC)
s3_client = self._s3()
s3_obj_version = s3_client.list_object_versions(
Bucket=parsed_uri.netloc, Prefix=parsed_uri.path[1:])
if "Versions" in s3_obj_version:
obj_versions = [(v["LastModified"], v["VersionId"])
for v in s3_obj_version["Versions"]]
else:
meta = s3_client.get_object(Bucket=parsed_uri.netloc,
Key=parsed_uri.path[1:])
obj_versions = [meta["LastModified"], meta["VersionId"]]
obj_versions = sorted(obj_versions,
key=lambda x: x[0],
reverse=True)
self._lock.acquire() # pylint: disable=consider-using-with
all_versions = self._versions.get(parsed_uri.geturl(),
OrderedDict())
concurrency = self._function_concurrency()
for date_version, version_id in obj_versions:
if date_version not in all_versions:
# Purge refresh during current period. Then, all AWS instance see the
# same data and wait the end of the current period to refresh.
# Else, it's may be possible to have two different versions if an
# new AWS Lambda instance was created after an updated version.
if not first_time or concurrency <= 1 or date_version < start_of_current_period:
all_versions[date_version] = version_id # Add a slot
else:
log.warning(
"Ignore the version '%s' ignore until the next period.\n"
+ "Then, all lambda instance are synchronized.",
version_id)
self._versions[
parsed_uri.geturl()] = all_versions # Lru and versions)
self._lock.release()
else:
self._versions[parsed_uri.geturl()] = {
datetime(1, 1, 1, tzinfo=pytz.UTC): "direct_file"
}
if self._periodic_refresh:
partial_refresh = functools.partial(
self._periodic_refresh_versions, parsed_uri, False)
self._timer = threading.Timer((next_time - now).seconds,
partial_refresh)
self._timer.daemon = True
self._timer.start()
def format_url(self, url):
parsed_url = urlparse(url, 'http')
site_location = parsed_url.netloc or parsed_url.path
path = parsed_url.path if parsed_url.netloc else ''
site_location = site_location.replace('www.', '')
parsed_url = ParseResult('http', site_location, path, *parsed_url[3:])
if not validators.url(parsed_url.geturl()):
raise Exception("Error!")
return parsed_url.geturl(), site_location
def _open_url(url: ParseResult) -> None:
"""Opens a URL."""
opener = config.commands.open.command
try:
url_str: str = url.geturl() if url.scheme else str(RelPath(url.geturl()).path)
LOGGER.debug('Opening "%s" with %s.', url_str, opener)
with open(os.devnull, "w", encoding="utf-8") as devnull:
subprocess.Popen( # pylint: disable=consider-using-with
[opener, url_str], stdout=devnull, stderr=devnull, stdin=devnull, close_fds=True
)
except FileNotFoundError as err:
LOGGER.error(err)
def fetch_api_description(url: ParseResult, credentials: Optional[Credentials],
insecure: bool):
"""Fetch the API description from the remote MAAS instance."""
session = bones.SessionAPI.fromURL(url.geturl(),
credentials=credentials,
insecure=insecure)
return session.description
def add():
if 'logged_in' in session and session['logged_in']:
link = request.form['link'].lower()
dest = request.form['dest']
p = urlparse(dest, 'https')
netloc = p.netloc or p.path
path = p.path if p.netloc else ''
p = ParseResult('https', netloc, path, *p[3:])
dest = p.geturl()
if re.match(r'[a-z0-9-]{1,30}', link) is not None and \
Shortlink.query.get(link) is None and \
validators.url(dest):
shortlink = Shortlink(link=link, dest=dest)
db.session.add(shortlink)
db.session.commit()
session['message'] = 'Added!'
return redirect(url_for('admin'))
else:
session['message'] = 'There was an issue. Please check your input.'
return redirect(url_for('admin'))
else:
return '403 Forbidden', 403
def __init__(self,
srctype,
origurl,
hstspreloadchecker,
domain=None,
autoExitFilename=None): #{{{
self.links = []
self.alerts = set()
self.srctype = srctype
self.domain = domain
logging.debug("LoginPageChecker set domain to {}".format(self.domain))
self.HSTSPreloadListChecker = hstspreloadchecker
self.autoExitFilename = autoExitFilename
# toplevel URLs that have no path (e.g. http://test.com)
# should be converted to end in / (http://test.com/)
# otherwise things are messed up. WebKit translates http://test.com to http://test.com/ automagically
# which messes up the redirect chain. So perform this translation even before Webkit does it.
urlparts = urlparse(origurl)
if urlparts.path == "":
urlparts = ParseResult(urlparts[0], urlparts[1], "/", urlparts[3],
urlparts[4], urlparts[5])
origurl = urlparts.geturl()
self.origurl = origurl
self.pwFields = {}
self.url = None
self.initclick = None
self.preclicks = []
self.resultFlag = False
self.redirectPageResources = {}
self.mainRedirectChain = []
def _get_service_url(self):
parsed_url = urlparse(self.base_url)
encoded_get_args = self.extra_queryparams
_service_url = ParseResult(parsed_url.scheme, parsed_url.netloc,
parsed_url.path, parsed_url.params,
encoded_get_args, parsed_url.fragment)
return _service_url.geturl()
def format_url(my_url, scheme='https', return_type="url"):
"""Takes input string to valid URL format
Args:
my_url (string): URL-like string
Returns:
string: Properly formatted URL
"""
if scheme not in URL_SCHEMES:
raise ValueError(f"'scheme' arg must be in one of {URL_SCHEMES}")
if return_type not in URL_RETURN_TYPES:
raise ValueError(
f"'return_type' arg must be one of {URL_RETURN_TYPES}")
p = urlparse(my_url, scheme=scheme)
netloc = p.netloc or p.path
path = p.path if p.netloc else ''
p = ParseResult(scheme, netloc, path, *p[3:])
url = p.geturl()
if not is_valid_url(url):
raise InvalidURL
if return_type == 'parse_result':
return p
return url
def __all_url() -> str:
"""Method returns the url used to fetch all data from John hopkins server
Returns:
str: URL for getting all Covid data
"""
url = urlparse("https://services1.arcgis.com")
path = "/0MSEUqKaxRlEPj5g/arcgis/rest/services/ncov_cases/FeatureServer/2/query"
query = {}
query["f"] = "json"
query["where"] = "Confirmed > 0"
query["returnGeometry"] = "false"
query["spatialRel"] = "esriSpatialRelIntersects"
query["outFields"] = "*"
query["orderByFields"] = "Confirmed desc"
query["resultOffset"] = "0"
query["resultRecordCount"] = "200"
query["cacheHint"] = "true"
url = ParseResult(
scheme=url.scheme,
netloc=url.hostname,
path=path,
query=urlencode(query),
params=url.params,
fragment=url.fragment,
)
return url.geturl()
def __country_url(object_id: str) -> str:
"""Method formats and encodes the URL for a specific country information regarding Covid
Args:
country (str): Country name e.g. "sweden"
Returns:
str: Formatted encoded URL for the requested country
"""
url = urlparse("https://services1.arcgis.com")
path = "/0MSEUqKaxRlEPj5g/arcgis/rest/services/ncov_cases/FeatureServer/2/query"
query = {}
query["f"] = "json"
query["where"] = f"OBJECTID = {object_id}"
query["returnGeometry"] = "false"
query["spatialRel"] = "esriSpatialRelIntersects"
query["outFields"] = "*"
query["resultOffset"] = "0"
query["resultRecordCount"] = "1"
query["cacheHint"] = "true"
url = ParseResult(
scheme=url.scheme,
netloc=url.hostname,
path=path,
query=urlencode(query),
params=url.params,
fragment=url.fragment,
)
return url.geturl()
def fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult,
workspace_dir: str, kind: Kind,
uuid: str) -> Dict[str, Any]:
"""Fetch a resource by its uuid. Return `None` if nothing is found.
Raise a RuntimeError exception when a error is detected within the
gatekeeper's API."""
url = urljoin(gatekeeper_endpoint.geturl(),
os.path.join(_get_path_from_kind(kind), uuid))
_LOGGER.info('Fetching a resource by uuid at %s', url)
auth = 'Bearer ' + _get_workspace_token(workspace_dir)
res_resp = requests.get(url,
headers={
'content-type': 'application/json',
'Authorization': auth
})
try:
res_resp.raise_for_status()
except requests.exceptions.HTTPError as exc_notfound:
_LOGGER.exception(
'Failed to retrieve a resource at %s '
'(status code = %d)', res_resp.url, res_resp.status_code)
if exc_notfound.response.status_code == 404:
return None
else:
raise
tmp = res_resp.json()
if not isinstance(tmp, dict) or len(tmp) <= 0:
exc = RuntimeError('The returned json is malformed: {}'.format(tmp))
_LOGGER.error('Error while fetching a resource using an uuid: %s', exc)
raise exc
_LOGGER.info('Succeed to retrieve the resource %s (status code = %d)',
res_resp.url, res_resp.status_code)
if kind.name in tmp: # the resource is boxed
tmp = tmp[kind.name]
return tmp
def fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult,
workspace_dir: str, kind: Kind,
uuid: str) -> Dict[str, Any]:
"""Fetch a resource by its uuid. Return `None` if nothing is found.
Raise a RuntimeError exception when a error is detected within the
gatekeeper's API."""
url = urljoin(gatekeeper_endpoint.geturl(),
os.path.join(_get_path_from_kind(kind), uuid))
_LOGGER.info('Fetching a resource by uuid at %s', url)
auth = 'Bearer ' + _get_workspace_token(workspace_dir)
res_resp = requests.get(url, headers={'content-type': 'application/json',
'Authorization': auth})
try:
res_resp.raise_for_status()
except requests.exceptions.HTTPError as exc_notfound:
_LOGGER.exception('Failed to retrieve a resource at %s '
'(status code = %d)', res_resp.url,
res_resp.status_code)
if exc_notfound.response.status_code == 404:
return None
else:
raise
tmp = res_resp.json()
if not isinstance(tmp, dict) or len(tmp) <= 0:
exc = RuntimeError('The returned json is malformed: {}'.format(tmp))
_LOGGER.error('Error while fetching a resource using an uuid: %s', exc)
raise exc
_LOGGER.info('Succeed to retrieve the resource %s (status code = %d)',
res_resp.url, res_resp.status_code)
if kind.name in tmp: # the resource is boxed
tmp = tmp[kind.name]
return tmp
async def async_setup_entry(hass, entry, async_add_entities):
"""Add solarlog entry."""
host_entry = entry.data[CONF_HOST]
device_name = entry.title
url = urlparse(host_entry, "http")
netloc = url.netloc or url.path
path = url.path if url.netloc else ""
url = ParseResult("http", netloc, path, *url[3:])
host = url.geturl()
try:
api = await hass.async_add_executor_job(SolarLog, host)
_LOGGER.debug("Connected to Solar-Log device, setting up entries")
except (OSError, HTTPError, Timeout):
_LOGGER.error(
"Could not connect to Solar-Log device at %s, check host ip address", host
)
return
# Create solarlog data service which will retrieve and update the data.
data = await hass.async_add_executor_job(SolarlogData, hass, api, host)
# Create a new sensor for each sensor type.
entities = []
for sensor_key in SENSOR_TYPES:
sensor = SolarlogSensor(entry.entry_id, device_name, sensor_key, data)
entities.append(sensor)
async_add_entities(entities, True)
return True
def _fetch_resource_by_uuid(gatekeeper_endpoint: ParseResult, path: str,
uuid: str) -> Dict[str, Any]:
"""Fetch a resource by its uuid. Return `None` if nothing is found.
Raise a RuntimeError exception when a error is detected within the
gatekeeper's API."""
url = urljoin(gatekeeper_endpoint.geturl(), os.path.join(path, uuid))
_LOGGER.info('Fetching a resource by uuid at %s', url)
res_resp = requests.get(url, headers={'content-type': 'application/json'})
try:
res_resp.raise_for_status()
except requests.exceptions.HTTPError as exc_notfound:
_LOGGER.exception(
'Failed to retrieve a resource at %s '
'(status code = %d)', res_resp.url, res_resp.status_code)
if exc_notfound.response.status_code == 404:
return None
else:
raise
tmp = res_resp.json()
if not isinstance(tmp, dict) or len(tmp) <= 0:
exc = RuntimeError('The returned json is malformed: {}'.format(tmp))
_LOGGER.error(exc)
raise exc
_LOGGER.info('Succeed to retrieve the resource %s (status code = %d)',
res_resp.url, res_resp.status_code)
return tmp
def get_past_league_data(league_name):
data_links = scrape_data_links()
try:
ninja_url = ParseResult(
scheme="https",
netloc="poe.ninja",
path=data_links[league_name],
params="",
query="",
fragment="",
)
with HTMLSession() as session:
response = session.get(ninja_url.geturl())
zip = zipfile.ZipFile(BytesIO(response.content))
zipped_contents = {}
for info in zip.infolist():
zipped_contents[info.filename.split(".")[-2]] = pd.read_csv(
StringIO(zip.read(info.filename).decode()), sep=";"
)
return zipped_contents
except KeyError:
raise KeyError(
f"{league_name} not a valid league name, try: {data_links.keys()}"
)
def urlparams(url_, fragment=None, query_dict=None, **query):
"""
Add a fragment and/or query parameters to a URL.
New query params will be appended to exising parameters, except duplicate
names, which will be replaced.
"""
url_ = urlparse(url_)
fragment = fragment if fragment is not None else url_.fragment
q = url_.query
new_query_dict = (QueryDict(smart_bytes(q), mutable=True)
if q else QueryDict('', mutable=True))
if query_dict:
for k, l in query_dict.lists():
new_query_dict[k] = None # Replace, don't append.
for v in l:
new_query_dict.appendlist(k, v)
for k, v in query.items():
# Replace, don't append.
if isinstance(v, list):
new_query_dict.setlist(k, v)
else:
new_query_dict[k] = v
query_string = urlencode([(k, v) for k, l in new_query_dict.lists()
for v in l if v is not None])
new = ParseResult(url_.scheme, url_.netloc, url_.path, url_.params,
query_string, fragment)
return new.geturl()
def fetch_gitlab_versions(url: ParseResult) -> List[Version]:
match = GITLAB_API.match(url.geturl())
if not match:
return []
domain = match.group("domain")
project_id = match.group("project_id")
gitlab_url = f"https://{domain}/api/v4/projects/{project_id}/repository/tags"
info(f"fetch {gitlab_url}")
resp = urllib.request.urlopen(gitlab_url)
json_tags = json.loads(resp.read())
if len(json_tags) == 0:
raise VersionError("No git tags found")
releases = []
tags = []
for tag in json_tags:
name = tag["name"]
assert isinstance(name, str)
if tag["release"]:
# TODO: has gitlab preleases?
releases.append(Version(name))
else:
tags.append(Version(name))
# if no release is found, use latest tag
if releases == []:
return tags
return releases
def __Select_Driver(self, drivers_link: ParseResult) -> ResultSet:
"""
__Select_Driver(drivers_link: ParseResult) -> ResultSet
具体版本号驱动列表
:param drivers_link: 具体版本号驱动链接
:return: ResultSet
"""
html = requests.get(url=drivers_link.geturl())
for retry in range(3):
if html.status_code == 200:
break
html = requests.get(url=drivers_link.geturl())
html.encoding = html.apparent_encoding
soup = BeautifulSoup(html.text, 'html.parser')
return soup.pre.select(selector='a')
def get_url(parent_url, url):
parsed_parent_url = urlparse(parent_url)
parsed_url = urlparse(url)
scheme = parsed_url.scheme
netloc = parsed_url.netloc
path = parsed_url.path
if len(scheme) == 0:
scheme = parsed_parent_url.scheme
if len(netloc) == 0:
netloc = parsed_parent_url.netloc
if path == '/':
path = ''
parsed_url = ParseResult(scheme=scheme,
netloc=netloc,
path=path,
params=parsed_url.params,
query=parsed_url.query,
fragment='')
return parsed_url.geturl()
def proc_addr(addr: str) -> Tuple[str, str, int]:
p = urlparse(addr, 'http')
netloc = p.netloc or p.path
if netloc.startswith(':'):
netloc = 'localhost' + netloc
path = p.path if p.netloc else ''
p = ParseResult('http', netloc, path, '', '', '')
return (p.geturl(), p.hostname, p.port)
def modify_site_name(site):
if not site:
return
site = urlparse(site, 'http')
netloc = site.netloc or site.path
path = site.path if site.netloc else ''
site = ParseResult(site.scheme, netloc, path, *site[3:])
return site.geturl()
def __init__(self, baseUrl: ParseResult):
self.baseUrl = baseUrl
self.queueManager = QueueManager(baseUrl.geturl())
self.reqHeaders = deepcopy(config.headers)
self.reqHeaders['User-Agent'] = config.userAgents["Google"]
self.supportedPlatforms = []
for platform in config.supportedPlatforms:
self.supportedPlatforms.append(
config.SocialPlatform(**config.supportedPlatforms[platform]))
def get_redirect_url(self):
parsed = urlparse(self.service)
query = parse_qs(parsed.query)
query['ticket'] = [self.ticket]
query = [((k, v) if len(v) > 1 else (k, v[0])) for k, v in query.items()]
parsed = ParseResult(parsed.scheme, parsed.netloc,
parsed.path, parsed.params,
urlencode(query), parsed.fragment)
return parsed.geturl()
def convert_url(website_link: str) -> str:
# Prepend http schema to url if needed
p = urlparse(website_link, 'http')
netloc = p.netloc or p.path
path = p.path if p.netloc else ''
# if not netloc.startswith('www.'):
# netloc = 'www.' + netloc
p = ParseResult(p.scheme, netloc, path, *p[3:])
return p.geturl()
def service_url(self):
parsed_url = urlparse(self.base_url)
encoded_get_args = self.extra_queryparams
service_url = ParseResult(parsed_url.scheme, parsed_url.netloc,
parsed_url.path, parsed_url.params,
encoded_get_args, parsed_url.fragment)
service_url = service_url.geturl() if not self.proxy_base else urljoin(
settings.SITEURL, reverse('service_proxy', args=[self.id]))
return service_url
def format_url(url):
"""
Formats url by adding 'http://' if necessary and deleting 'www.'
:param url: ulr to article or domain
:return: formatted url e.g. the following urls:
'http://www.google.pl/', 'google.pl/', 'google.pl/', 'www.google.pl/',
'http://google.pl/', 'https://www.google.pl/'
will be all formatted to: http://google.pl/
"""
parsed_url = urlparse(url, 'http')
netloc = parsed_url.netloc or parsed_url.path
path = parsed_url.path if parsed_url.netloc else ''
netloc = netloc.replace('www.', '')
parsed_url = ParseResult('http', netloc, path, *parsed_url[3:])
if not validators.url(parsed_url.geturl()):
raise ValueError('Provided url=' + url + ' is not valid')
return parsed_url.geturl()
def go_to_page(self, page):
u = urlparse(self.br.geturl())
data = dict(parse_qsl(u.query))
if page == 0:
del data['page']
else:
data['page'] = page
res = ParseResult(scheme=u.scheme, netloc=u.hostname, path=u.path, params=u.params, query=urlencode(data),
fragment=u.fragment)
self.br.open(res.geturl())
def createURL(category):
global queryURL
parsedResult = ParseResult(scheme='https',
netloc=baseURL,
path='',
params='',
query=("q=%s" % category),
fragment='')
queryURL = parsedResult.geturl()
def url_parse(self, site):
"""Parse the url into a valid url."""
parse = list(
urlparse(site if urlparse(site).scheme else '%s%s' % ('//', site)))
parse[0] = b'https'
for index, value in enumerate(parse):
parse[index] = value.decode('utf-8') if isinstance(value, bytes) \
else value
parse = ParseResult(*parse)
return parse.geturl()
def __init__(self, username=None, password=None, site=None, email=None):
self.username = username
self.password = password
parse = list(urlparse(site if urlparse(site).scheme else
'%s%s' % ('//', site)))
parse[0] = b'https'
for index, value in enumerate(parse):
parse[index] = value.decode('utf-8') if isinstance(value, bytes) \
else value
parse = ParseResult(*parse)
self.url = parse.geturl()
self.email = email
def parse(self, url, respar):
res = urlparse(url)
nl = res.path
#print(res)
if respar.path.rfind("/")>=0:
nl = respar.path[0:respar.path.rfind("/")+1] + res.path
res2 = ParseResult(scheme = res.scheme or respar.scheme,
netloc = res.netloc or respar.netloc,
path = nl,
params='', query='', fragment='')
url = res2.geturl()
#print("parse", url, res2, respar)
return (url, res2)
def download_key_http(address, port):
url = ParseResult(
scheme='http',
# This seems to work well enough with both IPv6 and IPv4
netloc="[[%s]]:%d" % (address, port),
path='/',
params='',
query='',
fragment='')
log.debug("Starting HTTP request")
data = requests.get(url.geturl(), timeout=5).content
log.debug("finished downloading %d bytes", len(data))
return data
def handle_authcode(request, client, redirection_uri, state=None):
parts = urlparse(redirection_uri.uri)
qparams = dict(parse_qsl(parts.query))
user_id = authenticated_userid(request)
auth_code = Oauth2Code(client, user_id)
db.add(auth_code)
db.flush()
qparams['code'] = auth_code.authcode
if state:
qparams['state'] = state
parts = ParseResult(
parts.scheme, parts.netloc, parts.path, parts.params,
urlencode(qparams), '')
return HTTPFound(location=parts.geturl())
def fetch_resource(gatekeeper_endpoint: ParseResult, workspace_dir: str,
kind: Kind, vendor: str, name: str,
version: str) -> Tuple[str, Dict[str, Any]]:
"""Fetch a resource and return the Json as a dictionary. Return `None` if
nothing is found. It raise a RuntimeError exception when a gatekeeper API
is dectected"""
url = urljoin(gatekeeper_endpoint.geturl(), _get_path_from_kind(kind))
_LOGGER.info('Fetching a %s resource by name at %s', kind, url)
query_params_raw = {'vendor': vendor, # Dict[Str, Str]
'name': name,
'version': version}
# We force the order of the query's parameters to lower the impact on tests
# when a key is added or removed
query_params = collections.OrderedDict(sorted(query_params_raw.items()))
auth = 'Bearer ' + _get_workspace_token(workspace_dir)
res_resp = requests.get(url, params=query_params,
headers={'content-type': 'application/json',
'Authorization': auth})
try:
res_resp.raise_for_status()
except requests.exceptions.HTTPError:
_LOGGER.exception('Failed to retrieve a resource at %s '
'(status code = %d)', res_resp.url,
res_resp.status_code)
# REMARK: if nothing is found, then the API return an empty [] and
# not 404
raise
tmp = res_resp.json()
if not isinstance(tmp, list):
exc = RuntimeError('The returned json is not boxed by a list')
_LOGGER.error('The GK API must return a list of resources: %s', exc)
raise exc
_LOGGER.info('Succeed to retrieve the resource %s (status code = %d): %s',
res_resp.url, res_resp.status_code, tmp[:20])
for elt in tmp:
this_uuid = elt['uuid']
if not this_uuid:
_LOGGER.warning('Ignoring an element without id or uuid: %s', elt)
continue
if kind.name in elt: # the resource is boxed
elt = elt[kind.name]
if all([elt['vendor'] == vendor, elt['name'] == name,
elt['version'] == version]):
return (this_uuid, elt)
return None
def get_direct_url(url, headers):
"""Gets the zip direct download link from the project download page"""
direct_download_url = href_from_link_text(url,
headers,
'Problems Downloading')
parsed_download_url = urlparse(direct_download_url)
if parsed_download_url.scheme not in ['http', 'https']:
# url is relative, and is missing the scheme and netloc
parsed_parent_url = urlparse(url)
parsed_download_url = ParseResult(parsed_parent_url.scheme,
parsed_parent_url.netloc,
parsed_download_url.path,
parsed_download_url.params,
parsed_download_url.query,
parsed_download_url.fragment)
direct_download_url = parsed_download_url.geturl()
direct_download_url = href_from_link_text(direct_download_url,
headers,
'direct link')
return direct_download_url
def __init__(self, parsed: ParseResult = None):
self.parsed = parsed
if parsed is None:
self.kind = TaskCreationType.Empty
self.url = None
return
path = parsed.path
scheme = parsed.scheme
url = parsed.geturl()
# subtaskInfo is set by Model
self.subtaskInfo = []
self.url = url
self.path = None
self.kind = None
if path.endswith(".torrent"):
if scheme == "":
self.kind = TaskCreationType.LocalTorrent
return
elif scheme in ("http", "https", "ftp"):
self.kind = TaskCreationType.RemoteTorrent
return
if path.endswith(".metalink") or path.endswith(".meta4"):
if scheme in ("http", "https", "ftp"):
self.kind = TaskCreationType.MetaLink
elif scheme == "ed2k":
self.kind = TaskCreationType.Emule
elif scheme == "magnet":
self.kind = TaskCreationType.Magnet
elif scheme in ("http", "https", "ftp"):
self.kind = TaskCreationType.Normal
def batch_raw_query(prometheus_endpoint: ParseResult,
start_timestamp: int,
end_timestamp: int,
step: datetime.timedelta,
query: str,
maxpts=11000) -> Iterable[bytes]:
"""Retrieve metrics from a Prometheus database"""
sstep = '{}s'.format(int(step.total_seconds()))
url = urljoin(prometheus_endpoint.geturl(), 'api/v1/query_range')
def sub(sub_start, sub_end):
"""sub"""
payload = [('start', sub_start),
('end', sub_end),
('step', sstep),
('query', query)]
req = requests.get(url, params=payload)
return req.content
delta = end_timestamp - start_timestamp
batch_size = min(delta // int(step.total_seconds()), maxpts) # type: int
for limits in _create_batches(start_timestamp, end_timestamp, batch_size):
sub_start, sub_end = limits
yield sub(sub_start, sub_end)
class Url(object):
"""An object based representation of a Url.
"""
@property
def scheme(self):
return self._parts.scheme
@property
def netloc(self):
return self._parts.netloc
@property
def hostname(self):
return self._parts.hostname
@property
def subdomain(self):
"""
Returns the subdomain for the URL.
With thanks: http://stackoverflow.com/questions/1189128/regex-to-extract-subdomain-from-url
"""
regex = r'(?:http[s]*\:\/\/)*(.*?)\.(?=[^\/]*\..{2,5})'
hostname = self.hostname
if not hostname:
return None
matches = re.match(regex, hostname)
return matches.group(1) if matches else None
@property
def port(self):
return self._parts.port
@property
def path(self):
return self._parts.path
def path_index(self, index=0):
try:
split_path = self.path.strip('/').split('/')
return split_path[index]
except:
return None
@property
def params(self):
return self._parts.params
@property
def query(self):
return self._parts.query
@property
def fragment(self):
return self._parts.fragment
@property
def username(self):
return self._parts.username
@property
def password(self):
return self._parts.password
def __init__(self, url):
"""Initialize the url object.
Create a new Url object from either a well formed url string,
a dict of key/values, or a ParseResult.
Args:
url (mixed): The value to generate the url from.
"""
if isinstance(url, ParseResult):
self._parts = url
elif isinstance(url, dict):
if 'hostname' in url and 'netloc' not in url:
netloc = url.pop('hostname')
if 'port' in url:
netloc += ':' + str(url.pop('port'))
url['netloc'] = netloc
if 'scheme' not in url:
url['scheme'] = 'http'
if 'username' in url:
url['netloc'] = '{0}:{1}@{2}'.format(
url.pop('username'), url.pop('password', ''), url['netloc'])
if 'params' not in url:
url['params'] = None
if 'fragment' not in url:
url['fragment'] = None
if 'path' not in url:
url['path'] = '/'
if 'query' not in url:
url['query'] = None
self._parts = ParseResult(**url)
elif isinstance(url, str):
self._parts = urlparse(url)
def assemble(self):
return self._parts.geturl()
def __str__(self):
return self.assemble()
def __init__(self,
username,
password,
site='https://tutor-qa.openstax.org',
email=None,
email_username=None,
email_password=None,
driver_type='chrome',
capabilities=None,
pasta_user=None,
wait_time=DEFAULT_WAIT_TIME,
opera_driver='',
existing_driver=None,
**kwargs):
"""
Base user constructor.
username (string): website username
password (string): website password
site (string): website URL
driver_type (string): web browser type
pasta_user (PastaSauce): optional API access for saucelabs
capabilities (dict): browser settings; copy object to avoid overwrite
Defaults:
DesiredCapabilities.ANDROID.copy()
DesiredCapabilities.CHROME.copy()
DesiredCapabilities.EDGE.copy()
DesiredCapabilities.FIREFOX.copy()
DesiredCapabilities.HTMLUNIT.copy()
DesiredCapabilities.HTMLUNITWITHJS.copy()
DesiredCapabilities.INTERNETEXPLORER.copy()
DesiredCapabilities.IPAD.copy()
DesiredCapabilities.IPHONE.copy()
DesiredCapabilities.ORERA.copy()
DesiredCapabilities.PHANTOMJS.copy()
DesiredCapabilities.SAFARI.copy()
Keys:
platform
browserName
version
javascriptEnabled
wait (int): standard time, in seconds, to wait for Selenium commands
opera_driver (string): Chromium location
"""
self.username = username
self.password = password
parse = list(
urlparse(
site if urlparse(site).scheme
else '%s%s' % ('//', site)
)
)
parse[0] = b'https'
for index, value in enumerate(parse):
parse[index] = value.decode('utf-8') if isinstance(value, bytes) \
else value
parse = ParseResult(*parse)
self.url = parse.geturl()
self.email = email
self.email_username = email_username
self.email_password = email_password
super(User, self).__init__(driver_type=driver_type,
capabilities=capabilities,
pasta_user=pasta_user,
wait_time=wait_time,
opera_driver=opera_driver,
existing_driver=existing_driver,
**kwargs)
