def _http_or_https(self, download_class, url, **kwargs):
"""
Build a downloader for http:// or https:// URLs.
Args:
download_class (:class:`~pulpcore.plugin.download.BaseDownloader`): The download
class to be instantiated.
url (str): The download URL.
kwargs (dict): All kwargs are passed along to the downloader. At a minimum, these
include the :class:`~pulpcore.plugin.download.BaseDownloader` parameters.
Returns:
:class:`~pulpcore.plugin.download.HttpDownloader`: A downloader that
is configured with the remote settings.
"""
options = {"session": self._session}
if self._remote.proxy_url:
options["proxy"] = self._remote.proxy_url
if self._remote.username and self._remote.password:
options["auth"] = aiohttp.BasicAuth(login=self._remote.username,
password=self._remote.password)
return download_class(url, **options, **kwargs)
async def test_proxy_http_auth(proxy_test_server, get_request) -> None:
url = 'http://aiohttp.io/path'
proxy = await proxy_test_server()
await get_request(url=url, proxy=proxy.url)
assert 'Authorization' not in proxy.request.headers
assert 'Proxy-Authorization' not in proxy.request.headers
auth = aiohttp.BasicAuth('user', 'pass')
await get_request(url=url, auth=auth, proxy=proxy.url)
assert 'Authorization' in proxy.request.headers
assert 'Proxy-Authorization' not in proxy.request.headers
await get_request(url=url, proxy_auth=auth, proxy=proxy.url)
assert 'Authorization' not in proxy.request.headers
assert 'Proxy-Authorization' in proxy.request.headers
await get_request(url=url, auth=auth, proxy_auth=auth, proxy=proxy.url)
assert 'Authorization' in proxy.request.headers
assert 'Proxy-Authorization' in proxy.request.headers
async def secure(self, user, passwd):
""" Setting the password... and remembering it for subsequent access """
if passwd:
params = {"enabled": 1, "username": user, "password": passwd}
self.myauth = aioh.BasicAuth(login=user, password=passwd)
try:
async with aioh.ClientSession(auth=self.myauth) as session:
async with session.request(
"get", "http://192.168.33.1/settings/login", params=params
) as resp:
logging.debug(resp.url)
logging.debug(
"Shelly: Response status was {}".format(resp.status)
)
if resp.status != 200:
self.myauth = None
try:
logging.debug(
"Shelly: Response was {}".format(await resp.text())
)
except:
pass
logging.debug(
"Shelly: Password %sset"
% ((self.myauth is None and "not") or "")
)
await asyncio.sleep(DELAY)
except Exception as e:
logging.debug(
"Shelly: Something really went wrong when setting user/password: {}".format(
e
)
)
await asyncio.sleep(0)
else:
await asyncio.sleep(0)
def __init__(self, hass, client_id, client_secret, redirect_url, scope,
api_auth_url, api_functions_url):
"""Initialize a NIBE API client."""
self.hass = hass
self.client_id = client_id
self.client_secret = client_secret
self.redirect_url = redirect_url
self.scope = scope
self.api_auth_url = api_auth_url
self.api_functions_url = api_functions_url
# hass.config.path(DEFAULT_AUTH_DATA_FILE)
self.auth_data_file_path = os.path.join(os.path.dirname(__file__),
DEFAULT_AUTH_DATA_FILE)
self.access_token = None
self.refresh_token = None
self.asyncio_lock = asyncio.Lock()
self.session = aiohttp.ClientSession(headers=DEFAULT_SESSION_HEADERS,
auth=aiohttp.BasicAuth(
self.client_id,
self.client_secret))
_LOGGER.info("Nibe Uplink lib version: {}".format(__version__))
async def upload_extension_attribute(session, url, user, passwd, ext_attr, semaphore):
mypath = dirname(realpath(__file__))
auth = aiohttp.BasicAuth(user, passwd)
headers = {'Accept': 'application/xml','Content-Type':'application/xml'}
# Get the script files within the folder, we'll only use script_file[0] in case there are multiple files
script_file = [ f.name for f in os.scandir(join('extension_attributes', ext_attr)) if f.is_file() and f.name.split('.')[-1] in supported_ea_extensions]
if script_file == []:
print('Warning: No script file found in extension_attributes/%s' % ext_attr)
return # Need to skip if no script.
with open(join(mypath, 'extension_attributes', ext_attr, script_file[0]), 'r') as f:
data=f.read()
async with semaphore:
with async_timeout.timeout(args.timeout):
template = await get_ea_template(session, url, user, passwd, ext_attr)
async with session.get(url + '/JSSResource/computerextensionattributes/name/' + template.find('name').text,
auth=auth, headers=headers) as resp:
template.find('input_type/script').text = data
if args.verbose:
print(ET.tostring(template))
print('response status initial get: ',resp.status)
if resp.status == 200:
put_url = url + '/JSSResource/computerextensionattributes/name/' + template.find('name').text
resp = await session.put(put_url, auth=auth, data=ET.tostring(template), headers=headers)
else:
post_url = url + '/JSSResource/computerextensionattributes/id/0'
resp = await session.post(post_url, auth=auth, data=ET.tostring(template), headers=headers)
if args.verbose:
print('response status: ',resp.status)
print('EA: ',ext_attr)
print('EA Name: ',template.find('name').text)
if resp.status in (201, 200):
print('Uploaded Extension Attribute: %s' % template.find('name').text)
else:
print('Error uploading script: %s' % template.find('name').text)
print('Error: %s' % resp.status)
return resp.status
def __init__(self,
controller,
address,
port,
username,
password,
https=False,
loop=False):
self.controller = controller
self._address = address
self._port = port
self._https = https # not implemented
self.loop = loop
# time outs
self._heart_beat = 30
self._request_timeout = 30
self._sleep_time = 10
self._websocket_connected = False
self._websocket_url = "ws://" + self._address + "/rest/subscribe"
self._http_connected = False
self._rest_url = "http://" + self._address + "/rest/"
self.session = None
self.auth = aiohttp.BasicAuth(username, password)
self.timeout = aiohttp.ClientTimeout(total=60,
connect=60,
sock_connect=60,
sock_read=60)
self.loop.run_until_complete(self.create_new_session())
self.keep_listening = True
async def execute(self, parameters, name, start, end):
query = self.build_query(name, start, end, parameters)
url = self.configuration["url"] + "/api/query"
credentials = self.configuration["credentials"].split(":", maxsplit=1)
auth = aiohttp.BasicAuth(credentials[0], credentials[1])
async with aiohttp.ClientSession(auth=auth) as session:
async with session.post(url, json=query) as r:
if r.status != 200:
raise DataFetchException(str(r.text))
result = await r.json()
# Convert the query result to be compliant with the Pandas compute
timeseries = []
for ts in result:
timeseries.append({
"metric": ts["metric"],
"tags": ts["tags"],
"dps": {int(k): v
for k, v in ts["dps"].items()},
})
return timeseries
def __init__(self, hass, device_info):
"""Initialize a generic camera."""
super().__init__()
self.hass = hass
self._authentication = device_info.get(CONF_AUTHENTICATION)
self._name = device_info.get(CONF_NAME)
self._still_image_url = device_info[CONF_STILL_IMAGE_URL]
self._still_image_url.hass = hass
self._limit_refetch = device_info[CONF_LIMIT_REFETCH_TO_URL_CHANGE]
self.content_type = device_info[CONF_CONTENT_TYPE]
username = device_info.get(CONF_USERNAME)
password = device_info.get(CONF_PASSWORD)
if username and password:
if self._authentication == HTTP_DIGEST_AUTHENTICATION:
self._auth = HTTPDigestAuth(username, password)
else:
self._auth = aiohttp.BasicAuth(username, password=password)
else:
self._auth = None
self._last_url = None
self._last_image = None
async def fetch_repositories(
self,
sess: aiohttp.ClientSession,
) -> AsyncIterator[str]:
api_url = self.registry_url / 'api' / 'v2.0'
registry_projects = self.registry_info['project']
rqst_args = {}
if self.credentials:
rqst_args['auth'] = aiohttp.BasicAuth(
self.credentials['username'],
self.credentials['password'],
)
repo_list_url: Optional[yarl.URL]
for project_name in registry_projects:
repo_list_url = (api_url / 'projects' / project_name /
'repositories').with_query({'page_size': '30'})
while repo_list_url is not None:
async with sess.get(repo_list_url,
allow_redirects=False,
**rqst_args) as resp:
items = await resp.json()
if isinstance(items, dict) and (errors := items.get(
'errors', [])):
raise RuntimeError(
f"failed to fetch repositories in project {project_name}",
errors[0]['code'], errors[0]['message'])
repos = [item['name'] for item in items]
for item in repos:
yield item
repo_list_url = None
next_page_link = resp.links.get('next')
if next_page_link:
next_page_url = cast(yarl.URL, next_page_link['url'])
repo_list_url = (self.registry_url.with_path(
next_page_url.path).with_query(
next_page_url.query))
async def getInstaclustrMetrics(cluster_id,
metrics_list,
auth={},
index=0,
dump_file=False):
auth_details = aiohttp.BasicAuth(login=auth.get("ic_user_name"),
password=auth.get("ic_api_key"))
target = ic_topic_metrics_url.format(cluster_id, ','.join(metrics_list))
session = aiohttp.ClientSession()
async with session.get(url=target, auth=auth_details) as response:
if response.status != 200 or response.headers[
'Content-Type'] != 'application/json':
logger.error(
'Missing metrics data from instaclustr - HTTP response code: {0}; \
HTTP Header Content-Type: {1}'.format(response.status,
response.headers['Content-Type']))
await session.close()
return None
metrics = await response.text()
# Dump metrics response as a JSON payload (for testing).
if dump_file:
await dump(metrics, 'instaclustr-{0}.json'.format(index))
await session.close()
return metrics
async def update_quotarule_qtree(
self,
spahali: int,
spasoli: int,
fihali: int,
fisoli: int,
rule_uuid,
) -> ClientResponse:
dataobj = {
"space": {"hard_limit": spahali, "soft_limit": spasoli},
"files": {"hard_limit": fihali, "soft_limit": fisoli},
}
headers = {"content-type": "application/json", "accept": "application/hal+json"}
async with self._session.patch(
f"{self.endpoint}/api/storage/quota/rules/{rule_uuid}",
auth=aiohttp.BasicAuth(self.user, self.password),
headers=headers,
json=dataobj,
ssl=False,
raise_for_status=True,
) as resp:
return await resp.json()
async def api_request(self,
method: str,
route: str,
*,
silent=False,
**kwargs):
if not self.is_api_running:
self.loop.create_task(self.check_api_running())
if not silent:
raise exceptions.DemocracivBotAPIError(
f"{config.NO} Internal API is not running, try again later."
)
auth = aiohttp.BasicAuth(token.API_USER, token.API_PASSWORD)
try:
async with self.session.request(method,
f"{self.BASE_API}/{route}",
auth=auth,
**kwargs) as response:
if response.status == 200:
return await response.json()
if response.status == 401:
logging.error(
"API_USER & API_PASSWORD in /bot/token.py does not match "
"auth['user'] and auth['password'] in /api/token.json - 401 Unauthorized"
)
if not silent:
raise exceptions.DemocracivBotAPIError(
f"{config.NO} Something went wrong.")
except aiohttp.ClientError:
if not silent:
raise
def __init__(self,
base_url,
loop=None,
session=None,
verify_ssl=True,
timeout=10,
auth=None):
if not isinstance(base_url, URL):
raise TypeError("base_url should be instance of URL")
if loop is None:
loop = asyncio.get_event_loop()
self.loop = loop
self.base_url = base_url
self.timeout = timeout
if isinstance(auth, str):
auth = aiohttp.BasicAuth(*auth.split(":"))
if auth is not None and not isinstance(auth, aiohttp.BasicAuth):
raise TypeError("auth should be str or aiohttp.BasicAuth")
self.auth = auth
if session is None:
session = aiohttp.ClientSession(
auth=auth,
raise_for_status=True,
connector=aiohttp.TCPConnector(use_dns_cache=True,
loop=loop,
verify_ssl=verify_ssl),
)
self.session = session
self.headers = {}
async def query(qurl, loop, page=1):
global proxy
conn = aiohttp.TCPConnector(limit=50)
count = len(proxy)
proxy_auth = None
proxy_host = None
if count > 0:
random_index = random.randint(0, count - 1)
proxy_info = proxy[random_index]
proxy_auth = aiohttp.BasicAuth(proxy_info['user'], proxy_info['pwd'])
proxy_host = proxy_info['host']
headers = {
"Cookie":
'''_qqq_uuid_="2|1:0|10:1494948386|10:_qqq_uuid_|56:NDUyY2RjZWRmM2ViZTU1MmRlODZjNDhlN2Q5ZTgyNjk1ODZjNDYxMQ==|8f70e6e7ecaa91941eb1acb75783419282af1715d4ef390c0554a9c65afea52e"; __cur_art_index=6400; _xsrf=2|bdadfff7|a64a26176df54659196af500457394c0|1495344490; Hm_lvt_2670efbdd59c7e3ed3749b458cafaa37=1494948394,1494949487,1495115207,1495344493; Hm_lpvt_2670efbdd59c7e3ed3749b458cafaa37=1495348175; _ga=GA1.2.875152845.1494948395; _gid=GA1.2.1955986889.1495348176; _gat=1''',
"User-Agent":
'''Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.87 Safari/537.36'''
}
async with aiohttp.ClientSession(headers=headers,
connector=conn,
loop=loop) as session:
async with session.get(qurl, proxy=proxy_host,
auth=proxy_auth) as resp:
cur_url = resp.url
pre_url = None
if resp.history:
pre_url = resp.history[0].url
cur_url = str(cur_url)[:-10]
cur_url = cur_url.strip('''/''')
pre_url = str(pre_url)[:-10]
pre_url = pre_url.strip(''''/''')
if pre_url and cur_url != pre_url and page != 1:
logging.warning('cur:%s||pre:%s' % (cur_url, pre_url))
return None
html = await resp.text()
return html
async def delete_bindings(self, *queues: Tuple[QueueName, ExchangeName]):
auth = (aiohttp.BasicAuth(login=self.http_api_url.user,
password=self.http_api_url.password)
if self.http_api_url.user and self.http_api_url.password else
None)
channel = await self.data_connection.channel()
try:
logger.info("Declaring exchanges...")
await asyncio.gather(*[
channel.declare_exchange(exchange, passive=True)
for _, exchange in queues
])
async with ClientSession(auth=auth) as session:
delete_tasks = {
self.delete_queue_bindings(
queue_name=queue,
exchange=exchange,
http_session=session,
channel=channel,
)
for queue, exchange in queues
}
logger.info(
"Deleting all bindings between the following: {}",
", ".join(f"{exchange} to {queue}"
for queue, exchange in queues),
)
await asyncio.gather(*delete_tasks)
finally:
try:
await channel.close()
except AMQPChannelError:
pass
async def xtest_proxy_from_env_https_with_auth(proxy_test_server,
get_request, mocker):
url = 'https://aiohttp.io/path'
proxy = await proxy_test_server()
auth = aiohttp.BasicAuth('user', 'pass')
mocker.patch.dict(os.environ, {'https_proxy':
str(proxy.url
.with_user(auth.login)
.with_password(auth.password))})
await get_request(url=url, trust_env=True)
assert len(proxy.requests_list) == 2
assert proxy.request.method == 'GET'
assert proxy.request.host == 'aiohttp.io'
assert proxy.request.path_qs == '/path'
assert 'Proxy-Authorization' not in proxy.request.headers
r2 = proxy.requests_list[0]
assert r2.method == 'CONNECT'
assert r2.host == 'aiohttp.io'
assert r2.path_qs == '/path'
assert r2.headers['Proxy-Authorization'] == auth.encode()
async def _set_door_state(self, session, door_id, door_key, new_state):
get_door_body = {
"auth": {
"cik": door_key,
"client_id": door_id
},
"calls": [
{
"arguments": [{
"alias": "dps1.desired_status"
}, new_state],
"id": 1,
"procedure": "write",
},
{
"arguments": [{
"alias": "dps1.desired_status_user"
}, self.username],
"id":
2,
"procedure":
"write",
},
],
}
get_door_headers = self.STATIC_HEADERS.copy()
get_door_headers["Content-Type"] = "application/json"
async with session.post(
self.DOOR_URL,
json=get_door_body,
headers=get_door_headers,
auth=aiohttp.BasicAuth(self.username, self.password),
) as response:
return await response.json()
async def test_proxy_http_auth(proxy_test_server, get_request) -> None:
url = "http://aiohttp.io/path"
proxy = await proxy_test_server()
await get_request(url=url, proxy=proxy.url)
assert "Authorization" not in proxy.request.headers
assert "Proxy-Authorization" not in proxy.request.headers
auth = aiohttp.BasicAuth("user", "pass")
await get_request(url=url, auth=auth, proxy=proxy.url)
assert "Authorization" in proxy.request.headers
assert "Proxy-Authorization" not in proxy.request.headers
await get_request(url=url, proxy_auth=auth, proxy=proxy.url)
assert "Authorization" not in proxy.request.headers
assert "Proxy-Authorization" in proxy.request.headers
await get_request(url=url, auth=auth, proxy_auth=auth, proxy=proxy.url)
assert "Authorization" in proxy.request.headers
assert "Proxy-Authorization" in proxy.request.headers
def __init__(self, hass, device_info):
super().__init__()
self.hass = hass
self._still_image_url = device_info[CONF_STILL_IMAGE_URL]
self._still_image_url.hass = hass
self._stream_source = device_info[CONF_STREAM_SOURCE]
self._limit_refetch = device_info[CONF_LIMIT_REFETCH_TO_URL_CHANGE]
self._frame_interval = 1 / device_info[CONF_FRAMERATE]
self._supported_features = SUPPORT_STREAM if self._stream_source else 0
self.content_type = device_info[CONF_CONTENT_TYPE]
self.verify_ssl = device_info[CONF_VERIFY_SSL]
username = device_info.get(CONF_USERNAME)
password = device_info.get(CONF_PASSWORD)
if username and password:
self._auth = aiohttp.BasicAuth(username, password=password)
else:
self._auth = None
self._last_url = None
self._last_image = None
async def check_texts(texts: List[Tuple[str, str]],
step: int = 50) -> List[Dict[str, str]]:
'''
Asyncronously sends texts to IBM entty and sentiment
recognition service in batches of step-size. Returns
results in List.
texts = [(uuid, text),...]
'''
result = []
index = 0
while index < len(texts):
progress(index, len(texts))
tasks = []
async with aiohttp.ClientSession(headers=HEADERS,
auth=aiohttp.BasicAuth(
'apikey', API_KEY)) as session:
for text in texts[index:index + step]:
task = asyncio.create_task(
check_text(text=text[1], uuid=text[0], session=session))
tasks.append(task)
try:
batch_result = await asyncio.gather(*tasks)
for doc_result in batch_result:
result.extend(doc_result)
index += step
# In case of network error or exceeding IBM limits
# We wait for 10 seconds and retry current batch.
except Exception as e:
sleep(10)
logging.exception(
f'Got exception processing batch {texts[index:index+step]}'
)
progress(index, len(texts))
print('\n')
return result
FULL_REBUILD_FLAG = os.getenv("FULL_REBUILD_FLAG")
MC_USER = os.getenv("MC_USERNAME")
MC_PWD = os.getenv("MC_PASSWORD")
SCHEMA = os.getenv("SCHEMA")
TABLE_PREFIX = os.getenv("TABLE_PREFIX")
URL = "https://secure.mcommons.com/api/"
ALL_ENDPOINTS = ["tags"]
RS_INCREMENTAL_KEYS = {"tags": None}
API_INCREMENTAL_KEYS = {"tags": None}
ENDPOINT_KEY = {1: {"tags": "tags"}, 0: {"tags": "tag"}}
MIN_PAGES = 2
MAX_PAGES = 20000
LIMIT = 500
AUTH = aiohttp.BasicAuth(MC_USER, password=MC_PWD)
# Mobile Commons API allows up to 160 concurrent connections but they asked us to reduce to 80 for now
SEMAPHORE = asyncio.BoundedSemaphore(80)
retries = Retry(total=3,
status_forcelist=[429, 500, 502, 503, 504],
backoff_factor=1)
retry_adapter = HTTPAdapter(max_retries=retries)
http = requests.Session()
http.mount("https://secure.mcommons.com/api/", retry_adapter)
def main():
async def post_lecture(req: aiohttp.web.Request) -> aiohttp.web.StreamResponse:
lecture = extract_lecture(req)
email = extract_verified_email(req)
admin = email is not None and cig.data.admin(email)
form = await req.post()
if not email:
# Process login form.
try:
email = normalize_email(str(form["email"]))
except KeyError:
raise aiohttp.web.HTTPBadRequest(reason="email required")
except ValueError as err:
return aiohttp.web.Response(text=cig.view.login(
lecture=lecture, error=str(err)).render(),
content_type="text/html")
token = hmac_email(req.app["secret"], email)
magic_link = req.app["base_url"].rstrip("/") + cig.templating.url(
req.match_info["lecture"], email=email, hmac=token)
print(magic_link)
email_text = "\n\n".join(line for line in [
f"Continue here: {magic_link}",
f"Admin interface: {magic_link}&admin=yes" if cig.data.
admin(email) else None,
"You can also bookmark this link for future lectures.",
f"---\nAutomated email on behalf of {lecture.lecturer} and team",
] if line is not None)
if not req.app["dev"]:
async with aiohttp.ClientSession() as session:
async with session.post(
f"https://api.eu.mailgun.net/v3/{req.app['mailgun_domain']}/messages",
auth=aiohttp.BasicAuth("api", req.app["mailgun_key"]),
data=
{
"from":
f"CIG Lectures <noreply@{req.app['mailgun_domain']}>",
"to": email,
"subject":
f"Register for the next {lecture.title} lecture (step 2/3)",
"text": email_text,
}) as res:
print("Response:", res.status, "-", await res.text())
return aiohttp.web.Response(text=cig.view.link_sent(
lecture=lecture,
email_text=email_text if req.app["dev"] else None).render(),
content_type="text/html")
else:
# Process registration form.
try:
event = cig.data.EVENTS[int(str(form["reserve"]))]
except (KeyError, ValueError):
pass
else:
name = str(form.get("name", email)).strip() if admin else email
if "@" in name:
name = name.lower()
if name and (admin or event.date == cig.db.now().date()):
req.app["db"].maybe_register(event=event.id,
name=name,
admin=admin)
# Process admin actions on registration form.
if admin:
try:
delete = int(str(form["delete"]))
name = str(form["name"])
except (KeyError, ValueError):
pass
else:
req.app["db"].delete(event=delete, name=name)
try:
restore = int(str(form["restore"]))
name = str(form["name"])
except (KeyError, ValueError):
pass
else:
req.app["db"].restore(event=restore, name=name)
return await get_lecture(req)
def from_connection_info(
cls,
info: credentials.ConnectionInfo,
) -> "APISession":
# Some SSL data are not accepted directly, so we have to use temp files.
tempfiles = _TempFiles()
ca_path: Optional[str]
certificate_path: Optional[str]
private_key_path: Optional[str]
if info.ca_path and info.ca_data:
raise credentials.LoginError("Both CA path & data are set. Need only one.")
elif info.ca_path:
ca_path = info.ca_path
elif info.ca_data:
ca_path = tempfiles[base64.b64decode(info.ca_data)]
else:
ca_path = None
if info.certificate_path and info.certificate_data:
raise credentials.LoginError("Both certificate path & data are set. Need only one.")
elif info.certificate_path:
certificate_path = info.certificate_path
elif info.certificate_data:
certificate_path = tempfiles[base64.b64decode(info.certificate_data)]
else:
certificate_path = None
if info.private_key_path and info.private_key_data:
raise credentials.LoginError("Both private key path & data are set. Need only one.")
elif info.private_key_path:
private_key_path = info.private_key_path
elif info.private_key_data:
private_key_path = tempfiles[base64.b64decode(info.private_key_data)]
else:
private_key_path = None
# The SSL part (both client certificate auth and CA verification).
context: ssl.SSLContext
if certificate_path and private_key_path:
context = ssl.create_default_context(
purpose=ssl.Purpose.CLIENT_AUTH,
cafile=ca_path)
context.load_cert_chain(
certfile=certificate_path,
keyfile=private_key_path)
else:
context = ssl.create_default_context(
cafile=ca_path)
if info.insecure:
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
# The token auth part.
headers: Dict[str, str] = {}
if info.scheme and info.token:
headers['Authorization'] = f'{info.scheme} {info.token}'
elif info.scheme:
headers['Authorization'] = f'{info.scheme}'
elif info.token:
headers['Authorization'] = f'Bearer {info.token}'
# The basic auth part.
auth: Optional[aiohttp.BasicAuth]
if info.username and info.password:
auth = aiohttp.BasicAuth(info.username, info.password)
else:
auth = None
# It is a good practice to self-identify a bit.
headers['User-Agent'] = f'kopf/unknown' # TODO: add version someday
# Generic aiohttp session based on the constructed credentials.
session = cls(
connector=aiohttp.TCPConnector(
limit=0,
ssl=context,
),
headers=headers,
auth=auth,
)
# Add the extra payload information. We avoid overriding the constructor.
session.server = info.server
session.default_namespace = info.default_namespace
session._tempfiles = tempfiles # for purging on garbage collection
session._discovery_lock = asyncio.Lock()
session._discovered_resources = {}
return session
def test_basic_auth_utf8(make_request) -> None:
req = make_request('get',
'http://python.org',
auth=aiohttp.BasicAuth('nkim', 'секрет', 'utf-8'))
assert 'AUTHORIZATION' in req.headers
assert 'Basic bmtpbTrRgdC10LrRgNC10YI=' == req.headers['AUTHORIZATION']
class FrostFetcher(fetcher.Fetcher):
__api_version = 'v0'
__response_format = 'jsonld'
__api_base_uri = 'https://frost.met.no'
__sources_uri = (
f'{__api_base_uri}/sources/{__api_version}.{__response_format}'
'?elements={0}&geometry=nearest(POINT({1} {2}))'
'&nearestmaxcount={3}&validtime={4}/{5}')
__observations_base_uri = f'{__api_base_uri}/observations'
__observations_uri = (
f'{__observations_base_uri}/{__api_version}.{__response_format}'
'?sources={0}&referencetime={1}/{2}&elements={3}')
# Currently unnecessary, but may be useful at some point
# __time_series_uri = (
# f'{__observations_base_uri}/availableTimeSeries/'
# f'{__api_version}.{__response_format}'
# '?sources={0}&referencetime={1}/{2}&elements={3}'
__time_format = '%Y-%m-%d'
__frost_auth = aiohttp.BasicAuth(config('FROST_CID'))
sources_headers = (
'id',
'type',
'name',
'short_name',
'country',
'country_code',
'wmo_id',
'latitude',
'longitude',
'masl',
# See comment in frost_initializer.py
# 'valid_from',
# 'valid_to',
'county',
'county_id',
'municipality',
'municipality_id',
'station_holders',
'external_ids',
'icao_codes',
'ship_codes',
'wigos_id')
observations_headers = ('source', 'element', 'time', 'reg_id', 'distance',
'value', 'orig_value', 'unit', 'code_table',
'level_type', 'level_unit', 'level_value',
'time_offset', 'time_resolution', 'time_series_id',
'performance_category', 'exposure_category',
'quality_code', 'control_info', 'data_version')
elements = [
'sum(precipitation_amount P1D)',
'over_time(precipitation_type P1D)',
'mean(wind_speed P1D)',
# This element can be used to calculate mean for day
# 'mean(wind_from_direction PT1H)',
'best_estimate_mean(air_temperature P1D)',
'mean(cloud_area_fraction P1D)'
]
nearestmaxcount = 5
days_before = 2
def fetch(self,
incidents: List[AvalancheIncident]) -> (Dict[str, pd.DataFrame]):
sources_df = pd.DataFrame(columns=type(self).sources_headers)
observations_df = pd.DataFrame(columns=type(self).observations_headers)
loop = asyncio.get_event_loop()
# Fetch for all incidents
dfs = loop.run_until_complete(
gather_with_semaphore(
10,
*(self.fetch_for_incident(incident)
for incident in incidents)))
# Convert the result into a usable structure
dfs2 = {'sources': [], 'observations': []}
for ds in dfs:
for d in ds:
for key, value in d.items():
dfs2[key].append(value)
for sources in dfs2['sources']:
sources_df = sources_df.append(sources, ignore_index=True)
for observations in dfs2['observations']:
observations_df = observations_df.append(observations,
ignore_index=True)
return {
'frost_sources': sources_df,
'frost_observations': observations_df
}
async def fetch_for_incident(
self,
incident: AvalancheIncident) -> (List[Dict[str, pd.DataFrame]]):
interval_start = (incident.time -
dt.timedelta(days=type(self).days_before)).strftime(
type(self).__time_format)
interval_end = (incident.time + dt.timedelta(days=1)).strftime(
type(self).__time_format)
# Fetch for all elements
return await asyncio.gather(*(self.fetch_for_element(
incident, element, interval_start, interval_end)
for element in type(self).elements))
async def fetch_for_element(self, incident: AvalancheIncident,
element: str, start: str,
end: str) -> Dict[str, pd.DataFrame]:
async with aiohttp.ClientSession(auth=type(self).__frost_auth) as s:
sources_response = await self.fetch_sources(
s, element, incident.coords_latlng[0],
incident.coords_latlng[1], start, end)
source_distances = {}
sources_df = pd.DataFrame(columns=type(self).sources_headers)
for source in sources_response:
source_distances[source['id']] = source['distance']
sources_df = sources_df.append(
self.__create_source_row(source), ignore_index=True)
# Fetch for all sources
observations_dfs = await gather_with_semaphore(
5,
*(self.fetch_observations(s, incident, source, start, end,
element, source_distances[source])
for source in source_distances.keys()))
observations_df = pd.DataFrame(
columns=type(self).observations_headers)
for obs_df in observations_dfs:
observations_df = observations_df.append(obs_df,
ignore_index=True)
return {'sources': sources_df, 'observations': observations_df}
async def fetch_sources(self, s: aiohttp.ClientSession, element: str,
latitude: float, longitude: float, start: str,
end: str) -> Dict[str, Any]:
url = type(self).__sources_uri.format(element, longitude, latitude,
type(self).nearestmaxcount,
start, end)
return (await get_with_retries(s, url, 5)).get('data')
async def fetch_observations(self, s: aiohttp.ClientSession,
incident: AvalancheIncident, source: str,
start: str, end: str, element: str,
distance: float) -> pd.DataFrame:
url = type(self).__observations_uri.format(source, start, end, element)
obs = (await get_with_retries(s, url, 5)).get('data')
if obs is None:
return None
return self.__create_obs_df(obs, source, incident.id, distance)
def __create_source_row(self, src: Dict[str, Any]) -> pd.DataFrame:
source_row = pd.DataFrame(
[[
src.get('id'),
src.get('@type'),
src.get('name'),
src.get('shortName'),
src.get('country'),
src.get('countryCode'),
src.get('wmoId'),
src['geometry']['coordinates'][1] if
(src.get('geometry') is not None) else None,
src['geometry']['coordinates'][0] if
(src.get('geometry') is not None) else None,
src.get('masl'),
# See comment in frost_initializer.py
# dt.datetime.fromisoformat(
# src.get('validFrom')[:-1]
# ),
# dt.datetime.fromisoformat(
# src.get('validTo')[:-1]
# ) if src.get('validTo') is not None else None,
src.get('county'),
src.get('countyId'),
src.get('municipality'),
src.get('municipalityId'),
(str(src.get('stationHolders'))
if src.get('stationHolders') is not None else None),
(str(src.get('externalIds'))
if src.get('externalIds') is not None else None),
(str(src.get('icaoCodes'))
if src.get('icaoCodes') is not None else None),
(str(src.get('shipCodes'))
if src.get('shipCodes') is not None else None),
src.get('wigosId')
]],
columns=type(self).sources_headers)
return source_row
def __create_observation_row(self, src: str, obs: Dict[str, Any],
obs_at_time: Dict[str, Any], id_: int,
distance: float) -> pd.DataFrame:
obs_row = pd.DataFrame([[
src,
obs.get('elementId'),
dt.datetime.fromisoformat(
obs_at_time.get('referenceTime')[:-1]), id_, distance,
obs.get('value'),
obs.get('origValue'),
obs.get('unit'),
obs.get('codeTable'), obs['level']['levelType'] if
(obs.get('level') is not None) else (None), obs['level']['unit'] if
(obs.get('level') is not None) else
(None), obs['level']['value'] if
(obs.get('level') is not None) else (None),
obs.get('timeOffset'),
obs.get('timeResolution'),
obs.get('timeSeriesId'),
obs.get('performanceCategory'),
obs.get('exposureCategory'),
obs.get('qualityCode'),
obs.get('controlInfo'),
obs.get('dataVersion')
]],
columns=type(self).observations_headers)
return obs_row
def __create_obs_df(self, obs_response: Dict[str, Any], src: str,
reg_id: int, distance: float) -> pd.DataFrame:
observations_df = pd.DataFrame(columns=type(self).observations_headers)
for obs_at_time in obs_response:
for obs in obs_at_time['observations']:
obs_row = self.__create_observation_row(
src, obs, obs_at_time, reg_id, distance)
observations_df = observations_df.append(obs_row,
ignore_index=True)
return observations_df
def test_basic_auth(make_request) -> None:
req = make_request('get',
'http://python.org',
auth=aiohttp.BasicAuth('nkim', '1234'))
assert 'AUTHORIZATION' in req.headers
assert 'Basic bmtpbToxMjM0' == req.headers['AUTHORIZATION']
async def test_auth_basicauth(auto_close, loop):
auth = aiohttp.BasicAuth('user', 'pass')
conn = auto_close(AIOHttpConnection(http_auth=auth, loop=loop))
assert conn.http_auth == auth
async def go():
async with aiohttp.ClientSession(
auth=aiohttp.BasicAuth("andrew", "password")) as session:
await fetch(session)
import aiohttp
import keyboards
from status import TEST
import config
from aiogram.dispatcher import FSMContext
from aiogram.contrib.fsm_storage.memory import MemoryStorage
import func
import sqlite as sql
import time
import asyncio
PROXY_URL = 'http://104.227.97.168:8000'
STOP = True
PROXY_AUTH = aiohttp.BasicAuth(login='******', password='******')
FIRST_POST = {'title': '', 'url': '', 'price': '', 'img': ''}
storage = MemoryStorage()
bot = Bot(token=config.API_TOKEN, proxy=PROXY_URL, proxy_auth=PROXY_AUTH)
#bot = Bot(token=config.API_TOKEN)
dp = Dispatcher(bot, storage=storage)
async def replace_link(link: str):
st = link
if link.startswith('https://m.avito.ru'):
st = link.replace('https://m.avito.ru', 'https://www.avito.ru')
def __init__(self, token):
self.token = token
self.auth = aiohttp.BasicAuth(login=self.token)
