async def scanImageFromQueue(queue, port):
session = arequests.Session()
for q in queue:
print(".... request for " + str(port) + ' ' + q)
session = arequests.Session()
url = 'http://lobs.local:' + str(port) + '/scan'
payload = {'image': q}
res = await session.post(url, data=payload)
if res.status_code != 200:
print("error " + url)
print(res.status_code)
async def get_transfers_with_filters(
self,
perPage=100,
recipient=None,
r_kind=None,
date_kind=None,
status=None,
_from=None,
_to=None,
):
result = await self.get_transfers(perPage=perPage)
if result[0]:
filter_params = {}
for key, value in {
"recipient": recipient,
"r_kind": r_kind,
"_from": _from,
"to": _to,
"date_kind": date_kind,
"status": status,
}.items():
if value:
filter_params[key] = value
meta = result[3]
tasks = []
async with requests_async.Session() as session:
for i in range(1, meta["pageCount"] + 1):
task = asyncio.ensure_future(
self.get_transfer_and_filter(perPage, i, session,
filter_params))
tasks.append(task)
responses = await asyncio.gather(*tasks)
return responses
async def request(self):
# Only make "Limit" number of Requests per minute (here 10 allowed) - 0 based index
async with self.limit:
# Get Authentication Token
TOKEN = await self.getToken()
# Start the Client Session and pass the Authentication Token Data in the Header
session = requests.Session()
session.headers.update({"Authorization": "Bearer= " + str(TOKEN)})
async with session:
# Output Object
result = {}
categories = []
# Get the Category list on Each Page
await self.getList(session, 1, categories)
# Throttle next request to maintain the rate limit
await asyncio.sleep(self.rate)
# For every category in the categories list, get the category api contents
for category in categories:
current = []
# Fills the current api category list
await self.getCategoryContents(session, 1, category,
current)
# Throttle the next request to maintain the rate limit
await asyncio.sleep(self.rate)
# Insert (category, apiList) pair into the result object
result[category] = current
return result
async def add_issue_to_project(self, issue_id, content_type='Issue'):
"""
Takes a Github issue id and adds the issue to a project board card.
Returns the response from Github API.
Note: Per Github, the API (and required 'Accept' headers) may change without notice.
See https://developer.github.com/v3/projects/cards/
"""
headers = {
"Authorization": "token {}".format(self.token),
"Accept": "application/vnd.github.inertia-preview+json"
}
data = {'content_id': issue_id, 'content_type': content_type}
payload = dumps(data)
async with requests.Session() as session:
try:
response = await session.post(self.project_url,
data=payload,
headers=headers)
response.raise_for_status()
return response.status_code
except requests.exceptions.HTTPError as errh:
return "An Http Error occurred:" + repr(errh)
except requests.exceptions.ConnectionError as errc:
return "An Error Connecting to the API occurred:" + repr(errc)
except requests.exceptions.Timeout as errt:
return "A Timeout Error occurred:" + repr(errt)
except requests.exceptions.RequestException as err:
return "An Unknown Error occurred" + repr(err)
async def addBulkPkgs(self, pkgs):
session = arequests.Session()
headers = {'Content-Type': 'application/json'}
pkgs_url = self._url + self._pkgPath + '/_bulk?pretty&refresh'
out = ''
image_path = f"{pkgs['registry']}/{pkgs['repo']}:{pkgs['tag']}"
for l in pkgs['pkgs']:
pkg_str = f"{l['name']}:{l['version']}"
name = self.createHash(image_path, pkg_str)
idx = {'index': {'_id': name}}
out += json.dumps(idx) + '\n'
l['index'] = name
l['image'] = image_path
l['flavor'] = pkgs['flavor']
l['registry'] = pkgs['registry']
l['repo'] = pkgs['repo']
l['tag'] = pkgs['tag']
out += json.dumps(l) + '\n'
res = await session.post(pkgs_url, headers=headers, data=out)
#print(res.text)
#print(res.status_code)
if res.status_code != 200 and res.status_code != 201:
logging.error(f"{self.i} Error writing pkgs for image {image_path}")
return False
logging.debug(f"{self.i} Done Writing pkgs in elastic for {image_path}")
return True
async def test_session(server):
url = "http://127.0.0.1:8000/"
with requests_async.Session() as session:
response = await session.get(url)
assert response.status_code == 200
assert response.json() == {"method": "GET", "url": url, "body": ""}
response = await session.post(url)
assert response.status_code == 200
assert response.json() == {"method": "POST", "url": url, "body": ""}
response = await session.put(url)
assert response.status_code == 200
assert response.json() == {"method": "PUT", "url": url, "body": ""}
response = await session.patch(url)
assert response.status_code == 200
assert response.json() == {"method": "PATCH", "url": url, "body": ""}
response = await session.delete(url)
assert response.status_code == 200
assert response.json() == {"method": "DELETE", "url": url, "body": ""}
response = await session.options(url)
assert response.status_code == 200
assert response.json() == {"method": "OPTIONS", "url": url, "body": ""}
response = await session.head(url)
assert response.status_code == 200
assert response.text == ""
def _create_session(self, conf: dict) -> requests.Session:
session = requests.Session()
session.verify = conf.pop("ssl.ca.location", None)
session.cert = self._configure_client_tls(conf)
session.auth = self._configure_basic_auth(conf)
return session
async def write(self, values: Dict[str, str]):
url = self._api_write
for key, value in values.items():
if key and value:
url += "&{0}={1}".format(key, value)
async with requests.Session() as session:
await session.get(url)
def __init__(self, url, user, passwd):
self._url = url
self._passwd = passwd
self._user = user
self._repos = []
self._tags = []
self._s = arequests.Session()
self._ss = requests.Session()
async def get_html(url: str) -> str:
while True:
async with request.Session() as session:
try:
response = await session.get(url=url)
break
except ConnectionError:
pass
return response.text
async def send(text):
if SLACK_URL is None or not text:
return None
async with requests.Session() as session:
await session.post(
SLACK_URL,
data=dumps({'text': text}),
headers={'Content-type': 'application/json'})
def __init__(self, key=None, secret=None, url=None):
self.key = key
self.secret = secret
self.session = requests.Session()
if url:
self.url = url
else:
self.url = "https://www.deribit.com"
def __init__(self, *, owner: str, repo: str, installation_id: str):
self.owner = owner
self.repo = repo
self.installation_id = installation_id
# NOTE: We must call `await session.close()` when we are finished with our session.
# We implement an async context manager this handle this.
self.session = http.Session()
self.session.headers[
"Accept"] = "application/vnd.github.antiope-preview+json,application/vnd.github.merge-info-preview+json"
async def test_session_redirection_disallowed(server):
url = "http://127.0.0.1:8000/redirect1"
async with requests_async.Session() as session:
response = await session.get(url, allow_redirects=False)
assert response.status_code == 302
response = await session.send(response.next, allow_redirects=False)
assert response.status_code == 302
response = await session.send(response.next, allow_redirects=False)
assert response.status_code == 200
assert response.url == "http://127.0.0.1:8000/redirect3"
async def tor_request_in_python_async(self, url):
import requests_async
tor = TorClient()
with tor.get_guard() as guard:
adapter = TorHttpAdapter(guard, 3, retries=RETRIES)
async with requests_async.Session() as s:
s.headers.update({'User-Agent': 'Mozilla/5.0'})
s.mount('http://', adapter)
s.mount('https://', adapter)
r = s.get(url, timeout=600)
self.log('<-- r', r)
return r
async def executeQueue(queue, port):
session = arequests.Session()
qtde = len(queue)
count = 0
for q in queue:
ss = time.perf_counter()
payload = {'image': q, 'push': 'true'}
url = f"http://lobs.local:{port}/scan"
res = await session.post(url, data=payload)
if res.status_code != 200:
print("error " + url)
print(res.status_code)
print(payload)
count += 1
ee = time.perf_counter() - ss
print(f" DONE {port} {count}/{qtde} {q} {ee:0.2f} seconds")
def __init__(self, clientname, PHPSESSID = None):
super().__init__(clientname)
self._ws = None
self.PHPSESSID = PHPSESSID
self._session = requests.Session()
self._is_running = False
self._event_loop = None
self._subscriptions = []
self._id_events = {}
self._send_data = []
self._last_send_time = 0.0
self._flushing_sends = False
self.subscribe(ConnectSubscription.instance)
self.subscribe(UserSubscription.instance)
self.subscribe(ChallengeSubscription(self))
self.subscribe(ListeningSubscription(self, "/game", GameSubscription))
def __init__(self, *, owner: str, repo: str, installation_id: str):
self.owner = owner
self.repo = repo
self.installation_id = installation_id
# NOTE: We must call `await session.close()` when we are finished with our session.
# We implement an async context manager this handle this.
self.session = http.Session()
self.session.headers[
"Accept"] = "application/vnd.github.antiope-preview+json,application/vnd.github.merge-info-preview+json"
if (conf.GITHUB_API_HEADER_NAME is not None
and conf.GITHUB_API_HEADER_VALUE is not None):
self.session.headers[
conf.GITHUB_API_HEADER_NAME] = conf.GITHUB_API_HEADER_VALUE
self.log = logger.bind(owner=self.owner,
repo=self.repo,
install=self.installation_id)
async def addVulnsFromQueue(self, image, vulns):
headers = {'Content-Type': 'application/json'}
session = arequests.Session()
for idx in range(len(vulns)):
id = self.createVulnId(vulns[idx])
es_url = self._url + self._vulnPath + '/_doc/' + id
logging.debug(f"{self.i} Going to inject vuln ({image}) {id}")
res = await session.post(es_url, headers=headers, auth=(self._user, self._passwd), data=json.dumps(vulns[idx]))
if res.status_code != 200 and res.status_code != 201:
# TODO: implement retry
print(es_url)
print(json.dumps(vulns[idx]))
print("error 09")
print(res.status_code)
print(res.text)
quit()
logging.debug(f"{self.i} ({res.status_code}) r: {id} {vulns[idx]['vulnId']} {vulns[idx]['version']}")
async def create_issue(title,
body,
labels=['feedback'],
milestone=None,
assignees=[]):
"""
Creates a Github issue via Github API v3 and returns the new issue id.
Note: Per Github, the API (and required 'Accept' headers) may change
without notice.
See https://developer.github.com/v3/issues/
"""
headers = {
"Authorization": "token {}".format(TOKEN),
"Accept": "application/vnd.github.v3+json"
}
data = {
'title': title,
'body': body,
'labels': labels,
'milestone': milestone,
'assignees': assignees
}
payload = dumps(data)
async with requests.Session() as session:
try:
response = await session.post(ISSUES_URL,
data=payload,
headers=headers)
response.raise_for_status()
response_content = loads(response.content)
issue_id = response_content['id']
issue_number = response_content['number']
return issue_id, issue_number
except requests.exceptions.HTTPError as errh:
raise errh
except requests.exceptions.ConnectionError as errc:
raise errc
except requests.exceptions.Timeout as errt:
raise errt
except requests.exceptions.RequestException as err:
raise err
async def main(self):
async with self.server_upper_limit:
AUTH_TOKEN = await self.fetchToken()
session = requests.Session()
session.headers.update(
{"Authorization": "Bearer= " + str(AUTH_TOKEN)})
async with session:
entire_data = {}
categories = []
await self.fetchAllCategories(session, categories, 1)
await asyncio.sleep(self.delay_rate)
for i in categories:
apiList = []
await self.fetchApiList(session, apiList, 1, i)
await asyncio.sleep(self.delay_rate)
entire_data[i] = apiList
return entire_data
async def addPkgsFromQueue(self, image, pkgs):
session = arequests.Session()
headers = {'Content-Type': 'application/json'}
for idx in range(len(pkgs)):
id = self.createPkgId(image, pkgs[idx])
es_url = self._url + self._pkgPath + '/_doc/' + id
payload = {
'image': image,
'pkg': pkgs[idx]['name'],
'version': pkgs[idx]['version']
}
logging.debug(f"{self.i} Going to inject pkg ({image}) {id}")
res = await session.post(es_url, headers=headers, auth=(self._user, self._passwd), data=json.dumps(payload))
if res.status_code != 200 and res.status_code != 201:
# TODO: implement retry
print("error 07")
quit()
logging.debug(f"{self.i} ({res.status_code}) r: {id} {pkgs[idx]['name']} {pkgs[idx]['version']}")
async def addBulkVulns(self, vulns):
session = arequests.Session()
headers = {'Content-Type': 'application/json'}
pkgs_url = self._url + self._vulnPath + '/_bulk?pretty&refresh'
out = ''
for v in vulns:
image_path = f"{v['registry']}/{v['repo']}:{v['tag']}"
name = self.createHash(image_path, v['vulnId'])
idx = {'index': {'_id': name}}
out += json.dumps(idx) + '\n'
out += json.dumps(v) + '\n'
res = await session.post(pkgs_url, headers=headers, data=out)
if res.status_code != 200 and res.status_code != 201:
logging.error(f"{self.i} Error writing vulns for image {image_path}")
return False
logging.debug(f"{self.i} Done Writing vulns in elastic for {image_path}")
return True
async def create_issue(self,
title,
body,
labels=['feedback'],
milestone=None,
assignees=[]):
"""
Creates a Github issue via Github API v3 and returns the new issue id.
Note: Per Github, the API (and required 'Accept' headers) may change without notice.
See https://developer.github.com/v3/issues/
"""
headers = {
"Authorization": "token {}".format(self.token),
"Accept": "application/vnd.github.v3+json"
}
data = {
'title': title,
'body': body,
'labels': labels,
'milestone': milestone,
'assignees': assignees
}
payload = dumps(data)
async with requests.Session() as session:
try:
response = await session.post(self.issues_url,
data=payload,
headers=headers)
response_content = loads(response.content)
issue_id = response_content['id']
response.raise_for_status()
return issue_id
except requests.exceptions.HTTPError as errh:
return "An Http Error occurred:" + repr(errh)
except requests.exceptions.ConnectionError as errc:
return "An Error Connecting to the API occurred:" + repr(errc)
except requests.exceptions.Timeout as errt:
return "A Timeout Error occurred:" + repr(errt)
except requests.exceptions.RequestException as err:
return "An Unknown Error occurred" + repr(err)
def get_new_session(self):
return requests.Session()
def __init__(self, chain: str, network: str):
self.url = self.URL.format(chain=chain, network=network)
self.session = requests.Session()
def __init__(self, url: str, *, auth: HTTPBasicAuth = None):
self.url = url
self.auth = auth
self.session = requests.Session()
def __init__(self, username: str = '', password: str = ''):
self.session = requests.Session()
self.session.auth = (username, password)
async def connect(self):
self.session = requests.Session()
async def _search_course(course_info_dict: dict):
async with requests_async.Session() as session:
return await session.post(url, course_info_dict)
