def offerfiles(self):
folder = os.path.expandvars(constants.DIRECTORY_TO_SHARE)
for filename in os.listdir(folder):
filepath = os.path.join(folder, filename)
if isfile(filepath):
with open(filepath, "rb") as f:
self.files.append({
"name":
filename,
"size":
os.path.getsize(filepath),
"hash":
hashlib.sha256(f.read()).hexdigest(),
})
rs = requests.post(
self.server + constants.REQ_OFFERFILES,
json={
"listen_port": self.listen_port,
"files": self.files
},
)
if 400 <= rs.status_code < 600:
raise HTTPError(
f"""HTTP {rs.status_code} - {rs.json()["error"]}""")
def download(data_location, data_dir, force_download=False):
""" Downloads a dataset from `data_location` to `data_dir`. """
req = requests.get(data_location, stream=True)
if req.status_code != 200:
logger.error(f'Could not download {data_location}: {req.status_code}')
raise HTTPError('data_location cannot be donwloaded')
target_filename = extract_file_name(req)
destination = os.path.join(data_dir, target_filename)
if os.path.exists(destination) and not force_download:
logger.warn(f'File {destination} already exists. Aborting.')
return destination
logger.info(f'File to download: {target_filename} from {data_location}')
os.makedirs(data_dir, exist_ok=True) # Assure data Dir
total_download_size = int(req.headers.get('Content-Length', 0))
download_block_size = 1024
with tqdm(total=total_download_size, unit='iB', unit_scale=True) as bar:
with open(destination, 'wb+') as datafile:
for block in req.iter_content(download_block_size):
datafile.write(block)
bar.update(len(block))
logger.info(f'File downloaded to {destination} successfully.')
return destination
def testAuthentication(self):
"""
Tests the parameters the user has provided in the ui to see if they are correct
"""
try:
self.response = self.session.get(self.url,
timeout=120,
verify=False)
except Exception:
pass
try:
self._get_auth()
if not self.response:
raise HTTPError(404, 'Not Found')
self.response.raise_for_status()
if self.auth:
return True, 'Success: Connected and Authenticated'
else:
return False, 'Failed to authenticate with {0}'.format(
self.name)
except Exception as error:
helpers.handle_requests_exception(error)
return False, '{0}'.format(error)
def download2(pmid_lists):
for pmid_list in pmid_lists:
print("———————" + pmid_list + "———————")
print()
api_url = build_api_url(pmid_list, retmode="xml")
res = requests.get(api_url)
if res.status_code != 200:
raise HTTPError(res.reason)
d = xmltodict.parse(res.text)
# with open("xml/tmp.xml", "w") as f:
# f.write(res.text)
# medline_json = pp.parse_medline_xml("xml/tmp.xml")
# for article in medline_json:
# with open("json/{}.json".format(article["pmid"]), "w") as f:
# f.write(json.dumps(article, indent=2))
articles = d["PubmedArticleSet"]["PubmedArticle"]
# When using xmltodict to convert a multiple-article request, the
# PubmedArticle tag becomes a list instead of a dict.
if isinstance(articles, dict):
articles = [articles]
write_articles(articles)
shutil.rmtree("xml")
def upload_to_presigned_url(presigned_url: str, file_pointer: IO):
# TODO optimize this further to deal with truly huge files and flaky internet connection.
upload_response = requests.put(presigned_url, file_pointer)
if not upload_response.ok:
raise HTTPError(
f"Tried to put a file to url, but failed with status {upload_response.status_code}. The detailed error was: {upload_response.text}"
)
def download_from_pmid_list(path, batch_size=1):
lines = []
for line in open("pmids/example.txt", "r"):
lines.append(line.strip())
pmid_batches = []
for batch in make_batches(lines, batch_size):
pmid_batches.append(batch)
pmid_lists = []
for batch in pmid_batches:
pmid_lists.append(",".join(batch))
print()
for pmid_list in pmid_lists:
print("———————" + pmid_list + "———————")
print()
api_url = build_api_url(pmid_list, retmode="xml")
res = requests.get(api_url)
if res.status_code != 200:
raise HTTPError(res.reason)
d = xmltodict.parse(res.text)
# with open("xml/tmp.xml", "w") as f:
# f.write(res.text)
# medline_json = pp.parse_medline_xml("xml/tmp.xml")
# for article in medline_json:
# with open("json/{}.json".format(article["pmid"]), "w") as f:
# f.write(json.dumps(article, indent=2))
articles = d["PubmedArticleSet"]["PubmedArticle"]
# When using xmltodict to convert a multiple-article request, the
# PubmedArticle tag becomes a list instead of a dict.
if isinstance(articles, dict):
articles = [articles]
for article in articles:
out = {"PubmedArticleSet": {"PubmedArticle": article}}
pmid = article["MedlineCitation"]["PMID"]["#text"]
xml = xmltodict.unparse(out, pretty=True, full_document=False)
# todo: write to /tmp/ instead?
with open("xml/{}.xml".format(pmid), "w") as f:
f.write(xml)
medline_json = pp.parse_medline_xml("xml/{}.xml".format(pmid))
with open("json/{}.json".format(pmid), "w") as f:
f.write(json.dumps(medline_json, indent=2))
shutil.rmtree("xml")
def raise_for_status(self):
if self.error_list is not None and self.url in self.error_list:
http_error_msg = "%s Client Error: %s for url: %s" % (
400,
"Simulate error",
self.url,
)
raise HTTPError(http_error_msg, response=self)
def _is_authenticated(self, headers):
"""
Make a test request to an authenticated endpoint to see if auth is good.
"""
TEST_ENDPOINT = self.base_url + "/print-jobs/" # used to test auth
response = requests.get(TEST_ENDPOINT, headers=headers)
if response.status_code == 200:
return True
elif response.status_code == 401:
logger.debug(
"Got HTTP 401 expried auth token so have to renew it.")
return False
elif response.status_code == 403:
raise HTTPError("Got HTTP 403 Unauthorized. Check credentials.")
else:
print("Response=", response.json())
raise HTTPError("Unexpected HTTP code " +
str(response.status_code))
def register(self):
rs = requests.post(
self.server + constants.REQ_REGISTER,
json={"listen_port": self.listen_port},
)
if 400 <= rs.status_code < 600:
raise HTTPError(
f"""HTTP {rs.status_code} - {rs.json()["error"]}""")
self.registered = True
def make_request(self, request_string: str) -> Iterable:
bsuir_api_responce = req.get(request_string)
if bsuir_api_responce.status_code == 200:
return bsuir_api_responce.json()
else:
logging.exception(
"GET request failed with status code {code}\nRequest:{req_str}".format(
code=bsuir_api_responce.status_code, req_str=request_string
)
)
raise HTTPError("")
def get_html_object(url: str, HttpHeader=None):
"""
result is a etree.HTML object
"""
response = requests.get(url, HttpHeader=None, timeout=3)
if response.status_code == 200:
# setting encoding
response.encoding = response.apparent_encoding
html = lxml.etree.HTML(response.text)
else:
html = None
raise HTTPError(f"Status code: {response.status_code} for {url}")
return html
def response_raise_error_if_any(response: requests.Response) -> None:
"""
Checks if there is any error while make a request.
If status 400+ then raises HTTPError with provided reason.
"""
try:
response.raise_for_status()
except requests.exceptions.HTTPError as e:
msg = response.text
try:
msg = response.json().get('detail')
except:
pass
raise HTTPError({"error": str(e), "reason": msg}) from None
def raise_for_status(self, resp: Response):
try:
resp.raise_for_status()
except HTTPError as e:
message = '\n'.join([str(a) for a in e.args])
body_str = str(resp.request.body)
resp_body = str(resp.text)
if len(body_str) > 2048:
body_str = body_str[:2048] + '...'
message += f'\nRequest url:\n{resp.request.url}' \
f'\nRequest headers:\n{resp.request.headers}' \
f'\nRequest body:\n{body_str}\n' \
f'\n' \
f'\nResponse body:\n{resp_body}\n'
raise HTTPError(message, response=resp)
def loginAPI(self):
urlParam = self.getUrlParamTail()
headers = self.headers.loginHeaders()
data = {
'username': self.username,
'password': self.password,
'login_submit': ''
}
response = self.session.post(
f'https://newsso.shu.edu.cn/login/{urlParam}',
headers=headers,
data=data)
if response.status_code != 200:
raise HTTPError('Cookies is not 200.')
return response
def get_REI_sale_items(items: List[REI_items]) -> Dict:
all_sale_items: Dict = {}
for item in items:
api_response = get_rei_response(
pathname=item.pathname, query_params=item.query_params
)
if api_response.status_code != 200:
raise HTTPError(
f"{api_response.status_code} {api_response.reason} for {api_response.url}"
)
if not api_response.text:
raise ValueError("No response text")
sale_items = PriceCheckerREI().get_sale_items(api_response.text)
all_sale_items[item.name] = {}
all_sale_items[item.name]["url"] = api_response.url
all_sale_items[item.name]["sale_items"] = sale_items
return all_sale_items
def get_headers(self, checkauth=True):
headers = {
'Cache-Control': 'no-cache',
'Authorization': 'Bearer ' + self.bearer_token,
}
if checkauth:
if not self._is_authenticated(headers):
logger.debug('Renewing bearer token...')
self.bearer_token = self._get_bearer_token(
client_key=self.client_key,
client_secret=self.client_secret)
headers["Authorization"] = 'Bearer ' + self.bearer_token
if self._is_authenticated(headers):
logger.debug("...bearer token successfully renewed.")
else:
raise HTTPError(
'Failed to renew auth token after 1 attempt')
return headers
def search(self):
print("Searching files")
rs = requests.post(self.server + constants.REQ_SEARCH,
json={"listen_port": self.listen_port})
if 400 <= rs.status_code < 600:
raise HTTPError(
f"""HTTP {rs.status_code} - {rs.json()["error"]}""")
files = rs.json()["files"]
if len(files) == 0:
print("No files found")
return
print(f"""{len(files)} file{"s" if len(files) > 1 else ""} found:""")
print(
tabulate(
[(
i,
f["name"],
naturalsize(f["size"], binary=True),
f["hash"],
len(f["peers"]),
) for i, f in enumerate(files)],
headers=["#", "Name", "Size", "Hash", "Peers"],
))
while True:
idx = input(
"Which file do you want to download? (Use # number or press c to cancel) "
)
if idx == "c":
return
try:
i = int(idx)
fr = files[i]
download_file(fr["name"], fr["hash"], fr["size"], fr["peers"])
return
except (ValueError, IndexError):
print("Invalid value")
def url_to_local_file(url: str) -> str:
"""Returns path to a local file, fetching remote files as needed."""
path: str
p: ParseResult = urlparse(url)
if p.scheme == 'file':
path = os.path.abspath(p.path)
else:
r: requests.Response = requests.get(url)
if r.status_code != 200:
raise HTTPError(r.status_code)
fd: int
#text: bool = r.headers.get('Content-Type', '').startswith('text')
fd, path = mkstemp(dir=ENV.SBNSIS_CUTOUT_CACHE)
outf: io.IOBase
with open(fd, 'wb') as outf:
outf.write(r.content)
# rw-rw-r--
os.chmod(path, 664)
return path
def cover(self) -> APIC:
"""Return the cover art of the album.
Raises:
HTTPError: if response status code is indicative of an error
Returns:
APIC: the album's cover
"""
response = requests.get(self.images[0].url)
if response.status_code != HTTPStatus.OK.value:
raise HTTPError(
"Failed to fetch %s" % (self.images[0].url, ),
response=response,
)
return APIC(
encoding=3,
mime="image/jpeg",
type=3,
desc="Cover",
data=response.content,
)
def raise_for_status(self):
if (status.HTTP_400_BAD_REQUEST <= self.status_code <=
status.HTTP_511_NETWORK_AUTHENTICATION_REQUIRED):
raise HTTPError("", response=self)
base_url = os.environ.get("IMM_STORE_URL")
headers = {
"Content-Type": "application/json",
"Authorization": "Basic {}".format(os.environ.get("IMM_STORE_TOKEN")),
}
extension_data = {
"title": metadata.get("name"),
"version": metadata.get("version"),
"url": metadata.get("url"),
"email": metadata.get("support"),
"author": metadata.get("vendor"),
"description": metadata.get("description"),
}
sys.stdout.write(
f"About to upload to {base_url} with extension {extension_data}")
response = requests.post(
"{}/api/v1/extensions/store/{}/upload".format(base_url,
metadata.get("extension")),
json=extension_data,
headers=headers,
)
if not response.ok:
if response.status_code == 401 or response.status_code == 403:
raise HTTPError("Unauthorised, check IMM_STORE_TOKEN for extension")
elif response.status_code == 400:
raise HTTPError(f"Badly formatted request: {response.json()}")
raise HTTPError(
f"Got response {response.status_code} from extension store")
def __maybe_error_on_response(cls, resp: Response) -> None:
if resp.status_code >= 400:
raise HTTPError(resp.json()["error"])