def _download_book_file(self, account, file_data, file_path):
"""Download book part file from Downpour and rename it."""
if os.path.isfile(file_path):
self.manager.warning(f'File "{file_path}" exists, skipping')
return
# Get download URL
file_url = self._get_download_url(account.session, file_data)
# Get target folder
out_folder = os.path.dirname(file_path)
# Check folder permissions
if not self._check_folder_permissions(out_folder):
return
# Open file download stream
stream = self.session(account.session).get(file_url, stream=True)
# Setup download progress bar data
progress_bar = {
'miniters': 1,
'desc': os.path.basename(file_path),
'total': int(stream.headers.get('content-length', 0))
}
# Read and download from file stream
with tqdm.wrapattr(open(file_path, 'wb'), 'write',
**progress_bar) as fh:
for chunk in stream.iter_content(chunk_size=4096):
fh.write(chunk)
# Check that the file was downloaded
if not os.path.isfile(file_path):
self.manager.error(f'Unable to download file: {file_path}')
def load_rlm_dnn_model():
# print(f'Finished loading random forest model', file=sys.stdout)
# global rlm_dnn_model
# with open('./models/dnn_morgan_all_data.pkl', 'rb') as pkl_file:
# dnn_model = pickle.load(pkl_file)
print(f'Loading RLM deep neural network model', file=sys.stdout)
rlm_dnn_model_path = f'{rlm_base_models_path}/dnn_morgan_all_data.pkl'
if path.exists(rlm_dnn_model_path):
with open(rlm_dnn_model_path, 'rb') as rlm_dnn_pkl_file:
rlm_dnn_model = pickle.load(rlm_dnn_pkl_file)
else:
rlm_dnn_pkl_url = f'{base_url}/dnn_morgan_all_data.pkl'
rlm_dnn_pkl_file_request = requests.get(rlm_dnn_pkl_url)
with tqdm.wrapattr(open(os.devnull, "wb"),
"write",
miniters=1,
desc=rlm_dnn_pkl_url.split('/')[-1],
total=int(
rlm_dnn_pkl_file_request.headers.get(
'content-length', 0))) as fout:
for chunk in rlm_dnn_pkl_file_request.iter_content(
chunk_size=4096):
fout.write(chunk)
with open(rlm_dnn_model_path, 'wb') as rlm_dnn_pkl_file:
rlm_dnn_pkl_file.write(rlm_dnn_pkl_file_request.content)
with open(rlm_dnn_model_path, 'rb') as rlm_dnn_pkl_file:
rlm_dnn_model = pickle.load(rlm_dnn_pkl_file)
return rlm_dnn_model
def download_file(url, filename):
response = requests.get(url, stream=True)
with tqdm.wrapattr(open(filename, "wb"), "write", miniters=1,
total=int(response.headers.get('content-length', 0)),
desc=filename) as fout:
for chunk in response.iter_content(chunk_size=4096):
fout.write(chunk)
def load_rlm_dnn_tokenizer():
# global rlm_tokenizer
# with open("./models/tokenizer.pickle",'rb') as tokfile:
# tokenizer = pickle.load(tokfile)
print(f'Loading RLM long-short term memory', file=sys.stdout)
rlm_dnn_tokfile_path = f'{rlm_base_models_path}/tokenizer.pickle'
if path.exists(rlm_dnn_tokfile_path):
with open(rlm_dnn_tokfile_path, 'rb') as rlm_dnn_tok_pkl_file:
rlm_dnn_tokenizer = pickle.load(rlm_dnn_tok_pkl_file)
else:
rlm_dnn_tokfile_url = f'{base_url}/tokenizer.pickle'
rlm_dnn_tokfile_request = requests.get(rlm_dnn_tokfile_url)
with tqdm.wrapattr(open(os.devnull, "wb"),
"write",
miniters=1,
desc=rlm_dnn_tokfile_url.split('/')[-1],
total=int(
rlm_dnn_tokfile_request.headers.get(
'content-length', 0))) as fout:
for chunk in rlm_dnn_tokfile_request.iter_content(chunk_size=4096):
fout.write(chunk)
with open(rlm_dnn_tokfile_path, 'wb') as rlm_dnn_tok_pkl_file:
rlm_dnn_tok_pkl_file.write(rlm_dnn_tokfile_request.content)
with open(rlm_dnn_tokfile_path, 'rb') as rlm_dnn_tok_pkl_file:
rlm_dnn_tokenizer = pickle.load(rlm_dnn_tok_pkl_file)
return rlm_dnn_tokenizer
def _download_image_file_and_return_file_path(temp_dir, image_type: str):
full_url = _find_url_of_latest_image(image_type)
print(f'Downloading LZMAFile from {full_url} to {temp_dir}')
path_to_lzma = temp_dir.joinpath('image.xz')
download_request = requests.get(full_url, stream=True)
content_length = int(download_request.headers.get('content-length'))
# https://stackoverflow.com/a/63831344/1040915
download_request.raw.read = functools.partial(download_request.raw.read,
decode_content=True)
with tqdm.wrapattr(download_request.raw, "read",
total=content_length) as tq:
with open(path_to_lzma, 'wb') as f:
copyfileobj(tq, f)
print(f'Finished downloading LZMAFile to {path_to_lzma}')
extract_dir = Path.home().joinpath('raspi_image')
extract_dir.mkdir(exist_ok=True)
extract_location = extract_dir.joinpath(
str(int(datetime.datetime.now().timestamp())))
# https://stackoverflow.com/a/72206813/1040915 - thanks, SO!
with lzma.open(path_to_lzma) as f_in:
with open(extract_location, 'wb') as f_out:
f_out.write(f_in.read())
path_to_lzma.unlink()
print(f'Extracted LZMAFile to {extract_location}')
return extract_location
def _rebuild_hmm(self, output, domains, options):
# fetch the resource from the URL in the ".ini" files
self.info(f"using fallback {options['url']}")
response = urllib.request.urlopen(options["url"])
# use `tqdm` to make a progress bar
pbar = tqdm.wrapattr(
response,
"read",
total=int(response.headers["Content-Length"]),
desc=os.path.basename(output),
leave=False,
)
# download to `output`
nsource = 0
nwritten = 0
with contextlib.ExitStack() as ctx:
dl = ctx.enter_context(pbar)
src = ctx.enter_context(gzip.open(dl))
dst = ctx.enter_context(open(output, "wb"))
for hmm in HMMFile(src):
nsource += 1
if hmm.accession.split(b".")[0] in domains:
nwritten += 1
hmm.write(dst, binary=True)
# log number of HMMs kept in final files
self.info(f"kept {nwritten} HMMs out of {nsource} in the source file")
def file_dl(url: str, fdst: str = "Temps"):
"""
This function download the files from the download link.
TQDM is used to keep track on the download progress,
This examples show how to use tqdm with shutil.copyfileobj.
Searching from the internet it is more common to find
for chunk in chunk_size:
f.write(chunk)
pbar.update(len(chunk))
:param url: download link
:param fdst: dst path
:return: shutil returns none, if you need data written use f.write().
"""
filename = get_file_name(url)
size = get_file_size(url)
save_to = set_save_to(filename, subdir=fdst)
with requests.get(url, stream=True) as r, open(save_to, "wb") as f, tqdm.wrapattr(
r.raw,
"read",
total=size,
unit="B",
unit_scale=True,
unit_divisor=1024,
desc=filename,
file=sys.stdout
) as raw:
shutil.copyfileobj(raw, f)
def load_gcnn_model():
# global rlm_gcnn_scaler, rlm_gcnn_features_scaler
# gcnn_scaler, gcnn_features_scaler = load_scalers('./models/gcnn_model.pt')
print(f'Loading RLM graph convolutional neural network model',
file=sys.stdout)
rlm_gcnn_scaler_path = f'{rlm_base_models_path}/gcnn_model.pt'
if path.exists(rlm_gcnn_scaler_path):
rlm_gcnn_scaler, _ = load_scalers(rlm_gcnn_scaler_path)
else:
rlm_gcnn_scaler_url = f'{base_url}/gcnn_model.pt'
rlm_gcnn_scaler_request = requests.get(f'{base_url}/gcnn_model.pt')
with tqdm.wrapattr(open(os.devnull, "wb"),
"write",
miniters=1,
desc=rlm_gcnn_scaler_url.split('/')[-1],
total=int(
rlm_gcnn_scaler_request.headers.get(
'content-length', 0))) as fout:
for chunk in rlm_gcnn_scaler_request.iter_content(chunk_size=4096):
fout.write(chunk)
with open(rlm_gcnn_scaler_path, 'wb') as rlm_gcnn_scaler_file:
rlm_gcnn_scaler_file.write(rlm_gcnn_scaler_request.content)
rlm_gcnn_scaler, _ = load_scalers(rlm_gcnn_scaler_path)
rlm_gcnn_model = load_checkpoint(rlm_gcnn_scaler_path)
return rlm_gcnn_scaler, rlm_gcnn_model
def download_source(path, url, download, checksum=None, verify=True):
if os.path.exists(path):
return
if not download:
raise RuntimeError(
f"Cannot find the data at {path}, but download was set to False")
with requests.get(url, stream=True, verify=verify) as r:
if r.status_code != 200:
r.raise_for_status()
raise RuntimeError(
f"Request to {url} returned status code {r.status_code}")
file_size = int(r.headers.get("Content-Length", 0))
desc = f"Download {url} to {path}"
if file_size == 0:
desc += " (unknown file size)"
with tqdm.wrapattr(r.raw, "read", total=file_size,
desc=desc) as r_raw, open(path, "wb") as f:
copyfileobj(r_raw, f)
if checksum is not None:
this_checksum = get_checksum(path)
if this_checksum != checksum:
raise RuntimeError(
"The checksum of the download does not match the expected checksum."
f"Expected: {checksum}, got: {this_checksum}")
print("Download successfull and checksums agree.")
else:
warn(
"The file was downloaded, but no checksum was provided, so the file may be corrupted."
)
def download_file(url: str, *, download_path: Optional[Path] = None) -> str:
"""Download to cwd with filename as last part of address, return filepath.
Returns
-------
str
Path to the downloaded file on disk
download_path
kwarg only which specifies the path to download the file to.
By default, None.
TODO
----
* cleanup on error
* allow specific file names
"""
fname = f"{url.split('/')[-1]}"
fpath = str(download_path.absolute()
) if download_path is not None else f"./{fname}"
response = requests.get(url, stream=True)
nbytes = int(response.headers.get("content-length", 0))
with tqdm.wrapattr(open(fpath, "wb"),
"write",
miniters=1,
desc=fname,
total=nbytes) as f:
for chunk in response.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
return fpath
def load_rlm_lstm_model():
rlm_lstm_model_path = f'{rlm_base_models_path}/lstm_all_data.h5'
if path.exists(rlm_lstm_model_path):
with open(rlm_lstm_model_path, 'rb') as rlm_lstm_pkl_file:
rlm_lstm_model = load_model(
rlm_lstm_pkl_file,
custom_objects=SeqSelfAttention.get_custom_objects())
else:
rlm_lstm_model_url = f'{base_url}/lstm_all_data.h5'
rlm_lstm_model_request = requests.get(rlm_lstm_model_url)
with tqdm.wrapattr(open(os.devnull, "wb"),
"write",
miniters=1,
desc=rlm_lstm_model_url.split('/')[-1],
total=int(
rlm_lstm_model_request.headers.get(
'content-length', 0))) as fout:
for chunk in rlm_lstm_model_request.iter_content(chunk_size=4096):
fout.write(chunk)
with open(rlm_lstm_model_path, 'wb') as rlm_lstm_pkl_file:
rlm_lstm_pkl_file.write(rlm_lstm_model_request.content)
with open(rlm_lstm_model_path, 'rb') as rlm_lstm_pkl_file:
rlm_lstm_model = load_model(
rlm_lstm_pkl_file,
custom_objects=SeqSelfAttention.get_custom_objects())
rlm_lstm_model._make_predict_function()
return rlm_lstm_model
def download(self, url):
"""Prepara i link di download."""
url, ll = url
if (self.verbose):
print(url)
file_name = url.split("/")[-1]
self.check_Path(
os.path.join(self.download_path,
file_name.split("_")[0]))
with open(
os.path.join(self.download_path,
file_name.split("_")[0], file_name), "wb"):
response = requests.get(url,
stream=True,
headers={"Referer": ll[0]})
with tqdm.wrapattr(open(
os.path.join(self.download_path,
file_name.split("_")[0], file_name), "wb"),
"write",
desc=url.split('/')[-1],
total=int(
response.headers.get('content-length',
0))) as fout:
for chunk in response.iter_content(chunk_size=4096):
fout.write(chunk)
def download(self, folder_path=None): # pragma: no cover
"""Download the asset to an indicated folder.
If tqdm is installed a progressbar will be shown.
:param folder_path: Folder path to download the asset, if left None,
the asset will be downloaded to the current
working directory.
:return: path to downloaded file.
"""
local_filename = urlparse(self['href'])[2].split('/')[-1]
if folder_path is not None:
folder_path += local_filename
response = requests.get(self['href'], stream=True)
try:
from tqdm import tqdm
with tqdm.wrapattr(open(folder_path if folder_path else local_filename, 'wb'), 'write', miniters=1,
total=int(response.headers.get('content-length', 0)),
desc=local_filename) as fout:
for chunk in response.iter_content(chunk_size=4096):
fout.write(chunk)
except ImportError:
with open(folder_path if folder_path else local_filename, 'wb') as f:
shutil.copyfileobj(response.raw, f)
return local_filename
def download(self, dir=None): # pragma: no cover
"""Download the asset to an indicated folder.
If tqdm is installed a progressbar will be shown.
:param dir: Directory path to download the asset, if left None,
the asset will be downloaded to the current
working directory.
:return: path to downloaded file.
"""
filename = urlparse(self['href'])[2].split('/')[-1]
if dir:
filename = os.path.join(dir, filename)
os.makedirs(os.path.dirname(filename), exist_ok=True)
try:
response = Utils.safe_request(self['href'], stream=True)
from tqdm import tqdm
with tqdm.wrapattr(open(filename, 'wb'),
'write',
miniters=1,
total=int(
response.headers.get('content-length', 0)),
desc=filename) as fout:
for chunk in response.iter_content(chunk_size=4096):
fout.write(chunk)
except ImportError:
with open(filename, 'wb') as f:
shutil.copyfileobj(response.raw, f)
return filename
def get_zip(self, base_url, symbol, date):
"""
Fetches a full day of trades from the Binance repository.
@param symbol: Symbol to fetch.
@param date: Day to download.
@return: Dataframe with full day.
"""
print_(["Fetching", symbol, date])
url = "{}{}/{}-aggTrades-{}.zip".format(base_url, symbol.upper(),
symbol.upper(), date)
df = pd.DataFrame(columns=[
"trade_id", "price", "qty", "timestamp", "is_buyer_maker"
])
column_names = [
"trade_id",
"price",
"qty",
"first",
"last",
"timestamp",
"is_buyer_maker",
]
if self.spot:
column_names.append("best_match")
try:
resp = urlopen(url)
file_tmp = BytesIO()
with tqdm.wrapattr(open(os.devnull, "wb"),
"write",
miniters=1,
total=getattr(resp, "length", None)) as fout:
for chunk in resp:
fout.write(chunk)
file_tmp.write(chunk)
with ZipFile(file_tmp) as my_zip_file:
for contained_file in my_zip_file.namelist():
tf = pd.read_csv(my_zip_file.open(contained_file),
names=column_names)
tf.drop(
errors="ignore",
columns=["first", "last", "best_match"],
inplace=True,
)
tf["trade_id"] = tf["trade_id"].astype(np.int64)
tf["price"] = tf["price"].astype(np.float64)
tf["qty"] = tf["qty"].astype(np.float64)
tf["timestamp"] = tf["timestamp"].astype(np.int64)
tf["is_buyer_maker"] = tf["is_buyer_maker"].astype(np.int8)
tf.sort_values("trade_id", inplace=True)
tf.drop_duplicates("trade_id", inplace=True)
tf.reset_index(drop=True, inplace=True)
if df.empty:
df = tf
else:
df = pd.concat([df, tf])
except Exception as e:
print("Failed to fetch", date, e)
return df
def upload_file(filename, api_key):
file_size = os.stat(filename).st_size
with open(filename, "rb") as f:
with tqdm.wrapattr(f, "read", total=file_size) as wrapped_file:
requests.put(
f"https://repo.labs.intellij.net/edu-tools/{filename}",
data=wrapped_file,
headers={"X-JFrog-Art-Api": api_key})
print()
def download_file(request, filename):
import shutil
file_size = int(request.headers["Content-length"])
with open(filename, 'wb') as f:
with tqdm.wrapattr(request.raw, "read",
total=file_size) as wrapped_data:
shutil.copyfileobj(wrapped_data, f)
print()
def _upload(path, upload_uri):
with open(path, 'rb') as f:
total_size = os.fstat(f.fileno()).st_size
f = tqdm.wrapattr(f, "read", desc='Uploading...', miniters=1, total=total_size, ascii=True)
with f as f_iter:
res = requests.put(
upload_uri,
data=ChunksIter(f_iter, total_size=total_size)
)
res.raise_for_status()
def upload(self, upload_url):
with open(self.realpath, "rb") as f:
total_size = os.fstat(f.fileno()).st_size
f = tqdm.wrapattr(f, "read", desc='正在上传', miniters=1, total=total_size, ascii=True)
with f as f_iter:
res = requests.put(
url=upload_url,
data=UploadChunksIterator(f_iter, total_size=total_size),
verify=False
)
res.raise_for_status()
def download_pdf_notices(result_df, folder=None, overwrite=False):
try:
dl_path = os.path.join(downloaded_notice_path, folder)
except:
dl_path = downloaded_notice_path
os.makedirs(dl_path, exist_ok=True)
info_df = result_df.copy()
file_names = info_df['secName'] + '_' \
+ info_df['announcementTime'].map(lambda x: x.replace('-', '')) + '_' \
+ info_df['announcementId'] + '_' \
+ info_df['announcementTitle'] + '.PDF'
download_urls = info_df['adjunctUrl']
total_records = info_df.shape[0]
print(f'将尝试下载 {total_records} 个公告文件。 ')
option = input('如需取消请输入n，留空或输入其他值将继续下载。 ')
if option.lower() == 'n':
print('用户取消下载！')
return
else:
print('开始下载...')
failure_num = 0
sleep(1)
for i in range(0, total_records):
url = download_urls[i]
name = file_names[i]
file_path = os.path.join(dl_path, name)
if os.path.exists(file_path) and (not overwrite):
print(f'公告 {name} 已存在，将跳过下载。')
continue
else:
try:
response = requests.get(url=url, stream=True)
with tqdm.wrapattr(
open(file_path, "wb"),
"write",
miniters=1,
desc=name,
total=int(response.headers.get('content-length',
0))) as f:
for chunk in response.iter_content(chunk_size=1024):
f.write(chunk)
except Exception as e:
print(f'下载第 {i + 1} 个公告 {name} 时出错！')
print(e)
failure_num += 1
continue
print(f'下载完成！下载成功 {total_records - failure_num} 个， 下载失败 {failure_num} 个。')
def download_track(tidal_session, track, folder):
media_url = tidal_session.get_media_url(track.id)
file_path = Path(folder) / track_file_name(track, media_url)
if file_path.exists():
return
with tqdm.wrapattr(open(file_path, 'wb+'),
"write",
miniters=1,
desc="Downloading {}".format(str(
file_path.name))) as fout:
for chunk in requests.get(media_url):
fout.write(chunk)
def download_file(
self,
out_dir: Path,
path: Path,
token: str,
endpoint_version: int,
**_,
) -> Path:
# Prep file_dir
file_dir = out_dir / path
# Skip if already exists
if file_dir.exists() and len(os.listdir(file_dir)):
return f"Skipped, files exist in {file_dir}"
file_dir.mkdir(parents=True, exist_ok=True)
# Download
logger.debug("Downloading %s...", path)
base_path = f"v{endpoint_version}/download"
url = f"{self.csdap_api_url}/{base_path}/{path.as_posix()}"
headers = {"authorization": f"Bearer {token}"}
with requests.get(url, stream=True, headers=headers) as response:
if not response.ok:
try:
msg = response.json()["detail"]
except (KeyError, json.JSONDecodeError):
msg = response.text
return f"Failed to download. {msg}"
# Determine filepath
filename = path.name
disposition = response.headers.get("Content-Disposition")
if disposition:
disposition_filename = re.findall("filename=(.+)", disposition)
if disposition_filename:
filename = disposition_filename[0]
filepath = file_dir / filename
# Write to local disk
tqdm_attrs = dict(
total=int(response.headers.get("content-size", 0)),
unit="iB",
unit_scale=True,
desc=f"Downloading {filepath.name}",
dynamic_ncols=True,
leave=False,
)
with filepath.open("wb") as f, tqdm.wrapattr(
response.raw, "read", **tqdm_attrs) as r_raw:
shutil.copyfileobj(r_raw, f)
return f"Downloaded file to {filepath}"
def _dowload_file(self, url: str, file_name: str):
raw_file = os.path.join(self._raw_folder, file_name)
response = requests.get(url, stream=True)
with tqdm.wrapattr(open(raw_file, "wb"),
"write",
total=int(response.headers.get('content-length',
0))) as target:
for chunk in response.iter_content(chunk_size=4096):
target.write(chunk)
print("Unzipping archive...")
with zipfile.ZipFile(raw_file, "r") as f:
f.extractall(self._raw_folder)
def download_video(self, link, f_name):
self.session_lock.acquire()
with self.session.get(link, stream=True) as r:
desc_name = os.path.sep.join(f_name.split(os.path.sep)[-2:])
with tqdm.wrapattr(open(f_name, "wb"), "write",
miniters=1, desc=desc_name,
total=int(r.headers.get('content-length'))) \
as f_out:
for chunk in r.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
f_out.write(chunk)
self.session_lock.release()
def upload(self):
with open(self.realpath, "rb") as f:
task_log_id = common.log('正在上传【%s】0%%' % self.filename, self.id, 'info')
with tqdm.wrapattr(f, "read", desc='正在上传【%s】' % self.filename, miniters=1,
initial=self.part_number * self.chunk_size,
total=self.filesize,
ascii=True
) as fs:
while self.part_number < len(self.part_upload_url_list):
upload_url = self.part_upload_url_list[self.part_number]['upload_url']
total_size = min(self.chunk_size, self.filesize)
fs.seek(self.part_number * total_size)
try:
res = requests.put(
url=upload_url,
data=common.read_in_chunks(fs, 16 * 1024, total_size),
verify=False,
timeout=None)
except Exception as e:
self.print(e, 'error')
self.part_upload_url_list = self.get_upload_url()
return self.upload()
if 400 <= res.status_code < 600:
common_get_xml_value = common.get_xml_tag_value(res.text, 'Message')
if common_get_xml_value == 'Request has expired.':
self.part_upload_url_list = self.get_upload_url()
continue
common_get_xml_value = common.get_xml_tag_value(res.text, 'Code')
if common_get_xml_value == 'PartNotSequential':
self.part_number -= 1
continue
elif common_get_xml_value == 'PartAlreadyExist':
pass
else:
self.print(res.text, 'error')
return False
self.part_number += 1
common.update_task_log(task_log_id,
'正在上传【%s】%.2f%%' % (
self.filename, ((self.part_number * total_size) / self.filesize) * 100))
udata = {
"part_number": self.part_number,
}
common.save_task(self.id, udata)
return True
def save_to_file(url, filename, id, info_message):
data_path = os.path.join(workdir, id)
data_id = filename.split("/")[-1].split("-")[1].split(".")[0]
rprint(info_message)
file_path = os.path.join(data_path, filename)
response = requests.get(url, stream=True)
with tqdm.wrapattr(
open(file_path, "wb"),
"write",
miniters=1,
desc=data_id,
total=int(response.headers.get("content-length", 0)),
) as fout:
for chunk in response.iter_content(chunk_size=4096):
fout.write(chunk)
fout.close()
def download_algoseek_data(self):
download_url = self.get_config_option("algoseek_data")
download_name = self.data_path + "/" + download_url.split("/")[-1]
if not os.path.exists(download_name):
zip_response = urlopen(download_url)
with tqdm.wrapattr(open(download_name, "wb"),
"write",
miniters=1,
desc="Downloading Algoseek Data",
total=getattr(zip_response, 'length',
None)) as out:
for chunk in zip_response:
out.write(chunk)
zip_file = ZipFile(download_name)
zip_file.extractall()
zip_file.close()
def download_file(r: requests.Response,
dest: Path,
name: str,
mtime: Optional[datetime] = None):
filename = dest / name
with tqdm.wrapattr(
open(filename, 'wb'),
'write',
miniters=1,
desc=str(filename),
total=int(r.headers.get('content-length', 0)),
) as f:
for chunk in r.iter_content(1024 * 1024 * 16):
f.write(chunk)
if mtime:
os.utime(filename, (datetime.now().timestamp(), mtime.timestamp()))
def upload(self):
with open(self.realpath, "rb") as f:
with tqdm.wrapattr(f,
"read",
desc='正在上传【%s】' % self.filename,
miniters=1,
initial=self.part_number * self.chunk_size,
total=self.filesize,
ascii=True) as fs:
while self.part_number < len(self.part_upload_url_list):
upload_url = self.part_upload_url_list[
self.part_number]['upload_url']
total_size = min(self.chunk_size, self.filesize)
fs.seek(self.part_number * total_size)
res = requests.put(url=upload_url,
data=common.read_in_chunks(
fs, 16 * 1024, total_size),
verify=False,
timeout=None)
if 400 <= res.status_code < 600:
common_get_xml_value = common.get_xml_tag_value(
res.text, 'Message')
if common_get_xml_value == 'Request has expired.':
self.part_upload_url_list = self.get_upload_url()
continue
common_get_xml_value = common.get_xml_tag_value(
res.text, 'Code')
if common_get_xml_value == 'PartAlreadyExist':
pass
else:
print_error(res.text)
res.raise_for_status()
self.part_number += 1
DATA['tasks'][
self.filepath_hash]['part_number'] = self.part_number
DATA['tasks'][
self.filepath_hash]['drive_id'] = self.drive_id
DATA['tasks'][self.filepath_hash]['file_id'] = self.file_id
DATA['tasks'][
self.filepath_hash]['upload_id'] = self.upload_id
DATA['tasks'][
self.filepath_hash]['chunk_size'] = self.chunk_size
common.save_task(DATA['tasks'])
def download_to_memory(
url: str,
show_progress: bool = False) -> Generator[BytesIO, None, None]:
stack = ExitStack()
try:
response = stack.enter_context(urllib.request.urlopen(url))
try:
total = int(response.getheader("Content-Length"))
except (TypeError, ValueError):
total = None
if show_progress:
response = stack.enter_context(
tqdm.wrapattr(response, "read", total=total))
memory_stream = stack.enter_context(BytesIO())
shutil.copyfileobj(response, memory_stream)
yield memory_stream
finally:
stack.close()
