def cleanup_space(main_dir=None,
size=1000,
percent=90,
never_delete=None,
what="media",
server="http://localhost:5279"):
"""Clean up space in the download drive when it is sufficiently full.
Parameters
----------
main_dir: str
It defaults to `$HOME`.
This is the main or root directory that holds both
the downloaded media files (mp4, mp3, mkv, etc.)
as well as the downloaded binary blobs.
On Linux, media files may go to `'$HOME/Downloads'`
and blobs are normally found in
`'$HOME/.locals/share/lbry/lbrynet/blobfiles'`
so `main_dir` would be `$HOME`, or `'/home/user'`
size: int, optional
It defaults to 1000.
Maximum size in GB of `main_dir`.
Ideally the downloaded media files and blobs never cross this limit.
percent: float, optional
It defaults to 90.
Percentage of `size` that indicates a soft limit
for the downloaded files.
After this limit is crossed it will try to free space in `main_dir`
by deleting older files and blobs, depending on the value
of `which_delete`.
never_delete: list of str, optional
It defaults to `None`.
If it exists it is a list with channel names.
The content produced by these channels will not be deleted
so the media files and blobs will remain in `main_dir`.
This is slow as it needs to perform an additional search
for the channel.
what: str, optional
It defaults to `'media'`, in which case only the full media file
(mp4, mp3, mkv, etc.) is deleted.
If it is `'blobs'` it will delete only the binary blobs.
If it is `'both'` it will delete both the media file
and the blobs.
As long as the blobs are present, the content can be seeded
to the network, and the full file can be restored.
That is, while the blobs exist the file is not completely deleted.
server: str, optional
It defaults to `'http://localhost:5279'`.
This is the address of the `lbrynet` daemon, which should be running
in your computer before using any `lbrynet` command.
Normally, there is no need to change this parameter from its default
value.
Returns
-------
bool
It returns `True` if the limit indicated by `size` and `percent`
was crossed by the downloaded files, and some of the older files
were successfully deleted to bring usage of `main_dir` within limits.
It returns `False` if there is a problem, or if the limit
was not crossed and thus there is nothing to clean up,
or if after going through all claims, it failed to clear
enough space to bring usage within limits.
"""
if not funcs.server_exists(server=server):
return False
if (not main_dir or not isinstance(main_dir, str) or main_dir == "~"
or not os.path.exists(main_dir)):
main_dir = os.path.expanduser("~")
print("Download directory should exist; "
f"set to main_dir='{main_dir}'")
if not isinstance(size, (int, float)) or size <= 0:
size = 1000
print("Max disk usage should be a positive number; "
f"set to size={size} GB")
if (not isinstance(percent, (int, float)) or percent <= 0
or percent > 100):
percent = 90
print("Percentage should be a positive number from 0 to 100; "
f"set to percent={percent} %")
if never_delete and not isinstance(never_delete, (list, tuple)):
print("Must be a list of channels that should never be deleted.")
print(f"never_delete={never_delete}")
return False
if (not isinstance(what, str) or what not in ("media", "blobs", "both")):
print(">>> Error: what can only be 'media', 'blobs', 'both'")
print(f"what={what}")
return False
limit_crossed = measure_usage(main_dir=main_dir,
size=size,
percent=percent)
if not limit_crossed:
print("Nothing to clean up.")
return False
sorted_items = sort.sort_items(server=server)
n_items = len(sorted_items)
for it, item in enumerate(sorted_items, start=1):
print(80 * "-")
out = "{:4d}/{:4d}, {}, ".format(it, n_items, item["claim_name"])
if never_delete:
channel = srch_ch.find_channel(cid=item["claim_id"],
full=False,
server=server)
if channel in never_delete:
print(out + f"item from {channel} will not be deleted. "
"Skipping.")
continue
print(out + "item will be deleted.")
clean.delete_single(cid=item["claim_id"], what=what, server=server)
limit_crossed = measure_usage(main_dir=main_dir,
size=size,
percent=percent)
if not limit_crossed:
print("Usage below limit. Stop deleting.")
print()
break
print()
if limit_crossed:
print(">>> Went through all downloaded claims, "
"and failed to clear enough space.")
print("Terminating.")
return False
return True
def redownload_blobs(uri=None, cid=None, name=None,
ddir=None, own_dir=True,
blobfiles=None, print_each=False,
server="http://*****:*****@MyChannel#3/some-video-name#2'
uri = '@MyChannel#3/some-video-name#2'
uri = 'some-video-name'
The URI is also called the `'canonical_url'` of the claim.
cid: str, optional
A `'claim_id'` for a claim on the LBRY network.
It is a 40 character alphanumeric string.
name: str, optional
A name of a claim on the LBRY network.
It is normally the last part of a full URI.
::
uri = 'lbry://@MyChannel#3/some-video-name#2'
name = 'some-video-name'
ddir: str, optional
It defaults to `$HOME`.
The path to the download directory.
own_dir: bool, optional
It defaults to `True`, in which case it places the downloaded
content inside a subdirectory named after the channel in `ddir`.
blobfiles: str, optional
It defaults to `'$HOME/.local/share/lbry/lbrynet/blobfiles'`.
The path to the directory where the blobs were downloaded.
This is normally seen with `lbrynet settings get`, under `'data_dir'`.
It can be any other directory if it is symbolically linked
to it, such as `'/opt/lbryblobfiles'`
print_each: bool, optional
It defaults to `False`.
If it is `True` it will not print all blobs
that belong to the claim, and whether each of them is already
in `blobfiles`.
server: str, optional
It defaults to `'http://localhost:5279'`.
This is the address of the `lbrynet` daemon, which should be running
in your computer before using any `lbrynet` command.
Normally, there is no need to change this parameter from its default
value.
Returns
-------
bool
It returns `True` if all blobs are already present in the system
so nothing needs to be downloaded.
It returns `False` if the item does not exist,
of if at least one blob was downloaded.
Bug
---
At the moment downloading missing blobs is not possible;
the command hangs and never timeouts.
::
lbrynet blob get <hash>
This bug is reported in lbryio/lbry-sdk, issue #2070.
If the bug is solved, `blob_get` could be called with the missing blob
hash to only get that piece.
"""
if not funcs.server_exists(server=server):
return False
blob_info = blobs.count_blobs(uri=uri, cid=cid, name=name,
blobfiles=blobfiles,
print_each=print_each,
server=server)
if "error_not_found" in blob_info:
return False
print(80 * "-")
if "error_no_sd_hash" in blob_info:
print(blob_info["error_no_sd_hash"]
+ "; start download from the start.")
elif blob_info["all_present"]:
print("All blobs files present, nothing to download.")
return True
else:
print("Blobs missing; redownload claim.")
print()
# If the bug #2070 is solved, this could be run.
# print("Blobs missing; redownload blobs")
# for blob in blob_info["missing"]:
# out = f"{blob[0]}, "
# blob_get(blob=blob[1], action="get", out=out,
# server=server)
# The missing blobs will only be downloaded if the media file
# is not present so we must make sure it is deleted.
# print("Blobs missing; redownload claim")
print("Ensure the media file is deleted.")
clean.delete_single(cid=blob_info["claim_id"], what="media",
server=server)
print()
dld.download_single(cid=blob_info["claim_id"],
ddir=ddir, own_dir=own_dir,
server=server)
return False
def remove_claims(start=1,
end=0,
file=None,
invalid=False,
what="media",
server="http://localhost:5279"):
"""Delete claims from a file, or delete the ones already present.
Parameters
----------
start: int, optional
It defaults to 1.
Operate on the item starting from this index in the internal list
of claims or in the claims provided by `file`.
end: int, optional
It defaults to 0.
Operate until and including this index in the internal list of claims
or in the claims provided by `file`.
If it is 0, it is the same as the last index.
file: str, optional
It defaults to `None`.
The file to read claims from. It is a comma-separated value (CSV)
list of claims, in which each row represents a claim,
and one element is the `'claim_id'` which can be used
with `delete_single` to delete that claim.
If `file=None` it will delete the claims obtained
from `sort_items` which should already be present
in the system fully or partially.
invalid: bool, optional
It defaults to `False`, in which case it will assume
the processed claims are still valid in the online database.
It will use `lbrynet claim search` to resolve the `claim_id`.
If it is `True` it will assume the claims are no longer valid,
that is, that the claims have been removed from the online database
and only exist locally.
In this case, it will use `lbrynet file list` to resolve
the `claim_id`.
Therefore this parameter is required if `file` is a document
containing 'invalid' claims, otherwise the claims won't be found
and won't be deleted.
what: str, optional
It defaults to `'media'`, in which case only the full media file
(mp4, mp3, mkv, etc.) is deleted.
If it is `'blobs'`, it will delete only the blobs.
If it is `'both'`, it will delete both the media file
and the blobs.
server: str, optional
It defaults to `'http://localhost:5279'`.
This is the address of the `lbrynet` daemon, which should be running
in your computer before using any `lbrynet` command.
Normally, there is no need to change this parameter from its default
value.
Returns
-------
list of bool
It returns a list of booleans, where each boolean represents
a deleted item; `True` if the claim was deleted successfully,
and `False` if it was not.
False
If there is a problem, non-existing claims, or non-existing file,
it will return `False`.
"""
if not funcs.server_exists(server=server):
return False
print(80 * "-")
if not file:
print("Remove claims from existing list")
sorted_items = sort.sort_items(server=server)
if not sorted_items:
print(">>> Error: no claims previously downloaded.")
return False
else:
if file and not isinstance(file, str) or not os.path.exists(file):
print("The file path must exist.")
print(f"file={file}")
return False
print("Remove claims from existing file")
sorted_items = srch.parse_claim_file(file=file)
print()
if not sorted_items:
print(">>> Error: the file must have a 'claim_id' "
"(40-character alphanumeric string); "
"could not parse the file.")
print(f"file={file}")
return False
n_items = len(sorted_items)
list_del_info = []
for it, item in enumerate(sorted_items, start=1):
if it < start:
continue
if end != 0 and it > end:
break
print(f"Claim {it}/{n_items}")
del_info = clean.delete_single(cid=item["claim_id"],
invalid=invalid,
what=what,
server=server)
list_del_info.append(del_info)
print()
return list_del_info
def download_missing_blobs(blobfiles=None,
ddir=None,
channel=None,
start=1,
end=0,
print_msg=False,
print_each=False,
server="http://*****:*****@MyChannel#5'`, `'MyChannel#5'`, `'MyChannel'`
If a simplified name is used, and there are various channels
with the same name, the one with the highest LBC bid will be selected.
Enter the full name to choose the right one.
start: int, optional
It defaults to 1.
Count the blobs from claims starting from this index
in the list of items.
end: int, optional
It defaults to 0.
Count the blobs from claims until and including this index
in the list of items.
If it is 0, it is the same as the last index in the list.
print_msg: bool, optional
It defaults to `True`, in which case it will print information
on the found claim.
If `print_msg=False`, it also implies `print_each=False`.
print_each: bool, optional
It defaults to `False`.
If it is `True` it will print all blobs
that belong to the claim, and whether each of them is already
in `blobfiles`.
server: str, optional
It defaults to `'http://*****:*****@"):
channel = "@" + channel
output = analyze_blobs(blobfiles=blobfiles,
channel=channel,
start=start,
end=end,
print_msg=print_msg,
print_each=print_each,
server=server)
if not output:
return False
# claims_complete = output[0]
claims_incomplete = output[1]
claims_no_sd_hash = output[2]
# claims_not_found = output[3]
# claims_other_error = output[4]
print()
if not (claims_incomplete or claims_no_sd_hash):
if channel:
print(f"All claims from {channel} have complete blobs "
"(data and 'sd_hash'). "
"Nothing will be downloaded.")
else:
print("All claims have complete blobs (data and 'sd_hash'). "
"Nothing will be downloaded.")
return False
n_claims_incomplete = len(claims_incomplete)
n_claims_no_sd_hash = len(claims_no_sd_hash)
info_get_incomplete = []
info_get_no_sd = []
if n_claims_incomplete:
print("Claims with incomplete blobs: "
f"{n_claims_incomplete} (will be redownloaded)")
else:
print("Claims with incomplete blobs: "
f"{n_claims_incomplete} (nothing to redownload)")
for it, info in enumerate(claims_incomplete, start=1):
blob_info = info["blob_info"]
uri = blob_info["canonical_url"]
cid = blob_info["claim_id"]
name = blob_info["name"]
print(f"Claim {it}/{n_claims_incomplete}")
# The missing blobs will only be downloaded if the media file
# is not present so we must make sure it is deleted.
print("Ensure the media file is deleted.")
clean.delete_single(uri=uri,
cid=cid,
name=name,
what="media",
server=server)
print()
info = dld.download_single(uri=uri,
cid=cid,
name=name,
ddir=ddir,
server=server)
info_get_incomplete.append(info)
print()
if n_claims_no_sd_hash:
print("Claims with no 'sd_hash' blobs: "
f"{n_claims_no_sd_hash} (will be redownloaded)")
else:
print("Claims with no 'sd_hash' blobs: "
f"{n_claims_no_sd_hash} (nothing to redownload)")
for it, info in enumerate(claims_no_sd_hash, start=1):
blob_info = info["blob_info"]
uri = blob_info["canonical_url"]
cid = blob_info["claim_id"]
name = blob_info["name"]
print(f"Claim {it}/{n_claims_no_sd_hash}")
# The missing blobs will only be downloaded if the media file
# is not present so we must make sure it is deleted.
print("Ensure the media file is deleted.")
clean.delete_single(uri=uri,
cid=cid,
name=name,
what="media",
server=server)
print()
info = dld.download_single(uri=uri,
cid=cid,
name=name,
ddir=ddir,
server=server)
info_get_no_sd.append(info)
print()
return info_get_incomplete, info_get_no_sd
def ch_cleanup(channel=None,
number=2,
what="media",
server="http://*****:*****@MyChannel#5'`, `'MyChannel#5'`, `'MyChannel'`
If a simplified name is used, and there are various channels
with the same name, the one with the highest LBC bid will be selected.
Enter the full name to choose the right one.
number: int, optional
It defaults to 2.
The number of items to keep from `channel`.
These will be the newest ones according to their `'release_time'`
or `'timestamp'`, if the former is missing.
what: str, optional
It defaults to `'media'`, in which case only the full media file
(mp4, mp3, mkv, etc.) is deleted.
If it is `'blobs'`, it will delete only the blobs.
If it is `'both'`, it will delete both the media file
and the blobs.
server: str, optional
It defaults to `'http://localhost:5279'`.
This is the address of the `lbrynet` daemon, which should be running
in your computer before using any `lbrynet` command.
Normally, there is no need to change this parameter from its default
value.
Returns
-------
list of bool
It returns a list of booleans, where each boolean represents
a deleted item; `True` if the claim was deleted successfully,
and `False` if it was not.
False
If there is a problem or non existing channel,
it will return `False`.
"""
if not funcs.server_exists(server=server):
return False
if not channel or not isinstance(channel, str):
print("Clean up items from a single channel.")
print(f"channel={channel}")
return False
if (number is None or number is False or not isinstance(number, int)
or number < 0):
number = 2
print("Number must be a positive integer, "
f"set to default value, number={number}")
if (not isinstance(what, str) or what not in ("media", "blobs", "both")):
print(">>> Error: what can only be 'media', 'blobs', 'both'")
print(f"what={what}")
return False
list_info_del = []
sorted_items = sort.sort_items(channel=channel, server=server)
if not sorted_items:
print()
return False
n_items = len(sorted_items)
remaining = n_items - 0
for it, item in enumerate(sorted_items, start=1):
if remaining <= number:
print(8 * "-")
print(f"Finished deleting; remaining {remaining}")
print()
break
print(f"Claim {it}/{n_items}")
del_info = clean.delete_single(cid=item["claim_id"],
what=what,
server=server)
list_info_del.append(del_info)
remaining = n_items - it
if remaining > number:
print()
if remaining == 0:
print(8 * "-")
print(f"Finished deleting; remaining {remaining}")
print()
return list_info_del