def prefetch_files(args, semaphore=None):
for infile_name in args.infile_names:
try:
prefetch_file(infile_name,
dry_run=args.dry_run,
refetch=args.refetch,
ignore_content_type=args.ignore_content_type,
gamedata_dir=args.gamedata_dir,
timeout=args.timeout,
semaphore=semaphore,
user_agent=args.user_agent)
except (FileNotFoundError, IllegalSavegameException, SystemExit):
print_err("Aborting.")
sys.exit(1)
def prefetch_files(args, semaphore=None):
for infile_name in args.infile_names:
try:
prefetch_file(
infile_name,
dry_run=args.dry_run,
refetch=args.refetch,
ignore_content_type=args.ignore_content_type,
gamedata_dir=args.gamedata_dir,
timeout=args.timeout,
semaphore=semaphore
)
except FileNotFoundError:
print_err("Aborting.")
sys.exit(1)
def prefetch_files(args, semaphore=None):
for infile_name in args.infile_names:
try:
prefetch_file(
infile_name,
dry_run=args.dry_run,
refetch=args.refetch,
ignore_content_type=args.ignore_content_type,
gamedata_dir=args.gamedata_dir,
timeout=args.timeout,
semaphore=semaphore
)
except FileNotFoundError:
print_err("Aborting.")
sys.exit(1)
def prefetch_files(args, semaphore=None):
print('Establishing TOR connection')
SOCKS_PORT = 9050
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, '127.0.0.1', SOCKS_PORT)
print("Done")
for infile_name in args.infile_names:
try:
prefetch_file(infile_name,
dry_run=args.dry_run,
refetch=args.refetch,
ignore_content_type=args.ignore_content_type,
gamedata_dir=args.gamedata_dir,
timeout=args.timeout,
semaphore=semaphore,
user_agent=args.user_agent)
except (FileNotFoundError, IllegalSavegameException, SystemExit):
print_err("Aborting.")
sys.exit(1)
def prefetch_file(filename,
refetch=False,
ignore_content_type=False,
dry_run=False,
gamedata_dir=GAMEDATA_DEFAULT,
timeout=5,
semaphore=None,
user_agent='TTS prefetch'):
try:
save_name = get_save_name(filename)
except Exception:
save_name = "???"
print("Prefetching assets for {file} ({save_name}).".format(
file=filename, save_name=save_name))
try:
urls = urls_from_save(filename)
except FileNotFoundError as error:
print_err("Error retrieving URLs from {filename}: {error}".format(
error=error, filename=filename))
raise
done = set()
for path, url in urls:
if semaphore and semaphore.acquire(blocking=False):
print("Aborted.")
return
# Some mods contain malformed URLs missing a prefix. I’m not
# sure how TTS deals with these. Let’s assume http for now.
if not urllib.parse.urlparse(url).scheme:
print_err("Warning: URL {url} does not specify a URL scheme. "
"Assuming http.".format(url=url))
fetch_url = "http://" + url
else:
fetch_url = url
# A mod might refer to the same URL multiple times.
if url in done:
continue
# To prevent downloading unexpected content, we check the MIME
# type in the response.
if is_obj(path, url):
content_expected = lambda mime: any(
map(mime.startswith,
('text/plain', 'application/binary',
'application/octet-stream', 'application/json',
'application/x-tgif')))
elif is_assetbundle(path, url):
content_expected = lambda mime: any(
map(mime.startswith,
('application/binary', 'application/octet-stream')))
elif is_image(path, url):
content_expected = lambda mime: mime in (
'image/jpeg', 'image/jpg', 'image/png',
'application/octet-stream', 'application/binary')
else:
errstr = "Do not know how to retrieve URL {url} at {path}.".format(
url=url, path=path)
raise ValueError(errstr)
outfile_name = os.path.join(gamedata_dir, get_fs_path(path, url))
# Check if the object is already cached.
if os.path.isfile(outfile_name) and not refetch:
done.add(url)
continue
print("{} ".format(url), end="", flush=True)
if dry_run:
print("dry run")
done.add(url)
continue
headers = {'User-Agent': user_agent}
request = urllib.request.Request(url=fetch_url, headers=headers)
try:
response = urllib.request.urlopen(request, timeout=timeout)
except urllib.error.HTTPError as error:
print_err("Error {code} ({reason})".format(code=error.code,
reason=error.reason))
continue
except urllib.error.URLError as error:
print_err("Error ({reason})".format(reason=error.reason))
continue
except socket.timeout as error:
print_err("Error ({reason})".format(reason=error))
continue
except http.client.HTTPException as error:
print_err("HTTP error ({reason})".format(reason=error))
continue
# Only for informative purposes.
length = response.getheader('Content-Length', 0)
length_kb = "???"
if length:
with suppress(ValueError):
length_kb = int(length) / 1000
size_msg = "({length} kb): ".format(length=length_kb)
print(size_msg, end="", flush=True)
content_type = response.getheader('Content-Type', '').strip()
is_expected = content_expected(content_type)
if not (is_expected or ignore_content_type):
print_err(
"Error: Content type {type} does not match expected type. "
"Aborting. Use --relax to ignore.".format(type=content_type))
sys.exit(1)
try:
with open(outfile_name, 'wb') as outfile:
outfile.write(response.read())
except FileNotFoundError as error:
print_err("Error writing object to disk: {}".format(error))
raise
# Don’t leave files with partial content lying around.
except:
with suppress(FileNotFoundError):
os.remove(outfile_name)
raise
else:
print("ok")
if not is_expected:
errmsg = ("Warning: Content type {} did not match "
"expected type.".format(content_type))
print_err(errmsg)
done.add(url)
if dry_run:
completion_msg = "Dry-run for {} completed."
else:
completion_msg = "Prefetching {} completed."
print(completion_msg.format(filename))
def prefetch_file(filename,
refetch=False,
ignore_content_type=False,
dry_run=False,
gamedata_dir=GAMEDATA_DEFAULT,
timeout=5,
semaphore=None):
print("Prefetching assets for {file}.".format(file=filename))
try:
urls = urls_from_save(filename)
except FileNotFoundError as error:
print_err("Error retrieving URLs from {filename}: {error}".format(
error=error,
filename=filename
))
raise
done = set()
for path, url in urls:
if semaphore and semaphore.acquire(blocking=False):
print("Aborted.")
return
# Some mods contain malformed URLs missing a prefix. I’m not
# sure how TTS deals with these. Let’s assume http for now.
if not urllib.parse.urlparse(url).scheme:
print_err("Warning: URL {url} does not specify a URL scheme. "
"Assuming http.".format(url=url))
fetch_url = "http://" + url
else:
fetch_url = url
# A mod might refer to the same URL multiple times.
if url in done:
continue
# To prevent downloading unexpected content, we check the MIME
# type in the response.
if is_obj(path, url):
content_expected = lambda mime: any(map(mime.startswith,
('text/plain',
'application/json')))
elif is_image(path, url):
content_expected = lambda mime: mime in ('image/jpeg',
'image/png')
else:
errstr = "Do not know how to retrieve URL {url} at {path}.".format(
url=url,
path=path
)
raise ValueError(errstr)
outfile_name = os.path.join(gamedata_dir, get_fs_path(path, url))
# Check if the object is already cached.
if os.path.isfile(outfile_name) and not refetch:
done.add(url)
continue
print("{} ".format(url), end="")
if dry_run:
print("dry run")
done.add(url)
continue
try:
response = urllib.request.urlopen(fetch_url, timeout=timeout)
except urllib.error.HTTPError as error:
print_err("Error {code} ({reason})".format(
code=error.code,
reason=error.reason)
)
continue
except urllib.error.URLError as error:
print_err("Error ({reason})".format(reason=error.reason))
continue
# Only for informative purposes.
length = response.getheader('Content-Length', 0)
length_kb = "???"
if length:
with suppress(ValueError):
length_kb = int(length) / 1000
size_msg = "({length} kb): ".format(length=length_kb)
print(size_msg, end="", flush=True)
content_type = response.getheader('Content-Type').strip()
is_expected = content_expected(content_type)
if not (is_expected or ignore_content_type):
print_err(
"Error: Content type {type} does not match expected type. "
"Aborting. Use --relax to ignore.".format(type=content_type)
)
sys.exit(1)
try:
with open(outfile_name, 'wb') as outfile:
outfile.write(response.read())
except FileNotFoundError as error:
print_err("Error writing object to disk: {}".format(error))
raise
# Don’t leave files with partial content lying around.
except:
with suppress(FileNotFoundError):
os.remove(outfile_name)
raise
else:
print("ok")
if not is_expected:
errmsg = ("Warning: Content type {} did not match "
"expected type.".format(content_type))
print_err(errmsg)
done.add(url)
if dry_run:
completion_msg = "Dry-run for {} completed."
else:
completion_msg = "Prefetching {} completed."
print(completion_msg.format(filename))
def backup_json(args):
try:
urls = urls_from_save(args.infile_name)
except FileNotFoundError as error:
errmsg = "Could not read URLs from '{file}': {error}".format(
file=args.infile_name,
error=error
)
print_err(errmsg)
sys.exit(1)
# Change working dir, since get_fs_path gives us a relative path.
orig_path = os.getcwd()
try:
os.chdir(args.gamedata_dir)
except FileNotFoundError as error:
errmsg = "Could not open gamedata directory '{dir}': {error}".format(
dir=args.gamedata_dir,
error=error
)
print_err(errmsg)
sys.exit(1)
# We also need to correct the the destination path now.
if args.outfile_name:
args.outfile_name = os.path.join(orig_path, args.outfile_name)
else:
outfile_basename = re.sub(
r"\.json$", "",
os.path.basename(args.infile_name)
)
args.outfile_name = os.path.join(orig_path, outfile_basename) + ".zip"
try:
zipfile = ZipFile(args.outfile_name, 'w',
dry_run=args.dry_run,
ignore_missing=args.ignore_missing)
except FileNotFoundError as error:
errmsg = "Could not write to Zip archive '{outfile}': {error}".format(
outfile=args.outfile_name,
error=error
)
print_err(errmsg)
sys.exit(1)
with zipfile as outfile:
for path, url in urls:
filename = get_fs_path(path, url)
try:
outfile.write(filename)
except FileNotFoundError as error:
errmsg = "Could not write {filename} to Zip ({error}).".format(
filename=filename,
error=error
)
print_err(errmsg, "Aborting.", sep="\n", end=" ")
if not args.dry_run:
print_err("Zip file is incomplete.")
else:
print_err()
sys.exit(1)
# Finally, include the save file itself.
orig_json = os.path.join(orig_path, args.infile_name)
outfile.write(orig_json, os.path.basename(args.infile_name))
# Store some metadata.
outfile.put_metadata(comment=args.comment)
if args.dry_run:
print("Dry run for {file} completed.".format(file=args.infile_name))
else:
print("Backed-up contents for {file} found in {outfile}.".format(
file=args.infile_name,
outfile=args.outfile_name
))