def test_files(self):
files = {}
for directory in ['input', 'expected']:
_, _, filenames = next(walk(f'{TEST_DIR}/{directory}'))
files[directory] = [
f'{TEST_DIR}/{directory}/{file}' for file in filenames
if search(r'([0-9])', file)
]
actuals = []
files['output'] = []
for index, filepath in enumerate(files['input']):
files['output'].append(
write_receipt(read_receipt(filepath),
filepath.replace('input', 'output')))
self.assertTrue(
compare(files['output'][index], files['expected'][index]))
def _handle_response(cache_path, pipeline, req_headers, res, temp_file):
body = res.read()
items = pipeline(get_lines(body.decode()))
for item in items:
temp_file.write(item + os.linesep)
temp_file.flush()
if cache_path.is_file() and compare(temp_file.name, cache_path):
logger.debug(modi + ": %s", req_headers.get(modi, "N/A"))
logger.debug(modl + ": %s", res.headers.get(modl, "N/A"))
verb = "unmodified"
elif cache_path.is_file():
shutil.copyfile(temp_file.name, cache_path)
verb = "updated"
else:
shutil.copyfile(temp_file.name, cache_path)
verb = "created"
logger.info("%s %s", cache_path, verb)
def mod_patch(modpack_name, add_to_whitelist=True, check_only=False):
changes_present = False
changeSet = set()
patch_name = ''.join(char
for char in modpack_name if char.isalnum()) + "Patch"
print(f"\nChecking for customizations in {modpack_name}...\n")
fileSet = {"duplicateFilesList.txt", "allFilesList.txt", "whitelist.txt"
} # To avoid copying these meta lists to the patch folder.
patch_created = False
whitelist = open(
f'mod{os.sep}{modpack_name}{os.sep}whitelist.txt').read().split("\n")
# Keep only the alphanumeric characters in the names in the whitelist, remove blank lines:
whitelist = [
''.join(char for char in list_entry if char.isalnum())
for list_entry in whitelist if list_entry
]
for entry in whitelist:
fileList = glob.glob(f'mod{os.sep}{entry}{os.sep}**', recursive=True)
for cur_file in fileList:
if not os.path.isfile(cur_file) or ".mod" in cur_file:
continue # Skip directory the folders themselves and mod descriptor files.
file_path = cur_file.split(os.sep)
file_path[1] = modpack_name # Change folder to the modpack.
path_within_mod = os.sep.join(file_path[2:])
if path_within_mod in fileSet:
# A file with this path has already been compared.
# The cur_file might be a conflict or maybe a duplicate.
# Either way, no checking is necessary, so go to the next file.
continue
try:
if compare(cur_file, os.sep.join(file_path)):
# The first mod on the whitelist that has this file has an identical file to the file in the modpack.
# This means no customization has been made, and nothing need be done.
fileSet.add(path_within_mod)
else:
# This file has been altered.
if check_only:
changes_present = True
if path_within_mod not in changeSet:
print(
f"File in modpack differs from file in mod: {os.sep.join(cur_file.split(os.sep)[1:])}"
)
changeSet.add(path_within_mod)
continue # Advance to next file without any copying.
# Copy the file to the patch folder.
print(
f"Adding customized file to patch: {path_within_mod}")
patch_created = True
fileSet.add(path_within_mod)
modified_file = os.sep.join(file_path)
file_path[1] = patch_name # Change folder to the patch.
target_path = os.sep.join(file_path)
target_dir = os.path.dirname(target_path)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
copy2(modified_file, target_path)
except Exception as e:
print(cur_file)
print(e)
# Check the modpack for unique files that are not present in any of the mods on the whitelist.
all_mod_files = glob.glob(f'mod{os.sep}{modpack_name}{os.sep}**',
recursive=True)
for cur_file in all_mod_files:
if not os.path.isfile(cur_file) or ".mod" in cur_file:
continue # Skip directory the folders themselves and mod descriptor files.
file_path = cur_file.split(os.sep)
path_within_mod = os.sep.join(file_path[2:])
if path_within_mod not in fileSet:
# Even after looking through all of the mods, we've never seen this file. It must be new.
if check_only:
changes_present = True
if path_within_mod not in changeSet:
print(f"File has been added to modpack: {path_within_mod}")
changeSet.add(path_within_mod)
continue # Advance to next file without any copying.
# Copy the new file to the patch.
print(f"Adding unique file to patch: {path_within_mod}")
patch_created = True
file_path = cur_file.split(os.sep)
file_path[1] = patch_name
target_path = os.sep.join(file_path)
target_dir = os.path.dirname(target_path)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
copy2(cur_file, target_path)
if check_only:
if changes_present:
return True
return False # Done checking for new changes - there were none!
if add_to_whitelist:
if patch_name not in whitelist:
whitelist.insert(0, patch_name)
with open(f"mod{os.sep}{modpack_name}{os.sep}whitelist.txt",
"w+") as f:
f.writelines([i + '\n' for i in whitelist])
whitelist = open(f'whitelist.txt').read().split("\n")
if patch_name not in whitelist:
whitelist.insert(0, patch_name)
with open("whitelist.txt", "w+") as f:
f.writelines([i + '\n' for i in whitelist])
if patch_created:
if not os.path.isfile(f"mod{os.sep}{patch_name}.mod"):
with open(f"mod{os.sep}{patch_name}.mod", "w+") as f:
f.writelines([
f"name=\"{patch_name}\"\n",
f"path=\"mod{os.sep}{patch_name}\"\n", "tags={\n",
"\t\"Gameplay\"\n", "}\n", "supported_version=\"2.7.*\"\n"
])
print(
f"\n\nCustomized files in \"{modpack_name}\" have been copied to \"{patch_name}\".{' The patch has been added to your whitelist.' if add_to_whitelist else ''}\n"
)
else:
print(
f"\n\nNo customized files found in \"{modpack_name}\". No patch created.\n"
)
# Return a bool indicating that a patch has indeed been created.
return patch_created
def make_modpack(modpack_name, overwrite):
print("\nAssembling modpack...")
fileIndex = dict()
whitelist = open('whitelist.txt').read().split('\n')
# Keep only the alphanumeric characters in the names in the whitelist, remove blank lines:
whitelist = [
''.join(char for char in list_entry if char.isalnum())
for list_entry in whitelist if list_entry
]
patch_created = False
if os.path.isdir(f"mod{os.sep}{modpack_name}") and not overwrite:
print(f"The {modpack_name} directory already exists...")
patch_name = ''.join(
char for char in modpack_name if char.isalnum()) + "Patch"
if os.path.isdir(f"mod{os.sep}{patch_name}"):
print(
"It looks like a modpack patch has already been created for this modpack. Checking for new changes."
)
new_changes_present = mod_patch(modpack_name, check_only=True)
if new_changes_present:
print(
f"\nIt looks like there are additional changes in your {modpack_name} folder aside from those saved in {patch_name}."
)
print(
f"This process will abort in order to prevent loss of changes in your {modpack_name} or overwriting of files in your {patch_name} folder."
)
print(
f"To force skip the patch creation process and overwrite files in the {modpack_name} folder, run \"installer.py -ovr\""
)
print(
f"To force the updating of the contents of {patch_name}, run \"make_mod_patch.py\" and then run \"installer.py\" again."
)
print(
"If the only differing files are in the modpack patch itself, you must run \"installer.py -ovr\" to push those files into the modpack."
)
return
else:
print(
"There are no additional changes in the modpack aside from those already in the patch."
)
else:
patch_created = mod_patch(modpack_name, add_to_whitelist=False)
print("Continuing to assemble modpack...\n")
for entry in whitelist:
fileList = glob.glob(f'mod{os.sep}{entry}{os.sep}interface{os.sep}**',
recursive=True)
for cur_file in fileList:
if not os.path.isfile(cur_file) or ".mod" in cur_file:
continue # Skip directory the folders themselves and mod descriptor files.
file_path = cur_file.split(os.sep)
file_path[1] = modpack_name # Change folder to the modpack.
path_within_mod = os.sep.join(file_path[2:])
if path_within_mod in fileIndex:
# There is already a file at this path in the modpack, it is either a conflict or a duplicate.
if compare(cur_file, os.sep.join(file_path)):
fileIndex[path_within_mod][0].append(
f"DUPLICATE: {entry.strip()}")
fileIndex[path_within_mod][2] += 1
continue # Skip duplicate files.
# File is a conflict:
fileIndex[path_within_mod][0].append(
f"CONFLICT: {entry.strip()}")
fileIndex[path_within_mod][1] += 1
file_path[
1] = f"{modpack_name}_conflicts!" # Copy the conflicting file to the conflicts folder.
fname, extension = file_path[-1].split('.')
file_path[-1] = fname + "." + extension + " " + entry.strip(
) + "." + extension
print(
f"Confict detected. Moving to {os.sep.join(file_path[1:])}."
)
else:
# First time we've seen a file at this path.
# First entry in the dict is for mod source, second is for conflict count, third is for repeat count.
fileIndex[path_within_mod] = [[(f"SELECTED: {entry.strip()}")],
0, 0]
# Make dirs if necessary:
target_path = os.sep.join(file_path)
target_dir = os.path.dirname(target_path)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# Copy the mod file to its destination.
copy2(cur_file, target_path)
print("\n\nDone copying files.")
if fileIndex:
with open(f'mod{os.sep}{modpack_name}{os.sep}whitelist.txt',
"w+") as f:
f.writelines([i + '\n' for i in whitelist])
conflicts_dict = dict()
duplicates_dict = dict()
for i in fileIndex:
if fileIndex[i][1]:
conflicts_dict[i] = fileIndex[i][0]
if fileIndex[i][2]:
duplicates_dict[i] = fileIndex[i][0]
fileIndex[i] = fileIndex[i][0]
# Output a list of all files.
with open(f"mod{os.sep}{modpack_name}{os.sep}allFilesList.txt",
"w+") as f:
f.write(json.dumps(fileIndex, indent=4))
if duplicates_dict:
# Output a list of duplicate files.
with open(
f"mod{os.sep}{modpack_name}{os.sep}duplicateFilesList.txt",
"w+") as f:
f.write(json.dumps(duplicates_dict, indent=4))
print(
f"Duplicate files listed in mod{os.sep}{modpack_name}{os.sep}duplicateFilesList.txt"
)
if conflicts_dict:
# Output a list of conflicting files.
with open(
f"mod{os.sep}{modpack_name}_conflicts!{os.sep}conflictingFilesList.txt",
"w+") as f:
f.write(json.dumps(conflicts_dict, indent=4))
print(
f"Conflicting files listed in mod{os.sep}{modpack_name}_conflicts!{os.sep}conflictingFilesList.txt"
)
else:
print("No conflicts!")
mod_descriptor_name = ''.join(char for char in modpack_name
if char.isalnum()).capitalize()
if not os.path.isfile(f"mod{os.sep}{mod_descriptor_name}.mod"):
with open(f"mod{os.sep}{mod_descriptor_name}.mod", "w+") as f:
f.writelines([
f"name=\"{modpack_name}\"\n", f"path=\"mod/{modpack_name}\"\n",
"tags={\n", "\t\"Gameplay\"\n", "}\n",
"supported_version=\"2.7.*\"\n"
])
if patch_created:
print(
"A modpack patch was created in order to preserve the customizations in the modpack folder."
)
print(
f"To revert the customizations to your modpack, add {patch_name} to your whitelist (at the top if you want to ensure your customizations are prioritized)."
)
print("Done!")
def test_copy01(self):
"""
Test our copy function
The copy function allows us to duplicate an existing NNTPContent
object without obstructing the original. Copied content is
always attached; so if the object falls out of scope; so does
the file.
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy')
assert (isdir(my_dir) is False)
assert (mkdir(my_dir) is True)
assert (isdir(my_dir) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath='myfile',
work_dir=my_dir,
)
# Content is attached by default
assert (obj.is_attached() is True)
obj.detach()
assert (obj.is_attached() is False)
new_dir = join(my_dir, 'copy')
assert (isdir(new_dir) is False)
# Create a copy of our object
obj_copy = obj.copy()
# Successfully loaded files are never attached reguardless
# of the original copy
assert (obj_copy.is_attached() is True)
# Reattach the original so it dies when this test is over
obj.attach()
assert (obj.is_attached() is True)
# Create a copy of our copy
obj_copy2 = obj_copy.copy()
assert (obj_copy2.is_attached() is True)
assert (isfile(obj_copy2.path()))
_path = obj_copy2.path()
del obj_copy2
assert (isfile(_path) is False)
assert (isfile(obj_copy.path()) is True)
_path = obj_copy.path()
del obj_copy
assert (isfile(_path) is False)
assert (isfile(obj.path()) is True)
_path = obj.path()
del obj
assert (isfile(_path) is False)
# now lets do a few more tests but with actual files this time
tmp_file = join(my_dir, '2MB.zip')
assert (self.touch(tmp_file, size='2MB', random=True) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath=tmp_file,
work_dir=my_dir,
)
assert (isfile(obj.path()) is True)
obj_copy = obj.copy()
assert (isfile(obj_copy.path()) is True)
stats = stat(obj_copy.path())
assert (bytes_to_strsize(stats['size']) == "2.00MB")
# Compare that our content is the same
assert (compare(obj_copy.path(), obj.path()) is True)
# note that the filenames are NOT the same so we are dealing with 2
# distinct copies here
assert (isfile(obj_copy.path()) is True)
assert (isfile(obj.path()) is True)
assert (obj.path() != obj_copy.path())
def test_maybe_download(self):
file_path = maybe_download("sample.txt", self.sample_download_location,
self.working_directory)
if not self.passed:
self.assertTrue(compare(file_path, self.sample_txt))
return
def test_copy01(self):
"""
Test our copy function
The copy function allows us to duplicate an existing NNTPContent
object without obstructing the original. Copied content is
always attached; so if the object falls out of scope; so does
the file.
"""
my_dir = join(self.tmp_dir, 'NNTPContent', 'test_copy')
assert(isdir(my_dir) is False)
assert(mkdir(my_dir) is True)
assert(isdir(my_dir) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath='myfile',
work_dir=my_dir,
)
# Content is attached by default
assert(obj.is_attached() is True)
obj.detach()
assert(obj.is_attached() is False)
new_dir = join(my_dir, 'copy')
assert(isdir(new_dir) is False)
# Create a copy of our object
obj_copy = obj.copy()
# Successfully loaded files are never attached reguardless
# of the original copy
assert(obj_copy.is_attached() is True)
# Reattach the original so it dies when this test is over
obj.attach()
assert(obj.is_attached() is True)
# Create a copy of our copy
obj_copy2 = obj_copy.copy()
assert(obj_copy2.is_attached() is True)
assert(isfile(obj_copy2.path()))
_path = obj_copy2.path()
del obj_copy2
assert(isfile(_path) is False)
assert(isfile(obj_copy.path()) is True)
_path = obj_copy.path()
del obj_copy
assert(isfile(_path) is False)
assert(isfile(obj.path()) is True)
_path = obj.path()
del obj
assert(isfile(_path) is False)
# now lets do a few more tests but with actual files this time
tmp_file = join(my_dir, '2MB.zip')
assert(self.touch(tmp_file, size='2MB', random=True) is True)
# Now create our NNTPContent object witin our directory
obj = NNTPContent(
filepath=tmp_file,
work_dir=my_dir,
)
assert(isfile(obj.path()) is True)
obj_copy = obj.copy()
assert(isfile(obj_copy.path()) is True)
stats = stat(obj_copy.path())
assert(bytes_to_strsize(stats['size']) == "2.00MB")
# Compare that our content is the same
assert(compare(obj_copy.path(), obj.path()) is True)
# note that the filenames are NOT the same so we are dealing with 2
# distinct copies here
assert(isfile(obj_copy.path()) is True)
assert(isfile(obj.path()) is True)
assert(obj.path() != obj_copy.path())
