def download(self, dest, sel_files=None):
count = 0
num = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
for urls, files in self.urls:
if not isinstance(urls, list):
urls = [urls]
if sel_files is not None:
files = set(files) & sel_files
for filename in files:
done = False
for link in urls:
with open(os.path.join(dest, filename), 'wb') as dl:
progress.start_task(float(num) / count, 1.0 / count, '%d/%d: %%s' % (num + 1, count))
if download(pjoin(link, filename), dl):
num += 1
done = True
progress.finish_task()
break
progress.finish_task()
if not done:
logging.error('Failed to download "%s"!', filename)
def download(self, dest, sel_files=None):
count = 0
num = 0
for u, files in self.urls:
if sel_files is not None:
files = set(files) & sel_files
count += len(files)
for urls, files in self.urls:
if not isinstance(urls, list):
urls = [urls]
if sel_files is not None:
files = set(files) & sel_files
for filename in files:
done = False
for link in urls:
with open(os.path.join(dest, filename), 'wb') as dl:
progress.start_task(
float(num) / count, 1.0 / count,
'%d/%d: %%s' % (num + 1, count))
if download(pjoin(link, filename), dl):
num += 1
done = True
progress.finish_task()
break
progress.finish_task()
if not done:
logging.error('Failed to download "%s"!', filename)
def set_files(self, files):
self.files = {}
if not isinstance(files, list):
logging.error('"%s"\'s file list has an unknown type.', self.name)
return False
elif len(files) > 0 and isinstance(files[0], dict):
for item in files:
self.files[item["filename"]] = PkgFile(item, self)
else:
for urls, file_items in files:
for name, info in file_items.items():
info = info.copy()
info["filename"] = name
info["urls"] = [pjoin(url, name) for url in urls]
self.files[name] = PkgFile(info, self)
def set_files(self, files):
self.files = {}
if not isinstance(files, list):
logging.error('"%s"\'s file list has an unknown type.', self.name)
return False
elif len(files) > 0 and isinstance(files[0], dict):
for item in files:
self.files[item['filename']] = PkgFile(item, self)
else:
for urls, file_items in files:
for name, info in file_items.items():
info = info.copy()
info['filename'] = name
info['urls'] = [pjoin(url, name) for url in urls]
self.files[name] = PkgFile(info, self)
def get_tree(self, parent=None, tree=None, prefix=''):
if parent is None:
parent = self.root
if tree is None:
tree = {}
for item, info in parent.contents.items():
if item in ('.', '..'):
continue
path = util.pjoin(prefix, item)
if isinstance(info, Directory):
self.get_tree(info, tree, path)
else:
tree[path] = info.content
return tree
def get_tree(self, parent=None, tree=None, prefix=''):
if parent is None:
parent = self.root
if tree is None:
tree = {}
for item, info in parent.contents.items():
if item in ('.', '..'):
continue
path = util.pjoin(prefix, item)
if isinstance(info, Directory):
self.get_tree(info, tree, path)
else:
tree[path] = info.content
return tree
def work(self, item):
id_, links, name, archive, tstamp = item
with tempfile.TemporaryDirectory() as dest:
if self.dl_path is None:
base_path = dest
else:
base_path = os.path.join(self.dl_path, self.dl_slug)
f_name = os.path.basename(name)
path = os.path.join(base_path, f_name)
idx = 1
while os.path.exists(path):
path = os.path.join(base_path, str(idx) + '_' + f_name)
idx += 1
res = self._download(links, path, tstamp)
for i, link in reversed(list(enumerate(links))):
for pref in self.rem_prefixes:
if link.startswith(pref):
del links[i]
if res == 304:
# Nothing changed.
self.post((id_, 'CACHE', None, 0))
elif res:
logging.info('Inspecting "%s"...', name)
progress.update(0.999, 'Inspecting "%s"...' % name)
csum, content = self._inspect_file(id_, archive, dest, path)
if csum != 'FAILED':
if self.dl_mirror is not None:
links.append(util.pjoin(self.dl_mirror, self.dl_slug, os.path.basename(path)))
self.post((id_, csum, content, os.path.getsize(path)))
else:
os.unlink(path)
self.post((id_, 'FAILED', None, 0))
else:
# None of the links worked!
self.post((id_, 'FAILED', None, 0))
def work(self, item):
id_, links, name, archive, tstamp = item
with tempfile.TemporaryDirectory() as dest:
if self.dl_path is None:
base_path = dest
else:
base_path = self.dl_path
f_name = os.path.basename(name)
path = os.path.join(base_path, f_name)
idx = 1
while os.path.exists(path):
path = os.path.join(base_path, str(idx) + '_' + f_name)
idx += 1
res = self._download(links, path, tstamp)
if res == 304:
# Nothing changed.
self.post((id_, 'CACHE', None, 0))
elif res:
logging.info('Inspecting "%s"...', name)
progress.update(1, 'Inspecting "%s"...' % name)
csum, content = self._inspect_file(id_, archive, dest, path)
if csum != 'FAILED':
if self.dl_mirror is not None:
links.append(
util.pjoin(self.dl_mirror, os.path.basename(path)))
self.post((id_, csum, content, os.path.getsize(path)))
else:
self.post((id_, 'FAILED', None, 0))
else:
# None of the links worked!
self.post((id_, 'FAILED', None, 0))
def import_tree(self, mod_tree):
mod_tree = self._flatten_tree(mod_tree)
for path, mod in mod_tree.items():
# First we need its ID.
# For now, I'll simply use the path with a prefix.
id_ = "FSOI#" + path
if id_ in self.mods:
# Let's update it but try to honour local modifications.
submods = ["FSOI#" + ".".join(smod.path) for smod in mod.submods]
dependencies = ["FSOI#" + path for path in mod.dependencies]
delete = [pjoin(mod.folder, path) for path in mod.delete]
entry = self.mods[id_]
for smod in submods:
if smod not in entry["submods"]:
entry["submods"].append(smod)
e_deps = entry["packages"][0]["dependencies"]
for path in dependencies:
if path not in e_deps:
e_deps.append(path)
del_files = []
for act in entry["actions"]:
if act["type"] == "delete":
del_files.extend(act["files"])
missing_del_files = set(delete) - set(del_files)
added = False
for act in entry["actions"]:
if act["type"] == "delete":
act["files"].extend(missing_del_files)
added = True
if not added:
entry["actions"].append({"type": "delete", "files": missing_del_files})
else:
# NOTE: I'm deliberatly dropping support for COPY and RENAME here.
entry = self.mods[id_] = {
"id": id_,
"title": mod.name,
"version": mod.version,
"description": mod.desc,
"logo": None,
"notes": mod.note,
"submods": ["FSOI#" + ".".join(smod.path) for smod in mod.submods],
"packages": [
{
"name": "Core files",
"notes": "",
"status": "required",
"dependencies": [
{"id": "FSOI#" + path, "version": "*", "packages": []} for path in mod.dependencies
],
"environment": [],
"files": [],
}
],
"actions": [{"type": "delete", "files": [pjoin(mod.folder, path) for path in mod.delete]}],
}
trail = "FSOI#"
e_deps = entry["packages"][0]["dependencies"]
for item in mod.path:
trail += item
if trail not in e_deps:
e_deps.append({"id": trail, "version": "*", "packages": []})
trail += "."
# Always keep the files in sync.
flist = self.mods[id_]["packages"][0]["files"] = []
for urls, filenames in mod.urls:
files = {}
flist.append((urls, files))
for name in filenames:
files[name] = {"is_archive": is_archive(name), "dest": mod.folder}
def import_tree(self, mod_tree):
mod_tree = self._flatten_tree(mod_tree)
for path, mod in mod_tree.items():
# First we need its ID.
# For now, I'll simply use the path with a prefix.
id_ = 'FSOI#' + path
if id_ in self.mods:
# Let's update it but try to honour local modifications.
submods = ['FSOI#' + '.'.join(smod.path) for smod in mod.submods]
dependencies = ['FSOI#' + path for path in mod.dependencies]
delete = [pjoin(mod.folder, path) for path in mod.delete]
entry = self.mods[id_]
for smod in submods:
if smod not in entry['submods']:
entry['submods'].append(smod)
e_deps = entry['packages'][0]['dependencies']
for path in dependencies:
if path not in e_deps:
e_deps.append(path)
del_files = []
for act in entry['actions']:
if act['type'] == 'delete':
del_files.extend(act['files'])
missing_del_files = set(delete) - set(del_files)
added = False
for act in entry['actions']:
if act['type'] == 'delete':
act['files'].extend(missing_del_files)
added = True
if not added:
entry['actions'].append({'type': 'delete', 'files': missing_del_files})
else:
# NOTE: I'm deliberatly dropping support for COPY and RENAME here.
entry = self.mods[id_] = {
'id': id_,
'title': mod.name,
'version': mod.version,
'description': mod.desc,
'logo': None,
'notes': mod.note,
'submods': ['FSOI#' + '.'.join(smod.path) for smod in mod.submods],
'packages': [
{
'name': 'Core files',
'notes': '',
'status': 'required',
'dependencies': [{'id': 'FSOI#' + path, 'version': '*', 'packages': []} for path in mod.dependencies],
'environment': [],
'files': []
}
],
'actions': [{
'type': 'delete',
'files': [pjoin(mod.folder, path) for path in mod.delete]
}]
}
trail = 'FSOI#'
e_deps = entry['packages'][0]['dependencies']
for item in mod.path:
trail += item
if trail not in e_deps:
e_deps.append({'id': trail, 'version': '*', 'packages': []})
trail += '.'
# Always keep the files in sync.
flist = self.mods[id_]['packages'][0]['files'] = []
for urls, filenames in mod.urls:
files = {}
flist.append((urls, files))
for name in filenames:
files[name] = {
'is_archive': is_archive(name),
'dest': mod.folder
}