def load(self):
result = []
for url in self.urls:
if is_local_file(url) and not os.path.isabs(url):
logging.debug('build norm_url from "{0}" and "{1}"'.format(
self.core.settings.zoo_home, url))
norm_url = os.path.join(self.core.settings.zoo_home, url)
norm_url = os.path.normpath(norm_url)
else:
norm_url = url
items = None
try:
fli = FeedLoaderFactory.create_feed_loader_item(norm_url)
items = fli.load_yaml()
except FeedLoaderDownload as ex:
self.core.error_manager.add_non_critical(ex)
except Exception as ex:
self.core.error_manager.add(ex)
if items:
if not isinstance(items, (Iterable, Sized)):
raise FeedLoaderError(
"Data loaded from {0} is not list".format(url))
logging.debug('loaded: {0} items'.format(len(items)))
result.extend(items)
return result
def download_single(self, file):
"""
Загржает одиночный файл. Если это локальный файл - просто копирует в кеш,
если урл - загружает в отдельном потоке.
:param file:
"""
url = file.file
headers = file.headers
cookies = file.cookies
if is_local_file(url):
# это локальный файл
local_file = url
shutil.copy(local_file, file.path)
logging.info('local file: {0}'.format(url))
else:
# урл
if not self.cache.is_in_cache(file.filename):
# в кеше нет
logging.info('begin download: {0}'.format(url))
# создаём поток для загрузки
t = DownloadThread(target=self.process_download,
name='Downloading of ' + url,
args=(url, file, headers, cookies))
self.threads.append(t)
t.daemon = True
# стратуем его
t.start()
else:
# уже есть в кеше
logging.info('{0} found in cache'.format(file))
def load(self):
result = []
for url in self.urls:
if is_local_file(url) and not os.path.isabs(url):
norm_url = os.path.join(self.core.settings.root, url)
norm_url = os.path.normpath(norm_url)
else:
norm_url = url
items = None
try:
fli = FeedLoaderFactory.create_feed_loader_item(norm_url)
items = fli.load_yaml()
except Exception as ex:
self.core.error_manager.add(ex)
if items:
if not isinstance(items, (Iterable, Sized)):
raise FeedLoaderError("Data loaded from {0} is not list".format(url))
logging.debug('loaded: {0} items'.format(len(items)))
result.extend(items)
return result
def download_single(self, file):
"""
Загржает одиночный файл. Если это локальный файл - просто копирует в кеш,
если урл - загружает в отдельном потоке.
:param file:
"""
url = file.file
headers = file.headers
cookies = file.cookies
if is_local_file(url):
# это локальный файл
local_file = url
shutil.copy(local_file, file.path)
logging.info('local file: {0}'.format(url))
else:
# урл
if not self.cache.is_in_cache(file.filename):
# в кеше нет
logging.info('begin download: {0}'.format(url))
# создаём поток для загрузки
t = DownloadThread(
target=self.process_download,
name='Downloading of ' + url,
args=(url, file, headers, cookies)
)
self.threads.append(t)
t.daemon = True
# стратуем его
t.start()
else:
# уже есть в кеше
logging.info('{0} found in cache'.format(file))
def create_feed_loader_item(url):
if not is_local_file(url):
return FeedLoaderUrl(url)
if os.path.isdir(url):
return FeedLoaderDir(url)
if os.path.isfile(url):
return FeedLoaderFile(url)
raise FeedLoaderDownload("Could not load url '{0}'".format(url))
def create_feed_loader_item(url):
if not is_local_file(url):
return FeedLoaderUrl(url)
if os.path.isdir(url):
return FeedLoaderDir(url)
if os.path.isfile(url):
return FeedLoaderFile(url)
raise Exception("Could not load url '{0}'".format(url))
def _patch(self, path, product):
"""
не исправлять url и абсалютные пути
:param path:
:return:
"""
if not is_local_file(path):
return path
if os.path.isabs(path):
return path
full_path = self.url
local_file_full_path = os.path.join(os.path.dirname(full_path), path)
return local_file_full_path
def _patch(self, path, product):
"""
не исправлять url и абсалютные пути
:param path:
:return:
"""
if not is_local_file(path):
return path
if os.path.isabs(path):
return path
full_path = self.url
local_file_full_path = os.path.join(os.path.dirname(full_path), path)
return local_file_full_path
def process_yaml(full_path, data: dict, src, dest):
"""
Processes icon and file link in yaml object.
:param full_path: file full path
:param data: yaml object
:param src: source folder
:param dest: destination folder
"""
for product in data:
# icon attr
if "icon" in product:
product["icon"] = _patch(full_path, product["icon"], src, dest)
if "description" in product:
product["description"] = Literal(product["description"])
if "find_installed_command" in product:
product["find_installed_command"] = Literal(
product["find_installed_command"])
if "install_command" in product:
product["install_command"] = Literal(product["install_command"])
if "uninstall_command" in product:
product["uninstall_command"] = Literal(
product["uninstall_command"])
if "upgrade_command" in product:
product["upgrade_command"] = Literal(product["upgrade_command"])
# files list
if "files" in product:
if product["files"] is None:
continue
for file in product["files"]:
path = file["file"]
if is_local_file(path):
# path path for local (in 'source') files, not hyperlinks
file["file"] = _patch(full_path, file["file"], src, dest)
def process_yaml(full_path, data: dict, src, dest):
"""
Processes icon and file link in yaml object.
:param full_path: file full path
:param data: yaml object
:param src: source folder
:param dest: destination folder
"""
for product in data:
# icon attr
if "icon" in product:
product["icon"] = _patch(full_path, product["icon"], src, dest)
# files list
if "files" in product:
if product["files"] is None:
continue
for file in product["files"]:
path = file["file"]
if is_local_file(path):
# path path for local (in 'source') files, not hyperlinks
file["file"] = _patch(full_path, file["file"], src, dest)
def __init__(self, url):
super().__init__(url)
if is_local_file(url):
raise Exception("Url '{0}' is not external url".format(url))
def __init__(self, url):
super().__init__(url)
if is_local_file(url):
raise Exception("Url '{0}' is not external url".format(url))
