def load(self): result = [] for url in self.urls: if is_local_file(url) and not os.path.isabs(url): logging.debug('build norm_url from "{0}" and "{1}"'.format( self.core.settings.zoo_home, url)) norm_url = os.path.join(self.core.settings.zoo_home, url) norm_url = os.path.normpath(norm_url) else: norm_url = url items = None try: fli = FeedLoaderFactory.create_feed_loader_item(norm_url) items = fli.load_yaml() except FeedLoaderDownload as ex: self.core.error_manager.add_non_critical(ex) except Exception as ex: self.core.error_manager.add(ex) if items: if not isinstance(items, (Iterable, Sized)): raise FeedLoaderError( "Data loaded from {0} is not list".format(url)) logging.debug('loaded: {0} items'.format(len(items))) result.extend(items) return result
def download_single(self, file): """ Загржает одиночный файл. Если это локальный файл - просто копирует в кеш, если урл - загружает в отдельном потоке. :param file: """ url = file.file headers = file.headers cookies = file.cookies if is_local_file(url): # это локальный файл local_file = url shutil.copy(local_file, file.path) logging.info('local file: {0}'.format(url)) else: # урл if not self.cache.is_in_cache(file.filename): # в кеше нет logging.info('begin download: {0}'.format(url)) # создаём поток для загрузки t = DownloadThread(target=self.process_download, name='Downloading of ' + url, args=(url, file, headers, cookies)) self.threads.append(t) t.daemon = True # стратуем его t.start() else: # уже есть в кеше logging.info('{0} found in cache'.format(file))
def load(self): result = [] for url in self.urls: if is_local_file(url) and not os.path.isabs(url): norm_url = os.path.join(self.core.settings.root, url) norm_url = os.path.normpath(norm_url) else: norm_url = url items = None try: fli = FeedLoaderFactory.create_feed_loader_item(norm_url) items = fli.load_yaml() except Exception as ex: self.core.error_manager.add(ex) if items: if not isinstance(items, (Iterable, Sized)): raise FeedLoaderError("Data loaded from {0} is not list".format(url)) logging.debug('loaded: {0} items'.format(len(items))) result.extend(items) return result
def download_single(self, file): """ Загржает одиночный файл. Если это локальный файл - просто копирует в кеш, если урл - загружает в отдельном потоке. :param file: """ url = file.file headers = file.headers cookies = file.cookies if is_local_file(url): # это локальный файл local_file = url shutil.copy(local_file, file.path) logging.info('local file: {0}'.format(url)) else: # урл if not self.cache.is_in_cache(file.filename): # в кеше нет logging.info('begin download: {0}'.format(url)) # создаём поток для загрузки t = DownloadThread( target=self.process_download, name='Downloading of ' + url, args=(url, file, headers, cookies) ) self.threads.append(t) t.daemon = True # стратуем его t.start() else: # уже есть в кеше logging.info('{0} found in cache'.format(file))
def create_feed_loader_item(url): if not is_local_file(url): return FeedLoaderUrl(url) if os.path.isdir(url): return FeedLoaderDir(url) if os.path.isfile(url): return FeedLoaderFile(url) raise FeedLoaderDownload("Could not load url '{0}'".format(url))
def create_feed_loader_item(url): if not is_local_file(url): return FeedLoaderUrl(url) if os.path.isdir(url): return FeedLoaderDir(url) if os.path.isfile(url): return FeedLoaderFile(url) raise Exception("Could not load url '{0}'".format(url))
def _patch(self, path, product): """ не исправлять url и абсалютные пути :param path: :return: """ if not is_local_file(path): return path if os.path.isabs(path): return path full_path = self.url local_file_full_path = os.path.join(os.path.dirname(full_path), path) return local_file_full_path
def process_yaml(full_path, data: dict, src, dest): """ Processes icon and file link in yaml object. :param full_path: file full path :param data: yaml object :param src: source folder :param dest: destination folder """ for product in data: # icon attr if "icon" in product: product["icon"] = _patch(full_path, product["icon"], src, dest) if "description" in product: product["description"] = Literal(product["description"]) if "find_installed_command" in product: product["find_installed_command"] = Literal( product["find_installed_command"]) if "install_command" in product: product["install_command"] = Literal(product["install_command"]) if "uninstall_command" in product: product["uninstall_command"] = Literal( product["uninstall_command"]) if "upgrade_command" in product: product["upgrade_command"] = Literal(product["upgrade_command"]) # files list if "files" in product: if product["files"] is None: continue for file in product["files"]: path = file["file"] if is_local_file(path): # path path for local (in 'source') files, not hyperlinks file["file"] = _patch(full_path, file["file"], src, dest)
def process_yaml(full_path, data: dict, src, dest): """ Processes icon and file link in yaml object. :param full_path: file full path :param data: yaml object :param src: source folder :param dest: destination folder """ for product in data: # icon attr if "icon" in product: product["icon"] = _patch(full_path, product["icon"], src, dest) # files list if "files" in product: if product["files"] is None: continue for file in product["files"]: path = file["file"] if is_local_file(path): # path path for local (in 'source') files, not hyperlinks file["file"] = _patch(full_path, file["file"], src, dest)
def __init__(self, url): super().__init__(url) if is_local_file(url): raise Exception("Url '{0}' is not external url".format(url))