def download_remote_sources(self): """ Method that iterates over all sources and downloads ones, which contain URL instead of just a file. :return: None """ try: # try to download old sources from Fedora lookaside cache LookasideCacheHelper.download('fedpkg', os.path.dirname(self.path), self.get_package_name()) except LookasideCacheError as e: logger.debug("Downloading sources from lookaside cache failed. " "Reason: '{}'.".format(str(e))) # filter out only sources with URL remote_files = [ source for source in self.sources if bool(urllib.parse.urlparse(source).scheme) ] # download any sources that are not yet downloaded for remote_file in remote_files: local_file = os.path.join(self.sources_location, os.path.basename(remote_file)) if not os.path.isfile(local_file): logger.debug( "File '%s' doesn't exist locally, downloading it.", local_file) try: DownloadHelper.download_file(remote_file, local_file) except DownloadError as e: raise RebaseHelperError( "Failed to download file from URL {}. " "Reason: '{}'. ".format(remote_file, str(e)))
def test_download_existing_file(self, url, content): """Test downloading existing file""" local_file = 'local_file' DownloadHelper.download_file(url, local_file) assert os.path.isfile(local_file) with open(local_file) as f: assert f.readline().strip() == content
def download_remote_sources(self): """ Method that iterates over all sources and downloads ones, which contain URL instead of just a file. :return: None """ try: # try to download old sources from Fedora lookaside cache LookasideCacheHelper.download('fedpkg', os.path.dirname(self.path), self.get_package_name()) except LookasideCacheError as e: logger.debug("Downloading sources from lookaside cache failed. " "Reason: '{}'.".format(str(e))) # filter out only sources with URL remote_files = [source for source in self.sources if bool(urllib.parse.urlparse(source).scheme)] # download any sources that are not yet downloaded for remote_file in remote_files: local_file = os.path.join(self.sources_location, os.path.basename(remote_file)) if not os.path.isfile(local_file): logger.debug("File '%s' doesn't exist locally, downloading it.", local_file) try: DownloadHelper.download_file(remote_file, local_file) except DownloadError as e: raise RebaseHelperError("Failed to download file from URL {}. " "Reason: '{}'. ".format(remote_file, str(e)))
def _get_initial_sources_list(self): """Function returns all sources mentioned in SPEC file""" # get all regular sources sources = [] tar_sources = [] sources_list = [x for x in self.spc.sources if x[2] == 1] remote_files_re = re.compile(r'(http:|https:|ftp:)//.*') for index, src in enumerate(sorted(sources_list, key=lambda source: source[1])): # src is type of (SOURCE, Index of source, Type of source (PAtch, Source) # We need to download all archives and only the one abs_path = os.path.join(self.sources_location, os.path.basename(src[0]).strip()) sources.append(abs_path) archive = [x for x in Archive.get_supported_archives() if src[0].endswith(x)] # if the source is a remote file, download it if archive: if remote_files_re.search(src[0]) and self.download and not os.path.isfile(abs_path): logger.debug("Tarball is not in absolute path {} " "trying to download one from URL {}".format(abs_path, src[0])) logger.info("Tarball is not in absolute path. Trying to download it from URL") try: DownloadHelper.download_file(src[0], abs_path) except DownloadError as e: raise RebaseHelperError("Failed to download file from URL {}. " "Reason: '{}'. ".format(src[0], str(e))) tar_sources.append(abs_path) return sources, tar_sources
def test_download_non_existing_file_FTP(self): """ Test downloading NON existing file via FTP :return: """ KNOWN_URL = "ftp://ftp.isc.org/isc/bind9/9.10.3-P5/srcid" LOCAL_FILE = os.path.basename(KNOWN_URL) with pytest.raises(DownloadError): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.isfile(LOCAL_FILE)
def test_download_non_existing_file_FTP(self): """ Test downloading NON existing file via FTP :return: """ KNOWN_URL = 'ftp://ftp.isc.org/isc/bind9/9.10.3-P5/srcid' LOCAL_FILE = os.path.basename(KNOWN_URL) with pytest.raises(DownloadError): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.isfile(LOCAL_FILE)
def test_download_existing_file_HTTP(self): """ Test downloading existing file via HTTP. """ KNOWN_URL = 'http://fedoraproject.org/static/hotspot.txt' LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = 'OK' DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.read().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_FTP(self): """ Test downloading existing file via FTP """ KNOWN_URL = "ftp://ftp.isc.org/isc/bind9/9.10.4-P1/srcid" LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = "SRCID=adfc588" DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.read().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_HTTP(self): """ Test downloading existing file via HTTP. """ KNOWN_URL = "http://fedoraproject.org/static/hotspot.txt" LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = "OK" DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.read().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_FTP(self): """ Test downloading existing file via FTP """ KNOWN_URL = 'ftp://ftp.isc.org/isc/bind9/9.10.4-P1/srcid' LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = 'SRCID=adfc588' DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.read().strip() == KNOWN_URL_CONTENT
def test_download_non_existing_file_HTTPS(self): """ Test downloading NON existing file via HTTPS :return: """ KNOWN_URL = 'https://ftp.isc.org/isc/bind9/9.10.3-P5/srcid' LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = 'SRCID=adfc588' with pytest.raises(DownloadError): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.isfile(LOCAL_FILE)
def test_download_non_existing_file_HTTPS(self): """ Test downloading NON existing file via HTTPS :return: """ KNOWN_URL = "https://ftp.isc.org/isc/bind9/9.10.3-P5/srcid" LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = "SRCID=adfc588" with pytest.raises(DownloadError): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.isfile(LOCAL_FILE)
def test_download_existing_file_HTTPS(self): """ Test downloading exiting file via HTTPS. """ KNOWN_URL = 'https://ftp.isc.org/isc/bind9/9.10.4-P1/srcid' LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = 'SRCID=adfc588' DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.read().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_of_unknown_length_FTP(self): """ Test downloading existing file of unknown length via FTP :return: """ KNOWN_URL = "ftp://ftp.gnupg.org/README" LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = "Welcome hacker!" DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.readline().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_of_unknown_length_FTP(self): """ Test downloading existing file of unknown length via FTP :return: """ KNOWN_URL = 'ftp://ftp.gnupg.org/README' LOCAL_FILE = os.path.basename(KNOWN_URL) KNOWN_URL_CONTENT = 'Welcome hacker!' DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.readline().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_of_unknown_length_HTTPS(self): """ Test downloading existing file of unknown length via HTTPS :return: """ COMMIT = "cf5ae2989a32c391d7769933e0267e6fbfae8e14" KNOWN_URL = "https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/patch/?id={}".format(COMMIT) LOCAL_FILE = "{}.patch".format(COMMIT) KNOWN_URL_CONTENT = "From {} Mon Sep 17 00:00:00 2001".format(COMMIT) DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.readline().strip() == KNOWN_URL_CONTENT
def test_download_existing_file_of_unknown_length_HTTPS(self): """ Test downloading existing file of unknown length via HTTPS :return: """ COMMIT = 'cf5ae2989a32c391d7769933e0267e6fbfae8e14' KNOWN_URL = 'https://git.kernel.org/cgit/linux/kernel/git/stable/linux-stable.git/patch/?id={}'.format( COMMIT) LOCAL_FILE = '{}.patch'.format(COMMIT) KNOWN_URL_CONTENT = 'From {} Mon Sep 17 00:00:00 2001'.format(COMMIT) DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert os.path.isfile(LOCAL_FILE) with open(LOCAL_FILE) as f: assert f.readline().strip() == KNOWN_URL_CONTENT
def test_keyboard_interrupt_situation(self, monkeypatch): """ Test that the local file is deleted in case KeyboardInterrupt is raised during the download """ KNOWN_URL = "https://ftp.isc.org/isc/bind9/9.10.4-P1/srcid" LOCAL_FILE = os.path.basename(KNOWN_URL) def interrupter(*args, **kwargs): raise KeyboardInterrupt # make sure that some function call inside tha actual download section raises the KeyboardInterrupt exception. monkeypatch.setattr("time.time", interrupter) with pytest.raises(KeyboardInterrupt): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.exists(LOCAL_FILE)
def test_keyboard_interrupt_situation(self, monkeypatch): """ Test that the local file is deleted in case KeyboardInterrupt is raised during the download """ KNOWN_URL = 'https://ftp.isc.org/isc/bind9/9.10.4-P1/srcid' LOCAL_FILE = os.path.basename(KNOWN_URL) def interrupter(*args, **kwargs): raise KeyboardInterrupt # make sure that some function call inside tha actual download section raises the KeyboardInterrupt exception. monkeypatch.setattr('time.time', interrupter) with pytest.raises(KeyboardInterrupt): DownloadHelper.download_file(KNOWN_URL, LOCAL_FILE) assert not os.path.exists(LOCAL_FILE)
def _get_initial_sources_list(self): """Function returns all sources mentioned in SPEC file""" # get all regular sources sources = [] tar_sources = [] sources_list = [x for x in self.spc.sources if x[2] == 1] remote_files_re = re.compile(r'(http:|https:|ftp:)//.*') for index, src in enumerate(sorted(sources_list, key=lambda source: source[1])): # src is type of (SOURCE, Index of source, Type of source (PAtch, Source) # We need to download all archives and only the one abs_path = os.path.join(self.sources_location, os.path.basename(src[0]).strip()) sources.append(abs_path) archive = [x for x in Archive.get_supported_archives() if src[0].endswith(x)] # if the source is a remote file, download it if archive: if remote_files_re.search(src[0]) and self.download: DownloadHelper.download_file(src[0], abs_path) tar_sources.append(abs_path) return sources, tar_sources
def _get_initial_sources_list(self): """ Function returns all sources mentioned in SPEC file """ # get all regular sources sources = [] sources_list = [x for x in self.spc.sources if x[2] == 1] remote_files_re = re.compile(r'(http:|https:|ftp:)//.*') for index, src in enumerate(sources_list): abs_path = os.path.join(self.sources_location, os.path.basename(src[0]).strip()) # if the source is a remote file, download it if remote_files_re.search(src[0]) and self.download: DownloadHelper.download_file(src[0], abs_path) # the Source0 has to be at the beginning! if src[1] == 0: sources.insert(0, abs_path) else: sources.append(abs_path) return sources
def _download_scratch_build(cls, task_list, dir_name): session = cls._session_maker() rpms = [] logs = [] for task_id in task_list: logger.info('Downloading packaged for %i taskID' % task_id) task = session.getTaskInfo(task_id) tasks = [task] for task in tasks: base_path = koji.pathinfo.taskrelpath(task_id) output = session.listTaskOutput(task['id']) for filename in output: logger.info('Downloading file %s' % filename) downloaded_file = os.path.join(dir_name, filename) DownloadHelper.download_file(cls.scratch_url + base_path + '/' + filename, downloaded_file) if filename.endswith('.rpm'): rpms.append(downloaded_file) if filename.endswith('build.log'): logs.append(downloaded_file) session.logout() return rpms, logs
def test_download_source(self): DownloadHelper.download_file(self.KNOWN_URL, self.LOCAL_FILE) assert os.path.isfile(self.LOCAL_FILE) with open(self.LOCAL_FILE) as f: assert f.read() == self.KNOWN_URL_CONTENT
def test_download_non_existing_file(self, url): """Test downloading NON existing file""" local_file = 'local_file' with pytest.raises(DownloadError): DownloadHelper.download_file(url, local_file) assert not os.path.isfile(local_file)