def fetch_archive(self, archive_url, strip_level, dest): try: archiver = archiver_for_path(archive_url) except ValueError: raise FetchError( f"Don't know how to unpack archive at url {archive_url}") with self._fetch(archive_url) as archive: if strip_level == 0: archiver.extract(archive, dest) else: with temporary_dir() as scratch: archiver.extract(archive, scratch) for dirpath, dirnames, filenames in os.walk(scratch, topdown=True): if dirpath != scratch: relpath = os.path.relpath(dirpath, scratch) relpath_components = relpath.split(os.sep) if len(relpath_components) == strip_level and ( dirnames or filenames): for path in dirnames + filenames: src = os.path.join(dirpath, path) dst = os.path.join(dest, path) shutil.move(src, dst) del dirnames[:] # Stops the walk.
def fetch(self, dest, rev=None): imported_repo = self._meta_tag_reader.get_imported_repo( self.import_path) if not imported_repo: raise FetchError( 'No <meta name="go-import"> tag found, so cannot fetch repo ' 'at {}'.format(self.import_path)) if imported_repo.vcs != 'git': # TODO: Support other vcs systems as needed. raise FetchError("Don't know how to fetch for vcs type {}.".format( imported_repo.vcs)) # TODO: Do this in a workunit (see https://github.com/pantsbuild/pants/issues/3502). logger.info('Cloning {} into {}'.format(imported_repo.url, dest)) repo = Git.clone(imported_repo.url, dest) if rev: repo.set_state(rev)
def root(self): imported_repo = self._meta_tag_reader.get_imported_repo( self.import_path) if imported_repo: return imported_repo.import_prefix else: raise FetchError( f'No <meta name="go-import"> tag found at {self.import_path}')
def _download(self, url): # TODO(jsirois): Wrap with workunits, progress meters, checksums. logger.info('Downloading {}...'.format(url)) with closing(self._session().get(url, stream=True)) as res: if res.status_code != requests.codes.ok: raise FetchError('Failed to download {} ({} error)'.format(url, res.status_code)) with temporary_file() as archive_fp: # NB: Archives might be very large so we play it safe and buffer them to disk instead of # memory before unpacking. for chunk in res.iter_content(chunk_size=self.get_options().buffer_size): archive_fp.write(chunk) archive_fp.close() res.close() yield archive_fp.name
def get_fetcher(self, import_path): for matcher, unexpanded_url_info in self._matchers: # Note that the url_formats are filled in in two stages. We match.expand them here, # and the ArchiveFetcher applies .format() later, when it knows the rev. match = matcher.match(import_path) if match: expanded_url_info = ArchiveFetcher.UrlInfo(match.expand(unexpanded_url_info.url_format), unexpanded_url_info.default_rev, unexpanded_url_info.strip_level) return ArchiveFetcher(import_path, match.group(0), expanded_url_info, ArchiveRetriever.global_instance()) if self.get_options().disallow_cloning_fetcher: raise FetchError('Cannot fetch {}. No archive match, and remote repo cloning ' 'disallowed.'.format(import_path)) return CloningFetcher(import_path, GoImportMetaTagReader.global_instance())