def _fetchProject(self, targetdir, clone_url, name, commits, skip_dirs): self.logger.info('Fetching project "{}"'.format(name)) try: workdir = tempfile.mkdtemp() except IOError: self.logger.error("Cannot create temporary working directory") return try: try: repo = git.Repo.clone_from(clone_url, workdir) except git.exc.GitCommandError: self.logger.error('Cannot clone git repository "{}" to "{}"'.format(clone_url, workdir)) return for commit in commits: self.logger.info('Processing commit "{}"'.format(commit)) pid = ProjectID.get(name, commit) archive = os.path.join(targetdir, pid) + ".tar.gz" prefix = os.path.join(commit, "") try: with open(archive, "wb") as f: repo.archive(f, commit, prefix, format="tar.gz") except (IOError, git.exc.GitCommandError): self.logger.warn('Cannot create archive "{}"'.format(archive)) continue try: repo.git.checkout(commit) except git.exc.GitCommandError: self.logger.warn('Cannot checkout to commit "{}"'.format(commit)) continue input_data = { "resource": workdir, "directories_to_skip": skip_dirs, "project": name, "commit": commit, "ipprefix": name, } try: plugin = GoSymbolsExtractor() if not plugin.setData(input_data): self.logger.warn("Failed to set input data to " + "GoSymbolsExtractor plugin") continue if not plugin.execute(): self.logger.warn("Failed to execute " + "GoSymbolsExtractor plugin") continue output_data = plugin.getData() if not output_data: self.logger.warn("Failed to get output data from " + "GoSymbolsExtractor plugin") continue except: self.logger.warn("Unhandled exception occured during " + "execution of GoSymbolsExtractor plugin") continue api = os.path.join(targetdir, pid) + "-api.json" try: with open(api, "wb") as f: json.dump(output_data[1], f) except IOError: self.logger.error('Cannot save exported API to file "{}"'.format(api)) continue finally: shutil.rmtree(workdir, ignore_errors=True)
def prepare(self, request): for f in request.funcargnames: if f.startswith('pair_'): pair = request.getfuncargvalue(f) break self.input_data = {} for i in range(2): pid = ProjectID.get(pair[i]['name'], pair[i]['commit']) api = os.path.join(configuration['testdatadir'], pid) + '-api.json' try: with open(api, 'rb') as f: self.input_data['exported_api_{}'.format(i + 1)] = json.load(f) except IOError: raise RuntimeError( 'Cannot load exported API from file "{}"'.format(api))
def prepare(self, request, project): pid = ProjectID.get(project["name"], project["commit"]) archive = os.path.join(configuration["testdatadir"], pid) + ".tar.gz" targetdir = os.path.join(configuration["workdir"], pid) try: with tarfile.open(archive, "r:gz") as tar: tar.extractall(targetdir) except IOError: raise RuntimeError('Cannot open archive "{}"'.format(archive)) except tarfile.TarError: raise RuntimeError('Cannot extract archive "{}"'.format(archive)) self.input_data = { "resource": targetdir, "directories_to_skip": project["directories_to_skip"], "project": project["name"], "commit": project["commit"], "ipprefix": project["name"], }
def prepare(self, request, repository): project = '{provider}.com/{username}/{project}'.format(**repository) pid = ProjectID.get(project, 'repository') archive = os.path.join(configuration['testdatadir'], pid) + '.tar.gz' targetdir = os.path.join(configuration['workdir'], pid) try: with tarfile.open(archive, 'r:gz') as tar: tar.extractall(targetdir) except IOError: raise RuntimeError('Cannot open archive "{}"'.format(archive)) except tarfile.TarError: raise RuntimeError('Cannot extract archive "{}"'.format(archive)) self.input_data = { 'repository': { 'provider': repository['provider'], 'username': repository['username'], 'project': repository['project'], }, 'resource': targetdir, 'commit': repository['commit'], }
def _fetchRepository(self, targetdir, clone_url, name): self.logger.info('Fetching repository "{}"'.format(name)) try: workdir = tempfile.mkdtemp() except IOError: self.logger.error("Cannot create temporary working directory") return try: try: repo = git.Repo.clone_from(clone_url, workdir) except git.exc.GitCommandError: self.logger.error('Cannot clone git repository "{}" to "{}"'.format(clone_url, workdir)) return pid = ProjectID.get(name, "repository") archive = os.path.join(targetdir, pid) + ".tar.gz" try: with tarfile.open(archive, "w:gz") as tar: tar.add(workdir, arcname="") except (IOError, tarfile.TarError): self.logger.error('Cannot create archive "{}"'.format(archive)) return finally: shutil.rmtree(workdir, ignore_errors=True)