def _fetchProject(self, targetdir, clone_url, name, commits, skip_dirs):
self.logger.info('Fetching project "{}"'.format(name))
try:
workdir = tempfile.mkdtemp()
except IOError:
self.logger.error("Cannot create temporary working directory")
return
try:
try:
repo = git.Repo.clone_from(clone_url, workdir)
except git.exc.GitCommandError:
self.logger.error('Cannot clone git repository "{}" to "{}"'.format(clone_url, workdir))
return
for commit in commits:
self.logger.info('Processing commit "{}"'.format(commit))
pid = ProjectID.get(name, commit)
archive = os.path.join(targetdir, pid) + ".tar.gz"
prefix = os.path.join(commit, "")
try:
with open(archive, "wb") as f:
repo.archive(f, commit, prefix, format="tar.gz")
except (IOError, git.exc.GitCommandError):
self.logger.warn('Cannot create archive "{}"'.format(archive))
continue
try:
repo.git.checkout(commit)
except git.exc.GitCommandError:
self.logger.warn('Cannot checkout to commit "{}"'.format(commit))
continue
input_data = {
"resource": workdir,
"directories_to_skip": skip_dirs,
"project": name,
"commit": commit,
"ipprefix": name,
}
try:
plugin = GoSymbolsExtractor()
if not plugin.setData(input_data):
self.logger.warn("Failed to set input data to " + "GoSymbolsExtractor plugin")
continue
if not plugin.execute():
self.logger.warn("Failed to execute " + "GoSymbolsExtractor plugin")
continue
output_data = plugin.getData()
if not output_data:
self.logger.warn("Failed to get output data from " + "GoSymbolsExtractor plugin")
continue
except:
self.logger.warn("Unhandled exception occured during " + "execution of GoSymbolsExtractor plugin")
continue
api = os.path.join(targetdir, pid) + "-api.json"
try:
with open(api, "wb") as f:
json.dump(output_data[1], f)
except IOError:
self.logger.error('Cannot save exported API to file "{}"'.format(api))
continue
finally:
shutil.rmtree(workdir, ignore_errors=True)
def prepare(self, request):
for f in request.funcargnames:
if f.startswith('pair_'):
pair = request.getfuncargvalue(f)
break
self.input_data = {}
for i in range(2):
pid = ProjectID.get(pair[i]['name'], pair[i]['commit'])
api = os.path.join(configuration['testdatadir'], pid) + '-api.json'
try:
with open(api, 'rb') as f:
self.input_data['exported_api_{}'.format(i + 1)] = json.load(f)
except IOError:
raise RuntimeError(
'Cannot load exported API from file "{}"'.format(api))
def prepare(self, request, project):
pid = ProjectID.get(project["name"], project["commit"])
archive = os.path.join(configuration["testdatadir"], pid) + ".tar.gz"
targetdir = os.path.join(configuration["workdir"], pid)
try:
with tarfile.open(archive, "r:gz") as tar:
tar.extractall(targetdir)
except IOError:
raise RuntimeError('Cannot open archive "{}"'.format(archive))
except tarfile.TarError:
raise RuntimeError('Cannot extract archive "{}"'.format(archive))
self.input_data = {
"resource": targetdir,
"directories_to_skip": project["directories_to_skip"],
"project": project["name"],
"commit": project["commit"],
"ipprefix": project["name"],
}
def prepare(self, request, repository):
project = '{provider}.com/{username}/{project}'.format(**repository)
pid = ProjectID.get(project, 'repository')
archive = os.path.join(configuration['testdatadir'], pid) + '.tar.gz'
targetdir = os.path.join(configuration['workdir'], pid)
try:
with tarfile.open(archive, 'r:gz') as tar:
tar.extractall(targetdir)
except IOError:
raise RuntimeError('Cannot open archive "{}"'.format(archive))
except tarfile.TarError:
raise RuntimeError('Cannot extract archive "{}"'.format(archive))
self.input_data = {
'repository': {
'provider': repository['provider'],
'username': repository['username'],
'project': repository['project'],
},
'resource': targetdir,
'commit': repository['commit'],
}
def _fetchRepository(self, targetdir, clone_url, name):
self.logger.info('Fetching repository "{}"'.format(name))
try:
workdir = tempfile.mkdtemp()
except IOError:
self.logger.error("Cannot create temporary working directory")
return
try:
try:
repo = git.Repo.clone_from(clone_url, workdir)
except git.exc.GitCommandError:
self.logger.error('Cannot clone git repository "{}" to "{}"'.format(clone_url, workdir))
return
pid = ProjectID.get(name, "repository")
archive = os.path.join(targetdir, pid) + ".tar.gz"
try:
with tarfile.open(archive, "w:gz") as tar:
tar.add(workdir, arcname="")
except (IOError, tarfile.TarError):
self.logger.error('Cannot create archive "{}"'.format(archive))
return
finally:
shutil.rmtree(workdir, ignore_errors=True)
