def plant_seed(self, seed_id=None, target_dir=None):
if not seed_id or not target_dir:
self.rage_quit("Missing arguments, seed plant failed")
seeds = self.get_seeds()
tagref = seeds.get(seed_id, None)
if not tagref:
self.rage_quit("Seed id {} not found".format(seed_id))
git = Git(tagref.repo.working_dir)
current_commit = str(Repo(tagref.repo.working_dir).commit())
cprint("Current commit: {}".format(current_commit), Fore.GREEN)
dirty = tagref.repo.is_dirty()
cprint("Working directory {}".format('dirty' if dirty else 'clean'), Fore.GREEN)
if dirty:
cprint("--> git stash", Fore.YELLOW)
git.stash()
cprint("--> git checkout {}".format(seed_id), Fore.YELLOW)
git.checkout(seed_id)
try:
cprint("Copying seed directory: {}".format(tagref.repo.working_dir), Fore.GREEN)
call(["cp", "-r", tagref.repo.working_dir, target_dir])
except OSError as error:
cprint("Copying directory failed:\n{}".format(error), Fore.RED)
finally:
if dirty:
cprint("--> git stash apply", Fore.YELLOW)
git.stash('apply')
cprint("--> git checkout {}".format(current_commit), Fore.YELLOW)
git.checkout(current_commit)
def clone(self, url=None, to_path=None, branch=None):
Log.info(Status["STAT_GET_PACKAGE"] % url)
g = Git(to_path)
g.clone(url or self.github_url)
if branch:
g.checkout(branch)
return True
def compare_git_commits(repo_path, branch, start_commit, end_commit):
# print 'Repo path: ' + repo_path + ' and branch: ' + branch
# print 'Starting commit: ' + start_commit + ', Ending commit: ' + end_commit
repo = Repo(repo_path)
git = Git(repo_path)
head = repo.heads[0]
git.checkout(start_commit)
cmd_str = 'java -jar bin/threadfix-endpoint-cli-2.4-SNAPSHOT-jar-with-dependencies.jar ' + pipes.quote(
repo_path) + ' -json 2>/dev/null > ' + pipes.quote(
make_attack_surface_filename(start_commit))
print 'About to generate start attack surface with command: ' + cmd_str
os.system(cmd_str)
git.checkout(end_commit)
cmd_str = 'java -jar bin/threadfix-endpoint-cli-2.4-SNAPSHOT-jar-with-dependencies.jar ' + pipes.quote(
repo_path) + ' -json 2>/dev/null > ' + pies.quote(
make_attack_surface_filename(end_commit))
print 'About to generate end attack surface with command: ' + cmd_str
os.system(cmd_str)
ret_val = diff_attack_surface_files(
make_attack_surface_filename(start_commit),
make_attack_surface_filename(end_commit))
return ret_val
def diff_it(role_dir: Path, rev1: str, rev2: str) -> Any:
g = Git(role_dir)
g.checkout(rev1)
role_v1 = Role.load(role_tmp_dir, role_base_dir)
g.checkout(rev2)
role_v2 = Role.load(role_tmp_dir, role_base_dir)
return [d for d in role_v1.diff(role_v2) if d]
def _try_git_checkout(repo, repo_url, ref=None, fetch=True):
"""Try to checkout a ``ref`` from ``repo``.
Local branches are created for multiple variations of the ``ref``,
including its refpath and hexpath (i.e. commit ID).
This is to locally "cache" references that would otherwise require
resolution upstream. We increase performance by creating local branches
for these other ``ref`` formats when the ``ref`` is fetched remotely for
the first time only.
:param repo: Git Repo object.
:param repo_url: URL of remote Git repo or path to local Git repo.
:param ref: branch, commit or reference in the repo to clone.
:param fetch: Whether to fetch the ``ref`` from remote before checkout or
to use the already-cloned local repo.
:raises GitException: If ``ref`` could not be checked out.
"""
try:
g = Git(repo.working_dir)
branches = [b.name for b in repo.branches]
LOG.debug('Available branches for repo_url=%s: %s', repo_url, branches)
if fetch:
LOG.debug('Fetching ref=%s from remote repo_url=%s', ref, repo_url)
# fetch_info is guaranteed to be populated if ref resolves, else
# a GitCommandError is raised.
fetch_info = repo.remotes.origin.fetch(ref)
hexsha = fetch_info[0].commit.hexsha.strip()
ref_path = fetch_info[0].remote_ref_path.strip()
# If ``ref`` doesn't match the hexsha/refpath then create a branch
# for each so that future checkouts can be performed using either
# format. This way, no future processing is required to figure
# out whether a refpath/hexsha exists within the repo.
_create_local_ref(
g, branches, ref=ref, newref=hexsha, reftype='hexsha')
_create_local_ref(
g, branches, ref=ref, newref=ref_path, reftype='refpath')
_create_or_checkout_local_ref(g, branches, ref=ref)
else:
LOG.debug(
'Checking out ref=%s from local repo_url=%s', ref, repo_url)
# Expect the reference to exist if checking out locally.
g.checkout(ref)
LOG.debug(
'Successfully checked out ref=%s for repo_url=%s', ref, repo_url)
except git_exc.GitCommandError as e:
LOG.exception(
'Failed to checkout ref=%s from repo_url=%s.', ref, repo_url)
raise exceptions.GitException(location=repo_url, details=e)
except Exception as e:
LOG.exception(
(
'Encountered unknown Exception during checkout of '
'ref=%s for repo_url=%s'), ref, repo_url)
raise exceptions.GitException(location=repo_url, details=e)
def prep_export(tag, repository):
tmpdir = mkdtemp()
Repo.clone_from(repository.working_dir, tmpdir)
log.debug('Cloned {} to {}'.format(repository.working_dir, tmpdir))
clone = Git(tmpdir)
clone.checkout(tag)
log.debug('Checked out tag {}'.format(tag))
return tmpdir
def checkout_tag(self, name):
repo = Repo('./')
repo.git.reset('--hard')
o = repo.remotes.origin
o.fetch()
g = Git('./')
g.checkout(name)
cbpi.notify("Checkout successful", "Please restart the system")
return ('', 204)
def checkout_source(self, git_path, checkout_options=None):
'''
Checkout source and switch to the right tag
'''
shutil.rmtree(self.workspace, ignore_errors=True)
os.mkdir(self.workspace)
os.chdir(self.workspace)
Repo.clone_from(git_path, self.target_name)
g = Git(self.source_path)
g.checkout(checkout_options)
def checkout_source(self, git_path, checkout_options = None):
'''
Checkout source and switch to the right tag
'''
shutil.rmtree(self.workspace, ignore_errors=True)
os.mkdir(self.workspace)
os.chdir(self.workspace)
Repo.clone_from(git_path, self.target_name)
g = Git(self.source_path)
g.checkout(checkout_options)
class GitUtil:
def __init__(self, git_url, results_dir=None, tag=None):
self._repopath = config['git']['repopath']
self._results_dir = results_dir
try:
shutil.rmtree(self._repopath)
except OSError as e:
print(e)
os.mkdir(self._repopath)
self.REPO = Repo.clone_from(git_url, self._repopath)
self.GIT = Git(self._repopath)
if tag:
self.GIT.checkout(tag)
@staticmethod
def purge_repository():
shutil.rmtree(config['git']['repopath'])
os.mkdir(config['git']['repopath'])
def copy_file_into_repo(self, file):
target = os.path.join(self._repopath, os.path.basename(file))
copyfile(file, target)
self.REPO.index.add([os.path.basename(target)])
def commit_and_push_source_code(self, message):
self.REPO.index.commit(message)
self.REPO.remotes.origin.push()
def tag_repository(self, tag_name):
tag = self.REPO.create_tag(tag_name)
self.REPO.remotes.origin.push(tag)
def create_new_file(self, filename, content):
target = os.path.join(self._repopath, filename)
with open(target, 'w') as file:
file.writelines(content)
self.REPO.index.add([os.path.basename(target)])
def copy_repofile_to_results(self, filename, execution_tag):
shutil.copyfile(
os.path.join(self._repopath, filename),
os.path.join(self._results_dir, 'pdf',
f"{execution_tag}_{filename}"))
def replace_repofile(self, src, target):
target = os.path.join(self._repopath, target)
os.remove(target)
shutil.copyfile(src, target)
def checkout(self, branch):
self.GIT.checkout(branch)
def _get_workdir(self):
"""Return an initialized workdir path.
If the stock links to a plain directory, the workdir is simply its path.
If the stock links to a git repository, the workdir will point to a
persistent lightweight checkout of the desired branch.
"""
if not self.branch:
return self.link
orig = Git(self.link)
checkout_path = self.paths.checkout
if not exists(checkout_path):
mkdir(checkout_path)
checkout = Git.init_create(checkout_path)
checkout.set_alternates(orig)
else:
checkout = Git(checkout_path)
def dup_branch(branch):
# checkout latest changes
commit = orig.rev_parse(branch)
if not commit:
raise self.Error("no such branch `%s' at %s" %
(branch, self.link))
checkout.update_ref("refs/heads/" + branch, commit)
dup_branch(self.branch)
checkout.checkout("-q", "-f", self.branch)
if exists(join(checkout_path, "arena.internals")):
dup_branch(self.branch + "-thin")
command = "cd %s && sumo-open" % commands.mkarg(checkout_path)
error = os.system(command)
if error:
raise self.Error("failed command: " + command)
return join(checkout_path, "arena")
# update tags
for tag in checkout.list_tags():
checkout.remove_tag(tag)
for tag in orig.list_tags():
try:
checkout.update_ref("refs/tags/" + tag, orig.rev_parse(tag))
except:
continue
return checkout_path
def clone_repo(repo_name, repo_version, repo_location):
"""Clone a repo and checkout the desired branch"""
Repo.clone_from(repo_name, repo_location)
repo = Git(repo_location)
repo.init()
version = define_version(repo, repo_version)
if DEBUG or config.DEBUG:
print("Checking out version: " + version)
if version != "master":
repo.checkout(version)
def fetch_and_checkout(config, repo):
"""Try to fetch the remote ref in the personal gerrit branch for
the running user."""
g = Git(repo.working_tree_dir)
try:
g.fetch(['origin', config._state_ref])
g.checkout('FETCH_HEAD')
except git.exc.GitCommandError as e:
if "couldn't find remote ref refs/personal" in e.stderr:
pass
else:
print(e)
del g
def _get_workdir(self):
"""Return an initialized workdir path.
If the stock links to a plain directory, the workdir is simply its path.
If the stock links to a git repository, the workdir will point to a
persistent lightweight checkout of the desired branch.
"""
if not self.branch:
return self.link
orig = Git(self.link)
checkout_path = self.paths.checkout
if not exists(checkout_path):
mkdir(checkout_path)
checkout = Git.init_create(checkout_path)
checkout.set_alternates(orig)
else:
checkout = Git(checkout_path)
def dup_branch(branch):
# checkout latest changes
commit = orig.rev_parse(branch)
if not commit:
raise self.Error("no such branch `%s' at %s" % (branch, self.link))
checkout.update_ref("refs/heads/" + branch, commit)
dup_branch(self.branch)
checkout.checkout("-q", "-f", self.branch)
if exists(join(checkout_path, "arena.internals")):
dup_branch(self.branch + "-thin")
command = "cd %s && sumo-open" % commands.mkarg(checkout_path)
error = os.system(command)
if error:
raise self.Error("failed command: " + command)
return join(checkout_path, "arena")
# update tags
for tag in checkout.list_tags():
checkout.remove_tag(tag)
for tag in orig.list_tags():
try:
checkout.update_ref("refs/tags/" + tag, orig.rev_parse(tag))
except:
continue
return checkout_path
def build_command(config_file, strict, site_dir, tags, default, latest):
"""Build the MkDocs documentation"""
cli.configure_logging(level=logging.INFO)
g = Git()
tags = tags or g.tag().splitlines()
log.info("Building %s to /", default)
g.checkout(default)
_build(_load_config(config_file, strict, site_dir), default, tags)
log.info("Building %s to /latest", latest)
g.checkout(default)
_build(_load_config(config_file, strict, site_dir), latest, tags, 'latest')
for tag in sorted(tags):
g.checkout(tag)
if not os.path.exists("mkdocs.yml"):
log.warning("Unable to build %s, as no mkdocs.yml was found", tag)
continue
site_dir = "v{0}".format(tag)
log.info("Building %s to /%s", tag, site_dir)
_build(_load_config(config_file, strict, site_dir), tag, tags,
site_dir)
g.checkout('master')
def build_command(config_file, strict, site_dir, tags, default, latest):
"""Build the MkDocs documentation"""
cli.configure_logging(level=logging.INFO)
g = Git()
tags = tags or g.tag().splitlines()
log.info("Building %s to /", default)
g.checkout(default)
_build(_load_config(config_file, strict, site_dir), default, tags)
log.info("Building %s to /latest", latest)
g.checkout(default)
_build(_load_config(config_file, strict, site_dir), latest, tags, 'latest')
for tag in sorted(tags):
g.checkout(tag)
if not os.path.exists("mkdocs.yml"):
log.warning("Unable to build %s, as no mkdocs.yml was found", tag)
continue
site_dir = "v{0}".format(tag)
log.info("Building %s to /%s", tag, site_dir)
_build(_load_config(config_file, strict, site_dir), tag, tags, site_dir)
g.checkout('master')
def collect_coverage_for_version(repo_name, git_repo, version, coverage):
g = Git(git_repo)
print(f"{repo_name} {version}")
repo_dir = git_repo.working_tree_dir
try:
with pushd(repo_dir):
git_repo.head.reference = git_repo.commit(version)
git_repo.head.reset(index=True, working_tree=True)
g.checkout(version)
implemented_rules = all_implemented_rules()
coverage.add_analyzer_version(repo_name, version,
implemented_rules)
except Exception as e:
print(f"{repo_name} {version} checkout failed: {e}")
raise
def cloneOrCheckout(base_path, repo): git = Git() repo_path = os.path.join(base_path, repo['name']) print (repo_path) try: os.chdir(repo_path) except OSError: os.chdir(base_path) git.clone(repo['base_url'] + repo['name']) os.chdir(repo_path) cloned = True finally: git.fetch() git.checkout(repo['branch']) os.chdir(base_path)
def run(self):
git_dir = os.path.join(self.docs_dir, ".git")
if os.path.exists(git_dir):
import shutil
shutil.rmtree(git_dir)
from git import Git
g = Git(self.docs_dir)
g.init()
g.checkout(b="main")
g.add(".")
g.commit("-m {}".format(self.commit_msg))
g.push("--force", self.repo, "main:gh-pages")
def git_clone(repo_url, ref):
# repo_url - URL of git repo to clone
# ref - branch, commit or reference in the repo to clone
if repo_url == '':
print("No repo URL")
try:
# Entering remote git URL and folder to save
repo = Repo.clone_from(repo_url, repodir)
# Getting selected branch files
repo.remotes.origin.fetch(ref)
# Working dir of Git
g = Git(repo.working_dir)
# Getting latest fetch
g.checkout('FETCH_HEAD')
except Exception:
print("Git clone failed.")
def git_module(source, module_path, name, tag=None):
check_path = check_directory(module_path)
logger.info(f"Checkout new module {name}")
try:
Repo.clone_from(source, f"{check_path}/{name}", progress=Progress())
except GitCommandError as error:
logger.debug(f"During checkout error {error} has been detected")
pass
git_version = Git(f"{check_path}/{name}")
if tag is not None:
logger.info(f"Getting {tag} for module {name}")
git_version.checkout(tag)
else:
logger.info(
f"{tag} hasn't present for module {name}, checkout master branch")
git_version.checkout("master")
class MyGit():
def __init__(self, path):
self.g = Git(path)
def update_force(self, config=None):
"""
强制更新 buckets
"""
if 'http.proxy' in config:
info_1 = self.g.config("--global", "http.proxy", config['git']['http.proxy'])
self.g.checkout('*')
pull_info = self.g.pull("--no-rebase")
self.g.config("--global", "--unset", "http.proxy")
else:
self.g.checkout('*')
pull_info = self.g.pull("--no-rebase")
return pull_info
def addSuperModuleCommit(self, id, hash, url, who, branch, project):
self.log.debug("branch: " + branch + ", project:" + project)
hasSuperModule = False
isSuperModuleBr = False
self.log.debug("Project names: " + str(self.config.projects))
projectNames = self.config.projects.keys()
for proj in projectNames:
self.log.debug("project: " + project + " proj: " + proj)
if project.lower() == proj:
hasSuperModule = True
break
for br in self.config.branches:
if branch == br:
isSuperModuleBr = True
break
self.log.debug("isSuperModuleBr: " + str(isSuperModuleBr) + " hasSuperModule: " + str(hasSuperModule))
if isSuperModuleBr and hasSuperModule:
self.log.debug("Git Profile Path: " + str(self.config.profile))
git = Git(self.config.profile)
self.checkoutTrackingBranch(git, branch)
git.pull()
git.submodule("update","--init")
gitSubmoduleProfile = {'git':self.config.superRepoPath + self.config.projects[project.lower()]}
gitSubmodule = Git(gitSubmoduleProfile)
self.log.debug("checking out hash: " + hash)
gitSubmodule.fetch()
if self.isOptOut(gitSubmodule, hash):
return
gitSubmodule.checkout(hash, True)
git.add(".")
commitMsg = "Auto checkin: " + self.getCommitMessage(gitSubmodule, hash) + "\nuser:"******"\nhash:" + hash + "\nproject: " + project
self.log.debug("commiting in super module: " + commitMsg)
git.commit(commitMsg)
self.log.debug("pushing super module to branch: " + branch)
git.push(branch)
else:
self.log.debug("No super module commit is required.")
def branch(branch_name):
# branch_name = 'remotes/ssh/' + branch_name
g = Git(PROJECT_DIR)
send('重新获取远程分支中.....')
g.fetch(REMOTE_NAME)
send('获取成功')
branch_names = g.branch('-a').split('\n')
branch_names = [_.strip() for _ in branch_names]
if branch_name not in branch_names:
return '你的这个branch啊，鹅母鸡鸭'
try:
send('重置分支')
g.reset('--hard')
send('切换分支中')
g.checkout(branch_name)
send('切换分支完成')
except Exception as e:
return str(e)
return branch_name
def branch(branch_name):
# branch_name = 'remotes/ssh/' + branch_name
g = Git(PROJECT_DIR)
send('重新获取远程分支中.....')
g.fetch(REMOTE_NAME)
send('获取成功')
branch_names = g.branch('-a').split('\n')
branch_names = [_.strip() for _ in branch_names]
if branch_name not in branch_names:
return '你的这个branch啊，鹅母鸡鸭'
try:
send('重置分支')
g.reset('--hard')
send('切换分支中')
g.checkout(branch_name)
send('切换分支完成')
except Exception as e:
return str(e)
return branch_name
def post(self):
parser = reqparse.RequestParser()
parser.add_argument('app_name', required=True)
parser.add_argument('git_url', required=True)
parser.add_argument('git_ref', required=True)
args = parser.parse_args()
if mongo.db.docker_apps.find_one({'app_name':args['app_name']}):
abort(500, message='App Already Exists')
r = requests.get(args['git_url'])
if r.status_code == int('200'):
git_clone_dir = git_base+args['app_name']
try:
if os.path.isdir(git_clone_dir):
shutil.rmtree(git_clone_dir, ignore_errors=True)
Repo.clone_from(args['git_url'], git_clone_dir, branch='master')
g = Git(git_clone_dir)
g.checkout(args['git_ref'])
except:
abort(500, message='Failed To Clone Git Repo')
try:
repo = Repo(git_clone_dir)
sha = repo.head.object.hexsha
short_sha = repo.git.rev_parse(sha, short=7)
response = [line for line in cli.build(path=git_clone_dir, tag=args['app_name']+'/'+short_sha+'/'+args['git_ref'])]
pprint(response) # Need to add to mongodb log database
except:
abort(500, message='Failed To Build Docker Container')
try:
mongo.db.docker_apps.create_index('app_name',unique=True)
mongo_insert = mongo.db.docker_apps.insert({'app_name':args['app_name'],'git_url':args['git_url']},{'_id': 0})
except:
abort(500, message='Database Updage Failed')
else:
abort(500, message="Invalid GIT URL")
return args
def compare_git_commits(repo_path, branch, start_commit, end_commit): # print 'Repo path: ' + repo_path + ' and branch: ' + branch # print 'Starting commit: ' + start_commit + ', Ending commit: ' + end_commit repo = Repo(repo_path) git = Git(repo_path) head = repo.heads[0] git.checkout(start_commit) cmd_str = 'java -jar bin/threadfix-endpoint-cli-2.4-SNAPSHOT-jar-with-dependencies.jar ' + pipes.quote(repo_path) + ' -json 2>/dev/null > ' + pipes.quote(make_attack_surface_filename(start_commit)) print 'About to generate start attack surface with command: ' + cmd_str os.system(cmd_str) git.checkout(end_commit) cmd_str = 'java -jar bin/threadfix-endpoint-cli-2.4-SNAPSHOT-jar-with-dependencies.jar ' + pipes.quote(repo_path) + ' -json 2>/dev/null > ' + pies.quote(make_attack_surface_filename(end_commit)) print 'About to generate end attack surface with command: ' + cmd_str os.system(cmd_str) ret_val = diff_attack_surface_files(make_attack_surface_filename(start_commit), make_attack_surface_filename(end_commit)) return ret_val
def checkout(repo,version,batch_mode): git_url=f"[email protected]:SonarSource/{repo}" git_repo=None g=Git(repo) if not os.path.exists(repo): git_repo=Repo.clone_from(git_url, repo) else: git_repo=Repo(repo) if batch_mode: os.chdir(repo) for tag in git_repo.tags: if not '-' in tag.name: print(f"{repo} {tag.name}") g.checkout(tag.name) dump_rules(repo,tag.name) os.chdir('..') else: g=Git(repo) g.checkout(version) os.chdir(repo)
class TempClone:
"""Create a clone in a temp dir used to write and push file changes"""
def __init__(self, url: str, branch: str=None):
import posixpath
self.path = join(gettempdir(), posixpath.basename(url))
if not isdir(self.path):
Repo.clone_from(url, self.path)
self.git = Git(self.path)
self.git.fetch()
if branch:
self.git.checkout(branch)
def write(self, path: str, content: str):
with open(join(self.path, path), 'w') as f:
f.write(content)
self.git.add(path)
self.git.commit(message="Automatic update of skill-metadata.json")
self.git.push()
def delete(self):
shutil.rmtree(self.path)
def buildForReal(site_branch):
# make sure there are no local mods outstanding
repo = Repo(os.getcwd())
if repo.is_dirty() or repo.untracked_files:
print "ERROR: Your working directory has outstanding changes."
print "Commit or stash them."
return
mygit = Git(os.getcwd())
cfg = config.load_config()
# sanity check that the version branches exist as named
for version in cfg['extra']['versions']:
print 'Verifying branch %s' % (version['branch'])
mygit.checkout(version['branch'])
# sanity check - only one latest
latest = False
for version in cfg['extra']['versions']:
if not latest and version['latest']:
print 'Latest is %s' % (version['branch'])
latest = True
elif latest and version['latest']:
print 'ERROR: More than one version is latest.'
print 'Only one version can be latest: True.'
print 'Check mkdocs.yml.'
return
mygit.checkout(site_branch)
print "Building site pages from: %s..." % (site_branch)
sh.rm('-rf', 'site')
sh.mkdocs('build', '--clean')
for version in cfg['extra']['versions']:
sh.git('checkout', version['branch'], '--', 'docs', 'mkdocs.yml')
deployVersion(version)
sh.git('reset', 'HEAD', '.')
sh.git('checkout', '--', '.')
sh.git('reset', 'HEAD', '.')
sh.git('checkout', '--', '.')
def get(self):
path = self.get_argument("path", "")
id = self.get_argument("id", "")
version = int(self.get_argument("version", 0))
if not id and not path:
self.finish({"id": "", "version": -1, "contents": {}})
return
if not id and path:
sha = hashlib.sha256()
sha.update(path.encode())
id = sha.hexdigest()
try:
tag = self.repo.tags["%s-%d" % (id, version)]
except IndexError:
self.finish({"id": "", "version": -1, "contents": {}})
return
past = self.repo.tags[-1]
git = Git(self.repo.working_tree_dir)
git.checkout(tag)
path = os.path.join(self.repo.working_tree_dir, id)
nb = os.path.join(path, "NOTEBOOK")
if os.path.exists(path):
if os.path.exists(nb):
nb = nbformat.read(nb, 4)
else:
nb = {}
else:
nb = {}
self.finish({"id": id, "version": version, "nb": nb})
git.checkout(past)
def git_clone(repo_url, branch='master'):
'''
:params repo_url - URL of git repo to clone
:params branch - branch of the repo to clone
Returns a path to the cloned repo
'''
if repo_url == '':
raise source_exceptions.GitLocationException(repo_url)
os.environ['GIT_TERMINAL_PROMPT'] = '0'
_tmp_dir = tempfile.mkdtemp(prefix='armada')
try:
repo = Repo.clone_from(repo_url, _tmp_dir, **{'branch': 'master'})
g = Git(repo.working_dir)
g.checkout(branch)
except Exception:
raise source_exceptions.GitLocationException(repo_url)
return _tmp_dir
def get(self):
path = self.get_argument('path', '')
id = self.get_argument('id', '')
version = int(self.get_argument('version', 0))
if not id and not path:
self.finish({'id': '', 'version': -1, 'contents': {}})
return
if not id and path:
sha = hashlib.sha256()
sha.update(path.encode())
id = sha.hexdigest()
try:
tag = self.repo.tags['%s-%d' % (id, version)]
except IndexError:
self.finish({'id': '', 'version': -1, 'contents': {}})
return
past = self.repo.tags[-1]
git = Git(self.repo.working_tree_dir)
git.checkout(tag)
path = os.path.join(self.repo.working_tree_dir, id)
nb = os.path.join(path, 'NOTEBOOK')
if os.path.exists(path):
if os.path.exists(nb):
nb = nbformat.read(nb, 4)
else:
nb = {}
else:
nb = {}
self.finish({'id': id, 'version': version, 'nb': nb})
git.checkout(past)
def git_clone(repo_url, ref='master', proxy_server=None):
'''Clone a git repository from ``repo_url`` using the reference ``ref``.
:params repo_url: URL of git repo to clone.
:params ref: branch, commit or reference in the repo to clone.
:params proxy_server: optional, HTTP proxy to use while cloning the repo.
:returns: Path to the cloned repo.
'''
if repo_url == '':
raise source_exceptions.GitException(repo_url)
os.environ['GIT_TERMINAL_PROMPT'] = '0'
_tmp_dir = tempfile.mkdtemp(prefix='armada')
try:
if proxy_server:
LOG.info('Cloning [%s] with proxy [%s]', repo_url, proxy_server)
repo = Repo.clone_from(repo_url, _tmp_dir,
config='http.proxy=%s' % proxy_server)
else:
LOG.info('Cloning [%s]', repo_url)
repo = Repo.clone_from(repo_url, _tmp_dir)
repo.remotes.origin.fetch(ref)
g = Git(repo.working_dir)
g.checkout('FETCH_HEAD')
except git_exc.GitCommandError as e:
LOG.exception('Encountered GitCommandError during clone.')
if 'Could not resolve proxy' in e.stderr:
raise source_exceptions.GitProxyException(proxy_server)
else:
raise source_exceptions.GitException(repo_url)
except Exception:
LOG.exception('Encountered unknown Exception during clone.')
raise source_exceptions.GitException(repo_url)
return _tmp_dir
def git_clone(repo_url, ref='master'):
'''
:params repo_url - URL of git repo to clone
:params ref - branch, commit or reference in the repo to clone
Returns a path to the cloned repo
'''
if repo_url == '':
raise source_exceptions.GitLocationException(repo_url)
os.environ['GIT_TERMINAL_PROMPT'] = '0'
_tmp_dir = tempfile.mkdtemp(prefix='armada')
try:
repo = Repo.clone_from(repo_url, _tmp_dir)
repo.remotes.origin.fetch(ref)
g = Git(repo.working_dir)
g.checkout('FETCH_HEAD')
except Exception:
raise source_exceptions.GitLocationException(repo_url)
return _tmp_dir
def plant_seed(self, seed_id=None, target_dir=None):
if not seed_id or not target_dir:
self.rage_quit("Missing arguments, seed plant failed")
seeds = self.get_seeds()
tagref = seeds.get(seed_id, None)
if not tagref:
self.rage_quit("Seed id {} not found".format(seed_id))
git = Git(tagref.repo.working_dir)
current_commit = str(Repo(tagref.repo.working_dir).commit())
cprint("Current commit: {}".format(current_commit), Fore.GREEN)
dirty = tagref.repo.is_dirty()
cprint("Working directory {}".format('dirty' if dirty else 'clean'),
Fore.GREEN)
if dirty:
cprint("--> git stash", Fore.YELLOW)
git.stash()
cprint("--> git checkout {}".format(seed_id), Fore.YELLOW)
git.checkout(seed_id)
try:
cprint(
"Copying seed directory: {}".format(tagref.repo.working_dir),
Fore.GREEN)
call(["cp", "-r", tagref.repo.working_dir, target_dir])
except OSError as error:
cprint("Copying directory failed:\n{}".format(error), Fore.RED)
finally:
if dirty:
cprint("--> git stash apply", Fore.YELLOW)
git.stash('apply')
cprint("--> git checkout {}".format(current_commit), Fore.YELLOW)
git.checkout(current_commit)
def temporary_macro(tag, macro, app, nevents):
app_map = {'BACCARAT': 'Bacc'}
macro_extras = Template(dedent("""
/control/getEnv SEED
/$app/randomSeed {SEED}
/$app/beamOn $nevents
exit
"""))
lzprod_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
git_dir = os.path.join(lzprod_root, 'git', 'TDRAnalysis')
macro = os.path.join(git_dir, macro)
git = Git(git_dir)
git.fetch('origin')
git.checkout(tag)
if not os.path.isfile(macro):
raise Exception("Macro file '%s' doesn't exist in tag %s" % (macro, tag))
with NamedTemporaryFile(prefix=os.path.splitext(os.path.basename(macro))[0] + '_',
suffix='.mac') as tmpfile:
with open(macro, 'rb') as macro_file:
tmpfile.write(macro_file.read())
tmpfile.write(macro_extras.safe_substitute(app=app_map.get(app, app), nevents=nevents))
tmpfile.flush()
yield tmpfile
def parameter(file):
''' parameter file is mandatory to run a2a !!! '''
try:
data = ruamel.yaml.load(open(file), ruamel.yaml.RoundTripLoader)
current_dir = '{0}{1}'.format(os.getcwd(), '/repositories/')
for item in data:
repo = None
branch = None
clone_dir = None
if ('repo' in data[item] and len(data[item]['repo']) != 0):
repo = data[item]['repo']
clone_dir = '{0}{1}'.format(
current_dir,
data[item]['repo'].split('/')[-1].split('.')[0])
if ('branch' in data[item] and len(data[item]['branch']) != 0):
branch = data[item]['branch']
if ('force_clone' in data[item]
and data[item]['force_clone'] is True):
shutil.rmtree(clone_dir, ignore_errors=True)
if (not repo is None and not branch is None
and not clone_dir is None):
repo = Repo.clone_from(data[item]['repo'], clone_dir)
repo.remotes.origin.fetch(data[item]['branch'])
git_cmd = Git(clone_dir)
git_cmd.checkout('FETCH_HEAD')
print(colored("[A2A COMMAND ID: {0}]".format(item), 'green'))
os.system(data[item]['command'])
except Exception as error:
print(colored("[ERROR] - {0}\n".format(error), 'red'))
import git
from git import Git
from shutil import copyfile
import datetime
# Grab todays date
now = datetime.datetime.now()
now = now.strftime("%Y-%m-%d")
# Git pull (cron job)
cloned_repo = git.cmd.Git("/women-in-tech-datasets/triketora")
cloned_repo1 = Git("/women-in-tech-datasets/triketora")
# Getting sha for historic commits since last successfull run
loginfo = cloned_repo.log('--format=format:%H', '--since=%s' % now, '--', 'data.txt')
# Converting it into an array
loginfo_array = loginfo.split('\n')
# Run only if there have been commits
ntp = open("/datasets/tracy_data/new_to_parse.txt")
if len(loginfo_array) > 1:
for hexsha in loginfo_array:
cloned_repo1.checkout(hexsha)
copyfile("/women-in-tech-datasets/triketora/data.txt", "/datasets/company_coder_gender_stats/data_%s.txt" % hexsha)
ntp.append("data_%s.txt" % hexsha)
ntp.close()
f = open("/datasets/tracy_data/success_runDate.txt", "w+")
f.write(now)
f.close()
def initialize(path, branch):
try:
repo = Repo(path)
git = Git(path)
except InvalidGitRepositoryError as err:
print "Not a valid Git repo: %s" % err
sys.exit("Run `git init %s` first" % err)
output = ['Initializing in %s [%s]' % (path, branch)]
current_branch = repo.active_branch
if branch in [b.name for b in repo.branches]:
output.append("error: A branch named '%s' already exists in this repo!"
% branch)
return output
output.append("--> Stashing current branch contents [%s]" % current_branch)
try:
cmd = git.stash(u=True)
for line in cmd.splitlines():
output.append(" %s" % line)
except GitCommandError as err:
output.append("error: %s" % err)
return output
output.append("--> Switching to branch '%s'" % branch)
try:
git.checkout(B=branch)
except GitCommandError as err:
output.append("error: %s" % err)
return output
output.append("--> Clearing current files and committing")
try:
files = os.listdir(path)
files.remove('.git')
for entry in files:
git.rm(entry, r=True, f=True)
cmd = git.commit(m="Clearing files in preparation for git-pm")
for line in cmd.splitlines():
output.append(" %s" % line)
except GitCommandError as err:
output.append("error: %s" % err)
return output
output.append("--> Creating git-pm file structure and committing")
try:
for directory in DIRECTORIES:
dir_path = os.path.join(path, directory)
gitkeep = os.path.join(dir_path, '.gitkeep')
os.mkdir(dir_path)
with open(gitkeep, 'a'):
os.utime(gitkeep, None)
cmd = git.commit(m="Created git-pm file structure")
for line in cmd.splitlines():
output.append(" %s" % line)
except GitCommandError as err:
output.append("error: %s" % err)
return output
output.append("--> Returning to previous branch and popping stash")
try:
git.checkout(current_branch)
git.stash("pop")
except GitCommandError as err:
output.append("error: %s" % err)
return output
return output
def build_command(config_file, strict, site_dir, branches, default_branch,
latest, logging_level):
"""Build the MkDocs documentation"""
# cli.configure_logging(level=logging.INFO)
global release_branches
logging.basicConfig(
stream=sys.stdout,
level=get_logging_level(logging_level),
format=
'%(asctime)s %(levelname)s [%(threadName)s] [%(filename)s:%(lineno)d] %(message)s'
)
g = Git()
repo = Repo()
branches = branches or g.branch('-r').splitlines()
all_branch_names = list(
map(lambda branch: branch.split("origin/")[1], branches))
active_branch = repo.active_branch.name
print("Active branch %s", active_branch)
print("Default branch %s", default_branch)
print("Latest branch %s", latest)
start_stashes_count = len(
re.findall("stash@{[0-9]{1,3}}:", repo.git.stash("list")))
repo.git.stash("save")
if active_branch != latest:
print("Checkout Default %s", active_branch)
g.checkout(default_branch)
default_config = _load_config(config_file, strict, site_dir)
versions = default_config.get("extra").get("versions")
formatedCSVersions = {}
jelasticVersions = []
for version in versions:
formatedCSVersions[unicode(version)] = versions[version]
if formatedCSVersions is not None:
release_branches = formatedCSVersions.keys()
jelasticVersions = formatedCSVersions.values()
if release_branches is not None:
release_branches = sorted(release_branches,
key=functools.cmp_to_key(version_compare))
jelasticVersions = sorted(jelasticVersions,
key=functools.cmp_to_key(version_compare))
default_version = next(iter(release_branches),
None) # release_branches[-1]
print("Default version %s", default_version)
print("Building %s to /", default_version)
_build(default_config, default_version, release_branches,
jelasticVersions)
for branch in release_branches:
if branch in all_branch_names: #branch != default_version and
g.checkout(branch)
g.pull()
if not os.path.exists("mkdocs.yml"):
log.warning(
"Unable to build %s, as no mkdocs.yml was found",
branch)
print("Unable to build %s, as no mkdocs.yml was found",
branch)
continue
site_dir = "{0}".format(branch)
log.info("Building %s to /%s", branch, "site/" + site_dir)
print("Building %s to /%s", branch, "site/" + site_dir)
_build(_load_config(config_file, strict, site_dir), branch,
release_branches, jelasticVersions, "site/" + site_dir)
# print("Selected Branches %s", default_config.get("versions").get("releases"))
print("Checkout branch %s", active_branch)
g.checkout("master")
end_stashes_count = len(
re.findall("stash@{[0-9]{1,3}}:", repo.git.stash("list")))
if end_stashes_count > start_stashes_count:
repo.git.stash("pop")
print("pop latest stash")
if version['latest']:
buildTo('latest')
def buildTo(branch):
print 'Building doc pages for: %s...' % (branch)
branchCfg = config.load_config()
if branchCfg['extra']['version'] != branch:
updateConfigVersion(branch)
sh.mkdocs('build', '--site-dir', 'site/%s' % (branch))
if branchCfg['extra']['version'] != branch:
sh.git('checkout', '--', 'mkdocs.yml')
def updateConfigVersion(branch):
updated = False
for line in fileinput.input('mkdocs.yml', inplace=True):
line = line.rstrip()
if line.find(" version:") == 0:
line = " version: '%s'" % (branch)
updated = True
print line
assert updated, "Could not fix the version field on %s" % (branch)
if __name__ == '__main__':
repo = Repo(os.getcwd())
branch = repo.active_branch
mygit = Git(os.getcwd())
try:
build()
finally:
mygit.checkout(branch.name)
def changegittag(directory,tag): g = Git(directory) g.checkout(tag)
import pipes
parser = OptionParser()
parser.add_option('--repolocation', dest='repolocation', help='Path to Git repository location')
parser.add_option('--branch', dest='branch', help='Branch in the Git repository')
parser.add_option('--project', dest='project', help='Name of the project')
(options, args) = parser.parse_args()
repo_path = options.repolocation
project = options.project
branch = 'master'
if options.branch:
branch = options.branch
print 'Repo path: ' + repo_path + ' and branch: ' + branch
repo = Repo(repo_path)
git = Git(repo_path)
head = repo.heads[0]
commits = list(repo.iter_commits(branch))
commits.reverse()
for commit in commits:
print 'Commit: ' + commit.hexsha + ' with date: ' + str(commit.committed_date)
git.checkout(commit.hexsha)
cmd_str = 'java -jar bin/threadfix-endpoint-cli-2.4-SNAPSHOT-jar-with-dependencies.jar ' + pipes.quote(repo_path) + ' -json > work/' + pipes.quote(project) + '_attacksurface_' + pipes.quote(str(commit.committed_date)) + '.json'
print 'About to generate attack surface with command: ' + cmd_str
os.system(cmd_str)
