def _clone_required_cookbooks(self): """ Clone the cookbooks required to install Tomcat """ script = [] # Tomcat script.append(git.clone_opscode_cookbook("java")) script.append(git.clone("git://github.com/abiquo/tomcat.git", "/var/chef/cookbooks/tomcat", "ajp")) # Monitoring script.append(git.clone_opscode_cookbook("build-essential")) script.append(git.clone_opscode_cookbook("apt")) script.append(git.clone_opscode_cookbook("xml")) script.append(git.clone_opscode_cookbook("mysql")) script.append(git.clone_opscode_cookbook("php")) script.append(git.clone_opscode_cookbook("python")) script.append(git.clone_opscode_cookbook("apache2")) script.append( git.clone("git://github.com/escapestudios/chef-newrelic.git", "/var/chef/cookbooks/newrelic")) script.append( git.clone("git://github.com/boundary/boundary_cookbooks.git", "/tmp/boundary")) # Use only the bprobe cookbook script.append(Statements.exec( "mv /tmp/boundary/bprobe /var/chef/cookbooks/")) return script
def __init__(self): self.log = log.Log('logs/oc-release.log') # Setup AMQP connection sslkey = os.path.normpath(os.path.join(os.getcwd(), 'keys/ockey.pem')) sslcert = os.path.normpath(os.path.join(os.getcwd(), 'keys/CIA-londeroth.org.pem')) amqp_params = pika.ConnectionParameters( host='amqp.nosebud.de', port=5671, virtual_host='openclonk', ssl=True, ssl_options={'keyfile': sslkey, 'certfile': sslcert}, credentials=pika.credentials.ExternalCredentials(), ) self.amqp_connection = pika.BlockingConnection(amqp_params) if not os.path.exists('openclonk'): self.log.write('Openclonk Repository does not exist. Cloning...\n') git.clone('git://git.openclonk.org/openclonk') self.log.write('Repository created\n') os.chdir('openclonk') # Register triggers self.queue = notifyqueue.NotifyQueue() self.pushtrigger = pushtrigger.PushTrigger(self.amqp_connection, self.queue, self.log) self.xmltrigger = xmltrigger.XMLTrigger(self.amqp_connection, self.queue, self.log)
def get_git(scheme,url,target,overwrite,tag): import git if os.path.exists(target + '/.git'): if not overwrite: return else: if len(scheme) == 1: giturl = url else: giturl = url[4:] git.clone(giturl,target) fs.goto(target) git.fetch() out = git.branch() for line in out.split('\n'): if not line: continue if line[0] != '*': continue out = line.split()[1] break #print out,tag if out != tag: lbranches,rbranches = git.branches() if tag in lbranches: git.checkout(tag) else: # git 1.5 does not put remotes/ like 1.6 does from exception import CommandFailure try: git.checkout('origin/'+tag,tag) except CommandFailure: git.checkout('remotes/origin/'+tag,tag) git.pull() fs.goback() return
def prepare_scoreboard_repo(url): path = get_github_path(url).split('/') scoreboard_owner = path[0] scoreboard_name = path[1] scoreboard_dir = '.score' clone(scoreboard_owner, scoreboard_name, False, scoreboard_dir) return scoreboard_dir
def _prepare_output_git(self, branch, input_work_dir, output_work_dir, output_reference): start_commit = self.output_branch_name(branch) new_branch = False try: git.clone( output_reference, output_work_dir, options=['--branch', self.output_branch_name(branch), '-q']) git.set_origin_url(self._output_tree, output_work_dir) try: git.commit_env_vars(self.output_branch_name(branch), tree=output_work_dir) except git.GitError: shutil.rmtree(output_work_dir) self.debug('clone incorrect - new branch?') raise Exception('clone incorrect - new branch?') except: # assume error was due to branch not found new_branch = True start_commit = self._check_new_branch(branch, input_work_dir, output_work_dir, output_reference) git.reset(['--hard', start_commit], tree=output_work_dir) try: git.fetch('refs/notes/mirror:refs/notes/mirror', tree=output_work_dir) except: pass return start_commit, new_branch
def start_get_hash(config, github, config_file): repo_owner = config['repo_owner'] for team in config['teams']: repo_name = config['teams'][team]['repo_name'] if repo_name == '-': continue print('[*] Get the commit hash of %s repo.' % repo_name) bug_branches = config['teams'][team]['bug_branches'] clone(repo_owner, repo_name) branches = bug_branches if len(bug_branches) > 0 \ else list_branches(repo_name) if "master" in branches: branches.remove("master") # Do not consider master branch for branch in branches: checkout(repo_name, branch) hash = get_latest_commit_hash(repo_name, int(time.time()), branch) config['teams'][team][branch] = hash rmdir(repo_name) with open(config_file, 'w') as outfile: json.dump(config, outfile, indent=4) print ('[*] Successfully write in %s' % config_file) return
def mirror(self): """ Start the mirroring process according to the configuration. """ cwd = os.getcwd() result = True with tempdir.tempdir() as tmpdir: input_work_dir = os.path.join(tmpdir, 'input') git.clone(self._input_tree, input_work_dir, options=self.reference + ['-q']) git.remote_update(gitdir=input_work_dir) git.submodule(['--quiet', 'init'], tree=input_work_dir) git.submodule(['--quiet', 'sync'], tree=input_work_dir) git.submodule(['--quiet', 'update'], tree=input_work_dir) if self.output_tree_pull: output_reference = self.output_tree_pull else: output_reference = os.path.join(tmpdir, 'outref') git.clone(self._output_tree, output_reference, options=['-q', '--mirror']) _branches = self._glob_branches(input_work_dir) for branch in _branches: try: if not self._mirror_one(branch, input_work_dir, output_reference): result = False finally: os.chdir(cwd) return result
def clone_or_pull_repo(k, repo_url, b_updte_repo, b_tag_after_update=True): # initialize repository info repo = { 'url': repo_url, 'name': repo_path.get_repo_name_from_url(repo_url), # repository name } # get project path repo_path_in_section = repo_path.get_local_repo_path(repo['name']) repo['path'] = os.path.abspath(repo_path_in_section) # just in case dir_backup = os.getcwd() # even if b_updte_repo is False, # if not cloned yet, do it now. if not os.path.exists(repo_path_in_section): print('clone_or_pull_repo(%2d) : clone %s' % (k, repo['url'])) git.clone(repo['url'], id=config['Admin']['id']) else: if b_updte_repo: print('clone_or_pull_repo(%2d) : pull %s' % (k, repo['url'])) fetch_and_reset(repo_path_in_section) # tag with time stamp after clone or pull tag_all_remote_branches(b_tag_after_update, os.path.abspath(repo_path_in_section), repo) # just in case os.chdir(dir_backup) return repo
def verify_injection(team, config_file): config = load_config(config_file) timeout = config["exploit_timeout"]["injection_phase"] repo_owner = config['repo_owner'] repo_name = config['teams'][team]['repo_name'] bug_branches = config['teams'][team]['bug_branches'] clone(repo_owner, repo_name) branches = bug_branches if len(bug_branches) > 0 \ else list_branches(repo_name) if "master" in branches: branches.remove("master") # master branch is not verification target for branch in branches: checkout(repo_name, branch) exploit_dir = get_exploit_dir(repo_name, branch, config, team) bug_branch_result, _ = \ verify_exploit(exploit_dir, repo_name, branch, timeout, config) checkout(repo_name, "master") master_result, _ = \ verify_exploit(exploit_dir, repo_name, "master", timeout, config) rmdir(exploit_dir) if master_result == False and bug_branch_result == True: print('[*] Successflly verified branch "%s".' % branch) elif bug_branch_result == True: print ('[*] Exploit for branch "%s" works, but it also works on ' \ 'master branch, which indicates some error.' % branch) sys.exit() else: print('[*] Failed to verify exploit in branch "%s".' % branch) sys.exit() rmdir(repo_name)
def get_next_commit(last_commit, defender, branch, config): repo_name = config['teams'][defender]['repo_name'] rmdir(repo_name) clone(config['repo_owner'], repo_name) next_commit_hash = get_next_commit_hash(repo_name, branch, last_commit) rmdir(repo_name) print next_commit_hash if next_commit_hash == '': return None else: return next_commit_hash
def verify_service(team, branch, service_port, host_port, config_file): config = load_config(config_file) repo_owner = config['repo_owner'] repo_name = config['teams'][team]['repo_name'] container_name = "%s-%s" % (repo_name, branch) clone(repo_owner, repo_name) docker_cleanup(container_name) checkout(repo_name, branch) setup(repo_name, container_name, int(service_port), int(host_port)) check_liveness(container_name, int(host_port)) docker_cleanup(container_name) rmdir(repo_name) sys.exit()
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry): docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) img_id = None if '{0}@{1}'.format(repository, ref) not in processed.keys(): logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) dst_folder = git.clone(repository, ref) if not 'Dockerfile' in os.listdir(dst_folder): raise RuntimeError('Dockerfile not found in cloned repository') logger.info('Building using dockerfile...') img_id, logs = client.build(path=dst_folder, quiet=True) rmtree(dst_folder, True) else: img_id = processed['{0}@{1}'.format(repository, ref)] logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag or 'latest')) client.tag(img_id, docker_repo, docker_tag) if push: logger.info('Pushing result to registry {0}'.format( registry or "default")) if registry is not None: docker_repo = '{0}/{1}'.format(registry, docker_repo) logger.info('Also tagging {0}'.format(docker_repo)) client.tag(img_id, docker_repo, docker_tag) client.push(docker_repo) return img_id
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry): docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) img_id = None dst_folder = None if '{0}@{1}'.format(repository, ref) not in processed.keys(): logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) if repository not in processed: rep, dst_folder = git.clone(repository, ref) processed[repository] = rep processed_folders.append(dst_folder) else: dst_folder = git.checkout(processed[repository], ref) if not 'Dockerfile' in os.listdir(dst_folder): raise RuntimeError('Dockerfile not found in cloned repository') logger.info('Building using dockerfile...') img_id, logs = client.build(path=dst_folder, quiet=True) else: img_id = processed['{0}@{1}'.format(repository, ref)] logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag or 'latest')) client.tag(img_id, docker_repo, docker_tag) if push: logger.info('Pushing result to registry {0}'.format( registry or "default")) if registry is not None: docker_repo = '{0}/{1}'.format(registry, docker_repo) logger.info('Also tagging {0}'.format(docker_repo)) client.tag(img_id, docker_repo, docker_tag) client.push(docker_repo) return img_id
def testCloneModifyPushToBranch(self): new_repo = clone(self.repo, utils.get_temporary_location()) branch = new_repo.createBranch('work') new_repo.checkout(branch) utils.commit_change(new_repo) new_repo.push(self.repo, fromBranch=branch, toBranch='work') self.assertTrue(self.repo.getBranchByName('work').getHead() == new_repo.getHead())
def api(): """Setups a project database using django on the remote server.""" wsgi_file = os.path.join(env.config.ApiServer.document_root, "django.wsgi") repo = "[email protected]:klooff/klooff-server.git" if not files.exists(wsgi_file): print "Can't file the project code at %s." % env.config.ApiServer.document_root query = "Do you want to clone it from github?" if prompt(query): git.clone(repo, env.config.ApiServer.document_root, env.config.branch) run("chmod 777 %s" % os.path.join(env.config.ApiServer.document_root, "media", "user_uploads")) else: print "Aborting." sys.exit(1) virtualenv() project.configure_api()
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry, repos_folder, logger): ''' Builds one line of a library file. repository: URL of the git repository that needs to be built ref: Git reference (or commit ID) that needs to be built docker_repo: Name of the docker repository where the image will end up. docker_tag: Tag for the image in the docker repository. namespace: Namespace for the docker repository. push: If the image should be pushed at the end of the build registry: URL to private registry where image should be pushed repos_folder: Directory where repositories should be cloned logger: Logger instance ''' dst_folder = None img_id = None commit_id = None if repos_folder: # Repositories are stored in a fixed location and can be reused dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix()) docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) if '{0}@{1}'.format(repository, ref) not in processed.keys(): # Not already built rep = None logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) if repository not in processed: # Repository not cloned yet rep, dst_folder = git.clone(repository, ref, dst_folder) processed[repository] = rep processed_folders.append(dst_folder) else: rep = processed[repository] if ref in rep.refs: # The ref already exists, we just need to checkout dst_folder = git.checkout(rep, ref) else: # ref is not present, try pulling it from the remote origin rep, dst_folder = git.pull(repository, rep, ref) if not 'Dockerfile' in os.listdir(dst_folder): raise RuntimeError('Dockerfile not found in cloned repository') commit_id = rep.head() logger.info('Building using dockerfile...') img_id, logs = client.build(path=dst_folder, quiet=True) else: logger.info('This ref has already been built, reusing image ID') img_id = processed['{0}@{1}'.format(repository, ref)] if ref.startswith('refs/'): commit_id = processed[repository].ref(ref) else: commit_id = ref logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag or 'latest')) client.tag(img_id, docker_repo, docker_tag) if push: logger.info('Pushing result to registry {0}'.format( registry or "default")) push_repo(img_id, docker_repo, registry=registry, logger=logger) return img_id, commit_id
def _git_clone(self,url,submodules=False): import fs,git target = fs.projects()+'/gaudi' if os.path.exists(target): log.info('Directory already exists, skipping clone to: %s'%target) return fs.goto(fs.projects(),True) git.clone(url,'gaudi') fs.goback() if submodules: fs.goto(os.path.join(fs.projects(),'gaudi'),True) git.submodule('init') git.submodule('update') fs.goback() return
def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry, repos_folder, logger): ''' Builds one line of a library file. repository: URL of the git repository that needs to be built ref: Git reference (or commit ID) that needs to be built docker_repo: Name of the docker repository where the image will end up. docker_tag: Tag for the image in the docker repository. namespace: Namespace for the docker repository. push: If the image should be pushed at the end of the build registry: URL to private registry where image should be pushed repos_folder: Directory where repositories should be cloned logger: Logger instance ''' dst_folder = None img_id = None commit_id = None if repos_folder: # Repositories are stored in a fixed location and can be reused dst_folder = os.path.join(repos_folder, docker_repo + _random_suffix()) docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) if '{0}@{1}'.format(repository, ref) not in processed.keys(): # Not already built rep = None logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) if repository not in processed: # Repository not cloned yet rep, dst_folder = git.clone(repository, ref, dst_folder) processed[repository] = rep processed_folders.append(dst_folder) else: rep = processed[repository] if ref in rep.refs: # The ref already exists, we just need to checkout dst_folder = git.checkout(rep, ref) else: # ref is not present, try pulling it from the remote origin rep, dst_folder = git.pull(repository, rep, ref) if not 'Dockerfile' in os.listdir(dst_folder): raise RuntimeError('Dockerfile not found in cloned repository') commit_id = rep.head() logger.info('Building using dockerfile...') img_id, logs = client.build(path=dst_folder, quiet=True) else: logger.info('This ref has already been built, reusing image ID') img_id = processed['{0}@{1}'.format(repository, ref)] if ref.startswith('refs/'): commit_id = processed[repository].ref(ref) else: commit_id = ref logger.info('Committing to {0}:{1}'.format(docker_repo, docker_tag or 'latest')) client.tag(img_id, docker_repo, docker_tag) if push: logger.info('Pushing result to registry {0}'.format(registry or "default")) push_repo(img_id, docker_repo, registry=registry, logger=logger) return img_id, commit_id
def clone(git_params, repo_path, logfile, workDir=None, clone_once=False, **kwargs): git = Git(git_params, repo_path, clone_once, logfile) git.clone(**kwargs) if git.repo_cloned: git.checkout() git.apply_patch(workDir) git.log()
def api(): """Setups a project database using django on the remote server.""" wsgi_file = os.path.join(env.config.ApiServer.document_root, "django.wsgi") repo = "[email protected]:klooff/klooff-server.git" if not files.exists(wsgi_file): print "Can't file the project code at %s." % env.config.ApiServer.document_root query = "Do you want to clone it from github?" if prompt(query): git.clone(repo, env.config.ApiServer.document_root, env.config.branch) run("chmod 777 %s" % os.path.join( env.config.ApiServer.document_root, "media", "user_uploads")) else: print "Aborting." sys.exit(1) virtualenv() project.configure_api()
def testCloneModifyPushToBranch(self): new_repo = clone(self.repo, utils.get_temporary_location()) prev_branch = self.repo.getCurrentBranch() self.repo.checkout(self.repo.createBranch('temp')) branch = new_repo.createBranch('work') new_repo.checkout(branch) utils.commit_change(new_repo) new_repo.push(self.repo, fromBranch=branch, toBranch='work') self.repo.checkout(prev_branch) self.assertTrue(self.repo.getBranchByName('work').getHead() == new_repo.getHead())
def _clone_or_checkout(self, url, ref, dst_folder, rep): if rep: try: # The ref already exists, we just need to checkout dst_folder = git.checkout(rep, ref) except git.GitException: # ref is not present, try pulling it from the remote origin rep, dst_folder = git.pull(url, rep, ref) return rep, dst_folder if dst_folder: rmtree(dst_folder) return git.clone(url, ref, dst_folder)
def _check_new_branch(self, branch, input_work_dir, output_work_dir, output_reference): # try to find merge-base in input tree, # assumes master branch is always mirrored # # this handles the case of having created a new branch, # and asking for that to be mirrored into the prune tree. base_id = git.merge_base('origin/' + self.master, 'origin/' + branch, tree=input_work_dir) git.clone(output_reference, output_work_dir, options=['-q']) git.set_origin_url(self._output_tree, output_work_dir) # try to find the merge-base or its parent/grandparent/... in the # output tree - since it should have been branched from the master # (or in any case we look at the merge-base between master and it) # this should exist - HOWEVER: some commits might not be mirrored, # so look for the *parent [with a reasonable limit] for offset in range(0, self.max_parents): search = git.rev_parse(rev='%s~%d' % (base_id, offset), tree=input_work_dir) self.debug('search for %s~%d=%s:' % (base_id, offset, search)) grep = '%s: %s' % (self._commit_id_prefix, search) out_commits = git.log(options=[ '--grep', grep, '--format=format:%H', 'origin/' + self.output_branch_name(self.master) ], tree=output_work_dir) out_commits = out_commits.split() if not out_commits or len(out_commits) > 1: self.debug('not found') continue start_commit = out_commits[0] self.debug('found at %s' % start_commit) return start_commit raise Exception('failed to find parent/grandparent/...')
def checkout(llvm_commit, clang_commit, compilerrt_commit): cwd = os.path.abspath(os.getcwd()) if not os.path.exists('llvm'): git.clone(llvm_git_repo, ['-n']) os.chdir('llvm') git.checkout(llvm_commit) os.chdir(os.path.join(cwd, 'llvm', 'tools')) if not os.path.exists('clang'): git.clone(clang_git_repo, ['-n']) os.chdir('clang') git.checkout(clang_commit) os.chdir(os.path.join(cwd, 'llvm', 'projects')) if not os.path.exists('compiler-rt'): git.clone(compilerrt_git_repo, ['-n']) os.chdir('compiler-rt') git.checkout(compilerrt_commit) os.chdir(cwd)
def _check_new_branch(self, branch, input_work_dir, output_work_dir, output_reference): # This handles the case of having created a new branch, # and asking for that to be mirrored into the prune tree. # # Try to find a starting point in the input tree. We assume # this will basically always succeed, since the master branch # is always mirrored, we _should_ find something (but can fail # if somebody created a branch without a merge-base.) # unfortunately we now need to do this first, to sort out which # branches we already know in the output git.clone(output_reference, output_work_dir, options=['-q']) self.debug("trying to find starting point for new branch %s" % branch) candidate_branches = [] for other in self._branches: if other == branch: continue out_branch = 'origin/' + self.output_branch_name(other) try: git.rev_parse(out_branch, tree=output_work_dir) except: self.debug(" branch %s doesn't exist in output %s (yet)" % (out_branch, output_work_dir)) continue candidate_branches.append(other) potential_merge_bases = [] for other in candidate_branches: try: base = git.merge_base('origin/' + other, 'origin/' + branch, tree=input_work_dir) potential_merge_bases.append(base) self.debug(" base to %s is %s" % (other, base)) except git.GitError: self.debug(" no base to %s" % (other, )) bases = git.independent_commits(potential_merge_bases, tree=input_work_dir) self.debug("found starting points %s" % (", ".join(bases))) assert len(bases) == 1, "No single merge base found: %r" % bases base_id = bases[0] base_branch = None for other in candidate_branches: if git.merge_base('origin/' + other, base_id, tree=input_work_dir) == base_id: base_branch = 'origin/' + self.output_branch_name(other) break assert base_branch, "This shouldn't happen, found no base branch?!" # try to find the merge-base or its parent/grandparent/... in the # output tree - since it should have been branched from the master # (or in any case we look at the merge-base between master and it) # this should exist - HOWEVER: some commits might not be mirrored, # so look for the *parent [with a reasonable limit] for offset in range(0, self.max_parents): search = git.rev_parse(rev='%s~%d' % (base_id, offset), tree=input_work_dir) self.debug('search for %s~%d=%s:' % (base_id, offset, search)) grep = '%s: %s' % (self._commit_id_prefix, search) out_commits = git.log( options=['--grep', grep, '--format=format:%H', base_branch], tree=output_work_dir) out_commits = out_commits.split() if not out_commits or len(out_commits) > 1: self.debug('not found') continue start_commit = out_commits[0] self.debug('found at %s' % start_commit) return start_commit raise Exception('failed to find parent/grandparent/...')
def install(idp_fqdn=FQDN): """ Installe un serveur d'indentité disponible à *idp_fqdn*. """ # DEPS lasso.install() mysql.install() ssl.install() git.install() apache.install() apache.add_mod_rewrite() apache.add_mod_ssl() apache.add_mod_wsgi() venv.install_virtualenv() runcmd('apt-get install -y python-ldap') runcmd('apt-get install -y python-mysqldb --force-yes') # DB db_name = idp_fqdn.replace('.', '_') mysql.setup_db(db_name) # WEB ssl.create_certificats(idp_fqdn) extra = {'venv': venv.get_path(idp_fqdn), } apache.add_vhost(idp_fqdn, 'idp_vhost.txt', extra) # SOURCE git.clone('git://git.auf.org/authentic2', idp_fqdn) git.checkout(idp_fqdn, 'master') # VIRTUALENV venv.mkenv(idp_fqdn) bin_pip = venv.get_bin_pip(idp_fqdn) with cd(git.home(idp_fqdn)): git.sudo("%s install -r requirements.txt" % bin_pip) git.sudo("%s install django-auth-ldap" % bin_pip) # WSGI data = { 'project_path': git.home(idp_fqdn), 'venv': venv.get_path(idp_fqdn), } filename = os.path.join(TPL_DIR, 'idp_wsgi.txt') destination = os.path.join(venv.get_bin_path(idp_fqdn), 'idp_wsgi.py') upload_template( filename, destination, context=data, use_sudo=True,) runcmd('chown %s:%s %s' % (git.GIT_USER, git.GIT_GROUP, destination,)) runcmd('chmod 644 %s' % (destination, )) # LOG file log_file = os.path.join(git.home(idp_fqdn), 'log.log') runcmd('touch %s' % log_file) runcmd('chmod g+w %s' % log_file) runcmd('chown %s:www-data %s' % (git.GIT_USER, log_file)) # CONF data.update({ 'db_name': db_name, 'db_user': db_name, 'db_password': db_name, }) filename = os.path.join(TPL_DIR, 'idp_local_settings.txt') destination = os.path.join( git.home(idp_fqdn), 'aufcustom', 'local_settings.py') upload_template( filename, destination, context=data, use_sudo=True,) runcmd('chown %s:%s %s' % (git.GIT_USER, git.GIT_GROUP, destination,)) # manage.py data.update({ 'venv': venv.get_path(idp_fqdn), }) filename = os.path.join(TPL_DIR, 'idp_manage.txt') destination = os.path.join( git.home(idp_fqdn), 'manage.py') upload_template( filename, destination, context=data, use_sudo=True,) runcmd('chown %s:%s %s' % (git.GIT_USER, git.GIT_GROUP, destination,)) runcmd('chmod +x %s' % (destination,)) git.sudo('%s syncdb --migrate --noinput' % (destination,)) git.sudo('%s collectstatic --noinput' % (destination,)) apache.restart()
logging.warning(traceback.format_exc()) finally: heapq.heappush(h, (time.time() + tasks[task], task)) if TRACE: print int(time.time()), " add task again: ", int(time.time()) + tasks[task], task else: # if TRACE: # print time.time(), " miss task: ", exec_time, task heapq.heappush(h, (exec_time, task)) time.sleep(1) if __name__ == "__main__": init_db() logging.info("Please wait: cloning %s to %s ..." % (config.GIT_REMOTE_PATH, config.GIT_WORK_DIR)) git.clone() git_update_remote_heads() logging.info("Please wait: Initial branch merging ...") git_merge_updated(limit=5) logging.info("Please wait: Initial jira task information upload ...") jira_get_statuses_resolutions_priorities() jira_update_new() loop = threading.Thread(target=main_loop) loop.start() web_.init_web(dbcon, jira_priority_map) # import utils.memory # print utils.memory.stacksize() # print utils.memory.memory() # print utils.memory.resident()
def verify_issue(defender, repo_name, issue_no, config, github, target_commit=None): timeout = config["exploit_timeout"]["exercise_phase"] repo_owner = config['repo_owner'] title, submitter, create_time, content = \ get_github_issue(repo_owner, repo_name, issue_no, github) # Issue convention: "exploit-[branch_name]" target_branch = title[8:] clone(repo_owner, repo_name) # Write the fetched issue content to temp file tmpfile = "/tmp/gitctf_%s.issue" % random_string(6) tmpdir = "/tmp/gitctf_%s.dir" % random_string(6) with open(tmpfile, "w") as f: f.write(content) # Decrypt the exploit mkdir(tmpdir) team = defender decrypt_exploit(tmpfile, config, team, tmpdir, submitter) rmfile(tmpfile) # Now iterate through branches and verify exploit # zchn: not sure about this, was: branches = list_branches(repo_name) bug_branches = config['teams'][team]['bug_branches'] branches = bug_branches + ['master'] if len(bug_branches) > 0 \ else list_branches(repo_name) candidates = [] if (target_branch in branches) and (target_commit is None): # Iterate through branches and collect candidates commit = get_latest_commit_hash(repo_name, create_time, target_branch) candidates.append((target_branch, commit)) verified_branch = None verified_commit = None log = 'About %s (exploit-service branch)\n' % title for (branch, commit) in candidates: if branch in title: result, log = verify_exploit(tmpdir, repo_name, commit, timeout, \ config, log=log) else: result, _ = verify_exploit(tmpdir, repo_name, commit, timeout, \ config) if result: verified_branch = branch verified_commit = commit break rmdir(tmpdir) rmdir(repo_name) if verified_branch is None: print("[*] The exploit did not work against branch '%s'" % \ target_branch) else: print("[*] The exploit has been verified against branch '%s'" % verified_branch) return (verified_branch, verified_commit, submitter, log)
import os from git import clone from prompt import interact from render import render_all from helper import schema_path template = 'https://github.com/deepankarm/pod-template.git' if __name__ == '__main__': try: repo_dir = clone(url=template) schema = schema_path(repo_dir=repo_dir) user_context = interact(filepath=schema) if not user_context: print('Something wrong') render_all(context=user_context, project_directory=os.path.join(repo_dir, user_context['project_dir'])) except KeyboardInterrupt: print('\nUser interrupted. Nothing to do. Bye!')
#提醒功能 def alert(result): print(result) ''' 可以在下面添加发邮件提醒功能 或通过其他通讯工具发送提醒的功能 ''' #开始定时器任务 def start_scheduler(seconds): scheduler = BlockingScheduler() print('Croner is working. Press Ctrl+{0} to exit'.format( 'c/Enter' if os.name == 'nt' else 'C')) test_task() scheduler.add_job(test_task, 'interval', seconds=int(seconds), id='test_job') try: scheduler.start() except (KeyboardInterrupt, SystemExit): scheduler.shutdown() if __name__ == '__main__': git.clone() start_scheduler(sys.argv[1])
def _sync(self, remote): """ Pushes all normal branches from a source repo to gerrit. @param remote - gerrit.Remote object """ # Only sync if source repo is provided. if not self.source: return # Only sync if heads and/or tags are specified if not self.heads and not self.tags: return msg = "Project %s: syncing with repo %s." % (self.name, self.source) logger.info(msg) print msg repo_dir = '~/tmp' repo_dir = os.path.expanduser(repo_dir) repo_dir = os.path.abspath(repo_dir) # Make Empty directory - We want this to stop and fail on OSError if not os.path.isdir(repo_dir): os.makedirs(repo_dir) logger.debug("Project %s: Created directory %s" % (self.name, repo_dir)) # Save the current working directory old_cwd = os.getcwd() try: # Change cwd to that repo os.chdir(repo_dir) uuid_dir = str(uuid4()) repo_dir = os.path.join(repo_dir, uuid_dir) # Do a git clone --bare <source_repo> git.clone(self.source, name=uuid_dir, bare=True) # Change to bare cloned directory os.chdir(uuid_dir) # Add remote named gerrit ssh_url = 'ssh://%s@%s:%s/%s' % (remote.username, remote.host, remote.port, self.name) git.add_remote('gerrit', ssh_url) # Push heads if self.heads: kwargs = {'all_': True} if self.force: kwargs['force'] = True git.push('gerrit', **kwargs) # Push tags if self.tags: kwargs = {'tags': True} if self.force: kwargs['force'] = True git.push('gerrit', **kwargs) ref_kwargs = self.ref_kwargs() # Grab origin refs origin_refset = git.remote_refs('origin', **ref_kwargs) # Grab gerrit refs gerrit_refset = git.remote_refs('gerrit', **ref_kwargs) # Find refs that should be removed. prune_refset = gerrit_refset - origin_refset if self.preserve_prefix: msg = "Project %s: Preserving refs with prefixes of %s" \ % (self.name, self.preserve_prefix) logger.debug(msg) print msg heads_prefix = "refs/heads/%s" % self.preserve_prefix tags_prefix = "refs/tags/%s" % self.preserve_prefix keep = lambda ref: not ref.startswith(heads_prefix) and \ not ref.startswith(tags_prefix) prune_refset = filter(keep, prune_refset) # Prefix each ref in refset with ':' to delete colonize = lambda ref: ':%s' % ref prune_refset = map(colonize, prune_refset) # Remove branches no longer needed if prune_refset: git.push('gerrit', refspecs=prune_refset) finally: # Change to old current working directory os.chdir(old_cwd) # Attempt to clean up created directory shutil.rmtree(repo_dir)
def _sync(self, remote): """ Pushes all normal branches from a source repo to gerrit. @param remote - gerrit.Remote object """ # Only sync if source repo is provided. if not self.source: return # Only sync if heads and/or tags are specified if not self.heads and not self.tags: return msg = "Project %s: syncing with repo %s." % (self.name, self.source) logger.info(msg) print msg repo_dir = '~/tmp' repo_dir = os.path.expanduser(repo_dir) repo_dir = os.path.abspath(repo_dir) # Make Empty directory - We want this to stop and fail on OSError if not os.path.isdir(repo_dir): os.makedirs(repo_dir) logger.debug( "Project %s: Created directory %s" % (self.name, repo_dir) ) # Save the current working directory old_cwd = os.getcwd() try: # Change cwd to that repo os.chdir(repo_dir) uuid_dir = str(uuid4()) repo_dir = os.path.join(repo_dir, uuid_dir) # Do a git clone --bare <source_repo> git.clone(self.source, name=uuid_dir, bare=True) # Change to bare cloned directory os.chdir(uuid_dir) # Add remote named gerrit ssh_url = 'ssh://%s@%s:%s/%s' % ( remote.username, remote.host, remote.port, self.name ) git.add_remote('gerrit', ssh_url) # Push heads if self.heads: kwargs = {'all_': True} if self.force: kwargs['force'] = True git.push('gerrit', **kwargs) # Push tags if self.tags: kwargs = {'tags': True} if self.force: kwargs['force'] = True git.push('gerrit', **kwargs) ref_kwargs = self.ref_kwargs() # Grab origin refs origin_refset = git.remote_refs('origin', **ref_kwargs) # Grab gerrit refs gerrit_refset = git.remote_refs('gerrit', **ref_kwargs) # Find refs that should be removed. prune_refset = gerrit_refset - origin_refset if self.preserve_prefix == PRESERVE_ALL_BRANCHES: msg = "Project %s: Preserving all refs" % self.name logger.debug(msg) print msg prune_refset = set([]) elif not self.preserve_prefix is None: msg = "Project %s: Preserving refs with prefixes of %s" \ % (self.name, self.preserve_prefix) logger.debug(msg) print msg heads_prefix = "refs/heads/%s" % self.preserve_prefix tags_prefix = "refs/tags/%s" % self.preserve_prefix keep = lambda ref: not ref.startswith(heads_prefix) and \ not ref.startswith(tags_prefix) prune_refset = filter(keep, prune_refset) # Prefix each ref in refset with ':' to delete colonize = lambda ref: ':%s' % ref prune_refset = map(colonize, prune_refset) # Remove branches no longer needed if prune_refset: git.push('gerrit', refspecs=prune_refset) finally: # Change to old current working directory os.chdir(old_cwd) # Attempt to clean up created directory shutil.rmtree(repo_dir)
from apps import app from git import verify_new_branch, clone, push, merge from k8s import apply_cluster, verify_existed from efs import efs_create_directory import sys emissor = sys.argv[1] ambiente = sys.argv[2] new_branch = sys.argv[3] def create_object(emissor, ambiente, new_branch): app_object = app(name_app = emissor, ambiente = ambiente, new_branch = new_branch) return app_object if __name__ == "__main__": verify_new_branch(new_branch, ambiente) clone(new_branch, ambiente) app_object = create_object(emissor, ambiente, new_branch) NEW_COMMIT = verify_existed(app_object.name_deployment, app_object.name_ingress, app_object.name_app, app_object.ambiente) NEW_PUSH = push(NEW_COMMIT, app_object.ambiente, app_object.new_branch) merge(app_object.ambiente, NEW_PUSH, app_object.new_branch) efs_create_directory(app_object.name_app, app_object.ambiente) apply_cluster(app_object.name_app, app_object.name_deployment, app_object.name_ingress, app_object.ambiente)
parser.add_argument('url') args = parser.parse_known_args() name = str(args[0].url).rsplit('/', maxsplit=1)[-1] if name[-4:] == '.git': name = name[:-4] path = Path.cwd() / name print(path) identities = configparser.ConfigParser() identities.read(identities_file_path, encoding='utf_8') result = find_identity(path, identities) if result.identity_key is None: print(Colors.yellow + 'Warning:' + Colors.default + ' No identity matches the current path, so no identity will be set.') returned = git.clone(args[0].url, args[1]) else: line = 'Selected ' + Colors.bold + result.identity_key + Colors.default + ' based on ' if result.keyword is not None: line += 'keyword "%s".' % result.keyword elif result.path is not None: line += 'path "%s" with weakness %s.' % (result.path, result.weakness) print(line) returned = git.clone(args[0].url, args[1] + ['--config', 'user.name=%s' % result.identity['name'], '--config', 'user.email=%s' % result.identity['email']]) exit(returned.returncode)