def _clone( self, workspace, ssh_pub_key, ssh_priv_key, ssh_user='******', ssh_pass='', ): ws = util.create_workspace(workspace, self.alias) keypair = Keypair( username=ssh_user, pubkey=ssh_pub_key, privkey=ssh_priv_key, passphrase=ssh_pass, ) try: cb = RemoteCallbacks(credentials=keypair) repo = clone_repository( url=self.url, path=ws, callbacks=cb, ) try: repo.checkout(self.refs) print(f'Cloned {repo} to {ws}') except Exception as err: raise errors.CannotFetchRef( f'Cannot fetch ref: {self.refs}' ) except Exception as err: raise errors.CannotCloneRepository( f'Cannot clone repository: {err}' )
def clone_repo(self, repo_from, repo_to): self.repo_from = repo_from self.repo_to = repo_to keypair = Keypair("git", self.public_key, self.private_key, "") callbacks = RemoteCallbacks(credentials=keypair) clone_repository(self.repo_from, self.repo_to, callbacks=callbacks)
def __init__(self, settings): self.repo = "https://github.com/" + settings[ 'nixbot.hydra_jobsets_repo'] self.repo_path = settings['nixbot.repo_dir'] self.user = settings['nixbot.bot_name'] self.token = settings['nixbot.github_token'] self.creds = RemoteCallbacks(UserPass(self.user, self.token))
def find_version(repo: Repository) -> str: keypair = construct_keypair() repo.remotes['origin'].fetch( ['template'], callbacks=RemoteCallbacks(credentials=keypair)) # Return the commit message for the HEAD of the "template" branch. return repo[repo.references.get( f'refs/remotes/origin/template').target].message
def push_changes(repo_path: Path): """Push the repository at repo_path to the remote named origin.""" repo = Repository(repo_path) remote = repo.remotes['origin'] creds = UserPass('Technical27', GITHUB_TOKEN) callback = RemoteCallbacks(credentials=creds) remote.connect(callbacks=callback) remote.push(['refs/heads/master:refs/heads/master'], callbacks=callback)
def _fetch_remote_template(self): # First try to pull it from the remote origin/TEMPLATE_BRANCH keypair = construct_keypair() self.repo.remotes['origin'].fetch( [TEMPLATE_BRANCH], callbacks=RemoteCallbacks(credentials=keypair)) self.repo.references.create( f'refs/heads/{TEMPLATE_BRANCH}', self.repo.references.get( f'refs/remotes/origin/{TEMPLATE_BRANCH}').target)
def push(repo, ref="refs/heads/master", remote_name="origin"): print("Pushing...") ssh_rsa_dir = str(Path.home()) + "/.ssh/" for remote in repo.remotes: if remote.name == remote_name: remote.credentials = Keypair("git", ssh_rsa_dir + "id_rsa.pub", ssh_rsa_dir + "id_rsa", "") callbacks = RemoteCallbacks(credentials=remote.credentials) remote.push([ref], callbacks=callbacks)
def clone_code(repo_url, clone_path, revision, branch=None): """branch 为 None, 默认用远端的 default branch""" cred = _get_credit(repo_url) cbs = RemoteCallbacks(cred, None) repo = clone_repository(repo_url, clone_path, bare=False, checkout_branch=branch, callbacks=cbs) repo.checkout('HEAD') obj = repo.revparse_single(revision) repo.checkout_tree(obj.tree)
def git_push(self, repo): if not pygit2.features & pygit2.GIT_FEATURE_SSH: print("libgit2/pygit2 cant use SSH") if repo in self.ahead: repo_obj = self.repos[repo]['repo'] origin = repo_obj.remotes['origin'] print(repo_obj.remotes["origin"].url) credentials = KeypairFromAgent(self.push_user) origin.credentials = credentials callbacks = RemoteCallbacks(credentials=credentials) try: origin.push([repo_obj.head.name], callbacks=callbacks) return True except GitError as e: return e return False
def clone_from_github(self) -> bool: try: callbacks = RemoteCallbacks( pygit2.UserPass(self.api_key, 'x-oauth-basic')) clone_repository(url='https://github.com/{}'.format( self.repo_name), path=self.old_repo, callbacks=callbacks, checkout_branch=self.branch) if os.path.exists(path='{}/.git'.format(self.old_repo)): return True else: return False except (GitError, ValueError) as error: print(error) return False
def pull(job): keybucket=os.environ['KEYS_BUCKET'] outputbucket=job['data']['outputArtifacts'][0]['location']['s3Location']['bucketName'] outputKey=job['data']['outputArtifacts'][0]['location']['s3Location']['objectKey'] repo_name = job['data']['pipelineContext']['pipelineName'] remote_url = job['data']['actionConfiguration']['configuration']['GitUrl'] branch = job['data']['actionConfiguration']['configuration']['Branch'] repo_path = '/tmp/%s' % repo_name creds = RemoteCallbacks( credentials=get_keys(keybucket), ) try: repository_path = discover_repository(repo_path) repo = Repository(repository_path) logger.info('found existing repo, using that...') except: logger.info('creating new repo for %s in %s' % (remote_url, repo_path)) repo = create_repo(repo_path, remote_url, creds) pull_repo(repo,remote_url,branch,creds) zipfile = zip_repo(repo_path, repo_name) push_s3(zipfile,repo_name,outputbucket,outputKey) commit = repo.head.get_object() revision = str(commit.id) commit_message = commit.message created = commit.commit_time currentRevision = { 'revision': revision, 'changeIdentifier': '???', 'created': created, 'revisionSummary': commit_message } if cleanup: logger.info('Cleanup Lambda container...') shutil.rmtree(repo_path) os.remove(zipfile) os.remove('/tmp/id_rsa') os.remove('/tmp/id_rsa.pub') return currentRevision
def clone_repo(repo, creds, branch, commit, s3bucket, key): """ Clone git repo, zip and upload to S3 """ # create folder tempfolder = tempfile.TemporaryDirectory() tempfolder_name = os.path.realpath(tempfolder.name) logger.info("Temp dir for clone: %s" % (tempfolder_name)) repo = clone_repository( url=repo, path=tempfolder_name, checkout_branch=branch, callbacks=RemoteCallbacks(credentials=creds) ) logger.info("Cloned") # switch to detached head for commit we want repo.checkout_tree(repo.get(commit)) logger.info("Switched to deatched head for: %s" % (commit)) # zip it up zipfile = tempfile.NamedTemporaryFile(suffix=".zip") zipfile_name = os.path.realpath(zipfile.name) logger.info("Zipping up to: %s" % (zipfile_name)) zf = ZipFile(zipfile_name, "w") for dirname, subdirs, files in os.walk(tempfolder_name): try: subdirs.remove('.git') except ValueError: pass zdirname = dirname[len(tempfolder_name)+1:] zf.write(dirname, zdirname) for filename in files: zf.write(os.path.join(dirname, filename), os.path.join(zdirname, filename)) zf.close() logger.info("Zip complete") # upload to s3 logger.info("Uploading to S3") s3_file_id = commit s3_key = "%s/%s.zip" % (key, s3_file_id) s3.upload_file(zipfile_name, s3bucket, s3_key) s3_key = "%s/latest.zip" % (key) s3.upload_file(zipfile_name, s3bucket, s3_key) logger.info("Upload complete.") return s3_key
def lambda_handler(event, context): keybucket = event['context']['key-bucket'] outputbucket = event['context']['output-bucket'] pubkey = event['context']['public-key'] # Source IP ranges to allow requests from, if the IP is in one of these the request will not be chacked for an api key ipranges = [] for i in event['context']['allowed-ips'].split(','): ipranges.append(ip_network(u'%s' % i)) # APIKeys, it is recommended to use a different API key for each repo that uses this function apikeys = event['context']['api-secrets'].split(',') ip = ip_address(event['context']['source-ip']) secure = False for net in ipranges: if ip in net: secure = True if 'X-Gitlab-Token' in event['params']['header'].keys(): if event['params']['header']['X-Gitlab-Token'] in apikeys: secure = True if 'X-Git-Token' in event['params']['header'].keys(): if event['params']['header']['X-Git-Token'] in apikeys: secure = True if 'X-Gitlab-Token' in event['params']['header'].keys(): if event['params']['header']['X-Gitlab-Token'] in apikeys: secure = True if 'X-Hub-Signature' in event['params']['header'].keys(): for k in apikeys: if 'use-sha256' in event['context']: k1 = hmac.new(str(k), str(event['context']['raw-body']), hashlib.sha256).hexdigest() k2 = str(event['params']['header']['X-Hub-Signature'].replace( 'sha256=', '')) else: k1 = hmac.new(str(k), str(event['context']['raw-body']), hashlib.sha1).hexdigest() k2 = str(event['params']['header']['X-Hub-Signature'].replace( 'sha1=', '')) if k1 == k2: secure = True # TODO: Add the ability to clone TFS repo using SSH keys try: # GitHub full_name = event['body-json']['repository']['full_name'] except KeyError: try: # BitBucket #14 full_name = event['body-json']['repository']['fullName'] except KeyError: try: # GitLab full_name = event['body-json']['repository'][ 'path_with_namespace'] except KeyError: try: # GitLab 8.5+ full_name = event['body-json']['project'][ 'path_with_namespace'] except KeyError: try: # BitBucket server full_name = event['body-json']['repository']['name'] except KeyError: # BitBucket pull-request full_name = event['body-json']['pullRequest'][ 'fromRef']['repository']['name'] if not secure: logger.error('Source IP %s is not allowed' % event['context']['source-ip']) raise Exception('Source IP %s is not allowed' % event['context']['source-ip']) # GitHub publish event if ('action' in event['body-json'] and event['body-json']['action'] == 'published'): branch_name = 'tags/%s' % event['body-json']['release']['tag_name'] repo_name = full_name + '/release' else: repo_name = full_name try: # branch names should contain [name] only, tag names - "tags/[name]" branch_name = event['body-json']['ref'].replace( 'refs/heads/', '').replace('refs/tags/', 'tags/') except: branch_name = 'master' try: # GitLab remote_url = event['body-json']['project']['git_ssh_url'] except Exception: try: remote_url = 'git@' + event['body-json']['repository'][ 'links']['html']['href'].replace('https://', '').replace( '/', ':', 1) + '.git' except: try: # GitHub remote_url = event['body-json']['repository']['ssh_url'] except: # Bitbucket try: for i, url in enumerate(event['body-json']['repository'] ['links']['clone']): if url['name'] == 'ssh': ssh_index = i remote_url = event['body-json']['repository']['links'][ 'clone'][ssh_index]['href'] except: # BitBucket pull-request for i, url in enumerate( event['body-json']['pullRequest']['fromRef'] ['repository']['links']['clone']): if url['name'] == 'ssh': ssh_index = i remote_url = event['body-json']['pullRequest']['fromRef'][ 'repository']['links']['clone'][ssh_index]['href'] repo_path = '/tmp/%s' % repo_name creds = RemoteCallbacks(credentials=get_keys(keybucket, pubkey), ) try: repository_path = discover_repository(repo_path) repo = Repository(repository_path) logger.info('found existing repo, using that...') except Exception: logger.info('creating new repo for %s in %s' % (remote_url, repo_path)) repo = create_repo(repo_path, remote_url, creds) pull_repo(repo, branch_name, remote_url, creds) zipfile = zip_repo(repo_path, repo_name) push_s3(zipfile, repo_name, outputbucket) if cleanup: logger.info('Cleanup Lambda container...') shutil.rmtree(repo_path) os.remove(zipfile) os.remove('/tmp/id_rsa') os.remove('/tmp/id_rsa.pub') return 'Successfully updated %s' % repo_name
def merge_push(pr, base, settings): MAIN_REPO = "https://github.com/" + settings['nixbot.repo'] PR_REPO = "https://github.com/" + settings['nixbot.pr_repo'] REPO_PATH = settings['nixbot.repo_dir'] user = settings['nixbot.bot_name'] token = settings['nixbot.github_token'] creds = RemoteCallbacks(UserPass(user, token)) path = os.path.join(REPO_PATH, "nixpkgs.git") try: log.info('Cloning {} to {}'.format(MAIN_REPO, path)) repo = pygit2.clone_repository(MAIN_REPO, path, bare=False, callbacks=creds) except ValueError: repo = pygit2.Repository(path) log.info('Fetching base repository including PRs') repo.remotes.add_fetch('origin', "+refs/pull/*/head:refs/remotes/origin/pr/*") repo.remotes['origin'].fetch(callbacks=creds) try: repo.create_remote('pr', PR_REPO) except: pass log.info('Checking out and resetting to PR base branch') repo.checkout('refs/heads/{}'.format(base)) repo.reset( repo.lookup_reference('refs/remotes/origin/{}'.format(base)).target, pygit2.GIT_RESET_HARD) base = repo.lookup_branch(base) log.info('Merging PR {} to base branch'.format(pr)) origin_pr = repo.lookup_reference('refs/remotes/origin/pr/{}'.format(pr)) repo.merge(origin_pr.target) log.info('Commiting merge') author = Signature(settings['nixbot.bot_name'], '*****@*****.**') tree = repo.index.write_tree() repo.create_commit(base.name, author, author, 'Merge PR #{}'.format(pr), tree, [repo.head.target, origin_pr.target]) repo.state_cleanup() remote = repo.remotes['pr'] pr_branch = 'pr-{}'.format(pr) log.info('Pushing merge to {} branch at {}'.format(pr_branch, PR_REPO)) repo.create_branch(pr_branch, repo.head.get_object(), True) repo.checkout('refs/heads/{}'.format(pr_branch)) remote.push(['+refs/heads/{b}:refs/heads/{b}'.format(b=pr_branch)], callbacks=creds) repo.state_cleanup() return repo
def auth( username=None, pubkey=None, privkey=None, password=None ): credentials = Keypair(username=username, pubkey=pubkey, privkey=privkey, passphrase=password) return RemoteCallbacks( credentials = credentials )
def lambda_handler(event, context): keybucket = event['context']['key-bucket'] outputbucket = event['context']['output-bucket'] pubkey = event['context']['public-key'] ### Source IP ranges to allow requests from, if the IP is in one of these the request will not be chacked for an api key ipranges = [] for i in event['context']['allowed-ips'].split(','): ipranges.append(ip_network(u'%s' % i)) ### APIKeys, it is recommended to use a different API key for each repo that uses this function apikeys = event['context']['api-secrets'].split(',') ip = ip_address(event['context']['source-ip']) secure = False for net in ipranges: if ip in net: secure = True if 'X-Gitlab-Token' in event['params']['header'].keys(): if event['params']['header']['X-Gitlab-Token'] in apikeys: secure = True if 'X-Git-Token' in event['params']['header'].keys(): if event['params']['header']['X-Git-Token'] in apikeys: secure = True if 'X-Gitlab-Token' in event['params']['header'].keys(): if event['params']['header']['X-Gitlab-Token'] in apikeys: secure = True if 'X-Hub-Signature' in event['params']['header'].keys(): for k in apikeys: if hmac.new( str(k), str(event['context']['raw-body']), hashlib.sha1).hexdigest() == str( event['params']['header']['X-Hub-Signature'].replace( 'sha1=', '')): secure = True if not secure: logger.error('Source IP %s is not allowed' % event['context']['source-ip']) raise Exception('Source IP %s is not allowed' % event['context']['source-ip']) try: repo_name = event['body-json']['project']['path_with_namespace'] except: repo_name = event['body-json']['repository']['full_name'] try: remote_url = event['body-json']['project']['git_ssh_url'] except: try: remote_url = 'git@' + event['body-json']['repository'][ 'links']['html']['href'].replace('https://', '').replace( '/', ':', 1) + '.git' except: remote_url = event['body-json']['repository']['ssh_url'] repo_path = '/tmp/%s' % repo_name creds = RemoteCallbacks(credentials=get_keys(keybucket, pubkey), ) try: repository_path = discover_repository(repo_path) repo = Repository(repository_path) logger.info('found existing repo, using that...') except: logger.info('creating new repo for %s in %s' % (remote_url, repo_path)) repo = create_repo(repo_path, remote_url, creds) pull_repo(repo, remote_url, creds) zipfile = zip_repo(repo_path, repo_name) push_s3(zipfile, repo_name, outputbucket) if cleanup: logger.info('Cleanup Lambda container...') shutil.rmtree(repo_path) shutil.rm(zipfile) shutil.rm('/tmp/id_rsa') shutil.rm('/tmp/id_rsa.pub') return 'Successfully updated %s' % repo_name
def job_runner(job): job = Job.loads(job) db = make_db() def log(line): db(db.jobs.id == job.id).update(logs=db.jobs.logs.coalesce('') + '{0}\n'.format(line)) log('[JOB #{0}] Started'.format(job.id)) db(db.jobs.id == job.id).update(status=JobStatus.STARTED) db.commit() container = None repo_path = None try: repo_name = str(uuid4()) repo_path = os.path.join(os.getcwd(), 'repos', repo_name) repos_path = os.path.dirname(repo_path) if config.REPO_HOST_PATH: repo_host_path = os.path.join(config.REPO_HOST_PATH, repo_name) else: repo_host_path = repo_path if not os.path.exists(repos_path): os.makedirs(repos_path) log('[JOB #{0}] git clone {1}'.format(job.id, job.ssh_url)) repo = clone_repository( job.ssh_url, repo_path, callbacks=RemoteCallbacks(credentials=Keypair( config.SSH_USERNAME, config.SSH_PUBKEY, config.SSH_PRIVKEY, config.SSH_PASSPHRASE), certificate=lambda *_: True)) log('[JOB #{0}] git checkout {1}'.format(job.id, job.commit_id)) commit = repo.get(job.commit_id) repo.checkout_tree(commit.tree) cfg_path = os.path.join(repo_path, '.microci.json') cfg = { 'dockerimg': config.DOCKER_IMAGE, 'command': '/bin/sh microci.sh' } if os.path.exists(cfg_path): log('[JOB #{0}] Load .microci.json'.format(job.id)) try: with open(cfg_path) as f: cfg.update(json.load(f)) except Exception: log('[JOB #{0}] {1}'.format(job.id, traceback.format_exc())) log('[JOB #{0}] docker run {1}'.format(job.id, cfg['dockerimg'])) client = DockerClient(base_url=config.DOCKER_URL) container = client.containers.run( cfg['dockerimg'], command=cfg['command'], working_dir='/repo', volumes=['{0}:/repo:rw'.format(repo_host_path)], detach=True) for line in container.logs(stdout=True, stderr=True, stream=True): log(line.decode().rstrip('\n')) retcode = container.wait() success = retcode == 0 log('[JOB #{0}] Returned {1}'.format(job.id, retcode)) status = JobStatus.SUCCEED if success else JobStatus.FAILED db(db.jobs.id == job.id).update(status=status) except Exception: log('[JOB #{0}] {1}'.format(job.id, traceback.format_exc())) db(db.jobs.id == job.id).update(status=JobStatus.ERRORED) db.commit() if container is not None: log('[JOB #{0}] Remove container'.format(job.id)) container.remove() if repo_path is not None and os.path.exists(repo_path): log('[JOB #{0}] Remove repository'.format(job.id)) rmtree(repo_path) db.close()
def create(event, context): """ Place your code to handle Create events here """ logger.info(event) physical_resource_id = 'myResourceId' response_data = {} source_url = event['ResourceProperties']['SourceRepoUrl'] source_branch = event['ResourceProperties']['SourceRepoBranch'] source_bucket = event['ResourceProperties']['SourceS3Bucket'] source_key = event['ResourceProperties']['SourceS3Key'] s3_zip_filename = source_key.split('/')[-1] dest_url = event['ResourceProperties']['DestRepoUrl'] repo_name = event['ResourceProperties']['DestRepoName'] username = event['ResourceProperties']['DestRepoName'] if len(username) >= 64: raise Exception('Username is longer than 64 chars') user_id, codecommit_username, password = get_codecommit_credentials( username, repo_name) try: creds = RemoteCallbacks( credentials=UserPass(codecommit_username, password)) if source_url != "": repo = pull_repo(source_url, source_branch) # Uncomment the next line if you want to update your ci files to a minimal default # setup_ci_config(repo) else: # Fetch source from S3 repo = create_repo('/tmp/s3source') r = requests.get('https://' + s3_region_url() + '/' + source_bucket + '/' + source_key, stream=True) if r.status_code == 200: with open('/tmp/' + s3_zip_filename, 'wb') as f: for chunk in r: f.write(chunk) else: raise Exception("cannot fetch zip, s3 returned %s: %s" % (r.status_code, r.reason)) zip = zipfile.ZipFile('/tmp/' + s3_zip_filename) zip.extractall(path='/tmp/s3source') author = Signature('Template Validation Pipeline Clone', '*****@*****.**') tree = repo.TreeBuilder().write() repo.create_commit('HEAD', author, author, 'initial commit', tree, []) index = repo.index index.add_all() index.write() tree = index.write_tree() repo.create_commit('refs/heads/%s' % source_branch, author, author, 'initial commit', tree, [repo.head.get_object().hex]) while not push_repo(repo, dest_url, creds, source_branch): logger.info("waiting for git credential propagation...") sleep(5) except Exception: logger.error("Unhandled exception: ", exc_info=1) raise delete_codecommit_credentials(user_id, username) return physical_resource_id, response_data