def get_credentials(args):
if args.password:
credentials = UserPass(args.username, args.password)
else:
credentials = Keypair(args.ssh_user, args.ssh_key + ".pub",
args.ssh_key, "")
return RemoteCallbacks(credentials=credentials)
def _clone(
self,
workspace,
ssh_pub_key,
ssh_priv_key,
ssh_user='******',
ssh_pass='',
):
ws = util.create_workspace(workspace, self.alias)
keypair = Keypair(
username=ssh_user,
pubkey=ssh_pub_key,
privkey=ssh_priv_key,
passphrase=ssh_pass,
)
try:
cb = RemoteCallbacks(credentials=keypair)
repo = clone_repository(
url=self.url,
path=ws,
callbacks=cb,
)
try:
repo.checkout(self.refs)
print(f'Cloned {repo} to {ws}')
except Exception as err:
raise errors.CannotFetchRef(
f'Cannot fetch ref: {self.refs}'
)
except Exception as err:
raise errors.CannotCloneRepository(
f'Cannot clone repository: {err}'
)
def clone_repo(self, repo_from, repo_to):
self.repo_from = repo_from
self.repo_to = repo_to
keypair = Keypair("git", self.public_key, self.private_key, "")
callbacks = RemoteCallbacks(credentials=keypair)
clone_repository(self.repo_from, self.repo_to, callbacks=callbacks)
def get_keys(keybucket, pubkey, update=False):
if not os.path.isfile('/tmp/id_rsa') or not os.path.isfile('/tmp/id_rsa.pub') or update:
logger.info('Keys not found on Lambda container, fetching from S3...')
enckey = s3.get_object(Bucket=keybucket, Key=key)['Body'].read()
privkey = kms.decrypt(CiphertextBlob=enckey)['Plaintext']
write_key('/tmp/id_rsa', privkey)
write_key('/tmp/id_rsa.pub', pubkey)
return Keypair('git', '/tmp/id_rsa.pub', '/tmp/id_rsa', '')
def test_ssh_key_aspath():
username = "******"
pubkey = Path("id_rsa.pub")
privkey = Path("id_rsa")
passphrase = "bad wolf"
cred = Keypair(username, pubkey, privkey, passphrase)
assert (username, pubkey, privkey, passphrase) == cred.credential_tuple
def construct_keypair(public_key_path: str = None, private_key_path: str = None,
passphrase: str = '') -> Keypair:
ssh_path = os.path.join(os.path.expanduser('~'), '.ssh')
if not public_key_path:
public_key_path = os.path.join(ssh_path, 'id_rsa.pub')
if not private_key_path:
private_key_path = os.path.join(ssh_path, 'id_rsa')
return Keypair("git", public_key_path, private_key_path, passphrase)
def test_ssh_key(self):
username = "******"
pubkey = "id_rsa.pub"
privkey = "id_rsa"
passphrase = "bad wolf"
cred = Keypair(username, pubkey, privkey, passphrase)
assert (username, pubkey, privkey, passphrase) == cred.credential_tuple
def push(repo, ref="refs/heads/master", remote_name="origin"):
print("Pushing...")
ssh_rsa_dir = str(Path.home()) + "/.ssh/"
for remote in repo.remotes:
if remote.name == remote_name:
remote.credentials = Keypair("git", ssh_rsa_dir + "id_rsa.pub",
ssh_rsa_dir + "id_rsa", "")
callbacks = RemoteCallbacks(credentials=remote.credentials)
remote.push([ref], callbacks=callbacks)
def credentials(self, url, username_from_url, allowed_types):
pubkey = self.config.ssh_pubkey
privkey = self.config.ssh_privkey
passphrase = self.config.ssh_passphrase
if self.count > 5:
raise Exception(
"5 times asking for credentials, looks like auth error")
else:
self.count = self.count + 1
return Keypair(username_from_url, pubkey, privkey, passphrase)
def get_credentials(args):
if args.password:
credentials = UserPass(args.username, args.password)
elif args.ssh_agent:
log.info('Using SSH agent with remote user: %s', args.ssh_user)
credentials = KeypairFromAgent(args.ssh_user)
else:
log.info('Using SSH user: %s, key: %s', args.ssh_user, args.ssh_key)
credentials = Keypair(args.ssh_user, args.ssh_key + ".pub",
args.ssh_key, "")
return RemoteCallbacks(credentials=credentials)
def credentials(self, url, username_from_url, allowed_types):
"""
The callback to return a suitable authentication method.
it supports GIT_CREDTYPE_SSH_KEY and GIT_CREDTYPE_USERPASS_PLAINTEXT
GIT_CREDTYPE_SSH_KEY with an ssh agent configured in the env variable SSH_AUTH_SOCK
or with id_rsa and id_rsa.pub in ~/.ssh (password must be the empty string)
GIT_CREDTYPE_USERPASS_PLAINTEXT from the env variables GIT_USERNAME and GIT_PASSWORD
"""
if credentials.GIT_CREDTYPE_SSH_KEY & allowed_types:
if "SSH_AUTH_SOCK" in environ:
# Use ssh agent for authentication
return KeypairFromAgent(username_from_url)
else:
ssh = join(expanduser('~'), '.ssh')
if "QUIT_SSH_KEY_HOME" in environ:
ssh = environ["QUIT_SSH_KEY_HOME"]
# public key is still needed because:
# _pygit2.GitError: Failed to authenticate SSH session:
# Unable to extract public key from private key file:
# Method unimplemented in libgcrypt backend
pubkey = join(ssh, 'id_rsa.pub')
privkey = join(ssh, 'id_rsa')
# check if ssh key is available in the directory
if isfile(pubkey) and isfile(privkey):
return Keypair(username_from_url, pubkey, privkey, "")
else:
raise Exception(
"No SSH keys could be found, please specify SSH_AUTH_SOCK or add keys to "
+ "your ~/.ssh/")
elif credentials.GIT_CREDTYPE_USERPASS_PLAINTEXT & allowed_types:
if "GIT_USERNAME" in environ and "GIT_PASSWORD" in environ:
return UserPass(environ["GIT_USERNAME"],
environ["GIT_PASSWORD"])
else:
raise Exception(
"Remote requested plaintext username and password authentication but "
+ "GIT_USERNAME or GIT_PASSWORD are not set.")
else:
raise Exception(
"Only unsupported credential types allowed by remote end")
def auth( username=None, pubkey=None, privkey=None, password=None ):
credentials = Keypair(username=username, pubkey=pubkey, privkey=privkey, passphrase=password)
return RemoteCallbacks( credentials = credentials )
def credentials(testrepo, url, username, allowed):
assert allowed & pygit2.GIT_CREDENTIAL_USERPASS_PLAINTEXT
return Keypair("git", "foo.pub", "foo", "sekkrit")
def credentials(url, username, allowed):
self.assertTrue(allowed & GIT_CREDTYPE_USERPASS_PLAINTEXT)
return Keypair("git", "foo.pub", "foo", "sekkrit")
def job_runner(job):
job = Job.loads(job)
db = make_db()
def log(line):
db(db.jobs.id == job.id).update(logs=db.jobs.logs.coalesce('') +
'{0}\n'.format(line))
log('[JOB #{0}] Started'.format(job.id))
db(db.jobs.id == job.id).update(status=JobStatus.STARTED)
db.commit()
container = None
repo_path = None
try:
repo_name = str(uuid4())
repo_path = os.path.join(os.getcwd(), 'repos', repo_name)
repos_path = os.path.dirname(repo_path)
if config.REPO_HOST_PATH:
repo_host_path = os.path.join(config.REPO_HOST_PATH, repo_name)
else:
repo_host_path = repo_path
if not os.path.exists(repos_path):
os.makedirs(repos_path)
log('[JOB #{0}] git clone {1}'.format(job.id, job.ssh_url))
repo = clone_repository(
job.ssh_url,
repo_path,
callbacks=RemoteCallbacks(credentials=Keypair(
config.SSH_USERNAME, config.SSH_PUBKEY, config.SSH_PRIVKEY,
config.SSH_PASSPHRASE),
certificate=lambda *_: True))
log('[JOB #{0}] git checkout {1}'.format(job.id, job.commit_id))
commit = repo.get(job.commit_id)
repo.checkout_tree(commit.tree)
cfg_path = os.path.join(repo_path, '.microci.json')
cfg = {
'dockerimg': config.DOCKER_IMAGE,
'command': '/bin/sh microci.sh'
}
if os.path.exists(cfg_path):
log('[JOB #{0}] Load .microci.json'.format(job.id))
try:
with open(cfg_path) as f:
cfg.update(json.load(f))
except Exception:
log('[JOB #{0}] {1}'.format(job.id, traceback.format_exc()))
log('[JOB #{0}] docker run {1}'.format(job.id, cfg['dockerimg']))
client = DockerClient(base_url=config.DOCKER_URL)
container = client.containers.run(
cfg['dockerimg'],
command=cfg['command'],
working_dir='/repo',
volumes=['{0}:/repo:rw'.format(repo_host_path)],
detach=True)
for line in container.logs(stdout=True, stderr=True, stream=True):
log(line.decode().rstrip('\n'))
retcode = container.wait()
success = retcode == 0
log('[JOB #{0}] Returned {1}'.format(job.id, retcode))
status = JobStatus.SUCCEED if success else JobStatus.FAILED
db(db.jobs.id == job.id).update(status=status)
except Exception:
log('[JOB #{0}] {1}'.format(job.id, traceback.format_exc()))
db(db.jobs.id == job.id).update(status=JobStatus.ERRORED)
db.commit()
if container is not None:
log('[JOB #{0}] Remove container'.format(job.id))
container.remove()
if repo_path is not None and os.path.exists(repo_path):
log('[JOB #{0}] Remove repository'.format(job.id))
rmtree(repo_path)
db.close()
def get_credentials(args):
if args.password:
return UserPass(args.username, args.password)
else:
return Keypair(args.ssh_user, args.ssh_key + ".pub", args.ssh_key, "")
