def get_travis_session(username, travis_ci_token, github_token):
travis_session = TravisPy(token=travis_ci_token)
try:
travis_session.repo('some_repo')
except TravisError:
logger.error("Travis session expired for {}. Please manually generate it by doing:\n{}"
.format(username, TOKEN_INSTRUCTION.format(github_token)))
exit(1)
else:
return travis_session
class Source():
def __init__(self, username, uri=None):
kwargs = {}
if uri: # leave as unspecified to use default
kwargs['uri'] = uri
self.client = TravisPy(**kwargs)
self.username = username
self.logger = logger.Logger('travis_ci')
def list_projects(self):
repos = self.client.repos(owner_name=self.username)
repos = filter(lambda x: x.active == True, repos)
return list(map(lambda x: x.slug.split('/')[1], repos))
def project_status(self, project, branch='master'):
try:
state = self.client.repo(self.username + '/' +
project).last_build_state
except Exception, e:
self.logger.log("Error while computing state for project '%s': %s",
project, str(e))
return STATUS.POLL_ERROR
if not state:
return STATUS.UNKNOWN
return _STATUS[state]
def on_message(self, headers, message):
logging.warn("New build id arrived: %s" % message)
# Remove output file after each run to avoid filling up the container.
self.conn.ack(headers.get('message-id'), headers.get('subscription'))
os.system("rm -rf clonedrepo")
# Fetch slug and branch name from travis given buildid.
reposlug = ""
repobranch = ""
selector = ""
if not SLUG_MODE:
t = TravisPy()
build = t.build(message)
repoid = build['repository_id']
repobranch = build.commit.branch
repo = t.repo(repoid)
reposlug = repo.slug
else:
# Expect message format 'selector,branchname,selector'
strList = message.split(",")
logging.warn(message)
logging.warn(strList)
reposlug = strList[0]
repobranch = strList[1]
selector = strList[2]
# Clone the repo branch
token = os.getenv('GITHUB_OAUTH') or ''
url = 'https://:@github.com/' + reposlug + '.git'
repo = None
branch_exist = True
logging.warn('Cloning url ' + url)
try:
repo = git.Repo.clone_from(url, 'clonedrepo', branch=repobranch)
except:
logging.warn("Branch " + str(repobranch) + " or the repo " +
str(reposlug) + " itself does not exist anymore")
branch_exist = False
report_error(
"Invalid repo or branch, double check if the input repo url is correct \n \n --STAMP/Dspot",
"", reposlug, repobranch)
# Check if project support dspot otherwise try autoconfiguring
# assuming that dspot configurations is in the top most directory of
# the project
timecap = '-{date:%Y-%m-%d-%H-%M-%S}'.format(
date=datetime.datetime.now())
if branch_exist:
if not run_Dspot_preconfig(reposlug, repobranch, selector):
logging.warn(
"PROJECT DOES NOT SUPPORT DSPOT, COMMENCING AUTOCONFIGURING"
)
run_Dspot_autoconfig(reposlug, repobranch, selector)
logging.warn("PIPELINE MESSAGE: DONE , AWAITING FOR NEW BUILD ID")
def on_message(self, headers, message):
logging.warning("New build id arrived: %s" % message)
# Remove output file after each run to avoid filling up the container.
self.conn.ack(headers.get('message-id'), headers.get('subscription'))
os.system("rm -rf clonedrepo")
# Fetch slug and branch name from travis given buildid.
t = TravisPy()
build = t.build(message)
repoid = build['repository_id']
repobranch = build.commit.branch
repo = t.repo(repoid)
reposlug = repo.slug
# Clone the repo branch
token = os.getenv('GITHUB_OAUTH') or ''
url = 'https://github.com/' + reposlug + '.git'
repo = None
branch_exist = True
logging.warning('Cloning url ' + url)
try:
repo = git.Repo.clone_from(url, 'clonedrepo', branch=repobranch)
except:
logging.warning("Branch " + str(repobranch) + " or the repo " +
str(reposlug) + " itself does not exist anymore")
branch_exist = False
# Check if project support dspot otherwise try autoconfiguring
# assuming that dspot configurations is in the top most directory of
# the project
POM_FILE = "clonedrepo/pom.xml"
timecap = '-{date:%Y-%m-%d-%H-%M-%S}'.format(
date=datetime.datetime.now())
if branch_exist:
if not run_Dspot_preconfig(POM_FILE, reposlug, timecap):
logging.warning("PROJECT DOES NOT SUPPORT DSPOT")
run_Dspot_autoconfig(reposlug, timecap)
# Commit build to github
if check_endprocess_gitcommit(repo):
templist = os.getenv("PUSH_URL").split('//')
pushurl = templist[0] + '//' + token + '@' + templist[1]
branch_name = reposlug.replace('/', "-") + timecap
logging.warning('Commit to git as new branch with name ' +
branch_name)
current = repo.git.checkout('-b', branch_name)
# current.checkout()
time.sleep(10)
repo.git.add(A=True)
repo.git.commit(m='update from dspot')
repo.git.push(pushurl, branch_name)
logging.warning("GITHUB NAME: " + os.getenv("GITHUB_USERNAME"))
logging.warning(
"PIPELINE MESSAGE: DONE , AWAITING FOR NEW BUILD ID")
if __name__ == '__main__':
# get the pelican repo and read the configuration
repo = get_pelican_repo()
reader = repo.config_reader()
# build the remote repo name with the github user, ex: vdorsonnens/pelican
reader.read()
github_username = reader.get_value('user', 'name')
remote_repo = '/'.join([github_username, PELICAN])
# connect to travis-ci
token = get_travis_token()
travis_client = TravisPy(token)
travis_repo = travis_client.repo(remote_repo)
success = False
if travis_repo.active:
print('Repository is already enabled')
sys.exit(0)
else:
print('Enabling repo...')
success = travis_repo.enable()
if success:
print('Operation successful')
sys.exit(0)
print('Failed operation')
def build_passed(repo):
t = TravisPy()
r = t.repo(repo)
while True:
yield r.last_build_state == 'passed'
def build_passed(repo='RPi-Distro/python-gpiozero', delay=3600):
t = TravisPy()
r = t.repo(repo)
while True:
yield r.last_build_state == 'passed'
sleep(delay) # Sleep an hour before hitting travis again
def build_passed(repo):
t = TravisPy()
r = t.repo(repo)
return r.last_build_state == 'passed'
def build_passed(repo='RPi-Distro/python-gpiozero', delay=3600):
t = TravisPy()
r = t.repo(repo)
while True:
yield r.last_build_state == 'passed'
sleep(delay) # Sleep an hour before hitting travis again
def build_passed(repo):
t = TravisPy()
r = t.repo(repo)
while True:
yield r.last_build_state == 'passed'
def test(self):
with self.assertRaises(TravisError):
travis = TravisPy(token='invalid_token')
travis.repo('some_repo')
class GitenbergTravisJob(GitenbergJob):
def __init__(self,
username,
password,
repo_name,
repo_owner,
update_travis_commit_msg,
tag_commit_message,
github_token=None,
access_token=None,
repo_token=None):
"""A subclass of GitenbergJob that adds travis-ci related functionality
Keyword arguments:
username -- GitHub username
password -- GitHub password
repo_name -- name of Gitenberg repo (e.g., "Moby-Dick--Or-The-Whale_2701")
repo_owner -- e.g., 'GITenberg' for https://github.com/gitenberg/At-the-Sign-of-the-Eagle_6218
update_travis_commit_msg -- the git commit message for the update commit
tag_commit_message -- the git commit message for the tagging commit
github_token (optional) -- a GitHub token that is ued to obtain travis access token
access_token (optional) -- the travis access token corresponding to GitHub account
repo_token (optional) -- the travis token for building the given repo
"""
super(GitenbergTravisJob,
self).__init__(username, password, repo_name, repo_owner,
update_travis_commit_msg, tag_commit_message)
self.username = username
self.password = password
self._github_token = github_token
self._access_token = access_token
# if access_token is given, use it
# the access token corresponds to the output of the travis cli operation
# `travis token`
if access_token is not None:
self.travis = TravisPy(access_token)
# otherwise, we can pass a GitHub token to travis
else:
self.travis = TravisPy.github_auth(self.github_token())
self._repo_token = repo_token
self._travis_repo_public_key = None
if self.gh_repo is not None:
self.travis_repo = self.travis.repo(self.repo_slug)
def public_key_for_travis_repo(self):
if self._travis_repo_public_key is None:
self._travis_repo_public_key = requests.get(
"https://api.travis-ci.org/repos/{}/{}/key".format(
self.repo_owner, self.repo_name)).json()['key']
return self._travis_repo_public_key
def github_token(self):
"""
use the GitHub token in the parameters; otherwise create a new GitHub token with the appropropriate permissions
"""
if self._github_token is not None:
return self._github_token
token_note = "token for travis {}".format(
datetime.datetime.utcnow().isoformat())
token = self.gh.authorize(self.username,
self.password,
scopes=('read:org', 'user:email',
'repo_deployment', 'repo:status',
'write:repo_hook'),
note=token_note)
self._github_token = token.token
return self._github_token
def repo_token(self, from_repo_owner='GITenberg', create_duplicate=False):
if self._repo_token is not None:
return self._repo_token
token_note = "automatic releases for {}/{}".format(
self.repo_owner, self.repo_name)
try:
token = self.gh.authorize(self.username,
self.password,
scopes=('public_repo'),
note=token_note)
except Exception as e:
raise e # for now
if self._authorization_description_already_exists(e):
# try again with a new description
if create_duplicate:
token_note += " " + datetime.datetime.utcnow().isoformat()
token = self.gh.authorize(self.username,
self.password,
scopes=('public_repo'),
note=token_note)
else:
raise Exception('repo token already exists for {}'.format(
self.repo_slug))
else:
raise e
self._repo_token = token.token
return self._repo_token
def delete_repo_token(self):
for auth in self.gh.iter_authorizations():
if auth.name == "automatic releases for {}/{}".format(
self.repo_owner, self.repo_name):
auth.delete()
return True
return False
def travis_encrypt(self, token_to_encrypt):
"""
return encrypted version of token_to_encrypt
"""
# token_to_encrypt has to be string
# if's not, assume it's unicode and enconde in utf-8
if isinstance(token_to_encrypt, unicode):
token_string = token_to_encrypt.encode('utf-8')
else:
token_string = token_to_encrypt
repo_public_key_text = self.public_key_for_travis_repo()
pubkey = repo_public_key_text.encode('utf-8')
if 'BEGIN PUBLIC KEY' in pubkey:
repo_public_key = serialization.load_pem_public_key(
pubkey, backend=default_backend())
elif 'BEGIN RSA PUBLIC KEY' in pubkey:
# http://stackoverflow.com/a/32804289
b64data = '\n'.join(pubkey.splitlines()[1:-1])
derdata = base64.b64decode(b64data)
repo_public_key = serialization.load_der_public_key(
derdata, default_backend())
else:
raise Exception('cannot parse repo public key')
ciphertext = repo_public_key.encrypt(token_string, padding.PKCS1v15())
return base64.b64encode(ciphertext)
def travis_last_build_status(self, refresh=False):
if refresh:
self.travis_repo = self.travis.repo(self.repo_slug)
try:
last_build = self.travis_repo.last_build
except KeyError:
return LastBuild(None, None, None, None, None, None, None)
else:
return LastBuild(self.travis_repo.last_build_number,
self.travis_repo.last_build_id,
self.travis_repo.last_build.created,
self.travis_repo.last_build_started_at,
self.travis_repo.last_build_finished_at,
self.travis_repo.last_build_duration,
self.travis_repo.last_build_state)
def status(self, refresh_travis=False):
_status = super(GitenbergTravisJob, self).status()
last_build = self.travis_last_build_status(refresh_travis)
_status.update({
'last_build_number': last_build.number,
'last_build_id': last_build.id,
'last_build_created': last_build.created,
'last_build_started_at': last_build.started_at,
'last_build_finished_at': last_build.finished_at,
'last_build_duration': last_build.duration,
'last_build_state': last_build.state,
})
return _status
@staticmethod
def _authorization_description_already_exists(e):
"""
Given an exception e when trying to create a token, is the exception the result of a duplicate description
"""
if (e.code == 422 and e.message == u'Validation Failed'
and (u'already_exists', u'description') in [
(error['code'], error['field']) for error in e.errors
]):
return True
else:
return False
class GitenbergTravisJob(GitenbergJob):
def __init__(self, username, password, repo_name, repo_owner,
update_travis_commit_msg,
tag_commit_message, github_token=None, access_token=None, repo_token=None):
super(GitenbergTravisJob, self).__init__(username, password, repo_name, repo_owner,
update_travis_commit_msg,
tag_commit_message)
self.username = username
self.password = password
self._github_token = github_token
self._access_token = access_token
# if access_token is given, use it
if access_token is not None:
self.travis = TravisPy(access_token)
else:
self.travis = TravisPy.github_auth(self.github_token())
self._repo_token = repo_token
self._travis_repo_public_key = None
if self.gh_repo is not None:
self.travis_repo = self.travis.repo(self.repo_slug)
def public_key_for_travis_repo(self):
if self._travis_repo_public_key is None:
self._travis_repo_public_key = requests.get("https://api.travis-ci.org/repos/{}/{}/key".format(self.repo_owner,
self.repo_name)).json()['key']
return self._travis_repo_public_key
def github_token(self):
if self._github_token is not None:
return self._github_token
token_note = "token for travis {}".format(datetime.datetime.utcnow().isoformat())
token = self.gh.authorize(self.username, self.password,
scopes=('read:org', 'user:email', 'repo_deployment',
'repo:status', 'write:repo_hook'), note=token_note)
self._github_token = token.token
return self._github_token
def repo_token(self, from_repo_owner='GITenberg', create_duplicate=False):
"""
"""
if self._repo_token is not None:
return self._repo_token
token_note = "automatic releases for {}/{}".format(self.repo_owner, self.repo_name)
try:
token = self.gh.authorize(self.username, self.password, scopes=('public_repo'), note=token_note)
except Exception as e:
raise e # for now
if self._authorization_description_already_exists(e):
# try again with a new description
if create_duplicate:
token_note += " " + datetime.datetime.utcnow().isoformat()
token = self.gh.authorize(self.username, self.password, scopes=('public_repo'), note=token_note)
else:
raise Exception('repo token already exists for {}'.format(self.repo_slug))
else:
raise e
self._repo_token = token.token
return self._repo_token
def delete_repo_token (self):
for auth in self.gh.iter_authorizations():
if auth.name == "automatic releases for {}/{}".format(self.repo_owner, self.repo_name):
auth.delete()
return True
return False
def travis_encrypt(self, token_to_encrypt):
"""
return encrypted version of token_to_encrypt
"""
# token_to_encrypt has to be string
# if's not, assume it's unicode and enconde in utf-8
if isinstance(token_to_encrypt, unicode):
token_string = token_to_encrypt.encode('utf-8')
else:
token_string = token_to_encrypt
repo_public_key_text = self.public_key_for_travis_repo()
pubkey = repo_public_key_text.encode('utf-8')
if 'BEGIN PUBLIC KEY' in pubkey:
repo_public_key = serialization.load_pem_public_key(pubkey,
backend=default_backend())
elif 'BEGIN RSA PUBLIC KEY' in pubkey:
# http://stackoverflow.com/a/32804289
b64data = '\n'.join(pubkey.splitlines()[1:-1])
derdata = base64.b64decode(b64data)
repo_public_key = serialization.load_der_public_key(derdata,
default_backend())
else:
raise Exception ('cannot parse repo public key')
ciphertext = repo_public_key.encrypt(
token_string,
padding.PKCS1v15()
)
return base64.b64encode(ciphertext)
def travis_last_build_status(self, refresh=False):
if refresh:
self.travis_repo = self.travis.repo(self.repo_slug)
try:
last_build = self.travis_repo.last_build
except KeyError:
return LastBuild (
None, None, None, None, None, None, None
)
else:
return LastBuild(
self.travis_repo.last_build_number,
self.travis_repo.last_build_id,
self.travis_repo.last_build.created,
self.travis_repo.last_build_started_at,
self.travis_repo.last_build_finished_at,
self.travis_repo.last_build_duration,
self.travis_repo.last_build_state
)
def status(self, refresh_travis=False):
_status = super(GitenbergTravisJob, self).status()
last_build = self.travis_last_build_status(refresh_travis)
_status.update({
'last_build_number': last_build.number,
'last_build_id': last_build.id,
'last_build_created': last_build.created,
'last_build_started_at': last_build.started_at,
'last_build_finished_at': last_build.finished_at,
'last_build_duration': last_build.duration,
'last_build_state': last_build.state,
})
return _status
@staticmethod
def _authorization_description_already_exists(e):
"""
Given an exception e when trying to create a token, is the exception the result of a duplicate description
"""
if (e.code == 422 and
e.message == u'Validation Failed' and
(u'already_exists', u'description') in [(error['code'], error['field']) for error in e.errors]):
return True
else:
return False
