def _last_build_hash_file_name(self):
if 'last_build_hash_file_name' in self._poller_config():
return _fortworth.join(
self._config.data_dir(),
self._poller_config()['last_build_hash_file_name'])
return _fortworth.join(self._config.data_dir(),
'last_build_hash.{}.json'.format(self._name))
def _data_file_name(self):
if 'artifact_poller_data_file_name' in self._poller_config():
return _fortworth.join(
self._config.data_dir(),
self._poller_config()['artifact_poller_data_file_name'])
return _fortworth.join(self._config.data_dir(),
'artifact-poller.{}.json'.format(self._name))
def _process_commit_hash(self, bodega_temp_dir):
""" Extract zipped commit-id json file into data dir """
file_name_base = _fortworth.join(bodega_temp_dir,
self._last_build_hash_artifact_name())
with _zipfile.ZipFile(file_name_base + '.zip', 'r') as zip_obj:
zip_obj.extractall(path=bodega_temp_dir)
self._last_build_commit_hash = _fortworth.read_json(
file_name_base + '.json')['commit-hash']
_shutil.move(file_name_base + '.json',
self._last_build_hash_file_name())
def download(self, data_dir, auth):
""" Download artifact to data_dir """
with _requests.get(self._download_url(), stream=True,
auth=auth) as req:
req.raise_for_status()
artifact_file_name = _fortworth.join(data_dir,
self.name() + '.zip')
with open(artifact_file_name, 'wb') as artifact_file:
for chunk in req.iter_content(
chunk_size=MAX_DOWNLOAD_CHUNK_SIZE):
artifact_file.write(chunk)
return self.name()
return None
def _push_to_bodega(self, wf_item, bodega_temp_dir):
""" Push an artifact to Bodega """
if not self._dry_run():
build_data = _fortworth.BuildData(self._repo_name(),
self._source_branch(),
wf_item.run_number(),
wf_item.html_url())
try:
_fortworth.bodega_put_build(bodega_temp_dir,
build_data,
service_url=self._bodega_url())
return _fortworth.join(self._bodega_url(), build_data.repo,
build_data.branch, str(build_data.id))
except _requests.exceptions.ConnectionError:
self._log.error('Bodega not running or invalid Bodega URL %s',
self._config['Local']['stagger_url'])
def __init__(self, home, data_dir=None, config_file=None):
self._home = home
self._log = _logging.getLogger(self.__class__.__name__)
self._process_list = []
config_file = config_file if config_file is not None else _fortworth.join(
home, 'wheedle.conf')
self._config = _config.Configuration(config_file, data_dir)
try:
_logging.basicConfig(
level=self._config['Logging']['default_log_level'],
format='%(asctime)s %(name)s - %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S %Z')
except ValueError as err:
raise _errors.ConfigFileError(self._config.config_file_name(),
'Logging', err)
self._log.info('Data directory: %s', self._config.data_dir())
def __init__(self, config_file_name, data_dir):
self._config_file_name = config_file_name
self._config = _cp.ConfigParser()
try:
self._read_config_file(config_file_name)
self._validate()
except _cp.ParsingError as err:
print('Config file error: {}'.format(err))
_fortworth.exit(1)
except _errors.PollerError as err:
print(err)
_fortworth.exit(1)
self._data_dir = data_dir if data_dir is not None else self._config['Local']['data_dir']
auth_token_file = _fortworth.join(self._data_dir,
self._config['GitHub']['gh_api_token_file_name'])
self._auth = (self._config['GitHub']['api_auth_uid'], self._read_token(auth_token_file))
def _push_to_stagger(self, workflow_metadata, bodega_artifact_list,
bodega_artifact_path):
""" Tag an artifact in Stagger """
if not self._dry_run():
stagger_artifact_list = {}
for artifact, bodega_file_name in bodega_artifact_list:
stagger_artifact_list[artifact.name()] = {
'type':
'file',
'update_time':
_gh_api.str_time_to_milli_ts(artifact.created_at()),
'url':
_fortworth.join(bodega_artifact_path,
bodega_file_name + '.zip'),
}
commit_url = None if self._source_repo_full_name() is None else \
'https://github.com/{}/commit/{}'.format(self._source_repo_full_name(),
self._last_build_commit_hash)
tag_data = {
'update_time':
_gh_api.str_time_to_milli_ts(workflow_metadata.updated_at()),
'build_id':
workflow_metadata.run_number(),
'build_url':
workflow_metadata.html_url(),
'commit_id':
self._last_build_commit_hash,
'commit_url':
commit_url,
'artifacts':
stagger_artifact_list,
}
try:
_fortworth.stagger_put_tag(self._repo_name(),
self._source_branch(),
self._stagger_tag(),
tag_data,
service_url=self._stagger_url())
except _requests.exceptions.ConnectionError:
self._log.error('Stagger not running or invalid Bodega URL %s',
self._stagger_url())
def _source_repo_full_name(self):
""" Convenience method to return source repository full name (owner/name) """
if 'source_repo_owner' not in self._poller_config():
return None
return _fortworth.join(self._poller_config()['source_repo_owner'],
self._poller_config()['source_repo_name'])
def _build_repo_full_name(self):
""" Convenience method to return build repository full name (owner/name) """
return _fortworth.join(self._poller_config()['build_repo_owner'],
self._poller_config()['build_repo_name'])
def full_name(self):
""" Get repository full name (owner/name) """
return _fortworth.join(self.owner, self.name)
def full_name(self):
""" Get repo full name """
return _fortworth.join(self.owner(), self.name())