def create_zip_artifact(root_dir, version, package, artifact_prefix,
artifact_suffix, binary_paths):
"""Builds the artifact for upload"""
logger.log_start("Creating artifact for package %s" % package)
artifact_root = "%s%s%s" % (
artifact_prefix,
version,
artifact_suffix,
)
artifact_name = "%s.zip" % artifact_root
artifact_name = artifact_name.replace("-", "_")
logger.log_info("Building artifact %s..." % artifact_name)
artifact_folder = "%s/%s/%s" % (root_dir, ARTIFACT_STAGING_DIR, package)
utils.execute(['mkdir', '-p', artifact_folder])
file_names = []
for path in binary_paths:
utils.execute(['cp', path, artifact_folder])
file_names.append(os.path.basename(path))
os.chdir(artifact_folder)
utils.execute(['zip', artifact_name] + file_names)
os.chdir(root_dir)
logger.log_done()
return {
'artifact_name': artifact_name,
'artifact_path': "%s/%s" % (artifact_folder, artifact_name)
}
def upload_bintray_artifact(version, package, repo, user_org, user, api_key, artifact_name, artifact_path, publish, override, continue_on_conflict):
"""Uploads the artifact to Bintray"""
logger.log_start("Uploading artifact to Bintray")
url = 'https://api.bintray.com/content/%s/%s/%s/%s/%s' % (user_org, repo, package, version, artifact_name)
parameters = {
'publish': publish,
'override': override
}
with open(artifact_path, "rb") as package_fp:
response = requests.put(
url,
auth=(user, api_key),
params=parameters,
data=package_fp
)
code = response.status_code
code_family = int(code) // 100
success = False
if code_family == 2 or code_family == 3:
logger.log_info("Bintray artifact uploaded!")
success = True
else:
logger.log_info("Bintray API response %s is not in 2xx or 3xx range" % code)
if continue_on_conflict:
logger.log_info("continue_on_conflict flag is true, not failing release...")
success = True
else:
success = False
logger.log_done()
return success
def create_bintray_version(version, package, repo, user_org, user, api_key):
"""Creates a new Bintray version for the package"""
logger.log_start("Creating Bintray version %s in package %s" % (version, package))
url = "https://api.bintray.com/packages/%s/%s/%s/versions" % (user_org, repo, package)
payload = {
'name': version,
'desc': 'Release of %s' % package
}
headers = {
'Content-Type': 'application/json'
}
response = requests.post(url, data=json.dumps(payload), headers=headers, auth=(user, api_key))
code = response.status_code
success = False
if code == 409:
logger.log_info("Bintray version %s already exists, skipping." % version)
success = True
else:
code_family = code // 100
if code_family == 2 or code_family == 3:
logger.log_info("Bintray Version created!")
success = True
else:
logger.log_info("Bintray API response %s is not 409 (package already exists) nor in 2xx or 3xx range" % code)
success = False
logger.log_done()
return success
def deploy_to_pypi():
"""Deploys the release to PyPi"""
logger.log_start("Deploying to PyPi")
os.chdir(TRAVIS_BUILD_DIR)
utils.execute("python setup.py register -r pypi", shell=True)
utils.execute("python setup.py sdist upload -r pypi", shell=True)
logger.log_info("Module deployed to PyPi!")
logger.log_done()
def check_version():
"""Fail deploy if tag version doesn't match version"""
logger.log_start("Checking versions")
if TRAVIS_TAG != _version.__version__:
sys.exit("Version extracted from project doesn't match the TRAVIS_TAG variable!")
else:
logger.log_info("Versions match!")
logger.log_done()
def deploy_to_pypi():
"""Deploys the release to PyPi"""
logger.log_start("Deploying to PyPi")
os.chdir(TRAVIS_BUILD_DIR)
utils.execute("python setup.py sdist bdist_wheel", shell=True)
utils.execute("twine upload dist/*", shell=True)
logger.log_info("Module deployed to PyPi!")
logger.log_done()
def check_version():
"""Fail deploy if tag version doesn't match version"""
logger.log_start("Checking versions")
if TRAVIS_TAG != _version.__build_version__:
sys.exit("Version extracted from project doesn't match the TRAVIS_TAG variable. TRAVIS_TAG: {}, __build_version__: {}!".format(TRAVIS_TAG, _version.__build_version__))
else:
logger.log_info("Versions match!")
logger.log_done()
def check_version(version, build_version):
"""Fail deploy if tag version doesn't match version"""
logger.log_start("Asserting versions match")
if version != build_version:
raise ValueError(
"Version extracted from build [%s] doesn't match declared in config [%s]"
% (build_version, version))
else:
logger.log_info("Version match!")
logger.log_done()
def write_config():
"""Writes an array of lines to the PyPi config file"""
logger.log_start("Writing ~/.pypirc file")
lines = [
'[distutils]\n', 'index-servers =\n',
' %s\n' % DEFAULT_REPO, '\n',
'[%s]\n' % DEFAULT_REPO,
'repository=%s\n' % DEFAULT_SERVER, 'username=snowplow\n',
'password=%s\n' % PYPI_PASSWORD
]
with open(PYPIRC_FILE, 'w') as outfile:
for line in lines:
outfile.write(line)
logger.log_info("The ~/.pypirc file has been written!")
logger.log_done()
def write_config():
"""Writes an array of lines to the PyPi config file"""
logger.log_start("Writing ~/.pypirc file")
lines = [
'[distutils]\n',
'index-servers =\n',
' %s\n' % DEFAULT_REPO,
'\n',
'[%s]\n' % DEFAULT_REPO,
'username=snowplow\n',
'password=%s\n' % PYPI_PASSWORD
]
with open(PYPIRC_FILE, 'w') as outfile:
for line in lines:
outfile.write(line)
logger.log_info("The ~/.pypirc file has been written!")
logger.log_done()
def create_asis_artifact(root_dir, version, package, artifact_prefix,
artifact_suffix, binary_paths):
"""Construct artifact name and perform no operations"""
logger.log_start("Creating artifact for package %s" % package)
if len(binary_paths) != 1 and type(binary_paths) != str:
raise ValueError(
"Invalid binary_paths for 'asis' artifact type. It must contain single item. %s items instead"
% len(binary_paths))
artifact_name = "%s%s%s" % (
artifact_prefix,
version,
artifact_suffix,
)
return {
'artifact_name': artifact_name,
'artifact_path': os.path.join(root_dir, binary_paths[0])
}
def create_artifact(root_dir, version, package, artifact_type, artifact_prefix,
artifact_suffix, binary_paths):
"""Builds the artifact for upload"""
logger.log_start("Creating artifact for package %s" % package)
if artifact_type == 'zip':
result = create_zip_artifact(root_dir, version, package,
artifact_prefix, artifact_suffix,
binary_paths)
elif artifact_type == 'asis':
result = create_asis_artifact(root_dir, version, package,
artifact_prefix, artifact_suffix,
binary_paths)
else:
raise ValueError(
"Invalid type specified; expected one of %s and got %s" % (
AVAILABLE_ARTIFACT_TYPES,
artifact_type,
))
return result
def execute_commands(commands):
"""Execute build commands"""
logger.log_start("Executing build commands")
for command in commands:
utils.execute([command], shell=True)
logger.log_done()
def main():
"""Main function entry point"""
parser = argparse.ArgumentParser(
description="Utility for creating and uploading zip packages.")
parser.add_argument("--config",
help="the path to the configuration yaml file")
parser.add_argument(
"--make-version",
action='store_true',
default=False,
help="makes a new version for the package (bintray-specific)")
parser.add_argument("--make-artifact",
action='store_true',
default=False,
help="makes the artifacts that will be uploaded")
parser.add_argument("--upload-artifact",
action='store_true',
default=False,
help="uploads the artifacts to the targets")
parser.add_argument(
"--check-version",
action='store_true',
default=False,
help="checks that the version specified matches the build")
parser.add_argument("--version",
action='version',
version=_version.__version__)
args = parser.parse_args()
# Parse args
if not args.config:
raise ValueError("A config must be passed to the program")
config = utils.parse_config(args.config)
if not args.make_version and not args.make_artifact and not args.upload_artifact and not args.check_version:
logger.log_start("No actions selected, quitting")
exit(0)
if not args.make_artifact and args.upload_artifact:
raise ValueError(
"Cannot upload artifact without first creating; please add '--make-artifact' to resolve..."
)
logger.log_header("Starting Package uploader...")
# Upload packages
for package in config["packages"]:
logger.log_start("Processing package %s" % package["name"])
# Check the versions
if args.check_version:
pack.check_version(package["version"], package["build_version"])
# Push to targets
for target in config['targets']:
if target['type'] == 'bintray':
bintray.deploy_to_bintray(args, config['local'], package,
target)
elif target['type'] == 'awss3':
s3.upload_to_s3(args, config['local'], package, target)
else:
raise ValueError(
"Invalid target specified; expected one of [bintray, awss3] and got %s"
% target["type"])
logger.log_footer("Finished processing package %s!" % package["name"])