def execute(arguments, settings, properties):
projects = settings['projects']
username = arguments.u
password = arguments.p
if not username:
raise PluginException("no username")
if not password:
raise PluginException("no password")
project = projects[0]
repository = project['repository'].format(urllib.quote(username),
urllib.quote(password))
# create an empty folder in tmp
tmp_folder = System.create_tmp(project['name'])
# clone the repository to the tmp_folder
result = Git.clone(repository, tmp_folder)
print("clone: " + str(result))
# retieve the branches
branches = Git.branches(tmp_folder)
for branch in branches:
print('branch: ' + branch)
def post(self):
try:
message = request.get_json(silent=True)
version = message.get('version')
# retrieve the settings, containing the projects to deploy.
data = open("harc.json")
settings = json.load(data)
# retrieve the project to deploy.
project = Settings.find_project(settings, message['project'])
print("project", project)
if not project:
message = "project urg not found."
response = dict()
response['status'] = 404
response['message'] = message
return response
# kill the daemon process, reflecting the server
daemon_name = project.get('daemon')
if Ps.is_active(daemon_name):
Ps.kill(daemon_name)
# set identifier, reflecting the checkout folder to build this release.
name = uuid.uuid4().hex
# create an empty folder in tmp
tmp_folder = System.create_tmp(name)
print(tmp_folder)
virtualenv = os.path.join(tmp_folder, 'env')
response = VirtualEnv.create(virtualenv)
print(response)
response = Pip.install_virtualenv(virtualenv,
project['repository'], version,
project['name'])
print(response)
response = Ps.start(virtualenv, daemon_name)
print(response)
# change to the working directory of the project.
# stop the system service
# remove the virtual environment
# install the project into the virtual environment
# start the system service
response = dict()
response['status'] = 200
return response
except:
response = Traceback.build()
response['status'] = 500
return response
from harc.system.System import System
from harc.system.Git import Git
username = '******'
password = '******'
repository = "https://'{0}':'{1}'@git-codecommit.eu-west-1.amazonaws.com/v1/repos/poc-elsevier-api"
repository = repository.format(username, password)
project_name = 'poc-elsevier-api'
# create an empty folder in tmp
tmp_folder = System.create_tmp(project_name)
print repository
print tmp_folder
# clone the repository to the tmp_folder
result = Git.clone(repository, tmp_folder)
print result
for fle in files:
if not fle['ContentType'] == 'application/x-directory':
path = fle['Key']
dirname = os.path.dirname(path)
filename = os.path.basename(path)
# move the file to the archive location.
bucket.copy_object(bucket_name, path, bucket_name,
os.path.join('archive', 'eb', now, filename))
bucket.delete_object(bucket_name, path)
# set identifier, reflecting the checkout folder to build this release.
name = uuid.uuid4().hex
# create an empty folder in tmp
tmp_folder = System.create_tmp(name)
# Copy the required projects into the temporary folder
for path in source_paths:
System.copy(path, os.path.join(tmp_folder, path.split("/")[-1]))
# set the filename and path of the zipped file to build
basename = "mdp_api_source_bundle_" + now.strip()
print(basename)
zip_filename = basename + ".zip"
zip_file = os.path.join(tmp_folder, zip_filename)
key = key_prefix + zip_filename
# add all the files in the temp folder to the zip file.
# reflecting the module and its dependencies
Zip.create(zip_file, tmp_folder)
def execute(arguments, settings, properties):
username = arguments.u
password = arguments.p
version = arguments.v
environment = arguments.e
profile_name = 'sandbox'
region_name = 'eu-west-1'
bucket_name = 'elsevier-mdp-dev-deploy'
key_prefix = 'eb/mdp_api/'
now = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
# if no environment is given sandbox is assumed.
if not environment:
environment = 'sandbox'
project = settings['project']
project_name = project['name']
# parse the url, when the scheme is http or https a username, password combination is expected.
url = urlparse(project['repository'])
repository = project['repository']
if url.scheme in ['http', 'https']:
if not username:
raise PluginException("no username")
if not password:
raise PluginException("no password")
# repository = url.scheme + "://'{0}':'{1}'@" + url.netloc + url.path
# repository = repository.format(quote(username), quote(password))
# retrieve aws profile_name to use, depending on the environment
profile_name = Settings.find_aws_profile_name(settings, environment)
region_name = Settings.find_aws_region_name(settings, environment)
# retrieve upload bucket name.
bucket_name = Settings.find_deploy_bucket_name(settings, environment)
# set identifier, reflecting the checkout folder to build this release.
name = uuid.uuid4().hex
# create an empty folder in tmp
build_folder = System.create_tmp(name)
module_name = project_name
module_version = None
module_repo = repository
module_type = 'pip'
module = PipUrl.build(module_name,
module_version,
module_repo,
username,
password,
no_dependencies=False)
print(build_folder)
if module_type == 'pip':
result = Pip.install(module, build_folder)
print(result)
# if module_type == 'yum':
# result = "which git || sudo yum install " + module_name + ' -y' + '\n'
# check for explicit dependencies that needs to be installed
if project['dependencies']:
for dependency in project['dependencies']:
module_name = dependency['name']
module_version = dependency.get('version')
module_repo = dependency.get('repository')
module_type = dependency.get('type')
module = PipUrl.build(module_name,
module_version,
module_repo,
username,
password,
no_dependencies=False)
if module_type == 'pip':
result = Pip.install(module, build_folder)
print(result)
# set the filename and path of the zipped file to build
basename = "mdp_api_source_bundle_" + now.strip()
print(basename)
zip_filename = basename + ".zip"
zip_file = os.path.join(build_folder, zip_filename)
key = key_prefix + zip_filename
# add all the files in the temp folder to the zip file.
# reflecting the module and its dependencies
Zip.create(zip_file, build_folder)
# upload the zipped file to aws
print('uploading', zip_file, "using profile", profile_name,
"into bucket ", bucket_name)
aws_bucket = AwsBucket(profile_name, region_name)
aws_bucket.upload(zip_file, bucket_name, key)
# create new application version on eb with the uploaded source bundle
client = boto3.client('elasticbeanstalk')
application_name = 'mdp-api-sandbox'
environment_name = application_name + '-env'
application_update_ts = now
version_label = 'harc_deployment_{}'.format(application_update_ts)
response = client.create_application_version(
ApplicationName=application_name,
VersionLabel=version_label,
Description='{} deployed on {}'.format(basename,
application_update_ts),
SourceBundle={
'S3Bucket': 'elsevier-mdp-dev-deploy',
'S3Key': key_prefix + zip_filename
},
AutoCreateApplication=False,
Process=True)
response = response.get('ApplicationVersion')
print(response)
while response.get('Status') == 'PROCESSING':
response = client.describe_application_versions(
ApplicationName=application_name,
VersionLabels=[version_label],
)
response = response.get('ApplicationVersions')[0]
print(response)
response = client.update_environment(
ApplicationName=response.get('ApplicationName'),
VersionLabel=response.get('VersionLabel'),
EnvironmentName=environment_name)
for fle in files:
if not fle['ContentType'] == 'application/x-directory':
path = fle['Key']
dirname = os.path.dirname(path)
filename = os.path.basename(path)
# move the file to the archive location.
bucket.copy_object(bucket_name, path, bucket_name,
os.path.join('archive', 'emr', now, filename))
bucket.delete_object(bucket_name, path)
# set identifier, reflecting the checkout folder to build this release.
name = uuid.uuid4().hex
# create an empty folder in tmp
tmp_folder = System.create_tmp(name)
# create the bootstrap.sh script.
f = open(os.path.join(tmp_folder, 'bootstrap.sh'), 'wb')
# todo process the dependencies here.
f.write('pip install awscli'.encode('utf-8'))
f.close()
# upload the bootstrap file to aws
bucket.upload(os.path.join(tmp_folder, 'bootstrap.sh'), bucket_name,
'emr/bootstrap/bootstrap.sh')
# print(file)
# print(basename(file['Key']))
# print(os.path.dirname(file['Key']))
# if not response['ContentType'] == 'application/x-directory':
settings['projects'] = projects
environment = dict()
environment['aws_profile_name'] = 'default'
environment['aws_bucket_name'] = 'elsevier-mdp-dev-source'
environment['aws_region_name'] = 'eu-west-1'
settings['dev'] = environment
project_name = "mdp_lambda"
filename = "register.py"
username = "******"
password = "******"
build_name = uuid.uuid4().hex
build_folder = System.create_tmp(build_name)
print(build_folder)
dependencies = Settings.list_dependencies(settings, project_name, filename)
for dependency in dependencies:
module_name = dependency['name']
module_version = dependency.get('version')
module_repo = dependency.get('repository')
module = module_name
if module_version:
module = module_name + "==" + module_version
if module_repo:
module = module_repo