def execute(arguments, settings, properties):
job_name = properties['job_name']
logger = Logger(StartService())
try:
s = arguments.s
environment = arguments.e
project = arguments.n
if not s:
raise PluginException("no service")
if not environment:
raise PluginException("no environment")
if not project:
raise PluginException("no project")
details = settings[environment]
for detail in details:
if detail['name'] == project:
services = detail['services']
for service in services:
if service['name'] == s:
service_password = service['password']
result = Systemctl.start(s, service_password)
logger.info(job_name, str(result))
except:
result = Traceback.build()
logger.fatal(job_name, result['message'], result['backtrace'])
def execute(arguments, settings, properties):
projects = settings['projects']
username = arguments.u
password = arguments.p
if not username:
raise PluginException("no username")
if not password:
raise PluginException("no password")
project = projects[0]
repository = project['repository'].format(urllib.quote(username),
urllib.quote(password))
# create an empty folder in tmp
tmp_folder = System.create_tmp(project['name'])
# clone the repository to the tmp_folder
result = Git.clone(repository, tmp_folder)
print("clone: " + str(result))
# retieve the branches
branches = Git.branches(tmp_folder)
for branch in branches:
print('branch: ' + branch)
def execute(arguments, settings, properties):
server_username = arguments.u
server_password = arguments.p
environment = arguments.e
if not server_username:
raise PluginException("no username")
if not server_password:
raise PluginException("no password")
if not environment:
environment = "dev"
print "using environment : " + environment
projects = settings['projects']
for project in projects:
detail = Settings.find_detail(settings, project['name'], environment)
server = detail['server']
if not detail:
message = "detail for project " + project['name'] + ' and environment ' + environment + " not found"
raise PluginException(message)
services = detail['services']
message = {"username": server_username, "password": server_password}
headers = {'Content-Type': 'application/json'}
response = post(server + '/login', json.dumps(message), headers=headers)
status_code = response.status_code
if status_code != 200:
message = "status_code: " + str(response.status_code) + " reason: " + response.reason + ", content: " + response.content
raise PluginException(message)
content = json.loads(response.content)
session = content['session']
for service in services:
message = {"command": "service:start", "session": session, "-s": service['name'], "-e": environment, "-n": project['name']}
headers = {'Content-Type': 'application/json'}
response = post(server + '/job', json.dumps(message), headers=headers)
status_code = response.status_code
if status_code != 200:
message = "service: " + service['name'] + ", status_code: " + str(response.status_code) + ", reason: " + response.reason + ", content: " + response.content
raise PluginException(message)
print "service: " + service['name'] + ", job_name: " + str(json.loads(response.content)['job_name'])
def push(repository, branch, folder):
statement = "cd " + folder + ";" + "git push --tags " + repository + " " + branch
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def clone(repository, folder):
statement = "git clone " + repository + " " + folder
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def start(service, password):
statement = "echo " + password + " | " + "sudo -kS systemctl start " + service
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def tag(version, folder):
statement = "cd " + folder + ";" + "git tag -a " + version + " -m '" + version + "'"
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def checkout_tag(folder, version):
statement = "cd " + folder + ";" + "git checkout " + version
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def commit(version, folder):
statement = "cd " + folder + ";" + "git commit -a --message='updated to version " + version + "'"
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def checkout_branch(branch, folder):
statement = "cd " + folder + ";" + "git checkout -b " + branch + " origin/" + branch
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def install(module, target_folder):
statement = "pip install " + module + " -t " + target_folder
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def install(folder, virtualenv):
statement = "cd " + folder + ";" + "source " + virtualenv + "/bin/activate ;" + " python setup.py install"
p = Popen([statement], stdout=PIPE, stderr=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def create(folder):
statement = 'virtualenv ' + folder
p = Popen([statement], stdout=PIPE, stderr=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def build(name,
version,
repository,
username,
password,
no_dependencies=True,
no_quote=False):
# retrieve the dependency details
module_name = name
module_version = version
module_repo = repository
module = module_name
if module_version:
module = module_name + "==" + module_version
if module_repo:
module = module_repo
module_url = urlparse(module_repo)
if module_url.scheme in ['http', 'https']:
if not username:
raise PluginException("no username")
if not password:
raise PluginException("no password")
module = "git+" + module_url.scheme + "://{0}:{1}@" + module_url.netloc + module_url.path + " --upgrade"
if no_dependencies:
module = module + " --no-dependencies"
if no_quote:
module = module.format(username, password)
if not no_quote:
module = module.format(quote(username), quote(password))
if module_version:
module = "git+" + module_url.scheme + "://{0}:{1}@" + module_url.netloc + module_url.path + '@' + module_version + " --upgrade"
if no_dependencies:
module = module + " --no-dependencies"
if no_quote:
module = module.format(username, password)
if not no_quote:
module = module.format(quote(username),
quote(password))
return module
def execute(arguments, settings, properties):
job_name = properties['job_name']
logger = Logger(GitDeploy())
try:
version = arguments.v
environment = arguments.e
project_name = arguments.n
if not version:
raise PluginException("no version")
if not project_name:
raise PluginException("no project")
# if no environment is given, dev is assumed.
if not environment:
environment = "dev"
logger.info(job_name, "using environment : " + environment)
project = Settings.find_project(settings, project_name)
if not project:
message = "project " + project_name + " not found"
raise PluginException(message)
details = settings[environment]
username = project['username']
password = project['password']
repository = project['repository'].format(urllib.quote(username),
urllib.quote(password))
for detail in details:
if detail['name'] == project['name']:
virtualenv = detail['virtualenv']
result = Pip.install(repository, version, project,
virtualenv)
logger.info(job_name, str(result))
except:
result = Traceback.build()
logger.fatal(job_name, result['message'], result['backtrace'])
def tags(folder):
statement = "cd " + folder + ";" + "git tag"
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
results = []
lines = output.split("\n")
for line in lines:
if line:
results.append(line)
return results
def start(virtualenv, daemon_name):
statement = "source " + virtualenv + "/bin/activate;" + \
daemon_name + "&"
p = Popen([statement], stdout=PIPE)
# pid = Popen([statement]).pid
# return pid
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
# pid = Popen(["/bin/mycmd", "myarg"]).pid
def branches(folder):
statement = "cd " + folder + ";" + "git ls-remote --heads origin"
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
output = output.decode('utf-8')
lines = output.split("\n")
results = []
for line in lines:
if line:
results.append(line.split("\t")[1].replace('refs/heads/', ''))
return results
def install_virtualenv(virtualenv, repository, version, project_name):
statement = "source " + virtualenv + "/bin/activate ;" + \
"pip install git+" + repository + "#egg=" + project_name + \
" --process-dependency-links"
if version:
statement = "source " + virtualenv + "/bin/activate ;" + \
"pip install git+" + repository + "@" + version + "#egg=" + project_name + \
" --process-dependency-links"
p = Popen([statement], stdout=PIPE, shell=True)
output, error = p.communicate()
if p.returncode != 0:
raise PluginException(error)
return output
def execute(arguments, settings):
raise PluginException("method not implemented")
for dependency in dependencies:
module_name = dependency['name']
module_version = dependency.get('version')
module_repo = dependency.get('repository')
module = module_name
if module_version:
module = module_name + "==" + module_version
if module_repo:
module = module_repo
module_url = urlparse(module_repo)
if module_url.scheme in ['http', 'https']:
if not username:
raise PluginException("no username")
if not password:
raise PluginException("no password")
module = "git+" + module_url.scheme + "://'{0}':'{1}'@" + module_url.netloc + module_url.path
module = module.format(quote(username), quote(password))
if module_version:
module = "git+" + module_url.scheme + "://'{0}':'{1}'@" + module_url.netloc + module_url.path + '@' + module_version + " --upgrade"
module = module.format(quote(username), quote(password))
print(module)
Pip.install(module, build_folder)
def execute(arguments, settings, properties):
username = arguments.u
password = arguments.p
version = arguments.v
environment = arguments.e
profile_name = 'sandbox'
region_name = 'eu-west-1'
bucket_name = 'elsevier-mdp-dev-deploy'
key_prefix = 'eb/mdp_api/'
now = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
# if no environment is given sandbox is assumed.
if not environment:
environment = 'sandbox'
project = settings['project']
project_name = project['name']
# parse the url, when the scheme is http or https a username, password combination is expected.
url = urlparse(project['repository'])
repository = project['repository']
if url.scheme in ['http', 'https']:
if not username:
raise PluginException("no username")
if not password:
raise PluginException("no password")
# repository = url.scheme + "://'{0}':'{1}'@" + url.netloc + url.path
# repository = repository.format(quote(username), quote(password))
# retrieve aws profile_name to use, depending on the environment
profile_name = Settings.find_aws_profile_name(settings, environment)
region_name = Settings.find_aws_region_name(settings, environment)
# retrieve upload bucket name.
bucket_name = Settings.find_deploy_bucket_name(settings, environment)
# set identifier, reflecting the checkout folder to build this release.
name = uuid.uuid4().hex
# create an empty folder in tmp
build_folder = System.create_tmp(name)
module_name = project_name
module_version = None
module_repo = repository
module_type = 'pip'
module = PipUrl.build(module_name,
module_version,
module_repo,
username,
password,
no_dependencies=False)
print(build_folder)
if module_type == 'pip':
result = Pip.install(module, build_folder)
print(result)
# if module_type == 'yum':
# result = "which git || sudo yum install " + module_name + ' -y' + '\n'
# check for explicit dependencies that needs to be installed
if project['dependencies']:
for dependency in project['dependencies']:
module_name = dependency['name']
module_version = dependency.get('version')
module_repo = dependency.get('repository')
module_type = dependency.get('type')
module = PipUrl.build(module_name,
module_version,
module_repo,
username,
password,
no_dependencies=False)
if module_type == 'pip':
result = Pip.install(module, build_folder)
print(result)
# set the filename and path of the zipped file to build
basename = "mdp_api_source_bundle_" + now.strip()
print(basename)
zip_filename = basename + ".zip"
zip_file = os.path.join(build_folder, zip_filename)
key = key_prefix + zip_filename
# add all the files in the temp folder to the zip file.
# reflecting the module and its dependencies
Zip.create(zip_file, build_folder)
# upload the zipped file to aws
print('uploading', zip_file, "using profile", profile_name,
"into bucket ", bucket_name)
aws_bucket = AwsBucket(profile_name, region_name)
aws_bucket.upload(zip_file, bucket_name, key)
# create new application version on eb with the uploaded source bundle
client = boto3.client('elasticbeanstalk')
application_name = 'mdp-api-sandbox'
environment_name = application_name + '-env'
application_update_ts = now
version_label = 'harc_deployment_{}'.format(application_update_ts)
response = client.create_application_version(
ApplicationName=application_name,
VersionLabel=version_label,
Description='{} deployed on {}'.format(basename,
application_update_ts),
SourceBundle={
'S3Bucket': 'elsevier-mdp-dev-deploy',
'S3Key': key_prefix + zip_filename
},
AutoCreateApplication=False,
Process=True)
response = response.get('ApplicationVersion')
print(response)
while response.get('Status') == 'PROCESSING':
response = client.describe_application_versions(
ApplicationName=application_name,
VersionLabels=[version_label],
)
response = response.get('ApplicationVersions')[0]
print(response)
response = client.update_environment(
ApplicationName=response.get('ApplicationName'),
VersionLabel=response.get('VersionLabel'),
EnvironmentName=environment_name)