def execute_script(step, script_name, exec_type, serv_as):
synchronous = 1 if exec_type.strip() == 'synchronous' else 0
server = getattr(world, serv_as)
script = Script.get_id(script_name)
LOG.info('Execute script id: %s, name: %s' % (script['id'], script_name))
server.scriptlogs.reload()
setattr(world, '_server_%s_last_scripts' % server.id, copy.deepcopy(server.scriptlogs))
LOG.debug('Count of complete scriptlogs: %s' % len(server.scriptlogs))
Script.script_execute(world.farm.id, server.farm_role_id, server.id, script['id'], synchronous, script['version'])
LOG.info('Script executed success')
def execute_script(step, script_name, exec_type, serv_as):
synchronous = 1 if exec_type.strip() == "synchronous" else 0
serv = getattr(world, serv_as)
script = Script.get_id(script_name)
LOG.info("Execute script id: %s, name: %s" % (script["id"], script_name))
serv.scriptlogs.reload()
setattr(world, "%s_script_count" % serv_as, len(serv.scriptlogs))
LOG.debug("Count of complete scriptlogs: %s" % len(serv.scriptlogs))
Script.script_execute(world.farm.id, serv.farm_role_id, serv.id, script["id"], synchronous, script["version"])
LOG.info("Script execute success")
def execute_script(context: dict, farm: Farm, server: Server, script_name: str,
is_local: bool = False, synchronous: bool = False, user=''):
path = None
if is_local:
path = script_name
script_id = None
else:
script_id = Script.get_id(script_name)
LOG.info('Execute script "%s" with id: %s' % (script_name, script_id))
server.scriptlogs.reload()
context['_server_%s_last_scripts' % server.id] = copy.deepcopy(server.scriptlogs)
context['_server_%s_last_script_name' % server.id] = script_name
Script.script_execute(farm.id, server.farm_role_id, server.id, script_id, int(synchronous), path=path, run_as=user)
LOG.info('Script executed success')
def add_role_to_farm(step, behavior=None, options=None, alias=None):
additional_storages = None
scripting = None
role_options = {
"base.hostname_format": "{SCALR_FARM_NAME}-{SCALR_ROLE_NAME}-{SCALR_INSTANCE_INDEX}"
}
if not behavior:
behavior = os.environ.get('RV_BEHAVIOR', 'base')
else:
behavior = behavior.strip()
if options:
for opt in [o.strip() for o in options.strip().split(',')]:
LOG.info('Inspect option: %s' % opt)
if 'redis processes' in opt:
redis_count = re.findall(r'(\d+) redis processes', options)[0].strip()
LOG.info('Setup %s redis processes' % redis_count)
role_options.update({'db.msr.redis.num_processes': int(redis_count)})
elif opt == 'scripts':
LOG.info('Setup scripting options')
script_id = Script.get_id('Linux ping-pong')['id']
scripting = json.loads(DEFAULT_SCRIPTING % {'SCRIPT_ID': script_id})
elif opt == 'storages':
LOG.info('Insert additional storages config')
additional_storages = {'configs': DEFAULT_ADDITIONAL_STORAGES.get(CONF.feature.driver.cloud_family, [])}
else:
LOG.info('Insert configs for %s' % opt)
role_options.update(DEFAULT_ROLE_OPTIONS.get(opt, {}))
if behavior == 'rabbitmq':
del(role_options['base.hostname_format'])
if behavior == 'tomcat6' and CONF.feature.dist.startswith('ubuntu'):
behavior = 'tomcat7'
if behavior == 'redis':
LOG.info('Insert redis settings')
role_options.update({'db.msr.redis.persistence_type': os.environ.get('RV_REDIS_SNAPSHOTTING', 'aof'),
'db.msr.redis.use_password': True})
if behavior in DATABASE_BEHAVIORS:
storages = DEFAULT_STORAGES.get(CONF.feature.driver.current_cloud, None)
if storages:
LOG.info('Insert main settings for %s storage' % CONF.feature.storage)
role_options.update(storages.get(CONF.feature.storage, {}))
LOG.debug('All farm settings: %s' % role_options)
role = world.add_role_to_farm(behavior, options=role_options, scripting=scripting,
storages=additional_storages, alias=alias)
LOG.debug('Save role object with name %s' % role.alias)
setattr(world, '%s_role' % role.alias, role)
def upload_scripts():
if CONF.scalr.te_id:
LOG.info("Upload scripts")
path_to_scripts = CONF.main.home / 'fixtures' / 'testusing' / 'scripts'
upload_counter = 0
for _, script_path in enumerate(os.listdir(str(path_to_scripts))):
with (path_to_scripts / script_path).open(mode='r') as f:
script = json.load(f)
exist_scripts = Script.get_id(name=script['name'])
if exist_scripts:
LOG.info("Script '%s' already exist" % (script['name']))
else:
IMPL.script.create(
name=script['name'],
description=script['description'],
content=script['content']
)
LOG.info("Script '%s' upload successfully" % (script['name']))
upload_counter += 1
LOG.info("Total number of uploaded scripts: %s" % upload_counter)
def execute_script(step, local, script_name, exec_type, serv_as):
synchronous = 1 if exec_type.strip() == 'synchronous' else 0
path = None
server = getattr(world, serv_as)
if local:
path = script_name
script_id = None
else:
script_id = Script.get_id(script_name)['id']
LOG.info('Execute script "%s" with id: %s' % (script_name, script_id))
server.scriptlogs.reload()
setattr(world, '_server_%s_last_scripts' % server.id,
copy.deepcopy(server.scriptlogs))
LOG.debug('Count of complete scriptlogs: %s' % len(server.scriptlogs))
Script.script_execute(world.farm.id,
server.farm_role_id,
server.id,
script_id,
synchronous,
path=path)
LOG.info('Script executed success')
def add_role_to_farm(step,
behavior=None,
saved_role=None,
options=None,
alias=None):
additional_storages = None
options = options or []
scripting = None
role_id = None
scaling_metrics = None
old_branch = CONF.feature.branch
role_options = {
"base.hostname_template":
"{SCALR_FARM_NAME}-{SCALR_ROLE_NAME}-{SCALR_INSTANCE_INDEX}"
}
if CONF.feature.dist.id == 'scientific-6-x' or (
CONF.feature.dist.id == 'centos-7-x'
and CONF.feature.driver.current_cloud == Platform.EC2):
DEFAULT_ROLE_OPTIONS['noiptables'] = {
"base.disable_firewall_management": False
}
if CONF.feature.dist.is_windows:
role_options[
"base.hostname_template"] = "{SCALR_FARM_NAME}-{SCALR_INSTANCE_INDEX}"
if saved_role:
role_id = getattr(world, '%s_id' % saved_role.strip())
if not behavior:
behavior = os.environ.get('RV_BEHAVIOR', 'base')
else:
behavior = behavior.strip()
if options:
for opt in [o.strip() for o in options.strip().split(',')]:
LOG.info('Inspect option: %s' % opt)
if opt == 'noiptables' and CONF.feature.driver.current_cloud in [
Platform.IDCF, Platform.CLOUDSTACK, Platform.RACKSPACE_US
]:
continue
if opt in ('branch_latest', 'branch_stable'):
CONF.feature.branch = opt.split('_')[1]
if 'redis processes' in opt:
redis_count = re.findall(r'(\d+) redis processes',
options)[0].strip()
LOG.info('Setup %s redis processes' % redis_count)
role_options.update(
{'db.msr.redis.num_processes': int(redis_count)})
elif opt == 'orchestration':
LOG.info('Setup scripting options')
script_pong_id = Script.get_id('Linux ping-pong')['id']
script_init_id = Script.get_id(
'Revizor orchestration init')['id']
script_sleep_10 = Script.get_id('Sleep 10')['id']
scripting = json.loads(
DEFAULT_ORCHESTRATION_SETTINGS % {
'SCRIPT_PONG_ID': script_pong_id,
'SCRIPT_INIT_ID': script_init_id,
'SCRIPT_SLEEP_10': script_sleep_10
})
elif opt == 'small_linux_orchestration':
LOG.debug('Add small orchestration for linux')
script_pong_id = Script.get_id('Linux ping-pong')['id']
script_last_reboot_id = Script.get_id(
'Revizor last reboot')['id']
scripting = json.loads(
SMALL_ORCHESTRATION_LINUX % {
'SCRIPT_PONG_ID': script_pong_id,
'SCRIPT_LAST_REBOOT_ID': script_last_reboot_id
})
elif opt == 'small_win_orchestration':
LOG.debug('Add small orchestration for windows')
script_pong_cmd_id = Script.get_id(
'Windows ping-pong. CMD')['id']
script_pong_ps_id = Script.get_id(
'Windows ping-pong. PS')['id']
scripting = json.loads(
SMALL_ORCHESTRATION_WINDOWS % {
'SCRIPT_PONG_CMD_ID': script_pong_cmd_id,
'SCRIPT_PONG_PS_ID': script_pong_ps_id
})
elif opt == 'failed_script':
script_id = Script.get_id('non-ascii-output')['id']
scripting = [{
"script_type": "scalr",
"script_id": script_id,
"script": "non-ascii-output",
"os": "linux",
"event": "BeforeHostUp",
"target": "instance",
"isSync": "1",
"timeout": "1200",
"version": "-1",
"params": {},
"order_index": "10",
"system": "",
"script_path": "",
"run_as": "root"
}]
role_options.update(DEFAULT_ROLE_OPTIONS.get(opt, {}))
elif opt == 'apachefix':
script_id = Script.get_id('CentOS7 fix apache log')['id']
scripting = [{
"script_type": "scalr",
"script_id": script_id,
"script": "CentOS7 fix apache log",
"os": "linux",
"event": "HostInit",
"target": "instance",
"isSync": "1",
"timeout": "1200",
"version": "-1",
"params": {},
"order_index": "20",
"system": "",
"script_path": "",
"run_as": "root"
}]
elif opt == 'storages':
LOG.info('Insert additional storages config')
if CONF.feature.dist.is_windows:
additional_storages = {
'configs':
DEFAULT_WINDOWS_ADDITIONAL_STORAGES.get(
CONF.feature.driver.cloud_family, [])
}
else:
if CONF.feature.driver.current_cloud == Platform.RACKSPACE_US:
additional_storages = {
'configs':
DEFAULT_ADDITIONAL_STORAGES.get(
Platform.RACKSPACE_US, [])
}
else:
additional_storages = {
'configs':
DEFAULT_ADDITIONAL_STORAGES.get(
CONF.feature.driver.cloud_family, [])
}
elif opt == 'scaling':
scaling_metrics = {
Metrics.get_id('revizor') or Metrics.add(): {
'max': '',
'min': ''
}
}
LOG.info('Insert scaling metrics options %s' % scaling_metrics)
elif opt == 'prepare_scaling_linux':
script_id = Script.get_id(
'Revizor scaling prepare linux')['id']
scripting = [{
"script_type": "scalr",
"script_id": script_id,
"script": "Revizor scaling prepare linux",
"os": "linux",
"event": "HostInit",
"target": "instance",
"isSync": "1",
"timeout": "1200",
"version": "-1",
"params": {},
"order_index": "20",
"system": "",
"script_path": "",
"run_as": "root"
}]
elif opt == 'prepare_scaling_win':
script_id = Script.get_id(
'Revizor scaling prepare windows')['id']
scripting = [{
"script_type": "scalr",
"script_id": script_id,
"script": "Revizor scaling prepare windows",
"os": "windows",
"event": "HostInit",
"target": "instance",
"isSync": "1",
"timeout": "1200",
"version": "-1",
"params": {},
"order_index": "20",
"system": "",
"script_path": "",
"run_as": ""
}]
elif opt.startswith('scaling'):
metric_name = DEFAULT_SCALINGS[opt]
metric_id = Metrics.get_id(metric_name)
scaling_metrics = {metric_id: {"max": "75", "min": "50"}}
elif 'chef-solo' in opt:
chef_opts = opt.split('-')
default_chef_solo_opts = DEFAULT_CHEF_SOLO.copy()
# Set common arguments
repo_url = DEFAULT_CHEF_SOLO_URL.get(chef_opts[2], '')
chef_attributes = json.dumps(
{'chef-solo': {
'result': opt.strip()
}})
private_key = ''
relative_path = ''
# Set arguments for repo url with branch
if chef_opts[-1] == 'branch':
repo_url = ''.join((repo_url, '@revizor-test'))
# Set arguments for private repo
elif chef_opts[2] == 'private':
relative_path = 'cookbooks'
private_key = open(
os.path.expanduser(CONF.main.private_key), 'r').read()
# Update default chef_solo options
default_chef_solo_opts.update({
"chef.cookbook_url":
repo_url,
"chef.relative_path":
relative_path,
"chef.ssh_private_key":
private_key,
"chef.attributes":
chef_attributes
})
# Update role options
role_options.update(default_chef_solo_opts)
elif 'chef' in opt:
option = DEFAULT_ROLE_OPTIONS.get(opt, {})
if option:
option['chef.server_id'] = ChefServer.get(
'https://api.opscode.com/organizations/webta').id
role_options.update(option)
else:
LOG.info('Insert configs for %s' % opt)
role_options.update(DEFAULT_ROLE_OPTIONS.get(opt, {}))
if behavior == 'rabbitmq':
del (role_options['base.hostname_template'])
if behavior == 'redis':
LOG.info('Insert redis settings')
role_options.update({
'db.msr.redis.persistence_type':
os.environ.get('RV_REDIS_SNAPSHOTTING', 'aof'),
'db.msr.redis.use_password':
True
})
if behavior in DATABASE_BEHAVIORS:
storages = DEFAULT_STORAGES.get(CONF.feature.driver.current_cloud,
None)
if storages:
LOG.info('Insert main settings for %s storage' %
CONF.feature.storage)
role_options.update(storages.get(CONF.feature.storage, {}))
LOG.debug('All farm settings: %s' % role_options)
role = world.add_role_to_farm(behavior,
options=role_options,
scripting=scripting,
storages=additional_storages,
alias=alias,
role_id=role_id,
scaling=scaling_metrics)
LOG.debug('Save role object with name %s' % role.alias)
setattr(world, '%s_role' % role.alias, role)
if 'branch_latest' in options or 'branch_stable' in options:
CONF.feature.branch = old_branch
def add_role_to_farm(step, behavior=None, options=None):
additional_storages = None
scripting = None
farm_options = {"base.hostname_format": "{SCALR_FARM_NAME}-{SCALR_ROLE_NAME}-{SCALR_INSTANCE_INDEX}"}
if not behavior:
behavior = os.environ.get("RV_BEHAVIOR", "base")
else:
behavior = behavior.strip()
options = options.strip() if options else None
if options:
for opt in [o.strip() for o in options.strip().split(",")]:
LOG.info("Inspect option: %s" % opt)
if "redis processes" in opt:
LOG.info("Add redis processes")
redis_count = re.findall(r"(\d+) redis processes", options)[0].strip()
farm_options.update({"db.msr.redis.num_processes": int(redis_count)})
elif opt == "scripts":
LOG.info("Add scripting")
script_id = Script.get_id("Linux ping-pong")["id"]
scripting = [
{
"script_id": script_id,
"script": "Linux ping-pong",
"params": {},
"target": "instance",
"version": "-1",
"timeout": "1200",
"issync": "1",
"order_index": "1",
"event": "HostInit",
"run_as": "root",
},
{
"script_id": script_id,
"script": "Linux ping-pong",
"params": {},
"target": "instance",
"version": "-1",
"timeout": "1200",
"issync": "1",
"order_index": "10",
"event": "BeforeHostUp",
"run_as": "revizor",
},
{
"script_id": script_id,
"script": "Linux ping-pong",
"params": {},
"target": "instance",
"version": "-1",
"timeout": "1200",
"issync": "1",
"order_index": "20",
"event": "HostUp",
"run_as": "ubuntu",
},
]
elif opt == "storages":
LOG.info("Add additional storages")
if CONF.main.driver in [Platform.EC2]:
LOG.info("Add storages from EC2")
additional_storages = {
"configs": [
{
"id": None,
"type": "ebs",
"fs": "ext3",
"settings": {"ebs.size": "1", "ebs.type": "standard", "ebs.snapshot": None},
"mount": True,
"mountPoint": "/media/ebsmount",
"reUse": True,
"status": "",
},
{
"id": None,
"type": "raid.ebs",
"fs": "ext3",
"settings": {
"raid.level": "10",
"raid.volumes_count": 4,
"ebs.size": "1",
"ebs.type": "standard",
"ebs.snapshot": None,
},
"mount": True,
"mountPoint": "/media/raidmount",
"reUse": True,
"status": "",
},
]
}
elif CONF.main.driver in [Platform.IDCF, Platform.CLOUDSTACK]:
LOG.info("Add storages from IDCF/CloudStack")
additional_storages = {
"configs": [
{
"id": None,
"type": "csvol",
"fs": "ext3",
"settings": {"csvol.size": "1"},
"mount": True,
"mountPoint": "/media/ebsmount",
"reUse": True,
"status": "",
},
{
"id": None,
"type": "raid.csvol",
"fs": "ext3",
"settings": {"raid.level": "10", "raid.volumes_count": 4, "csvol.size": "1"},
"mount": True,
"mountPoint": "/media/raidmount",
"reUse": True,
"status": "",
},
]
}
elif CONF.main.driver in [Platform.OPENSTACK, Platform.ECS]:
LOG.info("Add storages from OpenStack")
additional_storages = {
"configs": [
{
"id": None,
"type": "cinder",
"fs": "ext3",
"settings": {"cinder.size": "1"},
"mount": True,
"mountPoint": "/media/ebsmount",
"reUse": True,
"status": "",
"rebuild": False,
},
{
"id": None,
"type": "raid.cinder",
"fs": "ext3",
"settings": {"raid.level": "10", "raid.volumes_count": 4, "cinder.size": "1"},
"mount": True,
"mountPoint": "/media/raidmount",
"reUse": True,
"status": "",
"rebuild": False,
},
]
}
else:
LOG.info("Add %s" % opt)
farm_options.update(FARM_OPTIONS.get(opt, {}))
if behavior == "rabbitmq":
del (farm_options["base.hostname_format"])
if behavior == "tomcat6" and CONF.main.dist.startswith("ubuntu"):
behavior = "tomcat7"
if behavior == "redis":
LOG.info("Add redis settings")
farm_options.update(
{
"db.msr.redis.persistence_type": os.environ.get("RV_REDIS_SNAPSHOTTING", "aof"),
"db.msr.redis.use_password": True,
}
)
if behavior in ["mysql", "mysql2", "percona2", "mariadb", "postgresql", "redis", "mongodb", "percona"]:
storage = STORAGES.get(Platform.to_scalr(CONF.main.driver), None)
if storage:
LOG.info("Add main settings for %s storage" % CONF.main.storage)
farm_options.update(storage.get(CONF.main.storage, {}))
world.role_type = behavior
world.role_options = farm_options
world.role_scripting = scripting
LOG.debug("All farm settings: %s" % farm_options)
role = world.add_role_to_farm(
world.role_type, options=farm_options, scripting=scripting, storages=additional_storages
)
setattr(world, "%s_role" % world.role_type, role)
world.role = role
if behavior in ["mysql", "postgresql", "redis", "mongodb", "percona", "mysql2", "percona2", "mariadb"]:
db = Database.create(role)
if not db:
raise AssertionError("Database for role %s not found!" % role)
world.database_users = {}
world.db = db