def find_gen3_root_dir(dirname):
marked_root_dir = Search.upwards(dirname, 'root.json')
if marked_root_dir is not None:
return os.path.dirname(marked_root_dir)
config_root_dir = os.path.dirname(Search.upwards(dirname, 'config'))
infrastructure_root_dir = os.path.dirname(
Search.upwards(dirname, 'infrastructure'))
root_dir = config_root_dir or infrastructure_root_dir
if Search.isdir(root_dir, 'config') and Search.isdir(
root_dir, 'infrastructure'):
return root_dir
return None
def set_context(cwd=None, environment_obj=None):
e = environment_obj # shortcut
l = Environment() # local env
if e.GENERATION_CONTEXT_DEFINED:
return
e.GENERATION_CONTEXT_DEFINED = True
l.GENERATION_CONTEXT_DEFINED_LOCAL = True
if not e.GENERATION_TMPDIR:
# *******************************************
# * TODO: decide how to do temp dir cleanup *
# *******************************************
e.GENERATION_TMPDIR = tempfile.mkdtemp()
e.GENERATION_DATA_DIR = find_gen3_root_dir(cwd)
if not e.GENERATION_DATA_DIR:
raise Exception("Can't locate the root of the directory tree.")
if not e.GENERATION_NO_CMDB_CHECK:
if not upgrade_cmdb(e.GENERATION_DATA_DIR, '', '',
e.GENERATION_MAX_CMDB_UPGRADE_VERSION):
raise Exception('CMDB upgrade failed.')
if not cleanup_cmdb(e.GENERATION_DATA_DIR, '', '',
e.GENERATION_MAX_CMDB_UPGRADE_VERSION):
raise Exception('CMDB cleanup failed.')
e.CACHE_DIR = os.path.join(e.GENERATION_DATA_DIR, 'cache')
os.makedirs(e.CACHE_DIR, exist_ok=True)
l.TEMPLATE_COMPOSITES = ('account', 'fragment')
for composite in l.TEMPLATE_COMPOSITES:
key = f'COMPOSITE_{composite.upper()}'
e[key] = os.path.join(e.CACHE_DIR, f'{key.lower()}.ftl')
if (not e.GENERATION_USE_CACHE and not e.GENERATION_USE_FRAGMENTS_CACHE
or not Search.isfile(e.CACHE_DIR, 'composite_account.ftl')):
l[f'{composite.lower()}_array'] = []
# ********************************************
# * TODO: add legacy start fragments section *
# ********************************************
# Check if the current directory gives any clue to the context
# Accommodate both pre cmdb v2.0.0 where segment/environment in the config tree
# and post v2.0.0 where they are in the infrastructure tree
l.solutions_ancestor_dir = Search.upwards(cwd, 'solutions')
l.solutionsv2_ancestor_dir = Search.upwards(cwd, 'solutionsv2')
if not l.solutions_ancestor_dir and not l.solutionsv2_ancestor_dir:
l.infrastructure_dir = cwd.replace('settings', 'solutions')
l.infrastructure_dir = l.infrastructure_dir.replace(
'operations', 'infrastructure')
l.infrastructure_dir = l.infrastructure_dir.replace(
'config', 'infrastructure')
if os.path.isdir(l.infrastructure_dir):
cwd = l.infrastructure_dir
if Search.isfile(cwd, 'segment.json'):
e.LOCATION = e.LOCATION or 'segment'
e.SEGMENT = Search.basename(cwd)
if Search.isfile(cwd, 'environment.json', up=1):
cwd = Search.parent(cwd, up=1)
else:
e.ENVIRONMENT = e.SEGMENT
e.SEGMENT = 'default'
cwd = os.path.join(Search.parent(cwd, up=3), 'config')
if Search.isfile(cwd, 'environment.json'):
e.LOCATION = e.LOCATION or 'environment'
e.ENVIRONMENT = Search.basename(cwd)
cwd = os.path.join(Search.parent(cwd, up=3), 'config')
if Search.isfile(cwd, 'account.json'):
# account directory
# We check it before checking for a product as the account directory
# also acts as a product directory for shared infrastructure
# An account directory may also have no product information e.g.
# in the case of production environments in dedicated accounts.
e.LOCATION = e.LOCATION or 'account'
if Search.isfile(cwd, 'product.json'):
if e.LOCATION == 'account':
e.LOCATION = 'account|product'
else:
e.LOCATION = e.LOCATION or 'product'
e.PRODUCT = Search.basename(cwd)
if e.PRODUCT == 'config':
e.PRODUCT = Search.basename(cwd, up=1)
if Search.isfile(cwd, 'integrator.json'):
e.LOCATION = e.LOCATION or 'integrator'
e.INTEGRATOR = Search.basename(cwd)
if Search.isfile(cwd, 'root.json') or Search.isdir(
cwd, 'config') and Search.isdir(cwd, 'infrastructure'):
e.LOCATION = 'root'
cwd = e.GENERATION_DATA_DIR
if not e.ACCOUNT:
e.ACCOUNT = Search.basename(e.GENERATION_DATA_DIR)
# Analyse directory structure
find_gen3_dirs(e, e.GENERATION_DATA_DIR)
# Build the composite solution ( aka blueprint)
e.GENERATION_INPUT_SOURCE = e.GENERATION_INPUT_SOURCE or 'composite'
if e.GENERATION_INPUT_SOURCE == 'composite':
l.blueprint_alternate_dirs = (e.SEGMENT_SOLUTIONS_DIR,
e.ENVIRONMENT_SHARED_SOLUTIONS_DIR,
e.SEGMENT_SHARED_SOLUTIONS_DIR,
e.PRODUCT_SHARED_SOLUTIONS_DIR)
e.COMPOSITE_BLUEPRINT = os.path.join(e.CACHE_DIR,
'composite_blueprint.json')
if (not e.GENERATION_USE_CACHE and not e.GENERATION_USE_BLUEPRINT_CACHE
or not os.path.isfile(e.COMPOSITE_BLUEPRINT)):
l.blueprint_array = []
for blueprint_alternate_dir in l.blueprint_alternate_dirs:
if not blueprint_alternate_dir or not os.path.isdir(
blueprint_alternate_dir):
continue
l.blueprint_array += Search.match_files(
os.path.join('**', 'segment*.json'),
os.path.join('**', 'environment*.json'),
os.path.join('**', 'solution*.json'),
os.path.join('**', 'domains*.json'),
os.path.join('**', 'ipaddressgroups*.json'),
os.path.join('**', 'countrygroups*.json'),
root=blueprint_alternate_dir)
l.blueprint_array += Search.match_files(
os.path.join('**', 'domains*.json'),
os.path.join('**', 'ipaddressgroups*.json'),
os.path.join('**', 'countrygroups*.json'),
os.path.join('**', 'profiles*.json'),
os.path.join('**', 'product*.json'),
root=e.PRODUCT_DIR)
l.blueprint_array += Search.match_files(
os.path.join('**', 'domains*.json'),
os.path.join('**', 'ipaddressgroups*.json'),
os.path.join('**', 'countrygroups*.json'),
os.path.join('**', 'account*.json'),
root=e.ACCOUNT_DIR)
l.blueprint_array += Search.match_files(
os.path.join('**', 'domains*.json'),
os.path.join('**', 'ipaddressgroups*.json'),
os.path.join('**', 'countrygroups*.json'),
os.path.join('**', 'profiles*.json'),
os.path.join('**', 'tenant*.json'),
root=e.TENANT_DIR)
if not l.blueprint_array:
blueprint = {}
with open(e.COMPOSITE_BLUEPRINT, 'wt+') as f:
f.write('{}')
else:
# merging blueprint components
blueprint = {}
for blueprint_component_json in l.blueprint_array:
with open(blueprint_component_json, 'rt') as f:
deep_dict_update(blueprint, json.load(f))
with open(e.COMPOSITE_BLUEPRINT, 'wt+') as f:
json.dump(blueprint, f, indent=4, sort_keys=True)
# Extract key settings from the composite solution
tenant = blueprint.get('Tenant', {})
account = blueprint.get('Account', {})
product = blueprint.get('Product', {})
segment = blueprint.get('Segment', {})
e.TID = tenant.get('Id')
e.TENANT = tenant.get('Name')
e.AID = account.get('Id')
e.AWSID = account.get('AWSId')
e.ACCOUNT_REGION = account.get('Region')
e.PID = product.get('Id')
e.PRODUCT_REGION = product.get('Region')
e.DEPLOYMENTUNIT_REGION = product.get(e.DEPLOYMENT_UNIT,
{}).get('Region')
e.SID = segment.get('Id')
e.COMPONENT_REGION = e.DEPLOYMENTUNIT_REGION or e.PRODUCT_REGION
e.REGION = e.REGION or e.COMPONENT_REGION
# Perform a few consistency checks
if not e.REGION:
raise Exception(
"The region must be defined in the Product blueprint section."
)
l.BLUEPRINT_ACCOUNT = account.get('Name') or account.get('Id')
l.BLUEPRINT_PRODUCT = product.get('Name') or product.get('Id')
l.BLUEPRINT_SEGMENT = segment.get('Name') or segment.get('Id')
if e.ACCOUNT and l.BLUEPRINT_ACCOUNT != 'Account' and e.ACCOUNT != l.BLUEPRINT_ACCOUNT:
raise Exception(
f"Blueprint account of {l.BLUEPRINT_ACCOUNT} doesn't match expected value of {e.ACCOUNT}"
)
if e.PRODUCT and l.BLUEPRINT_PRODUCT != 'Product' and e.PRODUCT != l.BLUEPRINT_PRODUCT:
raise Exception(
f"Blueprint product of {l.BLUEPRINT_PRODUCT} doesn't match expected value of ${e.PRODUCT}"
)
if e.SEGMENT and l.BLUEPRINT_SEGMENT != 'Segment' and e.SEGMENT != l.BLUEPRINT_SEGMENT:
raise Exception(
f"Blueprint segment of {e.BLUEPRINT_SEGMENT} doesn't match expected value of {e.SEGMENT}"
)
# Set default AWS credentials if available (hook from Jenkins framework)
l.CHECK_AWS_ACCESS_KEY_ID = (e.AWS_ACCESS_KEY_ID
or e.ACCOUNT_TEMP_AWS_ACCESS_KEY_ID
or e[e.ACCOUNT_AWS_ACCESS_KEY_ID_VAR])
if l.CHECK_AWS_ACCESS_KEY_ID:
e.AWS_ACCESS_KEY_ID = l.CHECK_AWS_ACCESS_KEY_ID
l.CHECK_AWS_SECRET_ACCESS_KEY = (e.AWS_SECRET_ACCESS_KEY
or e.ACCOUNT_TEMP_AWS_SECRET_ACCESS_KEY
or e[e.ACCOUNT_AWS_SECRET_ACCESS_KEY_VAR])
if l.CHECK_AWS_SECRET_ACCESS_KEY:
e.AWS_SECRET_ACCESS_KEY = l.CHECK_AWS_SECRET_ACCESS_KEY
l.CHECK_AWS_SESSION_TOKEN = e.AWS_SESSION_TOKEN or e.ACCOUNT_TEMP_AWS_SESSION_TOKEN
if l.CHECK_AWS_SESSION_TOKEN:
e.AWS_SESSION_TOKEN = l.CHECK_AWS_SESSION_TOKEN
# Set the profile for IAM access if AWS credentials not in the environment
if not e.AWS_ACCESS_KEY_ID or not e.AWS_SECRET_ACCESS_KEY:
available_profiles = boto3.session.Session().available_profiles
if e.ACCOUNT and e.ACCOUNT in available_profiles:
e.AWS_DEFAULT_PROFILE = e.ACCOUNT
if e.AID and e.AID in available_profiles:
e.AWS_DEFAULT_PROFILE = e.AID
if e.AWSID and e.AWSID in available_profiles:
e.AWS_DEFAULT_PROFILE = e.AWSID
def test_search():
with tempfile.TemporaryDirectory() as temp_dir:
path = "/test/path/split"
assert Search.split_path(path) == ["/", "test", "path", "split"]
path = "test/path/split"
assert Search.split_path(path) == ["test", "path", "split"]
test_directory_fullpath = os.path.join(temp_dir, "test/directory")
test_directory_1_fullpath = os.path.join(temp_dir, "test/directory/1")
test_directory_2_fullpath = os.path.join(temp_dir, "test/directory/2")
test_filename_fullpath = os.path.join(test_directory_fullpath,
"node-0")
test_filename_1_fullpath = os.path.join(test_directory_1_fullpath,
"node-1")
test_filename_2_fullpath = os.path.join(test_directory_2_fullpath,
"node-2")
os.makedirs(test_directory_1_fullpath)
os.makedirs(test_directory_2_fullpath)
open(test_filename_fullpath, "w").close()
open(test_filename_1_fullpath, "w").close()
open(test_filename_2_fullpath, "w").close()
assert os.path.exists(test_filename_fullpath)
assert os.path.exists(test_filename_1_fullpath)
assert os.path.exists(test_filename_2_fullpath)
with pytest.raises(ValueError):
ContextSearch(os.path.relpath(temp_dir, test_directory_fullpath))
with pytest.raises(ValueError):
ContextSearch(test_filename_fullpath)
with pytest.raises(ValueError):
ContextSearch("/directory/doesnt/exist")
assert (Search.exists(test_directory_fullpath,
"node-0") == test_filename_fullpath)
assert (ContextSearch(test_directory_fullpath).exists("node-0") ==
test_filename_fullpath)
assert (Search.exists(test_directory_1_fullpath,
"node-1") == test_filename_1_fullpath)
assert (ContextSearch(test_directory_1_fullpath).exists("node-1") ==
test_filename_1_fullpath)
assert (Search.exists(test_directory_2_fullpath,
"node-2") == test_filename_2_fullpath)
assert (ContextSearch(test_directory_2_fullpath).exists("node-2") ==
test_filename_2_fullpath)
assert (Search.exists(test_directory_1_fullpath, "node-0",
up=1) == test_filename_fullpath)
assert (ContextSearch(test_directory_1_fullpath).exists(
"node-0", up=1) == test_filename_fullpath)
assert (Search.exists(test_directory_2_fullpath, "node-0",
up=1) == test_filename_fullpath)
assert (ContextSearch(test_directory_2_fullpath).exists(
"node-0", up=1) == test_filename_fullpath)
assert (Search.isfile(test_directory_fullpath,
"node-0") == test_filename_fullpath)
assert (ContextSearch(test_directory_fullpath).isfile("node-0") ==
test_filename_fullpath)
assert (Search.isfile(test_directory_1_fullpath, "node-0",
up=1) == test_filename_fullpath)
assert (ContextSearch(test_directory_1_fullpath).isfile(
"node-0", up=1) == test_filename_fullpath)
assert (Search.isdir(test_directory_fullpath,
os.path.basename(test_directory_1_fullpath)) ==
test_directory_1_fullpath)
assert (ContextSearch(test_directory_fullpath).isdir(
os.path.basename(test_directory_1_fullpath)) ==
test_directory_1_fullpath)
assert (Search.isdir(
test_directory_1_fullpath,
os.path.basename(test_directory_fullpath),
up=2,
) == test_directory_fullpath)
assert (ContextSearch(test_directory_1_fullpath).isdir(
os.path.basename(test_directory_fullpath),
up=2) == test_directory_fullpath)
assert (Search.upwards(test_directory_1_fullpath,
"node-0") == test_filename_fullpath)
assert (ContextSearch(test_directory_2_fullpath).upwards("node-0") ==
test_filename_fullpath)
assert Search.downwards(temp_dir,
"node-1") == [test_filename_1_fullpath]
assert ContextSearch(temp_dir).downwards("node-2") == [
test_filename_2_fullpath
]
assert Search.basename(test_directory_fullpath) == os.path.basename(
test_directory_fullpath)
assert ContextSearch(test_directory_fullpath).basename(
) == os.path.basename(test_directory_fullpath)
assert Search.basename(
test_directory_1_fullpath,
up=1) == os.path.basename(test_directory_fullpath)
assert ContextSearch(test_directory_2_fullpath).basename(
up=1) == os.path.basename(test_directory_fullpath)
assert Search.parent(test_directory_1_fullpath,
up=1) == test_directory_fullpath
assert (ContextSearch(test_directory_2_fullpath).parent(
up=1) == test_directory_fullpath)
assert Search.parent(test_directory_1_fullpath, up=3) == temp_dir
assert ContextSearch(test_directory_2_fullpath).parent(
up=3) == temp_dir
assert (Search.cut(
"/p/re/fix/path/to/something/suf/fix",
prefix="/p/re/fix",
suffix="suf/fix",
) == "path/to/something")
assert (Search.cut("/p/re/fix/path/to/something/suf/fix",
prefix="/p/re/fix") == "path/to/something/suf/fix")
assert (Search.cut("/p/re/fix/path/to/something/suf/fix",
suffix="suf/fix") == "/p/re/fix/path/to/something")
def test_search():
with tempfile.TemporaryDirectory() as temp_dir:
path = "/test/path/split"
assert Search.split_path(path) == ['/', 'test', 'path', 'split']
path = "test/path/split"
assert Search.split_path(path) == ['test', 'path', 'split']
test_directory_fullpath = os.path.join(temp_dir, 'test/directory')
test_directory_1_fullpath = os.path.join(temp_dir, 'test/directory/1')
test_directory_2_fullpath = os.path.join(temp_dir, 'test/directory/2')
test_filename_fullpath = os.path.join(test_directory_fullpath,
'node-0')
test_filename_1_fullpath = os.path.join(test_directory_1_fullpath,
'node-1')
test_filename_2_fullpath = os.path.join(test_directory_2_fullpath,
'node-2')
os.makedirs(test_directory_1_fullpath)
os.makedirs(test_directory_2_fullpath)
os.mknod(test_filename_fullpath)
os.mknod(test_filename_1_fullpath)
os.mknod(test_filename_2_fullpath)
assert os.path.exists(test_filename_fullpath)
assert os.path.exists(test_filename_1_fullpath)
assert os.path.exists(test_filename_2_fullpath)
with pytest.raises(ValueError):
ContextSearch(os.path.relpath(temp_dir, test_directory_fullpath))
with pytest.raises(ValueError):
ContextSearch(test_filename_fullpath)
with pytest.raises(ValueError):
ContextSearch("/directory/doesnt/exist")
assert Search.exists(test_directory_fullpath,
'node-0') == test_filename_fullpath
assert ContextSearch(test_directory_fullpath).exists(
'node-0') == test_filename_fullpath
assert Search.exists(test_directory_1_fullpath,
'node-1') == test_filename_1_fullpath
assert ContextSearch(test_directory_1_fullpath).exists(
'node-1') == test_filename_1_fullpath
assert Search.exists(test_directory_2_fullpath,
'node-2') == test_filename_2_fullpath
assert ContextSearch(test_directory_2_fullpath).exists(
'node-2') == test_filename_2_fullpath
assert Search.exists(test_directory_1_fullpath, 'node-0',
up=1) == test_filename_fullpath
assert ContextSearch(test_directory_1_fullpath).exists(
'node-0', up=1) == test_filename_fullpath
assert Search.exists(test_directory_2_fullpath, 'node-0',
up=1) == test_filename_fullpath
assert ContextSearch(test_directory_2_fullpath).exists(
'node-0', up=1) == test_filename_fullpath
assert Search.isfile(test_directory_fullpath,
'node-0') == test_filename_fullpath
assert ContextSearch(test_directory_fullpath).isfile(
'node-0') == test_filename_fullpath
assert Search.isfile(test_directory_1_fullpath, 'node-0',
up=1) == test_filename_fullpath
assert ContextSearch(test_directory_1_fullpath).isfile(
'node-0', up=1) == test_filename_fullpath
assert (Search.isdir(test_directory_fullpath,
os.path.basename(test_directory_1_fullpath)) ==
test_directory_1_fullpath)
assert (ContextSearch(test_directory_fullpath).isdir(
os.path.basename(test_directory_1_fullpath)) ==
test_directory_1_fullpath)
assert (Search.isdir(test_directory_1_fullpath,
os.path.basename(test_directory_fullpath),
up=2) == test_directory_fullpath)
assert (ContextSearch(test_directory_1_fullpath).isdir(
os.path.basename(test_directory_fullpath),
up=2) == test_directory_fullpath)
assert Search.upwards(test_directory_1_fullpath,
'node-0') == test_filename_fullpath
assert ContextSearch(test_directory_2_fullpath).upwards(
'node-0') == test_filename_fullpath
assert Search.downwards(temp_dir,
'node-1') == [test_filename_1_fullpath]
assert ContextSearch(temp_dir).downwards('node-2') == [
test_filename_2_fullpath
]
assert Search.basename(test_directory_fullpath) == os.path.basename(
test_directory_fullpath)
assert ContextSearch(test_directory_fullpath).basename(
) == os.path.basename(test_directory_fullpath)
assert Search.basename(
test_directory_1_fullpath,
up=1) == os.path.basename(test_directory_fullpath)
assert ContextSearch(test_directory_2_fullpath).basename(
up=1) == os.path.basename(test_directory_fullpath)
assert Search.parent(test_directory_1_fullpath,
up=1) == test_directory_fullpath
assert ContextSearch(test_directory_2_fullpath).parent(
up=1) == test_directory_fullpath
assert Search.parent(test_directory_1_fullpath, up=3) == temp_dir
assert ContextSearch(test_directory_2_fullpath).parent(
up=3) == temp_dir
assert (Search.cut("/p/re/fix/path/to/something/suf/fix",
prefix="/p/re/fix",
suffix="suf/fix") == "path/to/something")
assert (Search.cut("/p/re/fix/path/to/something/suf/fix",
prefix="/p/re/fix") == "path/to/something/suf/fix")
assert (Search.cut("/p/re/fix/path/to/something/suf/fix",
suffix="suf/fix") == "/p/re/fix/path/to/something")