def get_ts():
domains = get_domains().values()
if len(domains) != 1:
raise RuntimeError("Unexpected number of domains in drift config: %s" %
len(domains))
ts = domains[0]["table_store"]
return ts
def list_command(args):
# Enumerate subfolders at drift/config and see what's there
domains = get_domains(user_dir=args.user_dir)
if not domains:
print "No Drift configuration found at", config_dir(
'', user_dir=args.user_dir)
else:
for d in domains.values():
print _format_domain_info(d)
def create_command(args):
domain_info = get_domains(user_dir=args.user_dir).get(args.domain)
if domain_info:
print "The domain name '{}' is taken:".format(args.domain)
print _format_domain_info(domain_info)
sys.exit(1)
# Force s3 naming convention. The root folder name and domain name must match.
if args.source.startswith('s3://'):
# Strip trailing slashes
if args.source.endswith('/'):
args.source = args.source[:-1]
s3_backend = create_backend(args.source)
target_folder = s3_backend.folder_name.rsplit('/')[-1]
if target_folder != args.domain:
print "Error: For S3 source, the target folder name and domain name must match."
print "Target folder is '{}' but domain name is '{}'".format(
target_folder, args.domain)
print "Suggestion: {}".format(
args.source.replace(target_folder, args.domain))
sys.exit(1)
elif args.source.startswith('file://'):
# Expand user vars
args.source = args.source.replace('~', os.path.expanduser('~'))
# Get empty table store for Drift.
ts = get_drift_table_store()
ts.get_table('domain').add({
'domain_name': args.domain,
'origin': args.source,
'display_name': args.display_name or ''
})
# Save it locally
domain_folder = config_dir(args.domain, user_dir=args.user_dir)
local_store = create_backend('file://' + domain_folder)
local_store.save_table_store(ts)
print "New config for '{}' saved to {}.".format(args.domain, domain_folder)
print "Pushing to origin..."
result = push_to_origin(ts, _first=True)
if not result['pushed']:
print "Push failed. Reason:", result['reason']
print "Done."
def diff_command(args):
# Get local table store and its meta state
domain_info = get_domains(user_dir=args.user_dir).get(args.domain)
if domain_info is None:
click.secho("Configuration not found: {}".format(args.domain),
fg='red')
sys.exit(1)
local_ts = domain_info['table_store']
local_m1, local_m2 = local_ts.refresh_metadata()
# Get origin table store meta info
origin = local_ts.get_table('domain')['origin']
origin_backend = create_backend(origin)
origin_ts = origin_backend.load_table_store()
origin_meta = origin_ts.meta.get()
local_diff = ("Local store and scratch", local_m1, local_m2, False)
origin_diff = ("Local and origin", origin_meta, local_m2, args.details)
for title, m1, m2, details in local_diff, origin_diff:
diff = diff_meta(m1, m2)
if diff['identical']:
print title, "is clean."
else:
print title, "are different:"
print "\tFirst checksum: ", diff['checksum']['first'][:7]
print "\tSecond checksum:", diff['checksum']['second'][:7]
if diff['modified_diff']:
print "\tTime since pull: ", str(
diff['modified_diff']).split('.')[0]
print "\tNew tables:", diff['new_tables']
print "\tDeleted tables:", diff['deleted_tables']
print "\tModified tables:", diff['modified_tables']
if details:
# Diff origin
origin_ts = get_store_from_url(origin)
for table_name in diff['modified_tables']:
t1 = local_ts.get_table(table_name)
t2 = origin_ts.get_table(table_name)
tablediff = diff_tables(t1, t2)
print "\nTable diff for", table_name, "\n(first=local, second=origin):"
print json.dumps(tablediff, indent=4, sort_keys=True)
def push_command(args):
domain_info = get_domains(user_dir=args.user_dir).get(args.domain)
if not domain_info:
print "Can't push '{}'.".format(args.domain)
sys.exit(1)
ts = domain_info['table_store']
origin = ts.get_table('domain')['origin']
print "Pushing local config to source", origin
result = push_to_origin(ts, args.force)
if not result['pushed']:
print "Push failed. Reason:", result['reason']
print "Origin has changed. Use --force to force push."
if 'time_diff' in result:
print "Time diff", result['time_diff']
else:
print "Config pushed. Reason: ", result['reason']
local_store = create_backend('file://' + domain_info['path'])
local_store.save_table_store(ts)
def info():
"""List out all Drift configuration DB's that are active on this machine.
"""
domains = get_domains()
if not domains:
click.secho(
"No Drift configuration found on this machine. Run 'init' or 'create' "
"command to remedy.")
else:
ts, source = get_default_drift_config_and_source()
got_default = False
for domain_info in domains.values():
domain = domain_info['table_store'].get_table('domain')
is_default = domain['domain_name'] == ts.get_table(
'domain')['domain_name']
if is_default:
click.secho(domain['domain_name'] + " [DEFAULT]:",
bold=True,
nl=False)
got_default = True
else:
click.secho(domain['domain_name'] + ":", bold=True, nl=False)
click.secho(" \"{}\"".format(domain['display_name']), fg='green')
click.secho("\tOrigin: " + domain['origin'])
click.secho("\tLocal: " + domain_info['path'])
click.secho("")
if got_default:
if 'DRIFT_CONFIG_URL' in os.environ:
click.secho(
"The default config is specified using the 'DRIFT_CONFIG_URL' environment variable."
)
else:
click.secho(
"The config above is the default one as it's the only one cached locally in ~/.drift/config."
)
else:
click.secho("Note: There is no default config specified!")
def _pull_command(args):
for domain_name, domain_info in get_domains(
user_dir=args.user_dir).items():
if args.domain and args.domain != domain_name:
continue
result = pull_from_origin(domain_info['table_store'],
ignore_if_modified=args.ignore_if_modified,
force=args.force)
if not result['pulled']:
print "Pull failed for", domain_name, ". Reason:", result['reason']
if result['reason'] == 'local_is_modified':
print "Use --ignore-if-modified to overwrite local changes."
else:
print "Use --force to force a pull."
else:
if result['reason'] == 'pulled_from_origin':
local_backend = create_backend('file://' + domain_info['path'])
local_backend.save_table_store(result['table_store'])
print "Config for {} pulled. Reason: {}".format(
domain_name, result['reason'])
def cli(ctx, config_urls, verbose, test):
"""Generate settings for Zappa lambdas.
pass in bobo
"""
ctx.obj = Globals()
domains = get_domains()
if not config_urls:
config_urls = domains.keys() # Use all local configs
# 'config_urls' contains either urls or domain names. Needs some processing.
table_stores = []
for config_url in config_urls:
if config_url in domains:
# 'config_url' is a domain name. It should point to a locally stored
# config, which we must check if it has 'origin' defined.
ts = domains[config_url]['table_store']
config_url = ts.get_table('domain').get().get('origin')
if not config_url:
click.secho("No origin defined for {}".format(ts))
continue
parts = urlparse.urlsplit(config_url)
if parts.scheme != 's3':
click.secho("Origin must be on S3, but is {}".format(config_url))
else:
click.secho("Fetching {}".format(config_url))
table_stores.append(get_store_from_url(config_url))
# Template accepts list of tiers which contains the following attributes:
# S3_ORIGIN_URL: The config origin.
# TIER_NAME: Tier name.
# aws_region: The region where this tier is running.
# s3_bucket_region: The region of the config origin s3 bucket.
# bucket_name: Name of the bucket that holds the config.
# subnets: Array of subnet names.
# security_groups: Array of sg names.
tiers = {}
for ts in table_stores:
domain = ts.get_table('domain').get()
for tier in ts.get_table('tiers').find():
tier_name = tier['tier_name']
if 'organization_name' not in tier:
click.secho(
"Note: Tier {} does not define 'organization_name'. Skipping."
.format(tier_name))
continue
s3_origin_url = domain['origin']
if tier_name in tiers:
click.secho("Error: Duplicate tier names found. Tier '{}' is "
"defined in both of the following configs:".format(
tier_name),
fg='red')
click.secho("Config A: {}".format(s3_origin_url))
click.secho("Config B: {}".format(
tiers[tier_name]['s3_origin_url']))
click.secho(
"'{}' from config B will be skipped, but please fix!".
format(tier_name))
continue
if 'aws' not in tier or 'region' not in tier['aws']:
click.secho(
"Note: Tier {} does not define aws.region. Skipping.".
format(tier_name))
continue
click.secho("Processing {}".format(tier_name), bold=True)
# Figure out in which aws region this config is located
aws_region = tier['aws']['region']
parts = urlparse.urlsplit(s3_origin_url)
bucket_name = parts.hostname
s3_bucket_region = boto3.resource(
"s3").meta.client.get_bucket_location(
Bucket=bucket_name)["LocationConstraint"]
# If 'get_bucket_location' returns None, it really means 'us-east-1'.
s3_bucket_region = s3_bucket_region or 'us-east-1'
print "Connecting to AWS region {} to gather subnets and security group.".format(
aws_region)
ec2 = boto3.resource('ec2', aws_region)
filters = [{
'Name': 'tag:tier',
'Values': [tier_name]
}, {
'Name':
'tag:Name',
'Values': [
tier_name + '-private-subnet-1',
tier_name + '-private-subnet-2'
]
}]
subnets = list(ec2.subnets.filter(Filters=filters))
subnets = [subnet.id for subnet in subnets]
filters = [{
'Name': 'tag:tier',
'Values': [tier_name]
}, {
'Name': 'tag:Name',
'Values': [tier_name + '-private-sg']
}]
security_groups = list(ec2.security_groups.filter(Filters=filters))
security_groups = [sg.id for sg in security_groups]
# Sum it up
tier_args = {
's3_origin_url': s3_origin_url,
'tier_name': tier_name,
'organization_name': tier['organization_name'],
'aws_region': aws_region,
's3_bucket_region': s3_bucket_region,
'bucket_name': bucket_name,
'subnets': subnets,
'security_groups': security_groups,
'_ts': ts,
}
tiers[tier_name] = tier_args
if test:
_run_sanity_check(tiers)
return
env = Environment(
loader=PackageLoader('driftconfig', package_path='templates'))
template = env.get_template('zappa_settings.yml.jinja')
zappa_settings_text = template.render(tiers=tiers.values())
from driftconfig.cli import pretty
print pretty(zappa_settings_text, 'yaml')
with open('zappa_settings.yml', 'w') as f:
f.write(zappa_settings_text)
click.secho(
"zappa_settings.yml generated. Run 'zappa deploy' or zappa update'"
" for each of the tiers, or use the -all switch to zap them all.")
print pretty("Example: zappa update -s zappa_settings.yml -all", 'bash')