def main_clean(args=None):
# stand alone method to be used whenever we want to call the mongo_clean_status method independently
# Get configuration paths
conf_paths = get_config_paths(args.config)
# Get logger config file
get_log_conf(conf_paths['log'])
# Get main configuration and schema
config = ArgoConfig(conf_paths["main"], conf_paths["schema"])
# set up the mongo uri
section_tenant = "TENANTS:" + args.tenant
mongo_endpoint = config.get("MONGO", "endpoint")
mongo_uri = config.get(section_tenant,
"mongo_uri").fill(mongo_endpoint=mongo_endpoint,
tenant=args.tenant)
if args.job == "clean_ar":
argo_mongo_client = ArgoMongoClient(
args, config, ["service_ar", "endpoint_group_ar"])
argo_mongo_client.mongo_clean_ar(mongo_uri)
elif args.job == "clean_status":
argo_mongo_client = ArgoMongoClient(args, config, [
"status_metrics", "status_endpoints", "status_services",
"status_endpoint_groups"
])
argo_mongo_client.mongo_clean_status(mongo_uri)
def main(args=None):
# Get configuration paths
conf_paths = get_config_paths(args.config)
# Get logger config file
get_log_conf(conf_paths['log'])
# Get main configuration and schema
config = ArgoConfig(conf_paths["main"], conf_paths["schema"])
res = upload_recomputations(args.tenant, args.report, args.date, config)
if not res:
sys.exit(1)
def run_profile_update(args):
"""
Method to be executed when this is invoked as a command line script.
Reads tenant,report and config arguments from cli and instantiates an
ArgoProfileManager which does a profile update check
Args:
args: obj. command line arguments from arg parser
"""
# Get configuration paths
conf_paths = get_config_paths(args.config)
# Get logger config file
get_log_conf(conf_paths['log'])
# Get main configuration and schema
config = ArgoConfig(conf_paths["main"], conf_paths["schema"])
argo = ArgoProfileManager(config)
if args.tenant is not None:
# check for the following profile types
profile_type_checklist = [
"operations", "aggregations", "reports", "thresholds",
"recomputations"
]
reports = []
if args.report is not None:
reports.append(args.report)
else:
reports = config.get("TENANTS:" + args.tenant, "reports")
for report in reports:
for profile_type in profile_type_checklist:
argo.profile_update_check(args.tenant, report, profile_type)
else:
argo.upload_tenants_cfg()
argo.save_config(conf_paths["main"])
def run_ams_update(args):
"""
Runs when this script is called from CLI.
Runs for a specific tenant or all tenants in argo config
Reads argo configuration tenant list. For each tenant, creates missing tenant's definitions
and updates argo configuration with tenant's ams parameters
Args:
args: cli arguments
"""
# Get configuration paths
conf_paths = get_config_paths(args.config)
# Get logger config file
get_log_conf(conf_paths['log'])
# Get main configuration and schema
config = ArgoConfig(conf_paths["main"], conf_paths["schema"])
ams_token = config.get("AMS", "access_token")
ams_host = config.get("AMS", "endpoint").hostname
log.info("ams api used {}".format(ams_host))
tenant_list = config.get("API", "tenants")
ams = ArgoAmsClient(ams_host, ams_token)
if args.tenant is not None:
# Check if tenant exists in argo configuarion
if args.tenant not in tenant_list:
log.fatal(
"Tenant:{} is not in argo configuration - exiting...".format(
args.tenant))
sys.exit(1)
# Check only the tenant that user specified
check_tenants = [args.tenant]
else:
# Check all tenants that are specified in argo configuration
check_tenants = tenant_list
for tenant in check_tenants:
ams.check_project_exists(tenant)
missing = ams.check_tenant(tenant)
if is_tenant_complete(missing):
log.info("Tenant {} definition on AMS is complete!".format(tenant))
else:
ams.fill_missing(tenant, missing)
# Update tenant configuration
ams.update_tenant_configuration(tenant, config)
# Save changes to designated configuration file
config.save_as(config.conf_path)
def test_argod():
config_text = ArgoConfig.slurp_config_file(config.argo_conf)
network = 'mainnet'
is_testnet = False
genesis_hash = u'000005ece8d1964dc550f050f35c45398a4b9dda3158040e0519b9c230b91ed6'
for line in config_text.split("\n"):
if line.startswith('testnet=1'):
network = 'testnet'
is_testnet = True
genesis_hash = u'000002731816bccf90ab744347dc894cf484e3826b19f967b8d5f028c204a4f0'
creds = ArgoConfig.get_rpc_creds(config_text, network)
argod = ArgoDaemon(**creds)
assert argod.rpc_command is not None
assert hasattr(argod, 'rpc_connection')
# Argo testnet block 0 hash == 000002731816bccf90ab744347dc894cf484e3826b19f967b8d5f028c204a4f0
# test commands without arguments
info = argod.rpc_command('getinfo')
info_keys = [
'blocks',
'connections',
'difficulty',
'errors',
'protocolversion',
'proxy',
'testnet',
'timeoffset',
'version',
]
for key in info_keys:
assert key in info
assert info['testnet'] is is_testnet
# test commands with args
assert argod.rpc_command('getblockhash', 0) == genesis_hash
def run_tenant_check(args):
"""Run tenant/s check routine and update status json to argo-web-api
Args:
args (obj): command line arguments
"""
# Get configuration paths
conf_paths = get_config_paths(args.config)
# Get logger config file
get_log_conf(conf_paths['log'])
# Get main configuration and schema
config = ArgoConfig(conf_paths["main"], conf_paths["schema"])
# if argo_engine configuration is invalid return
if config.valid is False:
log.error("Argo engine not properly configured check file:{}".format(
conf_paths["main"]))
sys.exit(1)
# check for specific date or today
if args.date is not None:
target_date = args.date
else:
target_date = get_today()
# check for specific tenant or all of them
tenants = config.get("API", "tenants")
if args.tenant is not None:
if args.tenant in tenants:
tenants = [args.tenant]
else:
log.error("tenant {} not found".format(args.tenant))
return
check_tenants(tenants, target_date, args.days_back, config)
def test_config(self):
argo_conf = ArgoConfig()
argo_conf.load_conf(CONF_FILE)
argo_conf.load_schema(SCHEMA_FILE)
argo_conf.check_conf()
# get fixed HDFS -> user param (string)
self.assertEqual("foo", argo_conf.get("HDFS", "user"))
# get var TENANTS:TENANT_A -> reports param (list of strings)
self.assertEqual(["report1", "report2"],
argo_conf.get("TENANTS:TENANT_A", "reports"))
# get var TENANTS:TENANT_B -> mongo_uri (url)
self.assertEqual(
urlparse("mongodb://localhost:21017/argo_FOO"),
argo_conf.get("TENANTS:TENANT_B",
"mongo_uri").fill(mongo_uri=argo_conf.get(
"MONGO", "endpoint").geturl(),
tenant="FOO"))
# get var HDFS -> path_metric (template) and fill it with specific arguments
exp_result = urlparse(
"hdfs://localhost:2000/user/foobar/argo/tenants/wolf/mdata")
self.assertEqual(
exp_result,
argo_conf.get("HDFS",
"path_metric").fill(namenode="localhost:2000",
hdfs_user="******",
tenant="wolf"))
# fill template with different user argument
self.assertNotEqual(
exp_result,
argo_conf.get("HDFS",
"path_metric").fill(namenode="localhost:2000",
hdfs_user="******",
tenant="wolf"))
def test_update_cron(self):
config = ArgoConfig(CONF_FILE, SCHEMA_FILE)
# Test get_hourly
self.assertEquals("8 * * * *", get_hourly(8))
self.assertEquals("44 * * * *", get_hourly(44))
self.assertEquals("32 * * * *", get_hourly(32))
self.assertEquals("12 * * * *", get_hourly(12))
self.assertEquals("5 * * * *", get_hourly())
# Test get_daily
self.assertEquals("8 1 * * *", get_daily(1, 8))
self.assertEquals("44 3 * * *", get_daily(3, 44))
self.assertEquals("32 4 * * *", get_daily(4, 32))
self.assertEquals("12 5 * * *", get_daily(5, 12))
self.assertEquals("0 5 * * *", get_daily())
# Test gen_entry
self.assertEquals(
"#simple command\n5 * * * * root echo 12\n",
gen_entry(get_hourly(), "echo 12", "root", "simple command"))
self.assertEquals(
"#foo command\n8 12 * * * foo echo 1+1\n",
gen_entry(get_daily(12, 8), "echo 1+1", "foo", "foo command"))
# Test generation of ar cronjob for a specific tenant and report
expected = "#TENANT_A:report1 daily A/R\n"\
+ "5 5 * * * foo " + BATCH_AR + " -t TENANT_A -r report1 -d " + YESTERDAY + " -m upsert " + "-c "\
+ config.conf_path + "\n"
self.assertEquals(
expected,
gen_batch_ar(config, "TENANT_A", "report1", "daily", "foo",
"upsert"))
# Test generation of ar cronjob for a specific tenant and report
expected = "#TENANT_A:report1 hourly A/R\n"\
+ "5 * * * * " + BATCH_AR + " -t TENANT_A -r report1 -d " + TODAY + " -m insert " + "-c "\
+ config.conf_path + "\n"
self.assertEquals(
expected, gen_batch_ar(config, "TENANT_A", "report1", "hourly"))
# Test generation of ar cronjob for a specific tenant and report
expected = "#TENANT_B:report1 daily Status\n"\
+ "5 5 * * * foo " + BATCH_STATUS + " -t TENANT_B -r report1 -d " \
+ YESTERDAY + " -m upsert " + "-c "\
+ config.conf_path + "\n"
self.assertEquals(
expected,
gen_batch_status(config, "TENANT_B", "report1", "daily", "foo",
"upsert"))
# Test generation of status cronjob for a specific tenant and report
expected = "#TENANT_B:report1 hourly Status\n"\
+ "5 * * * * " + BATCH_STATUS + " -t TENANT_B -r report1 -d " + TODAY + " -m insert " + "-c "\
+ config.conf_path + "\n"
self.assertEquals(
expected, gen_batch_status(config, "TENANT_B", "report1",
"hourly"))
# Test generation of cronjobs for a tenant's reports
expected = "#Jobs for TENANT_A\n\n" \
+ "#TENANT_A:report1 hourly A/R\n" \
+ "5 * * * * " + BATCH_AR + " -t TENANT_A -r report1 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report1 daily A/R\n" \
+ "5 5 * * * " + BATCH_AR + " -t TENANT_A -r report1 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report1 hourly Status\n" \
+ "5 * * * * " + BATCH_STATUS + " -t TENANT_A -r report1 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report1 daily Status\n" \
+ "5 5 * * * " + BATCH_STATUS + " -t TENANT_A -r report1 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report2 hourly A/R\n" \
+ "5 * * * * " + BATCH_AR + " -t TENANT_A -r report2 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report2 daily A/R\n" \
+ "5 5 * * * " + BATCH_AR + " -t TENANT_A -r report2 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report2 hourly Status\n" \
+ "5 * * * * " + BATCH_STATUS + " -t TENANT_A -r report2 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_A:report2 daily Status\n" \
+ "5 5 * * * " + BATCH_STATUS + " -t TENANT_A -r report2 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "\n"
self.assertEquals(expected, gen_tenant_all(config, "TENANT_A"))
# Test generation of cronjobs for all tenants and all reports
expected2 = "#Jobs for TENANT_B\n\n" \
+ "#TENANT_B:report1 hourly A/R\n" \
+ "5 * * * * " + BATCH_AR + " -t TENANT_B -r report1 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report1 daily A/R\n" \
+ "5 5 * * * " + BATCH_AR + " -t TENANT_B -r report1 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report1 hourly Status\n" \
+ "5 * * * * " + BATCH_STATUS + " -t TENANT_B -r report1 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report1 daily Status\n" \
+ "5 5 * * * " + BATCH_STATUS + " -t TENANT_B -r report1 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report2 hourly A/R\n" \
+ "5 * * * * " + BATCH_AR + " -t TENANT_B -r report2 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report2 daily A/R\n" \
+ "5 5 * * * " + BATCH_AR + " -t TENANT_B -r report2 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report2 hourly Status\n" \
+ "5 * * * * " + BATCH_STATUS + " -t TENANT_B -r report2 -d " + TODAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "#TENANT_B:report2 daily Status\n" \
+ "5 5 * * * " + BATCH_STATUS + " -t TENANT_B -r report2 -d " + YESTERDAY + " -m insert -c " \
+ config.conf_path + "\n\n" \
+ "\n"
expected = expected + expected2
self.assertEquals(expected, gen_for_all(config))
def from_argo_conf(self, argo_dot_conf):
from argo_config import ArgoConfig
config_text = ArgoConfig.slurp_config_file(argo_dot_conf)
creds = ArgoConfig.get_rpc_creds(config_text, config.network)
return self(**creds)
"""
Set up defaults and read sentinel.conf
"""
import sys
import os
from argo_config import ArgoConfig
default_sentinel_config = os.path.normpath(
os.path.join(os.path.dirname(__file__), '../sentinel.conf'))
sentinel_config_file = os.environ.get('SENTINEL_CONFIG',
default_sentinel_config)
sentinel_cfg = ArgoConfig.tokenize(sentinel_config_file)
sentinel_version = "1.1.0"
min_argod_proto_version_with_sentinel_ping = 70207
def get_argo_conf():
home = os.environ.get('HOME')
argo_conf = os.path.join(home, ".argocore/argo.conf")
if sys.platform == 'darwin':
argo_conf = os.path.join(
home, "Library/Application Support/ArgoCore/argo.conf")
argo_conf = sentinel_cfg.get('argo_conf', argo_conf)
return argo_conf
def get_network():
return sentinel_cfg.get('network', 'mainnet')