def get_project_file_names(name):
try:
pwd = os.getcwd()
main_dir = pwd if name == '.' else os.path.join(
pwd,
name.lstrip('/').rstrip('/'))
project_name = os.path.basename(main_dir)
config_dir = Utility.CONSTS['config_dir']
config_file = Utility.project_config(project_name)
sub_files = ['.afctl_project', '.gitignore', 'requirements.txt']
sub_dirs = [
project_name, 'deployments', 'migrations', 'plugins', 'tests'
]
project_dirs = ['dags', 'commons']
return {
'main_dir': main_dir,
'project_name': project_name,
'config_dir': config_dir,
'config_file': config_file,
'sub_files': sub_files,
'sub_dirs': sub_dirs,
'project_dirs': project_dirs
}
except Exception as e:
raise AfctlParserException(e)
def act_on_configs(cls, args, project_name):
try:
# Setting global values.
if args.type == "global":
origin = args.o
token = args.t
version = args.v
if args.o is None and args.t is None and args.v is None:
origin = input("Git origin for deployment : ")
token = input("Personal access token : ")
version = input("Input airflow version : ")
if origin != '' and origin is not None:
Utility.update_config(
project_name, {'global': {
'git': {
'origin': origin
}
}})
if token != '' and token is not None:
Utility.update_config(
project_name,
{'global': {
'git': {
'access-token': token
}
}})
if version != '' and version is not None:
Utility.update_config(
project_name, {'global': {
'airflow_version': version
}})
# If adding or updating configs.
elif args.type == 'add' or args.type == 'update':
if args.d is None:
cls.parser.error(
colored(
"-d argument is required. Check usage. Run 'afctl config -h'",
'red'))
# Sanitize values.
configs, flag, msg = DeploymentConfig.validate_configs(args)
if flag:
cls.parser.error(colored(msg, 'red'))
else:
if args.type == 'update':
Utility.update_config(project_name, configs)
if args.type == 'add':
Utility.add_configs(['deployment', args.d],
project_name, configs)
# Showing configs
elif args.type == 'show':
Utility.print_file(Utility.project_config(project_name))
except Exception as e:
AfctlParserException(e)
def deploy_project(cls, args, config_file):
try:
if args.n is None:
return True, "-n is required. Check usage."
with open(Utility.project_config(config_file)) as file:
config = yaml.full_load(file)
project = config_file
origin = config['global']['git']['origin']
token = config['global']['git']['access-token']
if origin is None or origin == '':
return True, "Origin is not set for the project. Run 'afctl config global -o <origin>'"
params = QuboleUtils.generate_configs(config, args)
latest_commit_on_remote = QuboleUtils.fetch_latest_commit(
origin, params['branch'])
if token is None or token == '':
print(
"No personal access token found. The repository should be public."
)
if latest_commit_on_remote is None:
return True, "Unable to read latest commit on origin. Please make sure the current branch is present on origin."
print("Latest commit of {} on origin {} found.".format(
params['branch'], origin))
print("Deploying commit : {} on Qubole".format(
latest_commit_on_remote))
if token is not None and token != "":
origin = QuboleUtils.create_private_repo_url(origin, token)
qds_command = QuboleUtils.get_shell_command(
project, origin, params['branch'], latest_commit_on_remote)
command = QuboleUtils.run_qds_command(params['env'],
params['cluster'],
params['token'], qds_command)
if command.status != 'done':
return True, "Deployment failed on Qubole"
return False, ""
except Exception as e:
raise AfctlDeploymentException(e)
def validate_project(cls):
try:
project_name = None
project_path = None
pwd = os.getcwd()
# If any parent of pwd contains .afctl_project. If so then it should be the project.
project = Utility.find_project(pwd)
if project is None:
# Could not find .afctl_project
cls.parser.error(
colored("{} is not an afctl project.".format(pwd), 'red'))
else:
# Check is the dir containing .afctl_project has a config file
project_name = project[0]
project_path = project[1]
if not os.path.exists(Utility.project_config(project_name)):
cls.parser.error(
colored(
"Config file does not exists for {}".format(
project_name), 'red'))
return project_name, project_path
except Exception as e:
raise AfctlParserException(e)
def deploy_project(cls, args, config_file):
try:
print("Deploying afctl project to local")
with open(Utility.project_config(config_file)) as file:
config = yaml.full_load(file)
val = subprocess.call(['docker', 'info'])
if val != 0:
return True, "Docker is not running. Please start docker."
if args.d:
os.system("docker-compose -f {} up -d".format(
config['deployment']['local']['compose']))
else:
os.system("docker-compose -f {} up ".format(
config['deployment']['local']['compose']))
return False, ""
except Exception as e:
raise AfctlDeploymentException(e)
def test_return_project_config_file(self):
project = "test_project"
expected_path = os.path.join(PROJECT_CONFIG_DIR, project) + ".yml"
path = Utility.project_config(project)
assert path == expected_path