def import_verbs(flux_dir):
global verbs
# make sure flux-verbs find their modules
#sys.path.insert(0, flux_dir)
# verbs directory
verb_dir = util.get_verbs_dir(flux_dir)
if os.path.isdir(verb_dir):
# get all .py file in verb dir
verb_files = glob.glob(verb_dir + '/*.py')
if verb_files:
for verb_file in verb_files:
verb_name = util.split_name(verb_file)
if not verb_name.startswith('__'):
if is_python3:
spec = importlib.util.spec_from_file_location(
verb_name, verb_file)
verb_mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(verb_mod)
else:
#FIXME: Python2
fp, pathname, desc = imp.find_module(
verb_name, [verb_dir])
verb_mod = imp.load_module(verb_name, fp, pathname,
desc)
verbs[verb_name] = verb_mod
else:
log.error('no verb was found in `verbs` dir: `%s`' % verb_dir)
else:
log.error('`verbs` directory not found: `%s`' % verb_dir)
def push(proj_dir):
'''runs a "git push" in the provided git repo'''
exists()
cmd = 'git push'
try:
subprocess.check_call(cmd, cwd=proj_dir, shell=True)
except subprocess.CalledProcessError as e:
log.error('"%s" failed with "%s"' % (cmd, e.returncode))
def add(proj_dir, update=False):
'''run a "git add ." in the provided git repo'''
exists()
cmd = 'git add ' + '-u' if update else '.'
try:
subprocess.check_call(cmd, cwd=proj_dir, shell=True)
except subprocess.CalledProcessError as e:
log.error('"%s" failed with "%s"' % (cmd, e.returncode))
def run(flux_dir, cmd_line):
if not sdk_exe_exists(flux_dir):
log.error(
'emsdk script not found at "%s", please run "./flux emsdk install"'
% get_sdk_exe(flux_dir))
sdk_exe = get_sdk_exe(flux_dir)
sdk_dir = get_sdk_dir(flux_dir)
cmd = '%s %s' % (sdk_exe, cmd_line)
subprocess.call(cmd, cwd=sdk_dir, shell=True)
def commit(proj_dir, msg, allow_empty=False):
'''runs a "git commit -m msg" in the provided git repo'''
exists()
cmd = 'git commit ' + ('--allow-empty '
if allow_empty else '') + ('-m "%s"' % msg)
try:
subprocess.check_call(cmd, cwd=proj_dir, shell=True)
except subprocess.CalledProcessError as e:
log.error('"git commit" failed with "%s"' % (e.returncode))
def exists(flux_dir=None, verbose=True):
'''test if "git" is in the path'''
try:
subprocess.check_output(['git', '--version'])
return True
except (OSError, subprocess.CalledProcessError):
if verbose:
log.error(
'"git" not found in path, please run and fix `flux diag tools`'
)
return False
def run(flux_dir, proj_dir, args):
"""show help text"""
if len(args) > 0:
# show help for one verb
verb_name = args[0]
if verb_name in verb.verbs:
verb.verbs[verb_name].usage()
else:
log.error('unknown verb `%s`' % verb_name)
else:
# show generic help
usage()
def update_submodules(proj_dir):
'''runs a 'git submodule sync --recursive' followed by a "git submodule update --recursive" on the provided git repo, unconditionally (it will *not* check for local changes)'''
exists()
try:
subprocess.call('git submodule sync --recursive',
cwd=proj_dir,
shell=True)
subprocess.call('git submodule update --recursive',
cwd=proj_dir,
shell=True)
except subprocess.CalledProcessError:
log.error('failed to call "git submodule sync/update"!')
def checkout(proj_dir, revision):
'''checkout a specific revision hash of a repository'''
exists()
try:
output = subprocess.check_output('git checkout %s' % revision,
cwd=proj_dir,
shell=True).decode('utf-8')
update_submodules(proj_dir)
return output.split(':')[0] != 'error'
except subprocess.CalledProcessError:
log.error('failed to call "git checkout %s"' % revision)
return None
def parse_config(flux_dir):
'''evaluate the ".emscripten" config file and returns key/value pairs of content'''
config = {}
config_path = get_sdk_config_file(flux_dir)
os.environ['EM_CONFIG'] = config_path
try:
with open(config_path, 'r') as f:
config_text = f.read()
exec(config_text, config)
except Exception as e:
log.error(
'error in evaluating ".emscripten" config file at "%s" with "%s"' %
(config_path, str(e)))
return config
def run(flux_dir, proj_dir, args):
if len(args) > 0 and 'all' not in args:
for name in args:
if name in ['all', 'verbs']:
list_verbs(flux_dir)
if name in ['all', 'targets']:
list_targets(flux_dir)
# check errors
if name not in ['all', 'verbs', 'targets']:
log.error('unknown list name: `%s` \t- run `help list` for more informations' % name)
else:
# list all
list_verbs(flux_dir)
list_targets(flux_dir)
def install(flux_dir, sdk_version):
res = False
sdk_dir = get_sdk_dir(flux_dir)
if sdk_dir_exists(flux_dir):
res = update(flux_dir)
else:
res = clone(flux_dir)
if not res:
log.error('failed to install or update "emscripten" SDK')
if sdk_version is None:
sdk_version = EMSDK_DEFAULT_VERSION
log.colored(log.YELLOW,
'=== installing "emscripten" tools for "%s"' % sdk_version)
run(flux_dir, 'install --shallow --disable-assertions %s' % sdk_version)
activate(flux_dir, sdk_version)
def branches(proj_dir):
'''get a dictionary with all local branch names of a git repo as keys, and their remote branch names as value'''
exists()
branches = {}
try:
output = subprocess.check_output('git branch -vv',
cwd=proj_dir,
shell=True).decode('utf-8')
lines = output.splitlines()
for line in lines:
tokens = line[2:].split()
local_branch = tokens[0]
if re.compile("^\[.*(:|\])$").match(tokens[2]):
remote_branch = tokens[2][1:-1]
branches[local_branch] = remote_branch
except subprocess.CalledProcessError:
log.error('failed to call "git branch -vv"')
return branches
def list_targets(flux_dir):
log.colored(log.YELLOW, '===== targets =====')
# verbs directory
target_dir = util.get_targets_dir(flux_dir)
if os.path.isdir(target_dir):
# get all .py file in verb dir
target_files = sorted(glob.glob(target_dir + '/*.yml'))
if target_files:
width = max(len(util.split_name(x)) for x in target_files)
for target_file in target_files:
target_name = util.split_name(target_file)
if not target_name.startswith('__'):
#if target_name in verb.verbs:
log.text(log.style(log.CYAN, target_name).ljust(width+15)) # target['about']
else:
log.error('no target was found in `targets` dir: `%s`' % target_dir)
else:
log.error('`targets` directory not found: `%s`' % target_dir)
def run(flux_dir, proj_dir, args):
if len(args) > 0:
cmd = args[0]
if cmd == 'install':
sdk_version = None
if len(args) > 1:
sdk_version = args[1]
emsdk.install(flux_dir, sdk_version)
elif cmd == 'activate':
sdk_version = None
if len(args) > 1:
sdk_version = args[1]
emsdk.activate(flux_dir, sdk_version)
else:
log.error('emscripten SDK version expected (run "./flux emsdk list")')
elif cmd == 'list':
emsdk.list(flux_dir)
elif cmd == 'uninstall':
emsdk.uninstall(flux_dir)
elif cmd == 'show-config':
log.info('emscripten root directory: ' + emsdk.get_emscripten_dir(flux_dir))
log.info('emscripten config file: ' + emsdk.get_sdk_config_file(flux_dir))
emsdk.show_config(flux_dir)
else:
log.error('unknown subcommand "%s" (run "./flux help emsdk")' % cmd)
else:
log.error('expected a subcommand. run "./flux help emsdk" for help')
def parse_local_input(self, path):
'''parse local inputs'''
# check if already imported
if path in self.imported:
return
self.imported[path] = True
# assets
i = path.find('@/')
if i != -1:
src = path[:i]
if not os.path.exists(src):
log.error('asset `%s` not found' % src)
return
self.asset_files.append(path)
return
# get name and extension
name, ext = os.path.splitext(path)
name = util.split_name(path)
# filter by name
if name == '*':
path_dir = os.path.dirname(os.path.abspath(path))
# check dir
if not os.path.isdir(path_dir) and not path_dir.endswith('**'):
log.fatal('input directory `%s` not found' % path_dir)
return
# filter by extension
if ext == '.h':
self.include_dirs.append(path_dir)
self.cc_opts.append('-I"%s"' % path_dir)
self.cxx_opts.append('-I"%s"' % path_dir)
elif ext in ['.hh', '.hxx', '.hpp', '.h++']:
self.include_dirs.append(path_dir)
self.cxx_opts.append('-I"%s"' % path_dir)
elif ext in ['.a', '.lib']:
if self.toolchain == 'msvc':
self.library_dirs.append(path_dir)
self.ld_opts.append('-LIBPATH:"%s"' % path_dir)
else:
self.library_dirs.append(path_dir)
self.ld_opts.append('-L"%s"' % path_dir)
elif ext == '.dylib':
if self.toolchain == 'gcc':
self.library_dirs.append(path_dir)
self.ld_opts.append('-L"%s"' % path_dir)
elif ext == '.framework':
if self.toolchain == 'gcc':
self.library_dirs.append(path_dir)
self.ld_opts.append('-F"%s"' % path_dir)
elif ext in [
'.c', '.cc', '.cxx', '.cpp', '.c++', '.m', '.mm', '.asm',
'.s'
]:
srcs = []
if sys.version_info[:2] >= (3, 5):
srcs = glob.glob(os.path.join(path_dir, name + ext),
recursive=True)
else:
# recursive
if path_dir.endswith('**'):
path_dir = path_dir[:-3]
for dirpaths, _, files in sorted(os.walk(path_dir)):
for fn in fnmatch.filter(files, name + ext):
srcs.append(os.path.join(dirpaths, fn))
else:
srcs = glob.glob(os.path.join(path_dir, name + ext))
if srcs:
for src in sorted(srcs):
if src in self.imported:
continue
self.imported[src] = True
self.src_files.append(os.path.abspath(src))
else:
log.fatal('unrecognized input file filter `%s%s`' %
(name, ext))
return
# filter by extension
if ext == '.framework':
if self.toolchain == 'gcc':
if not os.path.isdir(path):
log.fatal('input framework not found "%s"' % path)
return
elif ext == '.flux':
log.fatal(
'input: flux module must be defined as a "system" input: `<%s>`'
% path)
return
elif '$(TARGET_ARCH' not in path:
if os.path.isdir(path):
self.asset_files.append(path)
return
elif not os.path.isfile(path):
log.fatal('input file not found "%s"' % path)
if ext in ['.h', '.hh', '.hxx', '.hpp', '.h++']:
pass # ignore
elif ext in [
'.c', '.cc', '.cxx', '.cpp', '.c++', '.m', '.mm', '.asm', '.s'
]:
self.src_files.append(os.path.abspath(path))
elif ext == '.java':
if self.target == 'android':
self.java_files.append(path)
elif ext == '.o':
self.obj_files.append(path)
elif ext == '.lib':
self.lib_files.append(path)
elif ext == '.a':
if self.toolchain == 'gcc':
self.lib_files.append('"%s"' % path)
elif ext in ['.so', '.dylib']:
if self.toolchain == 'gcc':
self.lib_files.append('"%s"' % path)
self.bin_files.append(path)
elif ext in ['.exe', '.dll']:
if self.target == 'windows':
self.bin_files.append(path)
elif ext == '.framework':
if self.toolchain == 'gcc':
#TODO: Ugly!
#parse_local_input(extract_dir(path)+'/*.framework')
#parse_system_input(strip_dir(path))
#self.bin_files.append(path)
pass
else:
self.asset_files.append(path)
def run(flux_dir, proj_dir, args):
flux_dir = util.fix_path(flux_dir)
proj_dir = util.fix_path(proj_dir)
curr_dir = proj_dir
# check space in project path
if ' ' in proj_dir:
log.warn(
'whitespace in project path detected, `flux` will not work correctly'
)
# import verbs from flux dir
verb.import_verbs(flux_dir)
# parse args
if len(args) <= 1:
usage()
else:
name = args[1]
args = args[2:]
# run available verb
if name in verb.verbs:
verb.verbs[name].run(flux_dir, proj_dir, args)
# for development stuff only
elif name == 'dev':
pass
elif name == 'build':
#log.text('===== test console output begin =====')
#log.trace('trace message')
#log.debug('debug message')
#log.info('info message')
#log.warn('warn message')
#log.error('error message')
#log.text('===== test console output end =====')
# set build opts
opts = BuildOpts()
args = opts.parse_opts(proj_dir, args)
# get target datas
#TODO: add user custom targets
# set target opts
target = Target(flux_dir, opts)
if target.toolchain == 'msvc':
# check msvc install
if target.find_msvc():
# update env vars
os.environ['PATH'] = os.environ[
'FLUX_MSVC_PATH'] + ';' + os.environ['PATH']
os.environ['INCLUDE'] = os.environ['FLUX_MSVC_INCLUDE']
os.environ['LIB'] = os.environ['FLUX_MSVC_LIB']
else:
log.fatal('MSVC installation not found!')
elif target.target == 'emscripten':
from mods.sdks import emsdk
if emsdk.sdk_dir_exists(flux_dir):
os.environ[
'PATH'] += os.pathsep + emsdk.get_emscripten_dir(
flux_dir)
if opts.verbose >= 3:
log.info('python %s.%s.%s' % (sys.version_info[:3]))
log.info('os.environ:')
for ev in sorted(
filter(lambda x: x.startswith('FLUX_'), os.environ)):
log.info('- %s: %s' %
(ev, log.YELLOW + os.environ[ev] + log.DEFAULT))
# if not arg, set project in current dir
if not len(args):
args += [proj_dir]
# for each projets
for arg in args:
print(log.YELLOW + ("===== `%s`" % arg) + log.DEFAULT)
arg = util.fix_path(arg)
path = os.path.join(proj_dir, arg)
#print(path)
proj = Project(flux_dir, path, opts)
# change to project dir
os.chdir(os.path.abspath(path))
# clean output dir
if opts.clean:
if opts.verbose >= 1:
log.info("cleaning `%s`" % proj.out_dir)
proj.clean()
# create project intermediate dirs
proj.make_dirs()
# make Info.plist file
if proj.target == 'macos' and proj.apptype == 'window':
proj.make_info_plist()
# parse project file
proj.parse_inputs()
#print(proj)
# depends
if proj.build in ['app']: # app only?
if len(proj.flux_libs):
sys_libs = []
for lib in proj.flux_libs:
# load dep project file
dep_dir = os.path.join(
util.get_workspace_dir(flux_dir), lib)
dep = Project(flux_dir, dep_dir, opts, True)
# module or library dep only
if dep.build in ['mod', 'lib']:
# change dir to dep dir
cd = os.getcwd()
os.chdir(os.path.abspath(dep_dir))
# parse dep project file
dep.parse_inputs()
# files stamp
out_file_stamp = None
gen_file_stamp = None
# check if module archive exists
if not os.path.exists(dep.out_file):
if not os.path.exists(dep.gen_file):
#create dep intermediate dirs
dep.make_dirs()
# generate ninja file for dep
if opts.verbose >= 1:
log.info('generate `%s`' %
dep.gen_file)
with open(dep.gen_file, 'w') as out:
dep.gen_ninja(out, opts, target)
else:
#TODO: regenerate ninja if 'dep.flux_file' is most recent than 'dep.out_file'
pass
# add 'dep.gen_file' to project
proj.ninja_files.append(dep.gen_file)
# build dep module
if os.path.exists(dep.gen_file):
if opts.verbose >= 1:
log.info(
'building `%s` %s' %
(dep.base_dir, log.BLUE +
'(dependency)' + log.DEFAULT))
dep.build_ninja(verbose=opts.verbose >= 3)
# add dep include dirs (abspath)
for inc in dep.include_dirs:
proj.cc_opts.append('-I"%s"' % inc)
proj.cxx_opts.append('-I"%s"' % inc)
# add dep module archive
proj.lib_files.append('"%s"' % dep.out_file)
# append system libs
for lib in dep.lib_files:
if lib not in sys_libs:
sys_libs.append(lib)
# return to main project
os.chdir(cd)
# add dep system libs at end
for lib in sys_libs:
if lib not in proj.lib_files:
proj.lib_files.append(lib)
# set rules vars
#rule_vars = target.get_rule_vars(proj)
# generate ninja in proj.out_dir
if opts.verbose >= 1:
log.info('generate `%s`' % proj.gen_file)
with open(proj.gen_file, 'w') as out: #StringIO()
proj.gen_ninja(out, opts, target)
# build project from ninja
if os.path.exists(proj.gen_file):
if opts.verbose >= 1:
log.info('building `%s`' % proj.base_dir)
proj.build_ninja(verbose=opts.verbose >= 3)
# copy assets files
proj.copy_assets(proj.asset_dir)
proj.copy_binaries(proj.out_dir)
# return to start dir
os.chdir(curr_dir)
else:
log.error('unknown verb `%s`' % name)