def __GetTimeOutForTask(cls, task):
"""Returns the timeout for the task.
Args:
task: string: The task for which the timeout should be prepared.
Returns:
int: The timeout in seconds.
"""
timeout = FileUtils.FileContents(task + '.timeout')
if not timeout:
timeout = FileUtils.FileContents(os.path.join(PipelineUtils.TaskDirName(task), 'timeout'))
if not timeout: return Flags.ARGS.timeout
timeout = re.sub('\s*', '', timeout)
timeout_parts = re.split('(\d+)', timeout)
if len(timeout_parts) < 3:
TermColor.Warning('Ignoring invalid timeout [%s] for task: %s' % (timeout, task))
return Flags.ARGS.timeout
timeout = float(timeout_parts[1])
annotation = timeout_parts[2]
if not annotation: return timeout
elif annotation == 'd': timeout *= 86400
elif annotation == 'h': timeout *= 3600
elif annotation == 'm': timeout *= 60
elif annotation == 'ms': timeout *= 0.001
elif annotation == 'us': timeout *= 0.000001
return timeout
def __init__(self):
"""Initialize the singleton instance."""
self._id = Flags.ARGS.id
self._pipeline_date = Flags.ARGS.date
# Set the src root.
self._pipeline_base_dir = FileUtils.GetAbsPathForFile(Flags.ARGS.root)
if not os.path.isdir(self._pipeline_base_dir):
TermColor.Fatal('Invalid Root directory: %s' % Flags.ARGS.root)
# Set the pipeline specific binary directory, if specified
self._pipeline_bin_dir = ''
if Flags.ARGS.bin_root:
self._pipeline_bin_dir = FileUtils.GetAbsPathForFile(
Flags.ARGS.bin_root)
# Set the pipeline utilities directory
self._pipeline_utils_dir = FileUtils.GetAbsPathForFile(
Flags.ARGS.utils_root)
# Create all necessary directories.
self._pipeline_output_dir = ''
self._pipeline_log_dir = ''
self._pipeline_publish_dir = Flags.ARGS.publish_root
self._subdirs = {}
self.__CreateInitialSubDirs()
self.PrintConfig()
def _WriteOutDirsStatus(cls, out_dirs_status):
"""Writes the status for each of the dirs in the dict.
Args:
out_dirs_status: dict {string, EXITCODE}: Dict of dir -> exit status.
"""
for k, v in out_dirs_status.items():
FileUtils.RemoveFiles([os.path.join(k, x) for x in Runner.EXITCODE_FILE.values()])
status_file = Runner.EXITCODE_FILE.get(v, '')
if not status_file: continue
FileUtils.CreateFileWithData(os.path.join(k, status_file))
def WriteMakefile(cls, specs, makefile):
"""Writes the auto make file for the given spec.
Args:
specs: List of dict of type {target, target_type, src, urls, ...}.
Each dict contains everything needed to build 'target'
makefile: The (auto) makefile to generate.
"""
f = open(makefile, 'w')
index = 0
for item in specs:
index += 1
target = item['_target']
target_bin = FileUtils.GetBinPathForFile(target)
f.write('\n# Srcs for %s\n' % target)
f.write('NGE2E_SRC_%d = %s\n' %
(index, str.join('\\\n ', item.get('src', set()))))
hostname = socket.gethostname()
# read in the template and insert the proper javascript
tmpl_f = open(Flags.ARGS.nge2e_template, 'r')
tmpl_js = cls.GenerateTemplateJs(item.get('src'), item.get('deps'))
tmpl = tmpl_f.read()
test_html_content = tmpl.replace(Flags.ARGS.nge2e_replace_str, tmpl_js)
timeout = item.get('timeout', Flags.ARGS.nge2e_timeout)
# Write the target.
f.write('\n%s' % target)
f.write(': $(NGE2E_SRC_%d)\n' % (index))
type = item.get('_type', 'invalid')
if type == 'nge2e_test':
# test name
name = item['name']
# write the test html
test_html_fn = os.path.join(FileUtils.GetWebTestHtmlDir(), '%s.html' % name)
test_html_f = open(test_html_fn, 'w')
test_html_f.write(test_html_content)
# the test url
test_url = 'https://%s%s/%s.html' % (hostname, FileUtils.GetWebTestHtmlUrlPath(), name)
f.write('\t@echo "Creating Test for %s "\n' % target)
f.write('\t@mkdir -p $(dir %s)\n' % target_bin)
f.write('\t@echo "# ng Unit Test for %s" > %s\n' % (target, target_bin))
f.write('\t@echo \'%s %s "%d" \' >> %s\n' %
(cls.PhantomJSCmd(), test_url, timeout, target_bin))
f.write('\tchmod 754 %s\n' % target_bin)
f.write('\t@ln -s -f %s $(BINDIR)/$(notdir $@)\n' % target_bin)
f.write('\t@mkdir -p $(dir %s)\n' % test_html_fn)
f.write('\t@\n')
f.write('\t@echo "Created: %s"\n' % target_bin)
f.write('\t@echo "Finished: $@"\n\n')
f.write('\n\n')
f.close()
def ResetMakeFiles(self):
"""Resets the main and auto make files."""
FileUtils.CreateFileWithData(self.__makefile_name)
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('cc'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('js'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('ng'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('nge2e'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('pkg'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('pkg_bin'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('pkg_sys'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('py'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('swig'))
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
all_tasks = []
dirs_to_publish = set()
publish_dir_to_task_map = {}
for set_tasks in tasks.values():
for task in set_tasks:
all_tasks += [task]
publish_dir = PipelineUtils.GetPublishDirForTask(task)
if not publish_dir: continue
dirs_to_publish |= set([publish_dir])
publish_dir_to_task_map[publish_dir] = (publish_dir_to_task_map.get(publish_dir, []) +
[publish_dir])
# Check if there are any directories to publish.
if not dirs_to_publish:
TermColor.Error('Did not find any dirs to publish. Do not forget to specify publish root '
'using --publish_root')
return ([], all_tasks)
# Run all the copy tasks.
successful_dirs = []; failed_dirs = []
for dir in dirs_to_publish:
publish_dir = cls._GetActualPublishDir(dir)
if not publish_dir:
failed_dirs += [publish_dir]
continue
(parent, name) = os.path.split(publish_dir)
TermColor.Info('Making current: %s' % publish_dir)
with FileUtils.PushDir(parent):
FileUtils.CreateLink('current', name)
successful_dirs += [publish_dir]
# Get the reverse mapping from dirs to tasks.
successful_tasks = []; failed_tasks = []
for i in successful_dirs:
successful_tasks += publish_dir_to_task_map.get(i, [])
for i in failed_dirs:
failed_tasks += publish_dir_to_task_map.get(i, [])
return (successful_tasks, failed_tasks)
def read_js_version(javascript_version_file=None):
'''Read the JS version stamp attached when building the production JS
bundles to improve frontend error reporting. If no version file was added,
we do not support versioning for this deployment.
'''
javascript_version_file = javascript_version_file or FileUtils.GetAbsPathForFile(
'web/public/build/version.txt')
# Only production has JS versions. If we are in production but the version
# file does not exist, FileUtils will not resolve the absolute path and
# will return None. Some deployments do not support versioning.
if not IS_PRODUCTION or not javascript_version_file:
return ''
return FileUtils.FileContents(javascript_version_file).strip()
def equal_dir_content(dir_a, dir_b):
'''
Shallow file check. Solely compares if all filenames in both directories
are equivalent.
NOTE(vinh): The name equal_dir_content could be interpreted as 'exact'
files, do we also check not just for name but for file hash as well? Or
is that doing too much?
'''
_validate_dir_path(dir_a)
_validate_dir_path(dir_b)
dir_a_files = set(
os.path.relpath(f, dir_a) for f in FileUtils.GetFilesInDir(dir_a))
dir_b_files = set(
os.path.relpath(f, dir_b) for f in FileUtils.GetFilesInDir(dir_b))
return dir_a_files == dir_b_files
def WorkHorse(cls, tasks):
"""Runs the workhorse for the command.
Args:
tasks: OrderedDict {int, set(string)}: Dict from priority to set of tasks to execute at the
priority. Note: the dict is ordered by priority.
Return:
(list, list): Returns a tuple of list in the form
(successful_tasks, failed_tasks) specifying tasks that succeeded and
ones that failed.
"""
all_tasks = []
dirs_to_import = {}
dir_to_task_map = {}
for set_tasks in tasks.values():
for task in set_tasks:
all_tasks += [task]
out_dir = PipelineUtils.GetOutDirForTask(task)
publish_dir = PipelineUtils.GetPublishCurrentDirForTask(task)
if not out_dir or not publish_dir: continue
dirs_to_import[publish_dir] = out_dir
dir_to_task_map[publish_dir] = (dir_to_task_map.get(publish_dir, []) + [publish_dir])
# Check if there are any directories to publish.
if not dirs_to_import:
TermColor.Error('Did not find any dirs to import. Do not forget to specify publish root '
'using --publish_root')
return ([], all_tasks)
# Create all the target dirs to import to.
for dir in dirs_to_import.values():
FileUtils.MakeDirs(dir)
# Run all the copy tasks.
successful_dirs = []; failed_dirs = []
args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeTask'),
list(dirs_to_import), list(dirs_to_import.values()))
dir_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size)
if not dir_res:
TermColor.Error('Could not process: %s' % all_tasks)
return ([], all_tasks)
for (res, dir) in dir_res:
if res == Importer.EXITCODE['SUCCESS']:
successful_dirs += [dir]
elif res == Importer.EXITCODE['FAILURE']:
failed_dirs += [dir]
else:
TermColor.Fatal('Invalid return %d code for %s' % (res, dir))
# Get the reverse mapping from dirs to tasks.
successful_tasks = []; failed_tasks = []
for i in successful_dirs:
successful_tasks += dir_to_task_map.get(i, [])
for i in failed_dirs:
failed_tasks += dir_to_task_map.get(i, [])
return (successful_tasks, failed_tasks)
def WriteMakefile(cls, spec, makefile):
src_root = FileUtils.GetSrcRoot()
name = spec['name']
with open(makefile, 'w') as f:
interface_files = {
x
for x in spec.get('src', set()) if x.endswith('.i')
}
for interface_file in interface_files:
wrapper_file = cls.__GetWrapperFileName(interface_file)
gen_out_dir = cls.__GetGenModuleDir(interface_file)
target_lib = cls.__GetLibFileName(interface_file, name)
target_lib_link = os.path.join(
gen_out_dir,
'_%s.so' % cls.__GetGenModuleName(interface_file))
#target_lib = target_lib.replace('.so', '_swig.so') # TODO(KK) fix this hack
assert gen_out_dir.startswith(
src_root) # otherwise we will loop forever
print("""
%(wrapper_file)s: %(interface_file)s
\t@mkdir -p %(gen_out_dir)s
\t@ln -s -f %(target_lib)s %(target_lib_link)s
\t@swig -c++ -python -o $@ -outdir %(gen_out_dir)s $<
\t@p=%(gen_out_dir)s; while [[ $$p != $(SRCROOT) ]]; do\
touch $$p/__init__.py; \
p=`dirname $$p`; \
done
""" % locals(),
file=f)
def __init__(self, debug=False, make_dir=None):
"""Generates the default make files.
Args:
make_dir: string: The directory in which make files are created.
cleanup: bool: Cleanup at the end.
"""
if not make_dir:
make_dir = os.path.join(FileUtils.GetBinDir(), '__build_files__')
self.__make_dir = make_dir
self.__debug = debug
# Create the dir if not already present.
if not os.path.exists(make_dir):
os.makedirs(make_dir)
(makefile_fd,
self.__makefile_name) = tempfile.mkstemp(prefix='makefile_',
suffix='.main.mak',
dir=make_dir)
# Detect supported compilers
self.__gcc_supported = self._CompilerSupported('gcc')
self.__clang_supported = self._CompilerSupported('clang')
# Generate dummy files.
self.ResetMakeFiles()
def WorkHorse(cls, rules):
"""Runs the workhorse for the command.
Args:
rules: list: List of rules to be handled.
Return:
(list, list): Returns a tuple of list in the form
(successful_rules, failed_rules) specifying rules that succeeded and
ones that failed.
"""
(successful_build, failed_build) = Builder.WorkHorse(rules)
# All our binaries assume they will be run from the source root.
os.chdir(FileUtils.GetSrcRoot())
pipe_output = len(successful_build) > 1
args = zip(itertools.repeat(cls), itertools.repeat('_RunSingeRule'),
successful_build, itertools.repeat(pipe_output))
rule_res = ExecUtils.ExecuteParallel(args, Flags.ARGS.pool_size)
successful_run = []; failed_run = []
for (res, rule) in rule_res:
if res == 1:
successful_run += [rule]
elif res == -1:
failed_run += [rule]
return (successful_run, failed_build + failed_run)
def __GetEnvVarsForTask(cls, task):
"""Returns the env vars for the task.
Args:
task: string: The task for which the envvar should be prepared.
Returns:
dict {string, string}: The dictionary of IDS to values.
"""
rel_path = PipelineUtils.GetTaskOutputRelativeDir(task)
vars = {}
for k, v in PipelineConfig.Instance().GetAllSubDirsForPath(rel_path).items():
vars[k] = v
prev_dir = FileUtils.GetPreviousDatedDir(v)
if not prev_dir: prev_dir = v
vars[k + '_PREV'] = prev_dir
vars.update(PipelineConfig.Instance().GetAllENVVars())
# Check if the task is critical or not.
task_options = cls.__GetTaskOptions(task)
if task_options[Runner.TASK_OPTIONS['ABORT_FAIL']]:
vars['PIPELINE_TASK_ABORT_FAIL'] = '1'
if task_options[Runner.TASK_OPTIONS['ALLOW_FAIL']]:
vars['PIPELINE_TASK_ALLOW_FAIL'] = '1'
return vars
def get_username_and_password(service,
section=None,
dir='static_data/push/auto/credentials'):
"""
Parse config file named `service`.cfg in dir.
Args:
service (string): Name of service to get credentials for. This should match
a file in dir (with .cfg extension)
section (string): section of config file to use. Defaults to same as
`service`
dir (stirng): directory (relative to source root) to look in for config file.
Returns:
(username, password) for the requested service
Raises:
ValueError if file does not exist
ConfigParser.NoSectionError if section does not exist
ConfigParser.NoOptionError if username or password is
not in the requested section
"""
config_parser = ConfigParser()
files_read = config_parser.read(
os.path.join(FileUtils.GetSrcRoot(), dir, '%s.cfg' % service))
if not files_read:
raise ValueError('No config file found for %s in %s' % (service, dir))
username = config_parser.get(section or service, 'username')
password = config_parser.get(section or service, 'password')
return username, password
def __get_cookie_salt():
CONFIG_FILE = 'static_data/push/auto/credentials/web.cfg'
config_parser = ConfigParser()
files_read = config_parser.read(
os.path.join(FileUtils.GetSrcRoot(), CONFIG_FILE))
if not files_read:
raise ValueError('config file not found: %s' % CONFIG_FILE)
return config_parser.get('web', 'user_cookie_salt')
def _ComputeTasks(cls, targets, ignore_list=[]):
"""Computes the tasks to be evaluate given the input targets.
Args:
targets: list: List of input targets.
ignore_list: list: List of strings to ignore.
Return:
dict{int, set(string)}: Dict from priority to set of tasks to execute at the priority.
"""
# First create a simple task list of priority string to task.
# Once all the tasks have been collected, then sort them to create an actual priority order.
tasks = {}
ignore_list += ['timeout']
for target in targets:
ignore = FileUtils.IgnorePath(target, ignore_list)
if ignore:
TermColor.Warning(
'Ignored target %s as anything with [%s] is ignored.' %
(target, ignore))
continue
recurse = False
if os.path.basename(target) == '...':
target = os.path.dirname(target)
if not target:
target = FileUtils.GetAbsPathForFile(os.getcwd())
if target.find(PipelineConfig.Instance().pipeline_base_dir(
)) != 0:
target = PipelineConfig.Instance().pipeline_base_dir()
recurse = True
abs_target = FileUtils.GetAbsPathForFile(target)
if not abs_target:
TermColor.Warning('[%s] is not a valid path' % (target))
continue
if os.path.isfile(abs_target):
cls.__AddFileToTasks(tasks, abs_target)
elif os.path.isdir(abs_target):
targets += FileUtils.GetFilesInDir(abs_target, recurse,
ignore_list)
else:
TermColor.Warning('[%s] is not supported' % (abs_target))
continue
return cls.__MergeTasks(tasks)
def WorkHorse(cls, rules):
"""Runs the workhorse for the command.
Args:
rules: list: List of rules to be handled.
Return:
(list, list): Returns a tuple of list in the form
(successful_rules, failed_rules) specifying rules that succeeded and
ones that failed.
"""
# Create the user friendly link to bin dir if it doesn't already exist.
FileUtils.CreateLink(FileUtils.GetEDir(), FileUtils.GetBinDir())
# Create the link to the web test directory if it exists
if os.path.exists(FileUtils.GetWebTestHtmlLink()):
FileUtils.CreateLink(FileUtils.GetWebTestHtmlLink(),
FileUtils.GetWebTestHtmlDir())
gen_makefile = GenMakefile(Flags.ARGS.debug)
gen_makefile.GenMainMakeFile()
(success_genmake,
failed_genmake) = gen_makefile.GenAutoMakeFileFromRules(
rules, Flags.ARGS.allowed_rule_types)
(success_make,
failed_make) = cls._MakeRules(success_genmake,
gen_makefile.GetMakeFileName())
return (success_make, failed_genmake + failed_make)
def __CreateInitialSubDirs(self):
"""Creates all necessary directories."""
# Check if we have to create any output directory
if not Flags.ARGS.out_dirs and Flags.ARGS.nolog_output: return
# Create the user friendly link to pipeline dir if it doesn't already exist.
FileUtils.CreateLink(FileUtils.GetPipelineLinkDir(),
FileUtils.GetPipelineDir())
# Create the output directory.
if Flags.ARGS.out_dirs or not Flags.ARGS.log_to_tmp:
self._pipeline_output_dir = os.path.join(
FileUtils.GetPipelineDir(), self._id)
FileUtils.MakeDirs(self._pipeline_output_dir)
# Create the log dir if required.
if not Flags.ARGS.nolog_output:
if Flags.ARGS.log_to_tmp:
log_root = os.path.join('/tmp', 'pipeline', self.pipeline_id())
else:
log_root = self._pipeline_output_dir
self._pipeline_log_dir = os.path.join(log_root, 'log',
self.pipeline_date())
FileUtils.MakeDirs(self._pipeline_log_dir)
# Create all the subdirs.
self._subdirs = {}
for i in Flags.ARGS.out_dirs:
subdir = os.path.join(self.pipeline_output_dir(), i)
FileUtils.MakeDirs(subdir)
self._subdirs['PIPELINE_' + i.upper() + '_DIR'] = subdir
def _create_app_internal(db, flask_config, instance_configuration,
zen_configuration_module):
# Create and configure the main Flask app. Perform any initializations that
# should happen before the site goes online.
# TODO(stephen): Is this the right place for this log to happen?
is_production = flask_config.IS_PRODUCTION
if is_production:
LOG.info('Zenysis is running in PRODUCTION mode')
app = Flask(__name__, static_folder='../public', static_url_path='')
# Misc app setup and settings.
app.secret_key = flask_config.SECRET_KEY
app.debug = not is_production
app.config.from_object(flask_config)
# Register the app with our db reference
db.init_app(app)
# Handle migrations before anyone uses the DB
migrations_directory = FileUtils.GetAbsPathForFile('web/server/migrations')
Migrate(app, db, migrations_directory)
# Only initialize the application if we are on the main processing thread.
# In debug mode when the app is started directly (not via gunicorn), the
# werkzeug reloader spawns a child process that gets restarted after a file
# change. The parent process is then not used.
if os.environ.get('SERVER_SOFTWARE', '').startswith('gunicorn') or (
app.debug and is_running_from_reloader()):
# NOTE(vedant): Not sure if this is the best way to accomplish this but it will at least
# prevent errors from being thrown during server start.
# NOTE(yitian): Initializing database seed values before app setup
# so that if new database values are added, app setup won't error.
initialize_database_seed_values(flask_config.SQLALCHEMY_DATABASE_URI)
with app.app_context():
_fail_if_schema_upgrade_needed()
_initialize_zenysis_module(app, zen_configuration_module)
_initialize_template_renderer(app)
_initialize_email_renderer(app)
_initialize_druid_context(app)
_initialize_geo_explorer(app)
_initialize_aqt_data(app)
_initialize_notification_service(app, instance_configuration)
_initialize_simple_cache(app)
_initialize_app(app, db, is_production)
# NOTE(stephen): The last thing we need to do when bootstrapping our app is
# dispose of the DB connection used to initialize the app. This connection
# *cannot* be shared across threads and disposing it prevents that from
# happening. After disposal, each thread will have a new connection
# established when they first access the DB.
with app.app_context():
db.engine.dispose()
return app
def __init__(self, rule_name):
"""initializes the state
Args:
rule_name (string) - the rule name
Raises:
UnsupportedRuleError: raises exception if the rule type is not yet
supported. add to the RULE_TYPES lists
"""
# Create the user friendly link to bin dir if it doesn't already exist.
FileUtils.CreateLink(FileUtils.GetEDir(), FileUtils.GetBinDir())
rule_name = Utils.RuleNormalizedName(rule_name)
Rules.LoadRule(rule_name)
self._rule = Rules.GetRule(rule_name)
if not self._rule['_type'] in Packager.RULE_PACKAGER:
err = 'Rule type %s not supported' % self._rule._type
TermColor.Error(err)
raise UnsupportedRuleError(err)
self._packager = Packager.RULE_PACKAGER[self._rule['_type']]
def make_package(cls, rule):
"""@override"""
name = rule['name']
if not 'rule' in rule or not rule['rule']:
err = 'no rule field for %s' % name
TermColor.Error(err)
raise Error(err)
if not 'ctrl' in rule or not rule['ctrl']:
err = 'no yaml ctrl field for %s' % name
TermColor.Error(err)
raise Error(err)
subprocess.check_call('flash build %s' % rule['rule'], shell=True)
# clean up the old package and create the working directory
workingdir = PkgUtils.create_working_dir(name)
# collect the files
files = {}
# the binary path
files[os.path.join(FileUtils.GetBinDir(), rule['rule'])] = \
os.path.basename(rule['rule'])
# the loop script
files[os.path.join(FileUtils.GetSrcRoot(),
Flags.ARGS.loop_script_path)] = 'loop'
# the control script
files[os.path.join(FileUtils.GetSrcRoot(),
Flags.ARGS.pkg_bin_ctrl_path)] = 'control'
# the yaml file
files[os.path.join(FileUtils.GetSrcRoot(),
rule['ctrl'])] = 'control.yaml'
# copy the files
for src, dest in files.items():
shutil.copy2(src, os.path.join(workingdir, dest))
# copy the shared files
CopyShared.copy(workingdir)
# import the package
packages = Packages(host=Flags.ARGS.pkg_host,
user=Flags.ARGS.pkg_user,
root=Flags.ARGS.pkg_repo)
# import the package
if Flags.ARGS.pkg_version_prefix:
return name, packages.f_import(workingdir, name,
Flags.ARGS.pkg_version_prefix)
else:
return name, packages.f_import(workingdir, name)
def __GetWrapperFileName(cls, src):
"""Returns the C++ wrapper file name for the src.
Args:
src: string: The interface file for which the output is generated.
Return:
string: The output file name.
"""
return FileUtils.GetBinPathForFile(src).replace('.i', '.swig.cc')
def TaskNormalizedName(cls, task):
"""Returns the normalized name for the task.
Args:
task: string: The task to normalize.
Return:
string: The Normalized name of the task.
"""
abs_path = FileUtils.GetAbsPathForFile(task)
if abs_path: return abs_path
return task
def CreateAllSubDirsForPath(self, path):
"""Creates all the subdirs for the given node.
Args:
node: string: The node for which the path needs to be created.
Returns:
dict {string, string}: The dictionary of SUBDIR IDS to actual paths.
"""
for k, v in self.GetAllSubDirsForPath(path).items():
FileUtils.MakeDirs(v)
def __GetGenModuleDir(cls, src):
"""Returns the python module name for the src.
Args:
src: string: The interface file for which the output is generated.
Return:
string: The directory name where the generated module will be outputted.
"""
return os.path.dirname(
src.replace(FileUtils.GetSrcRoot(), cls.GetSwigOutDir()))
def RuleRelativeName(cls, rule):
"""Returns the relative name for the rule w.r.t the src dir.
Args:
rule: string: The rule for which the relative name is required.
Return:
string: The relative name of the rule.
"""
if not rule: return None
return os.path.relpath(cls.RuleNormalizedName(rule),
FileUtils.GetSrcRoot())
def GetRulesFileForRule(cls, rule):
"""Returns the RULES file for the rule.
Args:
rule: string: The rule for which the file is needed.
Return:
string: The rules file if it exists.
"""
if not rule: return None
return FileUtils.GetAbsPathForFile(
os.path.join(os.path.dirname(rule), 'RULES'))
def RuleNormalizedName(cls, rule):
"""Returns the normalized name for the rule.
Args:
rule: string: The rule to normalize.
Return:
string: The Normalized name of the rule.
"""
if rule.find(FileUtils.GetSrcRoot()) == 0:
return os.path.normpath(rule)
rules_file = cls.GetRulesFileForRule(rule)
if rules_file:
return os.path.join(os.path.dirname(rules_file),
os.path.basename(rule))
# This does not have a rules file. Generally this happens for src files.
abs_path = FileUtils.GetAbsPathForFile(rule)
if abs_path: return abs_path
return rule
def __GetOutFileName(cls, src, out_suffix):
"""Returns the output file name for the src.
Args:
src: string: The src file for which the output is generated.
out_suffix: string: The suffix used to replace .proto in the src.
Return:
string: The output file name.
"""
return (src.replace(FileUtils.GetSrcRoot(), cls.GetProtoBufOutDir())
.replace('.proto', out_suffix))
def GenMainMakeFile(self):
"""Generates the main make file."""
f = open(self.GetMakeFileName(), 'w')
f.write('SRCROOT = %s\n' % FileUtils.GetSrcRoot())
f.write('BINDIR = %s\n' % FileUtils.GetBinDir())
f.write('AUTO_MAKEFILE_CC = %s\n' % self.GetAutoMakeFileName('cc'))
f.write('AUTO_MAKEFILE_JS = %s\n' % self.GetAutoMakeFileName('js'))
f.write('AUTO_MAKEFILE_NG = %s\n' % self.GetAutoMakeFileName('ng'))
f.write('AUTO_MAKEFILE_NGE2E = %s\n' %
self.GetAutoMakeFileName('nge2e'))
f.write('AUTO_MAKEFILE_PKG = %s\n' % self.GetAutoMakeFileName('pkg'))
f.write('AUTO_MAKEFILE_PKG_BIN = %s\n' %
self.GetAutoMakeFileName('pkg_bin'))
f.write('AUTO_MAKEFILE_PKG_SYS = %s\n' %
self.GetAutoMakeFileName('pkg_sys'))
f.write('AUTO_MAKEFILE_PY = %s\n' % self.GetAutoMakeFileName('py'))
f.write('AUTO_MAKEFILE_SWIG = %s\n' % self.GetAutoMakeFileName('swig'))
f.write('PROTOBUFDIR = %s\n' % ProtoRules.GetProtoBufBaseDir())
f.write('PROTOBUFOUTDIR = %s\n' % ProtoRules.GetProtoBufOutDir())
f.write('SWIGBUFOUTDIR = %s\n' % SwigRules.GetSwigOutDir())
f.write('\n')
makefile_template = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
self.GetMakeFileTemplate())
f.write(
'###############################################################\n'
)
f.write('#Template from: %s \n' % makefile_template)
f.write(
'###############################################################\n'
)
f.write(open(makefile_template).read())
f.write(
'\n###############################################################\n'
)
