def ResetMakeFiles(self):
"""Resets the main and auto make files."""
FileUtils.CreateFileWithData(self.__makefile_name)
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('cc'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('js'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('ng'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('nge2e'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('pkg'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('pkg_bin'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('pkg_sys'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('py'))
FileUtils.CreateFileWithData(self.GetAutoMakeFileName('swig'))
def _WriteOutDirsStatus(cls, out_dirs_status):
"""Writes the status for each of the dirs in the dict.
Args:
out_dirs_status: dict {string, EXITCODE}: Dict of dir -> exit status.
"""
for k, v in out_dirs_status.items():
FileUtils.RemoveFiles([os.path.join(k, x) for x in Runner.EXITCODE_FILE.values()])
status_file = Runner.EXITCODE_FILE.get(v, '')
if not status_file: continue
FileUtils.CreateFileWithData(os.path.join(k, status_file))
def main():
# Required flags
Flags.PARSER.add_argument(
'--data_files',
type=str,
required=True,
nargs='+',
help='Path to JSON data files to be ingested',
)
# Optional flags that override default values
Flags.PARSER.add_argument(
'--datasource_name',
type=str,
default='',
help='Optional datasource name. If unspecified, '
'one will be generated.',
)
Flags.PARSER.add_argument(
'--task_template_file',
type=str,
default='',
help='Optional indexing template to use',
)
Flags.PARSER.add_argument('--metrics_spec_file',
type=str,
default='',
help='Optional metrics spec to use')
Flags.PARSER.add_argument(
'--tuning_config_file',
type=str,
default='',
help='Optional task tuning config to use',
)
Flags.PARSER.add_argument(
'--task_hash_dir',
type=str,
default=DEFAULT_TASK_HASH_DIR,
help='Directory where indexing task hashes are '
'stored',
)
Flags.PARSER.add_argument(
'--output_task_id_file',
type=str,
default='',
help='File to store the indexing task ID in',
)
Flags.PARSER.add_argument(
'--force',
action='store_true',
default=False,
help='Force the datasource to be created even if '
'a datasource already exists with the same '
'data',
)
Flags.PARSER.add_argument(
'--dry_run',
action='store_true',
default=False,
help='Issue a "noop" indexing task and skip '
'building a new datasource',
)
Flags.PARSER.add_argument(
'--min_data_date',
type=str,
default=DEFAULT_MIN_DATA_DATE_STR,
help='Optional earliest data date string: YYYY-MM-DD',
)
Flags.PARSER.add_argument(
'--max_data_date',
type=str,
default=DEFAULT_MAX_DATA_DATE_STR,
help='Optional latest data date string: YYYY-MM-DD',
)
Flags.InitArgs()
# Create deterministic version number so that we can differentiate the
# current live datasources even if they have the same datasource name.
# NOTE(stephen): For some weird reason, this string version value has to
# resolve to a value less than the task "lock" version, which is the
# formatted timestamp that the druid indexing task actually began. This is
# dumb. https://github.com/druid-io/druid/pull/3559
version = TODAY.strftime('%Y-%m-%d.%H%M%S')
indexing_task = build_indexing_task(version)
indexing_task.print_overview()
print('')
(cur_datasource, cur_version) = get_current_datasource_for_site()
if (not Flags.ARGS.force and cur_datasource
and cur_version and not task_contains_new_data(
indexing_task, cur_datasource, cur_version)):
print('##### Skipping indexing since existing datasource '
'contains the same data specified in this task. #####')
print('##### Current datasource: %s #####' % cur_datasource)
print('##### Current version: %s #####' % cur_version)
# TODO(stephen): Switch to the log library so that we can specify
# a loglevel as a flag. Then I won't have to comment out potentially
# useful debug statements.
# print 'Current task hash:'
# print indexing_task.get_task_hash()
return 0
dry_run = Flags.ARGS.dry_run
task_id = run_task(indexing_task, dry_run)
if not task_id:
return 1
if not dry_run:
store_task_hash(indexing_task)
output_task_id_file = Flags.ARGS.output_task_id_file
if output_task_id_file:
FileUtils.CreateFileWithData(output_task_id_file, task_id)
print('Successfully started indexing task. Task ID: %s' % task_id)
return 0
def _RunSingeRule(cls, rule):
"""Runs a Single Rule.
Args:
rule: string: The rule to run.
Return:
(int, string): Returns a tuple of the result status and the rule.
The status is '1' for success, '0' for 'ignore', '-1' for fail.
"""
TermColor.Info('Generating dependencies for %s' %
Utils.RuleDisplayName(rule))
start = time.time()
gr = digraph.digraph()
gr.add_node(rule)
nodes = [rule]
while len(nodes):
node = nodes.pop(0)
# The rule has already been processed. We assume if the node has outgoing
# edges, the we already processed it.
if gr.node_order(node) > 0: continue
# Add the dependencies of the rule to the graph.
if not Rules.LoadRule(node) or not Rules.GetRule(node):
TermColor.Warning(
'Could not load dependency %s for target %s ' %
(Utils.RuleDisplayName(node), Utils.RuleDisplayName(rule)))
return (-1, rule)
node_data = Rules.GetRule(node)
for dep in node_data.get('dep', set()):
nodes += [dep]
# Add the dep to the graph.
if not gr.has_node(dep): gr.add_node(dep)
if not gr.has_edge([node, dep]): gr.add_edge([node, dep])
# Now we have the graph, lets render it.
try:
dt = dot.write(gr)
dt = dt.replace('"%s";' % rule, ('"%s" [style=filled];' % rule), 1)
dt = dt.replace(FileUtils.GetSrcRoot(), '')
depgrah_file_name = cls.__GetDepGraphFileNameForRule(rule)
if Flags.ARGS.mode == 'gv':
gvv = gv.readstring(dt)
gv.layout(gvv, 'dot')
gv.render(gvv, 'pdf', depgrah_file_name)
if not Flags.ARGS.quiet:
subprocess.call('gv %s &' % depgrah_file_name, shell=True)
elif Flags.ARGS.mode == 'text':
FileUtils.CreateFileWithData(depgrah_file_name, dt)
TermColor.Info(
'Generated dependency graph (%d nodes) for %s at %s \tTook %.2fs'
% (len(gr.nodes()), Utils.RuleDisplayName(rule),
depgrah_file_name, (time.time() - start)))
return (1, rule)
except Exception as e:
TermColor.Error('Failed to render %s. Error: %s' %
(Utils.RuleDisplayName(rule), e))
if type(e) == KeyboardInterrupt: raise e
return (-1, rule)
