def set_options(self, options):
for c in options:
key, value = c.split('=')
try:
setattr(self.options, key.strip(), value.strip())
except AttributeError:
fatal(D_SCRIPT, 'Invalid option: {0}={1}'.format(key, value))
def set_options(self, options):
for c in options:
key, value = c.split('=')
try:
setattr(self.options, key.strip(), value.strip())
except AttributeError:
fatal(D_SCRIPT, 'Invalid option: {0}={1}'.format(key, value))
def compile(self):
""" Compile script in the specified working directory. """
# Save active script instance and set this one as active
work_dir = self.output_directory
# Add nest path and path to script to Python module path to allow
# for importing modules outside of $PYTHONPATH
sys.path.insert(0, os.path.abspath(os.path.dirname(work_dir)))
# Load built-ins if specified on command line. If built-ins are
# not automatically loaded by the Script object, then the user must
# load them manually in their Weaver scripts using the standard
# Python import facilities.
if self.import_builtins:
self._import('abstraction', ABSTRACTIONS)
self._import('dataset', DATASETS)
self._import('function', FUNCTIONS)
self._import('nest', NESTS)
self._import('options', OPTIONS)
self._import('stack', STACKS)
# Execute nest
with Nest(work_dir, wrapper=self.engine_wrapper) as nest:
with self.options:
try:
self.function(*self.arguments)
nest.compile()
except Exception as e:
fatal(D_SCRIPT, 'Error compiling script: {0}'.format(e), print_traceback=True)
if self.execute_dag:
debug(D_SCRIPT, 'Executing generated DAG {0} with {1}'.format(
nest.dag_path, nest.path))
nest.execute(self.engine_arguments, exit_on_failure=True)
def make_function(func_name, *func_args):
try:
function = WORKBENCH_FUNCTIONS[func_name](*func_args)
except KeyError:
fatal(D_USER, 'Invalid function {0}'.format(func_name))
return function
def make_function(func_name, *func_args):
try:
function = WORKBENCH_FUNCTIONS[func_name](*func_args)
except KeyError:
fatal(D_USER, 'Invalid function {0}'.format(func_name))
return function
def __init__(self, args):
self.path = None
self.force = False # Ignore warnings
self.import_builtins = True # Load built-ins
self.output_directory = os.curdir # Where to create artifacts
self.start_time = time.time() # Record beginning of compiling
self.options = Options()
self.nested_abstractions = False
self.inline_tasks = 1
self.execute_dag = False
self.globals = {}
self.engine_wrapper = None
self.engine_arguments = None
self.include_symbols = False
self.normalize_paths = True
args = collections.deque(args)
while args:
arg = args.popleft()
try:
if arg.startswith('-'):
self.SCRIPT_OPTIONS_TABLE[arg](self, args)
else:
self.path = arg
self.arguments = list(args)
args.clear()
except (IndexError, KeyError):
fatal(D_SCRIPT, 'invalid command line option: {0}'.format(arg))
if self.normalize_paths:
self.output_directory = os.path.abspath(self.output_directory)
debug(D_SCRIPT, 'path = {0}'.format(self.path))
debug(D_SCRIPT, 'force = {0}'.format(self.force))
debug(D_SCRIPT,
'import_builtins = {0}'.format(self.import_builtins))
debug(D_SCRIPT,
'output_directory = {0}'.format(self.output_directory))
debug(D_SCRIPT, 'start_time = {0}'.format(self.start_time))
debug(D_SCRIPT, 'options = {0}'.format(self.options))
debug(D_SCRIPT,
'nested_abstractions = {0}'.format(self.nested_abstractions))
debug(D_SCRIPT, 'inline_tasks = {0}'.format(self.inline_tasks))
debug(D_SCRIPT, 'execute_dag = {0}'.format(self.execute_dag))
debug(D_SCRIPT,
'engine_wrapper = {0}'.format(self.engine_wrapper))
debug(D_SCRIPT,
'engine_arguments = {0}'.format(self.engine_arguments))
debug(D_SCRIPT,
'normalize_paths = {0}'.format(self.normalize_paths))
if self.path is None:
self.show_usage()
def __init__(self, args):
self.path = None
self.force = True # Ignore warnings
self.import_builtins = True # Load built-ins
self.output_directory = os.curdir # Where to create artifacts
self.start_time = time.time() # Record beginning of compiling
self.options = Options()
self.nested_abstractions = False
self.inline_tasks = 1
self.execute_dag = False
self.globals = {}
self.engine_wrapper = None
self.engine_arguments = None
self.include_symbols = False
self.normalize_paths = True
args = collections.deque(args)
while args:
arg = args.popleft()
try:
if arg.startswith('-'):
self.SCRIPT_OPTIONS_TABLE[arg](self, args)
else:
self.path = arg
self.arguments = list(args)
args.clear()
except (IndexError, KeyError):
fatal(D_SCRIPT, 'invalid command line option: {0}'.format(arg))
if self.normalize_paths:
self.output_directory = os.path.abspath(self.output_directory)
debug(D_SCRIPT, 'path = {0}'.format(self.path))
debug(D_SCRIPT, 'force = {0}'.format(self.force))
debug(D_SCRIPT, 'import_builtins = {0}'.format(self.import_builtins))
debug(D_SCRIPT, 'output_directory = {0}'.format(self.output_directory))
debug(D_SCRIPT, 'start_time = {0}'.format(self.start_time))
debug(D_SCRIPT, 'options = {0}'.format(self.options))
debug(D_SCRIPT, 'nested_abstractions = {0}'.format(self.nested_abstractions))
debug(D_SCRIPT, 'inline_tasks = {0}'.format(self.inline_tasks))
debug(D_SCRIPT, 'execute_dag = {0}'.format(self.execute_dag))
debug(D_SCRIPT, 'engine_wrapper = {0}'.format(self.engine_wrapper))
debug(D_SCRIPT, 'engine_arguments = {0}'.format(self.engine_arguments))
debug(D_SCRIPT, 'normalize_paths = {0}'.format(self.normalize_paths))
if self.path is None:
self.show_usage()
def _query(self, filters, **parameters):
cursor = None
try:
if self.db_conn is None:
self.connect()
try:
fields = parameters['fields']
except KeyError:
fields = self.db_fields
try:
limit = int(parameters['limit'])
except KeyError:
limit = None
try:
path = parameters['path']
except KeyError:
path = self.path
cursor = self.db_conn.cursor()
query = self.db_query_format.format(fields=','.join(fields),
table=self.db_table,
filters=' AND '.join(filters))
if limit:
query = '{0} LIMIT {1}'.format(query, limit)
debug(D_DATASET, 'Executing SQL query: {0}'.format(query))
cursor.execute(query)
for row in cursor.fetchall():
yield MakeFile(path(self, row), self.nest)
except Exception as e:
fatal(D_DATASET,
'Unable to perform SQL query: {0}'.format(e),
print_traceback=True)
finally:
if cursor:
cursor.close()
if not self.db_conn_keep_alive:
self.disconnect()
raise StopIteration
def _query(self, filters, **parameters):
cursor = None
try:
if self.db_conn is None:
self.connect()
try:
fields = parameters['fields']
except KeyError:
fields = self.db_fields
try:
limit = int(parameters['limit'])
except KeyError:
limit = None
try:
path = parameters['path']
except KeyError:
path = self.path
cursor = self.db_conn.cursor()
query = self.db_query_format.format(
fields = ','.join(fields),
table = self.db_table,
filters = ' AND '.join(filters))
if limit:
query = '{0} LIMIT {1}'.format(query, limit)
debug(D_DATASET, 'Executing SQL query: {0}'.format(query))
cursor.execute(query)
for row in cursor.fetchall():
yield MakeFile(path(self, row), self.nest)
except Exception as e:
fatal(D_DATASET, 'Unable to perform SQL query: {0}'.format(e), print_traceback=True)
finally:
if cursor:
cursor.close()
if not self.db_conn_keep_alive:
self.disconnect()
raise StopIteration
def __iter__(self):
# Generate the cache under any of the following conditions:
#
# 1. Cache file does not exist
# 2. Cache file exists, is older than compile start time, and we are
# forced to do so
debug(D_DATASET, 'Iterating on Dataset {0}'.format(self))
if os.path.exists(self.cache_path):
# If cache file is made after we started compiling, then it is
# valid, so don't bother generating.
if CurrentScript().start_time <= os.stat(self.cache_path).st_ctime:
debug(D_DATASET, 'Loading Dataset {0}'.format(self))
return (MakeFile(f.strip(), self.nest) \
for f in open(self.cache_path, 'r'))
message = 'Cache file {0} already exists'.format(self.cache_path)
if CurrentScript().force:
warn(D_DATASET, message)
else:
fatal(D_DATASET, message)
debug(D_DATASET, 'Generating Dataset {0}'.format(self))
return self._generate()
def __iter__(self):
# Generate the cache under any of the following conditions:
#
# 1. Cache file does not exist
# 2. Cache file exists, is older than compile start time, and we are
# forced to do so
debug(D_DATASET, 'Iterating on Dataset {0}'.format(self))
if os.path.exists(self.cache_path):
# If cache file is made after we started compiling, then it is
# valid, so don't bother generating.
if CurrentScript().start_time <= os.stat(self.cache_path).st_ctime:
debug(D_DATASET, 'Loading Dataset {0}'.format(self))
return (MakeFile(f.strip(), self.nest) \
for f in open(self.cache_path, 'r'))
message = 'Cache file {0} already exists'.format(self.cache_path)
if CurrentScript().force:
warn(D_DATASET, message)
else:
fatal(D_DATASET, message)
debug(D_DATASET, 'Generating Dataset {0}'.format(self))
return self._generate()
tasks = int(tasks)
bytes = int(bytes)
arguments = map(int, func_args)
function = make_function(func_name, *arguments)
inputs = []
for input in range(tasks):
inputs.append(generate_input_file(bytes))
Map(function, inputs, '{BASE_WOEXT}.output')
WORKBENCH_PATTERNS = {
'chained' : run_chained,
'concurrent': run_concurrent,
'fanout' : run_fanout,
'fanin' : run_fanin,
'map' : run_map,
}
# WorkBench Main Dispatch
Arguments = CurrentScript().arguments
try:
WORKBENCH_PATTERNS[Arguments[0]](*Arguments[1:])
except KeyError:
fatal(D_USER, 'Invalid pattern: {0}'.format(Arguments[0]), print_traceback=True)
except IndexError:
fatal(D_USER, 'No pattern specified', print_traceback=True)
tasks = int(tasks)
bytes = int(bytes)
arguments = map(int, func_args)
function = make_function(func_name, *arguments)
inputs = []
for input in range(tasks):
inputs.append(generate_input_file(bytes))
Map(function, inputs, '{BASE_WOEXT}.output')
WORKBENCH_PATTERNS = {
'chained' : run_chained,
'concurrent': run_concurrent,
'fanout' : run_fanout,
'fanin' : run_fanin,
'map' : run_map,
}
# WorkBench Main Dispatch
Arguments = CurrentScript().arguments
try:
WORKBENCH_PATTERNS[Arguments[0]](*Arguments[1:])
except KeyError:
fatal(D_USER, 'Invalid pattern: {0}'.format(Arguments[0]), print_traceback=True)
except IndexError:
fatal(D_USER, 'No pattern specified', print_traceback=True)