def setUp(self):
set_compmake_status(compmake_status_embedded)
use_filesystem('priorities')
set_namespace('priorities')
# make sure everything was clean
for key in storage.db.keys('*'):
storage.db.delete(key)
# clear the variable holding the result
TestOrder.order = []
def comp(command, *args, **kwargs):
''' Main method to define a computation.
Extra arguments:
:arg:job_id: sets the job id (respects job_prefix)
:arg:extra_dep: extra dependencies (not passed as arguments)
'''
if compmake.compmake_status == compmake_status_slave:
return None
# Check that this is a pickable function
try:
pickle.dumps(command)
except:
msg = ('Cannot pickle %r. Make sure it is not a lambda function or a '
'nested function. (This is a limitation of Python)' % command)
raise SerializationError(msg)
args = list(args) # args is a non iterable tuple
# Get job id from arguments
job_id_key = 'job_id'
if job_id_key in kwargs:
# make sure that command does not have itself a job_id key
#available = command.func_code.co_varnames
argspec = inspect.getargspec(command)
if job_id_key in argspec.args:
msg = ("You cannot define the job id in this way because 'job_id' "
"is already a parameter of this function.")
raise UserError(msg)
job_id = kwargs[job_id_key]
if job_prefix:
job_id = '%s-%s' % (job_prefix, job_id)
del kwargs[job_id_key]
if job_id in jobs_defined_in_this_session:
raise UserError('Job %r already defined.' % job_id)
else:
job_id = generate_job_id(command)
jobs_defined_in_this_session.add(job_id)
if 'extra_dep' in kwargs:
extra_dep = collect_dependencies(kwargs['extra_dep'])
del kwargs['extra_dep']
else:
extra_dep = set()
children = collect_dependencies([args, kwargs])
children.update(extra_dep)
all_args = (command, args, kwargs)
command_desc = command.__name__
c = Job(job_id=job_id, children=list(children), command_desc=command_desc)
# TODO: check for loops
for child in children:
child_comp = get_job(child)
if not job_id in child_comp.parents:
child_comp.parents.append(job_id)
set_job(child, child_comp)
if job_exists(job_id):
# OK, this is going to be black magic.
# We want to load the previous job definition,
# however, by unpickling(), it will start
# __import__()ing the modules, perhaps
# even the one that is calling us.
# What happens, then is that it will try to
# add another time this computation recursively.
# What we do, is that we temporarely switch to
# slave mode, so that recursive calls to comp()
# are disabled.
if compmake_config.check_params: #@UndefinedVariable
old_status = compmake_status
set_compmake_status(compmake_status_slave)
old_computation = get_job(job_id)
set_compmake_status(old_status)
assert False, 'update for job_args'
same, reason = old_computation.same_computation(c)
if not same:
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-redefined', job_id=job_id , reason=reason)
# XXX TODO clean the cache
else:
publish('job-already-defined', job_id=job_id)
else:
# We assume everything's ok
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-defined', job_id=job_id)
else:
set_job(job_id, c)
set_job_args(job_id, all_args)
publish('job-defined', job_id=job_id)
assert job_exists(job_id)
assert job_args_exists(job_id)
return Promise(job_id)
def setUp(self):
set_compmake_status(compmake_status_embedded)
use_filesystem('unpickable')
set_namespace('unpickable')
for key in storage.db.keys('*'):
storage.db.delete(key)
def main():
setproctitle('compmake')
parser = OptionParser(version=version)
parser.add_option("--slave", action="store_true",
default=False, dest="slave",
help="[internal] Runs compmake in slave mode.")
parser.add_option("--redis_events", action="store_true",
default=False, dest="redis_events",
help="[internal] Relays events using Redis.")
config_populate_optparser(parser)
(options, args) = parser.parse_args()
initialize_backend()
# We load plugins after we parsed the configuration
from compmake import plugins #@UnusedImport
if options.redis_events:
if not compmake_config.db == 'redis': #@UndefinedVariable
error('Cannot use redis_events without redis.')
sys.exit(-2)
from compmake.storage.redisdb import RedisInterface
# register an handler that will capture all events
def handler(event):
RedisInterface.events_push(event)
remove_all_handlers()
register_handler("*", handler)
if not options.slave:
# XXX make sure this is the default
set_compmake_status(compmake_status_interactive)
# TODO: add command namespace
# TODO: add command "load"
if not args:
user_error('I expect at least one parameter (module name)')
sys.exit(-2)
module_name = args[0]
args = args[1:]
if module_name.endswith('.py') or (module_name.find('/') > 0):
warning('You passed a string "%s" which looks like a filename.' %
module_name)
module_name = module_name.replace('/', '.')
module_name = module_name.replace('.py', '')
warning('However, I need a module name. I will try with "%s".' %
module_name)
set_namespace(module_name)
compmake.is_it_time = True
try:
__import__(module_name)
except Exception as e:
error('Error while trying to import module "%s": %s' %
(module_name, e))
traceback.print_exc(file=sys.stderr)
sys.exit(-5)
# TODO: BUG: XXX: remove old jobs those in defined_this_section
else:
set_compmake_status(compmake_status_slave)
if not args:
user_error('I expect at least one parameter (namespace name)')
sys.exit(-2)
module_name = args.pop(0)
set_namespace(module_name)
if args:
try:
# XXX is this redudant?
# compmake_config.interactive = False
retcode = interpret_commands(args)
# print "Exiting with retcode %s" % retcode
sys.exit(retcode)
except UserError as e:
user_error(e)
sys.exit(-6)
else:
retcode = interactive_console()
sys.exit(retcode)
