def before_experiment_run(self, parameter_type):
if parameter_type == "input" and "path" in dir(self.__clone_url):
self.subobjects["clone-url"] = self.__clone_url
Type.before_experiment_run(self, parameter_type)
self.__clone_url = self.__clone_url.path
else:
Type.before_experiment_run(self, parameter_type)
def __init__(self, default_name=""):
InputParameter.__init__(self)
OutputParameter.__init__(self)
Type.__init__(self)
self.__object_name = default_name
self.__enclosing_directory = os.path.abspath(os.curdir)
self.__force_enclosing_directory = False
def __init__(self, clone_url = None, ref = "refs/heads/master", shallow=False,
branches=None, tags=None):
"""clone_url: where to the git archive from
This might either be a string or a object with a path attribute
ref: which git reference to checkout
shallow: do a shallow copy (using git-archive).
branches: Also fetch other branches. Use branches=True for all branches and branches=REGEX for a filtered view.
tags: Also fetch other tags. Use tags=True for all tags and tags=REGEX for a filtered view.
The git archive will be cloned to self.name (which is the
key in the input parameters dict)"""
Type.__init__(self)
InputParameter.__init__(self)
Directory_op_with.__init__(self)
self.__clone_url = clone_url
self.__ref = ref
self.__shallow = shallow
self.__value = None
self.__hash = None
# Include branches and tags into the metadata-hash
self.__filter_refs = {"branches": branches, "tags": tags}
if (branches or tags ) and shallow:
raise RuntimeError("Shallow clone and branch/tag checkout is not supported.")
def __init__(self, default_name=""):
InputParameter.__init__(self)
OutputParameter.__init__(self)
Type.__init__(self)
self.__object_name = default_name
self.__enclosing_directory = os.path.abspath(os.curdir)
self.__force_enclosing_directory = False
def __init__(self, path = "sqlite3.db"):
InputParameter.__init__(self)
OutputParameter.__init__(self)
Type.__init__(self)
self.__database_path = path
self.__database_connection = None
def before_experiment_run(self, parameter_type):
# When experiment run as input, just run the normal input handlers
if parameter_type == "input":
Type.before_experiment_run(self, "input")
return
for (name, inp) in self.inputs.items():
if type(inp) == LambdaType:
continue
ret = inp.inp_extract_cmdline_parser(self.__opts, self.__args)
if ret:
(self.__opts, self.__args) = ret
# After all input parameters are parsed. Execute the
# calculated input parameters
for (name, inp) in self.inputs.items():
if type(inp) != LambdaType:
continue
inp = inp(self)
inp.name = name
self.subobjects[name] = inp
self.inputs[name] = inp
self.subobjects.update()
# Now set up the experiment tmp directory
self.__setup_tmp_directory()
for obj in self.inputs.values():
obj.before_experiment_run("input")
self.__calculate_metadata()
for obj in self.outputs.values():
obj.before_experiment_run("output")
def before_experiment_run(self, parameter_type):
Type.before_experiment_run(self, parameter_type)
assert parameter_type in ["input", "output"]
args = {"db": self.__database_name,
"host": self.__database_host}
if self.__database_user:
args["user"] = self.__database_user
if self.__database_password:
args["passwd"] = self.__database_password
default_file = os.path.join(os.environ["HOME"], ".my.cnf")
if os.path.exists(default_file):
args["read_default_file"] = default_file
self.__database_connection = MySQLdb.connect(**args)
if parameter_type == "output":
try:
self.create_table("metadata", [("experiment", "varchar(256)"), ("metadata", "text")],
keys = ["experiment"],
conflict_strategy = "REPLACE")
except _mysql_exceptions.OperationalError as e:
# Metadata table was already generated
pass
self.execute("REPLACE INTO metadata(experiment, metadata) values(?, ?)",
self.dynamic_experiment.experiment_identifier,
str(self.dynamic_experiment.metadata))
def before_experiment_run(self, parameter_type):
# When experiment run as input, just run the normal input handlers
if parameter_type == "input":
Type.before_experiment_run(self, "input")
return
for (name, inp) in self.inputs.items():
if type(inp) == LambdaType:
continue
ret = inp.inp_extract_cmdline_parser(self.__opts, self.__args)
if ret:
(self.__opts, self.__args) = ret
# After all input parameters are parsed. Execute the
# calculated input parameters
for (name, inp) in self.inputs.items():
if type(inp) != LambdaType:
continue
inp = inp(self)
inp.name = name
self.subobjects[name] = inp
self.inputs[name] = inp
self.subobjects.update()
# Now set up the experiment tmp directory
self.__setup_tmp_directory()
for obj in self.inputs.values():
obj.before_experiment_run("input")
self.__calculate_metadata()
for obj in self.outputs.values():
obj.before_experiment_run("output")
def before_experiment_run(self, parameter_type):
if parameter_type == "input" and "path" in dir(self.__clone_url):
self.subobjects["clone-url"] = self.__clone_url
Type.before_experiment_run(self, parameter_type)
self.__clone_url = self.__clone_url.path
else:
Type.before_experiment_run(self, parameter_type)
def before_experiment_run(self, parameter_type):
if parameter_type == "input" and "path" in dir(self.__filename):
self.subobjects["filename"] = self.__filename
Type.before_experiment_run(self, parameter_type)
self.__filename = self.__filename.path
else:
Type.before_experiment_run(self, parameter_type)
self.__filename = os.path.abspath(self.__filename)
def after_experiment_run(self, parameter_type):
Type.before_experiment_run(self, parameter_type)
assert parameter_type in ["input", "output"]
self.__database_connection = None
self.__disconnect(self.path)
if parameter_type == "output":
# Remove execute and write permissions for file
new_mode = os.stat(self.path).st_mode & (stat.S_IROTH | stat.S_IRGRP | stat.S_IRUSR)
os.chmod(self.path, new_mode)
def __init__(self, filename = None):
"""The default_filename is either a string to a file. Or a
object with a path attribute (e.g. a :class:`~versuchung.files.File`)"""
Type.__init__(self)
InputParameter.__init__(self)
Directory_op_with.__init__(self)
self.__filename = filename
self.__value = None
def before_experiment_run(self, parameter_type):
if parameter_type == "input" and "path" in dir(self.__filename):
self.subobjects["filename"] = self.__filename
Type.before_experiment_run(self, parameter_type)
self.__filename = self.__filename.path
else:
Type.before_experiment_run(self, parameter_type)
self.__filename = os.path.abspath(self.__filename)
def __init__(self, filename=None):
"""The default_filename is either a string to a file. Or a
object with a path attribute (e.g. a :class:`~versuchung.files.File`)"""
Type.__init__(self)
InputParameter.__init__(self)
Directory_op_with.__init__(self)
self.__filename = filename
self.__value = None
def before_experiment_run(self, parameter_type):
# Add database object as an
self.subobjects["database"] = self.__db
Type.before_experiment_run(self, parameter_type)
if parameter_type == "output":
self.read_only = False
self.__db.create_table(self.table_name, self.__fields,
keys = self.__keys,
conflict_strategy = self.__conflict_strategy)
def __init__(self, database = None, host = "localhost", user = None, password = None):
InputParameter.__init__(self)
OutputParameter.__init__(self)
Type.__init__(self)
assert database != None, "Please give a database name to database connection"
self.__database_name = database
self.__database_host = os.environ.get("MYSQL_HOST", None) or host
self.__database_user = os.environ.get("MYSQL_USER", None) or user
self.__database_password = os.environ.get("MYSQL_PASSWORD", None) or password
self.__database_connection = None
def __init__(self, default_experiment_instance=None):
"""The constructor of an experiment just fills in the
necessary attributes but has *no* sideeffects on the outside
world.
:param default_experiment_instance: If used as input
parameter, this is the default result set used. For
example
``"SimpleExperiment-aeb298601cdc582b1b0d8260195f6cfd"``
:type default_experiment_instance: str.
"""
Type.__init__(self)
InputParameter.__init__(self)
self.title = self.__class__.__name__
self.static_experiment = self
self.__experiment_instance = default_experiment_instance
self.__metadata = None
# Copy input and output objects
self.inputs = JavascriptStyleDictAccess(
copy.deepcopy(self.__class__.inputs))
self.i = self.inputs
self.outputs = JavascriptStyleDictAccess(
copy.deepcopy(self.__class__.outputs))
self.o = self.outputs
if default_experiment_instance != None:
self.base_directory = os.path.join(os.curdir,
self.__experiment_instance)
self.base_directory = os.path.realpath(self.base_directory)
else:
self.base_directory = os.path.realpath(os.curdir)
# Sanity checking for input parameters.
for (name, inp) in self.inputs.items():
# Lambdas are resolved, when the experiment is really started
if type(inp) == LambdaType:
continue
if not isinstance(inp, InputParameter):
print("%s cannot be used as an input parameter" % name)
sys.exit(-1)
self.subobjects[name] = inp
for (name, outp) in self.outputs.items():
if not isinstance(outp, OutputParameter):
print("%s cannot be used as an output parameter" % name)
sys.exit(-1)
self.subobjects[name] = outp
def __init__(self, fields, keys = None, db = None, conflict_strategy = "FAIL" ):
self.read_only = True
InputParameter.__init__(self)
OutputParameter.__init__(self)
Type.__init__(self)
self.__keys = keys
self.__fields = self.__field_typify(["experiment"] + fields)
self.__conflict_strategy = conflict_strategy
if not db:
self.__db = Database()
else:
self.__db = db
def __init__(self, default_experiment_instance = None):
"""The constructor of an experiment just fills in the
necessary attributes but has *no* sideeffects on the outside
world.
:param default_experiment_instance: If used as input
parameter, this is the default result set used. For
example
``"SimpleExperiment-aeb298601cdc582b1b0d8260195f6cfd"``
:type default_experiment_instance: str.
"""
Type.__init__(self)
InputParameter.__init__(self)
self.title = self.__class__.__name__
self.static_experiment = self
self.__experiment_instance = default_experiment_instance
self.__metadata = None
# Copy input and output objects
self.inputs = JavascriptStyleDictAccess(copy.deepcopy(self.__class__.inputs))
self.i = self.inputs
self.outputs = JavascriptStyleDictAccess(copy.deepcopy(self.__class__.outputs))
self.o = self.outputs
if default_experiment_instance != None:
self.base_directory = os.path.join(os.curdir, self.__experiment_instance)
self.base_directory = os.path.realpath(self.base_directory)
else:
self.base_directory = os.path.realpath(os.curdir)
# Sanity checking for input parameters.
for (name, inp) in self.inputs.items():
# Lambdas are resolved, when the experiment is really started
if type(inp) == LambdaType:
continue
if not isinstance(inp, InputParameter):
print("%s cannot be used as an input parameter" % name)
sys.exit(-1)
self.subobjects[name] = inp
for (name, outp) in self.outputs.items():
if not isinstance(outp, OutputParameter):
print("%s cannot be used as an output parameter" % name)
sys.exit(-1)
self.subobjects[name] = outp
def __init__(self, clone_url=None, ref="refs/heads/master", shallow=False):
"""clone_url: where to the git archive from
This might either be a string or a object with a path attribute
ref: which git reference to checkout
shallow: do a shallow copy (using git-archive).
The git archive will be cloned to self.name (which is the
key in the input parameters dict)"""
Type.__init__(self)
InputParameter.__init__(self)
Directory_op_with.__init__(self)
self.__clone_url = clone_url
self.__ref = ref
self.__shallow = shallow
self.__value = None
self.__hash = None
def __init__(self, clone_url = None, ref = "refs/heads/master", shallow = False):
"""clone_url: where to the git archive from
This might either be a string or a object with a path attribute
ref: which git reference to checkout
shallow: do a shallow copy (using git-archive).
The git archive will be cloned to self.name (which is the
key in the input parameters dict)"""
Type.__init__(self)
InputParameter.__init__(self)
Directory_op_with.__init__(self)
self.__clone_url = clone_url
self.__ref = ref
self.__shallow = shallow
self.__value = None
self.__hash = None
def __init__(self, default_experiment_instance=None, title=None):
"""The constructor of an experiment just fills in the
necessary attributes but has *no* sideeffects on the outside
world.
:param default_experiment_instance: If used as input
parameter, this is the default result set used. For
example
``"SimpleExperiment-aeb298601cdc582b1b0d8260195f6cfd"``
:type default_experiment_instance: str.
"""
Type.__init__(self)
InputParameter.__init__(self)
self.title = title or self.__class__.__name__
self.static_experiment = self
if default_experiment_instance and not os.path.exists(
default_experiment_instance):
default_experiment_instance = None
self.__reinit__(default_experiment_instance)
def before_experiment_run(self, parameter_type):
Type.before_experiment_run(self, parameter_type)
assert parameter_type in ["input", "output"]
if parameter_type == "input":
# Ensure the path does exist
if not os.path.exists(self.path):
raise RuntimeError("Database not found: %s" % self.path)
self.__database_connection = self.__connect(self.path)
if parameter_type == "output":
try:
self.create_table("metadata", [("experiment", "text"), ("metadata", "text")],
keys = ["experiment"],
conflict_strategy = "REPLACE")
except sqlite3.OperationalError as e:
# Metadata table was already generated
pass
self.execute("INSERT INTO metadata(experiment, metadata) values(?, ?)",
self.dynamic_experiment.experiment_identifier,
str(self.dynamic_experiment.metadata))
