class BundleMercurialRepository(Task):
task_namespace = "law.mercurial"
repo_path = luigi.Parameter(
description="the path to the repository to bundle")
exclude_files = CSVParameter(default=[],
description="patterns of files to exclude")
include_files = CSVParameter(
default=[],
description="patterns of files to force-include, "
"takes precedence over .hgignore")
custom_checksum = luigi.Parameter(default=NO_STR,
description="a custom checksum to use")
def __init__(self, *args, **kwargs):
super(BundleMercurialRepository, self).__init__(*args, **kwargs)
self.repo_path = os.path.expandvars(
os.path.expanduser(os.path.abspath(self.repo_path)))
self._checksum = None
@property
def checksum(self):
if self.custom_checksum != NO_STR:
return self.custom_checksum
if self._checksum is None:
cmd = [
rel_path(__file__, "repository_checksum.sh"), self.repo_path
]
code, out, _ = interruptable_popen(cmd, stdout=subprocess.PIPE)
if code != 0:
raise Exception("repository checksum calculation failed")
self._checksum = out.strip()
return self._checksum
def output(self):
return LocalFileTarget("{}_{}.tgz".format(
os.path.basename(self.repo_path), self.checksum))
@log
def run(self):
with self.output().localize("w") as tmp:
self.bundle(tmp.path)
def bundle(self, dst_path):
cmd = [
rel_path(__file__, "bundle_repository.sh"), self.repo_path,
get_path(dst_path)
]
cmd += [" ".join(self.exclude_files)]
cmd += [" ".join(self.include_files)]
code = interruptable_popen(cmd)[0]
if code != 0:
raise Exception("repository bundling failed")
class GLiteWorkflow(BaseRemoteWorkflow):
workflow_proxy_cls = GLiteWorkflowProxy
glite_workflow_run_decorators = None
glite_ce = CSVParameter(default=[],
significant=False,
description="target glite computing "
"element(s)")
exclude_params_branch = {"glite_ce"}
exclude_db = True
@abstractmethod
def glite_output_directory(self):
return None
@abstractmethod
def glite_bootstrap_file(self):
return None
def glite_stageout_file(self):
return None
def glite_workflow_requires(self):
return OrderedDict()
def glite_output_postfix(self):
self.get_branch_map()
return "_{}To{}".format(self.start_branch, self.end_branch)
def glite_output_uri(self):
return self.glite_output_directory().url()
def glite_delegate_proxy(self, endpoint):
return delegate_voms_proxy_glite(endpoint,
stdout=sys.stdout,
stderr=sys.stderr,
cache=True)
def glite_create_job_manager(self):
return GLiteJobManager()
def glite_create_job_file_factory(self):
return GLiteJobFileFactory()
def glite_job_config(self, config, job_num, branches):
return config
def glite_use_local_scheduler(self):
return True
def glite_cmdline_args(self):
return []
class ARCWorkflow(BaseRemoteWorkflow):
workflow_proxy_cls = ARCWorkflowProxy
arc_workflow_run_decorators = None
arc_ce = CSVParameter(default=[], significant=False, description="target arc computing "
"element(s)")
exclude_params_branch = {"arc_ce"}
exclude_db = True
@abstractmethod
def arc_output_directory(self):
return None
@abstractmethod
def arc_bootstrap_file(self):
return None
def arc_stageout_file(self):
return None
def arc_workflow_requires(self):
return OrderedDict()
def arc_output_postfix(self):
self.get_branch_map()
return "_{}To{}".format(self.start_branch, self.end_branch)
def arc_output_uri(self):
return self.arc_output_directory().url()
def arc_create_job_manager(self):
return ARCJobManager()
def arc_create_job_file_factory(self):
return ARCJobFileFactory()
def arc_job_config(self, config, job_num, branches):
return config
def arc_use_local_scheduler(self):
return True
def arc_cmdline_args(self):
return []
class ArcWorkflow(BaseRemoteWorkflow):
workflow_proxy_cls = ArcWorkflowProxy
arc_ce = CSVParameter(default=[],
significant=False,
description="target arc computing "
"element(s)")
exclude_params_branch = {"arc_ce"}
exclude_db = True
@abstractmethod
def arc_output_directory(self):
return None
@abstractmethod
def arc_bootstrap_file(self):
return None
def arc_stageout_file(self):
return None
def arc_workflow_requires(self):
return OrderedDict()
def arc_output_postfix(self):
# TODO (riga): use start/end branch by default?
return ""
def arc_output_uri(self):
return self.arc_output_directory().url()
def arc_create_job_manager(self):
return ArcJobManager()
def arc_create_job_file_factory(self):
return ArcJobFileFactory()
def arc_job_config(self, config, job_num, branches):
return config
def arc_use_local_scheduler(self):
return True
class ARCWorkflow(BaseRemoteWorkflow):
workflow_proxy_cls = ARCWorkflowProxy
arc_workflow_run_decorators = None
arc_job_manager_defaults = None
arc_job_file_factory_defaults = None
arc_ce = CSVParameter(default=[],
significant=False,
description="target arc computing "
"element(s)")
exclude_params_branch = {"arc_ce"}
exclude_index = True
@abstractmethod
def arc_output_directory(self):
return None
@abstractmethod
def arc_bootstrap_file(self):
return None
def arc_wrapper_file(self):
return law_src_path("job", "bash_wrapper.sh")
def arc_stageout_file(self):
return None
def arc_workflow_requires(self):
return OrderedDict()
def arc_output_postfix(self):
self.get_branch_map()
return "_{}To{}".format(self.start_branch, self.end_branch)
def arc_output_uri(self):
return self.arc_output_directory().url()
def arc_create_job_manager(self, **kwargs):
kwargs = merge_dicts(self.arc_job_manager_defaults, kwargs)
return ARCJobManager(**kwargs)
def arc_create_job_file_factory(self, **kwargs):
# job file fectory config priority: config file < class defaults < kwargs
get_prefixed_config = self.workflow_proxy.get_prefixed_config
cfg = {
"dir":
get_prefixed_config("job", "job_file_dir"),
"mkdtemp":
get_prefixed_config("job", "job_file_dir_mkdtemp", type=bool),
"cleanup":
get_prefixed_config("job", "job_file_dir_cleanup", type=bool),
}
kwargs = merge_dicts(cfg, self.arc_job_file_factory_defaults, kwargs)
return ARCJobFileFactory(**kwargs)
def arc_job_config(self, config, job_num, branches):
return config
def arc_post_submit_delay(self):
return self.poll_interval * 60
def arc_use_local_scheduler(self):
return True
def arc_cmdline_args(self):
return []
class BundleGitRepository(Task):
task_namespace = "law.git"
exclude_files = CSVParameter(default=[],
description="patterns of files to exclude")
include_files = CSVParameter(
default=[],
description="patterns of files to force-include, "
"takes precedence over .gitignore")
custom_checksum = luigi.Parameter(default=NO_STR,
description="a custom checksum to use")
def __init__(self, *args, **kwargs):
super(BundleGitRepository, self).__init__(*args, **kwargs)
self._checksum = None
@abstractmethod
def get_repo_path(self):
return
@property
def checksum(self):
if self.custom_checksum != NO_STR:
return self.custom_checksum
if self._checksum is None:
checksum_script = rel_path(__file__, "scripts",
"repository_checksum.sh")
cmd = [checksum_script, self.get_repo_path()]
code, out, _ = interruptable_popen(cmd, stdout=subprocess.PIPE)
if code != 0:
raise Exception("repository checksum calculation failed")
self._checksum = out.strip()
return self._checksum
def output(self):
repo_base = os.path.basename(self.get_repo_path())
return LocalFileTarget("{}_{}.tgz".format(repo_base, self.checksum))
@log
def run(self):
with self.output().localize("w") as tmp:
self.bundle(tmp.path)
def bundle(self, dst_path):
cmd = "{} \"{}\" \"{}\" \"{}\" \"{}\"".format(
rel_path(__file__, "scripts", "bundle_repository.sh"),
self.get_repo_path(),
get_path(dst_path),
" ".join(self.exclude_files),
" ".join(self.include_files),
)
code = interruptable_popen(cmd, shell=True, executable="/bin/bash")[0]
if code != 0:
raise Exception("repository bundling failed")