def test_auto_namespace_not_matching_2(self):
luigi.auto_namespace(scope='incorrect_namespace')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='incorrect_namespace')
self.assertEqual(MyTask.get_task_namespace(), '')
def test_auto_namespace_not_matching_2(self):
luigi.auto_namespace(scope='incorrect_namespace')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='incorrect_namespace')
self.assertEqual(MyTask.get_task_namespace(), '')
def test_auto_namespace_global(self):
luigi.auto_namespace()
class MyTask(luigi.Task):
pass
luigi.namespace()
self.assertEqual(MyTask.get_task_namespace(), self.this_module)
def test_auto_namespace_global(self):
luigi.auto_namespace()
class MyTask(luigi.Task):
pass
luigi.namespace()
self.assertEqual(MyTask.get_task_namespace(), self.this_module)
def test_task_ids_using_requries(self):
class ParentTask(luigi.Task):
my_param = luigi.Parameter()
luigi.namespace('blah')
@requires(ParentTask)
class ChildTask(luigi.Task):
pass
luigi.namespace('')
child_task = ChildTask(my_param='hello')
self.assertEqual(str(child_task), 'blah.ChildTask(my_param=hello)')
self.assertIn(ParentTask(my_param='hello'), luigi.task.flatten(child_task.requires()))
def go_mynamespace(self):
luigi.namespace("mynamespace")
class Foo(luigi.Task):
p = luigi.IntParameter()
class Bar(Foo):
task_namespace = "othernamespace" # namespace override
class Baz(Bar): # inherits namespace for Bar
pass
luigi.namespace()
return collections.namedtuple('mynamespace', 'Foo Bar Baz')(Foo, Bar, Baz)
def go_mynamespace(self):
luigi.namespace("mynamespace")
class Foo(luigi.Task):
p = luigi.IntParameter()
class Bar(Foo):
task_namespace = "othernamespace" # namespace override
class Baz(Bar): # inherits namespace for Bar
pass
luigi.namespace()
return collections.namedtuple('mynamespace', 'Foo Bar Baz')(Foo, Bar, Baz)
def test_task_ids_using_requries_2(self):
# Here we use this decorator in a unnormal way.
# But it should still work.
class ParentTask(luigi.Task):
my_param = luigi.Parameter()
decorator = requires(ParentTask)
luigi.namespace('blah')
class ChildTask(luigi.Task):
pass
luigi.namespace('')
ChildTask = decorator(ChildTask)
child_task = ChildTask(my_param='hello')
self.assertEqual(str(child_task), 'blah.ChildTask(my_param=hello)')
self.assertIn(ParentTask(my_param='hello'), luigi.task.flatten(child_task.requires()))
def test_task_ids_using_requries(self):
class ParentTask(luigi.Task):
my_param = luigi.Parameter()
luigi.namespace('blah')
@requires(ParentTask)
class ChildTask(luigi.Task):
pass
luigi.namespace('')
child_task = ChildTask(my_param='hello')
self.assertEqual(str(child_task), 'blah.ChildTask(my_param=hello)')
self.assertIn(ParentTask(my_param='hello'),
luigi.task.flatten(child_task.requires()))
def test_externalize_same_id_with_luigi_namespace(self):
# Dependent on the new behavior from spotify/luigi#1953
luigi.namespace('lets.externalize')
class MyTask(luigi.Task):
def run(self):
pass
luigi.namespace()
task_normal = MyTask()
task_ext_1 = luigi.task.externalize(MyTask())
task_ext_2 = luigi.task.externalize(MyTask)()
self.assertEqual(task_normal.task_id, task_ext_1.task_id)
self.assertEqual(task_normal.task_id, task_ext_2.task_id)
self.assertEqual(str(task_normal), str(task_ext_1))
self.assertEqual(str(task_normal), str(task_ext_2))
def test_externalize_same_id_with_luigi_namespace(self):
# Dependent on the new behavior from spotify/luigi#1953
luigi.namespace('lets.externalize')
class MyTask(luigi.Task):
def run(self):
pass
luigi.namespace()
task_normal = MyTask()
task_ext_1 = luigi.task.externalize(MyTask())
task_ext_2 = luigi.task.externalize(MyTask)()
self.assertEqual(task_normal.task_id, task_ext_1.task_id)
self.assertEqual(task_normal.task_id, task_ext_2.task_id)
self.assertEqual(str(task_normal), str(task_ext_1))
self.assertEqual(str(task_normal), str(task_ext_2))
def test_task_ids_using_inherits_kwargs(self):
class ParentTask(luigi.Task):
my_param = luigi.Parameter()
luigi.namespace('blah')
@inherits(parent=ParentTask)
class ChildTask(luigi.Task):
def requires(self):
return self.clone(ParentTask)
luigi.namespace('')
child_task = ChildTask(my_param='hello')
self.assertEqual(str(child_task), 'blah.ChildTask(my_param=hello)')
self.assertIn(ParentTask(my_param='hello'),
luigi.task.flatten(child_task.requires()))
def test_with_scope(self):
luigi.namespace('wohoo', scope='task_test')
luigi.namespace('bleh', scope='')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='task_test')
luigi.namespace(scope='')
self.assertEqual(MyTask.get_task_namespace(), 'wohoo')
def test_with_scope_not_matching(self):
luigi.namespace('wohoo', scope='incorrect_namespace')
luigi.namespace('bleh', scope='')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='incorrect_namespace')
luigi.namespace(scope='')
self.assertEqual(MyTask.get_task_namespace(), 'bleh')
def test_with_scope_not_matching(self):
luigi.namespace('wohoo', scope='incorrect_namespace')
luigi.namespace('bleh', scope='')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='incorrect_namespace')
luigi.namespace(scope='')
self.assertEqual(MyTask.get_task_namespace(), 'bleh')
def test_with_scope(self):
luigi.namespace('wohoo', scope='task_test')
luigi.namespace('bleh', scope='')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='task_test')
luigi.namespace(scope='')
self.assertEqual(MyTask.get_task_namespace(), 'wohoo')
def test_task_ids_using_requries_2(self):
# Here we use this decorator in a unnormal way.
# But it should still work.
class ParentTask(luigi.Task):
my_param = luigi.Parameter()
decorator = requires(ParentTask)
luigi.namespace('blah')
class ChildTask(luigi.Task):
pass
luigi.namespace('')
ChildTask = decorator(ChildTask)
child_task = ChildTask(my_param='hello')
self.assertEqual(str(child_task), 'blah.ChildTask(my_param=hello)')
self.assertIn(ParentTask(my_param='hello'),
luigi.task.flatten(child_task.requires()))
def test_auto_namespace_scope(self):
luigi.auto_namespace(scope='task_test')
luigi.namespace('bleh', scope='')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='task_test')
luigi.namespace(scope='')
self.assertEqual(MyTask.get_task_namespace(), self.this_module)
def test_auto_namespace_scope(self):
luigi.auto_namespace(scope='task_test')
luigi.namespace('bleh', scope='')
class MyTask(luigi.Task):
pass
luigi.namespace(scope='task_test')
luigi.namespace(scope='')
self.assertEqual(MyTask.get_task_namespace(), self.this_module)
def test_uses_latest_namespace(self):
luigi.namespace('a')
class _BaseTask(luigi.Task):
pass
luigi.namespace('b')
class _ChildTask(_BaseTask):
pass
luigi.namespace() # Reset everything
child_task = _ChildTask()
self.assertEqual(child_task.task_family, 'b._ChildTask')
self.assertEqual(str(child_task), 'b._ChildTask()')
def test_uses_latest_namespace(self):
luigi.namespace('a')
class _BaseTask(luigi.Task):
pass
luigi.namespace('b')
class _ChildTask(_BaseTask):
pass
luigi.namespace() # Reset everything
child_task = _ChildTask()
self.assertEqual(child_task.task_family, 'b._ChildTask')
self.assertEqual(str(child_task), 'b._ChildTask()')
# -*- coding: utf-8 -*-
"""
Example showing docker sandboxing.
"""
import os
from random import random
import luigi
import law
from law.sandbox.docker import DockerSandbox
luigi.namespace("example.docker")
class CreateNumbers(law.SandboxTask):
n_nums = luigi.IntParameter(
default=100, description="amount of random numbers to be generated")
def output(self):
return law.LocalFileTarget("data/docker/numbers_%i.txt" % self.n_nums)
def run(self):
with self.output().open("w") as f:
for _ in range(self.n_nums):
f.write("%s\n" % random())
class BinNumbers(law.SandboxTask):
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from unittest import TestCase
import luigi
import luigi.notifications
from luigi import postgres
luigi.notifications.DEBUG = True
luigi.namespace('postgres_test')
"""
Typical use cases that should be tested:
* Daily overwrite of all data in table
* Daily inserts of new segment in table
* (Daily insertion/creation of new table)
* Daily insertion of multiple (different) new segments into table
"""
# to avoid copying:
import datetime
from glob import glob
import logging
import os
from os.path import join
import luigi
from luigi.task import flatten_output, flatten, getpaths
from luigi.util import requires
from bioluigi.tasks import cutadapt, fastqc
from bioluigi.tasks.utils import RemoveTaskOutputOnFailureMixin, CreateTaskOutputDirectoriesBeforeRunMixin
from bioluigi.scheduled_external_program import ScheduledExternalProgramTask
logger = logging.getLogger('luigi-interface')
luigi.namespace('rnaseq_variant_pipeline')
class core(luigi.Config):
genome = luigi.Parameter()
annotations = luigi.Parameter()
star_index_dir = luigi.Parameter()
output_dir = luigi.Parameter(description='Output directory')
filtered_dir = luigi.Parameter(
description='Output directory for filtered variants')
tmp_dir = luigi.Parameter()
cfg = core()
import os
import sys
import re
import collections
import itertools
import six
import luigi
import law
import tabulate
from hltp.base import Task, TaskWithSummary, CMSSWSandbox, BrilSandbox
from hltp.util import expand_pset, fwlite_loop, text_to_process
luigi.namespace("hltp", scope=__name__)
class GetDatasetLFNs(TaskWithSummary, CMSSWSandbox):
dataset = luigi.Parameter(description="the dataset (no pattern!) to query")
check_for_patterns = ["dataset"]
def output(self):
return self.local_target("{}.json".format(self.dataset.replace("/", "_")))
@law.decorator.notify
@law.wlcg.ensure_voms_proxy
@law.decorator.localize
def run(self):
"""
HGCAL simulation tasks.
"""
__all__ = ["GSDTask", "RecoTask", "NtupTask"]
import os
import random
import law
import luigi
from hgc.tasks.base import Task, HTCondorWorkflow
from hgc.util import cms_run_and_publish, log_runtime
luigi.namespace("sim", scope=__name__)
class GeneratorParameters(Task):
n_events = luigi.IntParameter(
default=10, description="number of events to generate per task")
n_tasks = luigi.IntParameter(
default=1, description="number of branch tasks to create")
gun_type = luigi.ChoiceParameter(
default="closeby",
choices=["flatpt", "closeby"],
description="the type of the particle gun")
gun_min = luigi.FloatParameter(
default=1.0,
description="minimum value of the gun, either in "
# Copyright (c) 2012 Spotify AB
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
import luigi
luigi.namespace("mynamespace")
class Foo(luigi.Task):
p = luigi.Parameter()
class Bar(Foo):
task_namespace = "othernamespace" # namespace override
luigi.namespace()
"""
Plotting tasks.
"""
__all__ = ["PlotTask"]
import law
import luigi
from hgc.tasks.base import Task
from hgc.tasks.simulation import NtupTask
luigi.namespace("plot", scope=__name__)
class PlotTask(Task):
n_events = NtupTask.n_events
def requires(self):
return NtupTask.req(self, n_tasks=1)
def output(self):
return law.SiblingFileCollection([
self.local_target("eta_phi_{}.png".format(i))
for i in range(self.n_events)
])
__all__ = ["GSDTask", "RecoTask", "HarvestingTask", "ValidationPlotsTask"]
import os
import random
import law
import luigi
from hgc.tasks.base import Task
from hgc.util import cms_run_and_publish, log_runtime
luigi.namespace("vali", scope=__name__)
class SimTask(Task):
version = None
distance = luigi.FloatParameter(default=7.5, description="")
thickness = luigi.IntParameter(default=200, description="")
def store_parts(self):
parts = super(SimTask, self).store_parts()
parts += (self.thickness, '%g'%self.distance)
return parts
from django import template
from django.template.loader import get_template
from django.conf import settings
from pycarol import *
from pycarol.carol import *
from pycarol.tasks import *
from pycarol.query import *
from pycarol.auth.ApiKeyAuth import *
from pycarol.staging import *
from pycarol.connectors import Connectors
from pycarol.storage import Storage
namespace = 'main'
luigi.namespace(namespace)
class runMe(luigi.WrapperTask):
def requires(self):
print(">>>>> RUNNING THE METHOD REQUIRES")
return execution()
class execution(luigi.Task):
print(">>>>> RUNNING THE METHOD EXECUTION")
finished = False
def run(self):
print(">>>>> RUNNING THE METHOD RUN INSIDE EXECUTION")
osvarslist = ['FILENAME','FUNCTIONNAME','WORKERS','CAROLUSERNAME','CAROLPASSWORD','CAROLDOMAIN','CAROLCONNECTORID','CAROLAPPNAME','CAROLAPPVERSION','CAROLAPPOAUTH','REQUIREMENTS','URL','FILEPATH','BRANCHNAME','PYCAROL','LOGIN','DEBUG','RESULTSPATH','RAWPATH','SENTPATH']
apiAuth = ApiKeyAuth(api_key = os.environ['CAROLAPPOAUTH'])
connectorId = os.environ['CAROLCONNECTORID']
__all__ = ["ConverterTask", "MergeConvertedFiles"]
import os
import law
import luigi
from hgc.tasks.base import HTCondorWorkflow
from hgc.tasks.simulation import GeneratorParameters, ParallelProdWorkflow, NtupTask
from hgc.tasks.software import CompileConverter, CompileDeepJetCore
from hgc.util import hadd_task
luigi.namespace("gnn", scope=__name__)
class ConverterTask(ParallelProdWorkflow):
previous_task = ("ntup", NtupTask)
def workflow_requires(self):
reqs = super(ConverterTask, self).workflow_requires()
reqs["converter"] = CompileConverter.req(self)
return reqs
def requires(self):
reqs = super(ConverterTask, self).requires()
reqs["converter"] = CompileConverter.req(self)
return reqs
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import luigi
luigi.namespace("mynamespace")
class Foo(luigi.Task):
p = luigi.IntParameter()
class Bar(Foo):
task_namespace = "othernamespace" # namespace override
class Baz(Bar): # inherits namespace for Bar
pass
luigi.namespace()
from unittest import TestCase
import luigi
import luigi.notifications
from luigi import postgres
luigi.notifications.DEBUG = True
luigi.namespace('postgres_test')
"""
Typical use cases that should be tested:
* Daily overwrite of all data in table
* Daily inserts of new segment in table
* (Daily insertion/creation of new table)
* Daily insertion of multiple (different) new segments into table
"""
# to avoid copying:
class CopyToTestDB(postgres.CopyToTable):
host = 'localhost'
database = 'spotify'
user = '******'
password = '******'
class TestPostgresTask(CopyToTestDB):
table = 'test_table'
"UploadCMSSW",
"CMSSWSandboxTask",
"InstallCMSSW",
]
import os
import re
import math
import law
import luigi
law.contrib.load("cms", "git", "htcondor", "root", "slack", "tasks",
"telegram", "wlcg")
luigi.namespace("base", scope=__name__)
def user_parameter(significant=False):
return luigi.Parameter(
default=os.environ["HGC_GRID_USER"],
significant=significant,
description=
"the user executing the task, only for visual purposes on central luigi "
"schedulers")
class Task(law.Task):
"""
Custom base task.
"""
