def test_stop_getting_new_work(self): d = DummyTask() self.w.add(d) self.assertFalse(d.complete()) try: self.w.handle_interrupt(signal.SIGUSR1, None) except AttributeError: raise unittest.SkipTest('signal.SIGUSR1 not found on this system') self.w.run() self.assertFalse(d.complete())
def test_disabled_shutdown_hook(self): w = Worker(scheduler=self.sch, keep_alive=True, no_install_shutdown_handler=True) with w: try: # try to kill the worker! os.kill(os.getpid(), signal.SIGUSR1) except AttributeError: raise unittest.SkipTest('signal.SIGUSR1 not found on this system') # try to kill the worker... AGAIN! t = SuicidalWorker(signal.SIGUSR1) w.add(t) w.run() # task should have stepped away from the ledge, and completed successfully despite all the SIGUSR1 signals self.assertEqual(list(self.sch.task_list('DONE', '').keys()), [t.task_id])
def setUp(self): try: from luigi.sqs_history import SqsHistory, SqsTaskHistory, SqsWorkerHistory except ImportError as e: raise unittest.SkipTest( 'Could not test WorkerTaskGlobalEventHandlerTests: %s' % e) # Replace _config method with one that uses our dummy queue. def fake_config(s, *args): s._queue = DummyQueue() SqsHistory._config = fake_config # InstanceCache.disable() self.sch = CentralPlannerScheduler(retry_delay=100, remove_delay=1000, worker_disconnect_delay=10) self.w = Worker(scheduler=self.sch, worker_id='X') self.w2 = Worker(scheduler=self.sch, worker_id='Y') self.time = time.time
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # pylint: disable=F0401 from time import sleep from helpers import unittest try: import redis except ImportError: raise unittest.SkipTest('Unable to load redis module') from luigi.contrib.redis_store import RedisTarget HOST = 'localhost' PORT = 6379 DB = 15 PASSWORD = None SOCKET_TIMEOUT = None MARKER_PREFIX = 'luigi_test' EXPIRE = 5 class RedisTargetTest(unittest.TestCase): """ Test touch, exists and target expiration""" def test_touch_and_exists(self):
listener = socket.socket() listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) listener.bind(('localhost', 2134)) listener.listen(1) sys.stdout.write('ready') sys.stdout.flush() conn = listener.accept()[0] conn.sendall(b'hello') """ try: x = subprocess.check_output("ssh %s -S none -o BatchMode=yes 'echo 1'" % working_ssh_host, shell=True) if x != b'1\n': raise unittest.SkipTest('Not able to connect to ssh server') except Exception: raise unittest.SkipTest('Not able to connect to ssh server') class TestRemoteContext(unittest.TestCase): def setUp(self): self.context = RemoteContext(working_ssh_host) def tearDown(self): try: self.remote_server_handle.terminate() except Exception: pass def test_check_output(self):
database = 'spotify' user = os.getenv('POSTGRES_USER', 'spotify') password = '******' try: import psycopg2 conn = psycopg2.connect( user=user, host=host, database=database, password=password, ) conn.close() except Exception: raise unittest.SkipTest('Unable to connect to postgres') # to avoid copying: class CopyToTestDB(postgres.CopyToTable): host = host database = database user = user password = password class TestPostgresTask(CopyToTestDB): table = 'test_table' columns = (('test_text', 'text'), ('test_int', 'int'),
# See the License for the specific language governing permissions and # limitations under the License. # from helpers import unittest import luigi.contrib.batch as batch from helpers import skipOnTravis from nose.plugins.attrib import attr try: import boto3 client = boto3.client('batch') except ImportError: raise unittest.SkipTest('boto3 is not installed. BatchTasks require boto3') class MockBotoBatchClient(object): def describe_job_queues(self): return { 'jobQueues': [{ 'jobQueueName': 'test_queue', 'state': 'ENABLED', 'status': 'VALID' }] } def list_jobs(self, jobQueue='', jobStatus=''): return {'jobSummaryList': [{'jobName': 'test_job', 'jobId': 'abcd'}]}
DOC_TYPE = 'esindex_test_type' MARKER_INDEX = 'esindex_luigi_test_index_updates' MARKER_DOC_TYPE = 'esindex_test_entry' def _create_test_index(): """ Create content index, if if does not exists. """ es = elasticsearch.Elasticsearch(connection_class=Urllib3HttpConnection, host=HOST, port=PORT, http_auth=HTTP_AUTH) if not es.indices.exists(INDEX): es.indices.create(INDEX) try: _create_test_index() except Exception: raise unittest.SkipTest('Unable to connect to ElasticSearch') class ElasticsearchTargetTest(unittest.TestCase): """ Test touch and exists. """ def test_touch_and_exists(self): """ Basic test. """ target = ElasticsearchTarget(HOST, PORT, INDEX, DOC_TYPE, 'update_id', http_auth=HTTP_AUTH) target.marker_index = MARKER_INDEX target.marker_doc_type = MARKER_DOC_TYPE delete() self.assertFalse(target.exists(), 'Target should not exist before touching it')
def get_client(self): if six.PY3: raise unittest.SkipTest("snakebite doesn't work on Python yet.") return luigi.contrib.hdfs.SnakebiteHdfsClient()
from boto.s3 import key from botocore.exceptions import ClientError from mock import patch from helpers import skipOnTravisAndGithubActions, unittest, with_config from luigi.contrib.s3 import (DeprecatedBotoClientException, FileNotFoundException, InvalidDeleteException, S3Client, S3Target) from luigi.target import MissingParentDirectory from moto import mock_s3, mock_sts from target_test import FileSystemTargetTestMixin import pytest if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3): # spulec/moto#308 raise unittest.SkipTest('moto mock doesn\'t work with python3.4') AWS_ACCESS_KEY = "XXXXXXXXXXXXXXXXXXXX" AWS_SECRET_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" AWS_SESSION_TOKEN = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" def create_bucket(): conn = boto3.resource('s3', region_name='us-east-1') # We need to create the bucket since this is all in Moto's 'virtual' AWS account conn.create_bucket(Bucket='mybucket') return conn @pytest.mark.aws
def setUp(self): try: self.history = DbTaskHistory() except Exception: raise unittest.SkipTest( 'DBTaskHistory cannot be created: probably no MySQL available')
# See the License for the specific language governing permissions and # limitations under the License. # """This is an integration test for the GCS-luigi binding. This test requires credentials that can access GCS & access to a bucket below. Follow the directions in the gcloud tools to set up local credentials. """ from helpers import unittest try: import googleapiclient.errors import oauth2client except ImportError: raise unittest.SkipTest('Unable to load googleapiclient module') import os import tempfile import unittest from luigi.contrib import gcs from target_test import FileSystemTargetTestMixin from nose.plugins.attrib import attr # In order to run this test, you should set these to your GCS project/bucket. # Unfortunately there's no mock PROJECT_ID = os.environ.get('GCS_TEST_PROJECT_ID', 'your_project_id_here') BUCKET_NAME = os.environ.get('GCS_TEST_BUCKET', 'your_test_bucket_here') TEST_FOLDER = os.environ.get('TRAVIS_BUILD_ID', 'gcs_test_folder') CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default()
import luigi import logging from luigi.contrib.docker_runner import DockerTask import pytest logger = logging.getLogger('luigi-interface') try: import docker from docker.errors import ContainerError, ImageNotFound client = docker.from_env() client.version() except ImportError: raise unittest.SkipTest('Unable to load docker module') except Exception: raise unittest.SkipTest('Unable to connect to docker daemon') tempfile.tempdir = '/tmp' # set it explicitely to make it work out of the box in mac os local_file = NamedTemporaryFile() local_file.write(b'this is a test file\n') local_file.flush() class SuccessJob(DockerTask): image = "busybox:latest" name = "SuccessJob" class FailJobImageNotFound(DockerTask):
$ ESINDEX_TEST_PORT=9201 ESINDEX_TEST_HTTP_AUTH='admin:admin' nosetests test/_esindex_test.py """ # pylint: disable=C0103,E1101,F0401 import collections import datetime import os from helpers import unittest try: from luigi.contrib import esindex from elasticsearch.connection import Urllib3HttpConnection from luigi.contrib.esindex import CopyToIndex, ElasticsearchTarget except ImportError: raise unittest.SkipTest('Unable to import esindex') import luigi HOST = os.getenv('ESINDEX_TEST_HOST', 'localhost') PORT = os.getenv('ESINDEX_TEST_PORT', 9200) HTTP_AUTH = os.getenv('ESINDEX_TEST_HTTP_AUTH', None) INDEX = 'esindex_luigi_test' DOC_TYPE = 'esindex_test_type' MARKER_INDEX = 'esindex_luigi_test_index_updates' MARKER_DOC_TYPE = 'esindex_test_entry' def _create_test_index(): """ Create content index, if if does not exists. """ es = elasticsearch.Elasticsearch(connection_class=Urllib3HttpConnection,
from luigi.contrib.mongodb import MongoCellTarget, MongoRangeTarget from nose.plugins.attrib import attr HOST = 'localhost' PORT = 27017 INDEX = 'luigi_test' COLLECTION = 'luigi_collection' try: import pymongo mongo_client = pymongo.MongoClient(HOST, PORT) mongo_client.server_info() except ImportError: raise unittest.SkipTest('Unable to load pymongo module') except Exception: raise unittest.SkipTest('Unable to connect to local mongoDB instance') @attr('contrib') class MongoCellTargetTest(unittest.TestCase): """ MongoCellTarget unittest on local test database """ def setUp(self): """ Fill test database with fake data """ self.mongo_client = pymongo.MongoClient(HOST, PORT) self.collection = self.mongo_client[INDEX][COLLECTION]