def test_stop_getting_new_work(self):
d = DummyTask()
self.w.add(d)
self.assertFalse(d.complete())
try:
self.w.handle_interrupt(signal.SIGUSR1, None)
except AttributeError:
raise unittest.SkipTest('signal.SIGUSR1 not found on this system')
self.w.run()
self.assertFalse(d.complete())
def test_disabled_shutdown_hook(self):
w = Worker(scheduler=self.sch, keep_alive=True, no_install_shutdown_handler=True)
with w:
try:
# try to kill the worker!
os.kill(os.getpid(), signal.SIGUSR1)
except AttributeError:
raise unittest.SkipTest('signal.SIGUSR1 not found on this system')
# try to kill the worker... AGAIN!
t = SuicidalWorker(signal.SIGUSR1)
w.add(t)
w.run()
# task should have stepped away from the ledge, and completed successfully despite all the SIGUSR1 signals
self.assertEqual(list(self.sch.task_list('DONE', '').keys()), [t.task_id])
def setUp(self):
try:
from luigi.sqs_history import SqsHistory, SqsTaskHistory, SqsWorkerHistory
except ImportError as e:
raise unittest.SkipTest(
'Could not test WorkerTaskGlobalEventHandlerTests: %s' % e)
# Replace _config method with one that uses our dummy queue.
def fake_config(s, *args):
s._queue = DummyQueue()
SqsHistory._config = fake_config
# InstanceCache.disable()
self.sch = CentralPlannerScheduler(retry_delay=100,
remove_delay=1000,
worker_disconnect_delay=10)
self.w = Worker(scheduler=self.sch, worker_id='X')
self.w2 = Worker(scheduler=self.sch, worker_id='Y')
self.time = time.time
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=F0401
from time import sleep
from helpers import unittest
try:
import redis
except ImportError:
raise unittest.SkipTest('Unable to load redis module')
from luigi.contrib.redis_store import RedisTarget
HOST = 'localhost'
PORT = 6379
DB = 15
PASSWORD = None
SOCKET_TIMEOUT = None
MARKER_PREFIX = 'luigi_test'
EXPIRE = 5
class RedisTargetTest(unittest.TestCase):
""" Test touch, exists and target expiration"""
def test_touch_and_exists(self):
listener = socket.socket()
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listener.bind(('localhost', 2134))
listener.listen(1)
sys.stdout.write('ready')
sys.stdout.flush()
conn = listener.accept()[0]
conn.sendall(b'hello')
"""
try:
x = subprocess.check_output("ssh %s -S none -o BatchMode=yes 'echo 1'" %
working_ssh_host,
shell=True)
if x != b'1\n':
raise unittest.SkipTest('Not able to connect to ssh server')
except Exception:
raise unittest.SkipTest('Not able to connect to ssh server')
class TestRemoteContext(unittest.TestCase):
def setUp(self):
self.context = RemoteContext(working_ssh_host)
def tearDown(self):
try:
self.remote_server_handle.terminate()
except Exception:
pass
def test_check_output(self):
database = 'spotify'
user = os.getenv('POSTGRES_USER', 'spotify')
password = '******'
try:
import psycopg2
conn = psycopg2.connect(
user=user,
host=host,
database=database,
password=password,
)
conn.close()
except Exception:
raise unittest.SkipTest('Unable to connect to postgres')
# to avoid copying:
class CopyToTestDB(postgres.CopyToTable):
host = host
database = database
user = user
password = password
class TestPostgresTask(CopyToTestDB):
table = 'test_table'
columns = (('test_text', 'text'),
('test_int', 'int'),
# See the License for the specific language governing permissions and
# limitations under the License.
#
from helpers import unittest
import luigi.contrib.batch as batch
from helpers import skipOnTravis
from nose.plugins.attrib import attr
try:
import boto3
client = boto3.client('batch')
except ImportError:
raise unittest.SkipTest('boto3 is not installed. BatchTasks require boto3')
class MockBotoBatchClient(object):
def describe_job_queues(self):
return {
'jobQueues': [{
'jobQueueName': 'test_queue',
'state': 'ENABLED',
'status': 'VALID'
}]
}
def list_jobs(self, jobQueue='', jobStatus=''):
return {'jobSummaryList': [{'jobName': 'test_job', 'jobId': 'abcd'}]}
DOC_TYPE = 'esindex_test_type'
MARKER_INDEX = 'esindex_luigi_test_index_updates'
MARKER_DOC_TYPE = 'esindex_test_entry'
def _create_test_index():
""" Create content index, if if does not exists. """
es = elasticsearch.Elasticsearch(connection_class=Urllib3HttpConnection, host=HOST, port=PORT, http_auth=HTTP_AUTH)
if not es.indices.exists(INDEX):
es.indices.create(INDEX)
try:
_create_test_index()
except Exception:
raise unittest.SkipTest('Unable to connect to ElasticSearch')
class ElasticsearchTargetTest(unittest.TestCase):
""" Test touch and exists. """
def test_touch_and_exists(self):
""" Basic test. """
target = ElasticsearchTarget(HOST, PORT, INDEX, DOC_TYPE, 'update_id', http_auth=HTTP_AUTH)
target.marker_index = MARKER_INDEX
target.marker_doc_type = MARKER_DOC_TYPE
delete()
self.assertFalse(target.exists(),
'Target should not exist before touching it')
def get_client(self):
if six.PY3:
raise unittest.SkipTest("snakebite doesn't work on Python yet.")
return luigi.contrib.hdfs.SnakebiteHdfsClient()
from boto.s3 import key
from botocore.exceptions import ClientError
from mock import patch
from helpers import skipOnTravisAndGithubActions, unittest, with_config
from luigi.contrib.s3 import (DeprecatedBotoClientException, FileNotFoundException,
InvalidDeleteException, S3Client, S3Target)
from luigi.target import MissingParentDirectory
from moto import mock_s3, mock_sts
from target_test import FileSystemTargetTestMixin
import pytest
if (3, 4, 0) <= sys.version_info[:3] < (3, 4, 3):
# spulec/moto#308
raise unittest.SkipTest('moto mock doesn\'t work with python3.4')
AWS_ACCESS_KEY = "XXXXXXXXXXXXXXXXXXXX"
AWS_SECRET_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
AWS_SESSION_TOKEN = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
def create_bucket():
conn = boto3.resource('s3', region_name='us-east-1')
# We need to create the bucket since this is all in Moto's 'virtual' AWS account
conn.create_bucket(Bucket='mybucket')
return conn
@pytest.mark.aws
def setUp(self):
try:
self.history = DbTaskHistory()
except Exception:
raise unittest.SkipTest(
'DBTaskHistory cannot be created: probably no MySQL available')
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""This is an integration test for the GCS-luigi binding.
This test requires credentials that can access GCS & access to a bucket below.
Follow the directions in the gcloud tools to set up local credentials.
"""
from helpers import unittest
try:
import googleapiclient.errors
import oauth2client
except ImportError:
raise unittest.SkipTest('Unable to load googleapiclient module')
import os
import tempfile
import unittest
from luigi.contrib import gcs
from target_test import FileSystemTargetTestMixin
from nose.plugins.attrib import attr
# In order to run this test, you should set these to your GCS project/bucket.
# Unfortunately there's no mock
PROJECT_ID = os.environ.get('GCS_TEST_PROJECT_ID', 'your_project_id_here')
BUCKET_NAME = os.environ.get('GCS_TEST_BUCKET', 'your_test_bucket_here')
TEST_FOLDER = os.environ.get('TRAVIS_BUILD_ID', 'gcs_test_folder')
CREDENTIALS = oauth2client.client.GoogleCredentials.get_application_default()
import luigi
import logging
from luigi.contrib.docker_runner import DockerTask
import pytest
logger = logging.getLogger('luigi-interface')
try:
import docker
from docker.errors import ContainerError, ImageNotFound
client = docker.from_env()
client.version()
except ImportError:
raise unittest.SkipTest('Unable to load docker module')
except Exception:
raise unittest.SkipTest('Unable to connect to docker daemon')
tempfile.tempdir = '/tmp' # set it explicitely to make it work out of the box in mac os
local_file = NamedTemporaryFile()
local_file.write(b'this is a test file\n')
local_file.flush()
class SuccessJob(DockerTask):
image = "busybox:latest"
name = "SuccessJob"
class FailJobImageNotFound(DockerTask):
$ ESINDEX_TEST_PORT=9201 ESINDEX_TEST_HTTP_AUTH='admin:admin' nosetests test/_esindex_test.py
"""
# pylint: disable=C0103,E1101,F0401
import collections
import datetime
import os
from helpers import unittest
try:
from luigi.contrib import esindex
from elasticsearch.connection import Urllib3HttpConnection
from luigi.contrib.esindex import CopyToIndex, ElasticsearchTarget
except ImportError:
raise unittest.SkipTest('Unable to import esindex')
import luigi
HOST = os.getenv('ESINDEX_TEST_HOST', 'localhost')
PORT = os.getenv('ESINDEX_TEST_PORT', 9200)
HTTP_AUTH = os.getenv('ESINDEX_TEST_HTTP_AUTH', None)
INDEX = 'esindex_luigi_test'
DOC_TYPE = 'esindex_test_type'
MARKER_INDEX = 'esindex_luigi_test_index_updates'
MARKER_DOC_TYPE = 'esindex_test_entry'
def _create_test_index():
""" Create content index, if if does not exists. """
es = elasticsearch.Elasticsearch(connection_class=Urllib3HttpConnection,
from luigi.contrib.mongodb import MongoCellTarget, MongoRangeTarget
from nose.plugins.attrib import attr
HOST = 'localhost'
PORT = 27017
INDEX = 'luigi_test'
COLLECTION = 'luigi_collection'
try:
import pymongo
mongo_client = pymongo.MongoClient(HOST, PORT)
mongo_client.server_info()
except ImportError:
raise unittest.SkipTest('Unable to load pymongo module')
except Exception:
raise unittest.SkipTest('Unable to connect to local mongoDB instance')
@attr('contrib')
class MongoCellTargetTest(unittest.TestCase):
""" MongoCellTarget unittest on local test database """
def setUp(self):
"""
Fill test database with fake data
"""
self.mongo_client = pymongo.MongoClient(HOST, PORT)
self.collection = self.mongo_client[INDEX][COLLECTION]
