Example #1
0
    def test_init(self):
        x = LocalManager()
        self.assertListEqual(x.locals, [])
        self.assertTrue(x.ident_func)

        ident = lambda: 1
        loc = Local()
        x = LocalManager([loc], ident_func=ident)
        self.assertListEqual(x.locals, [loc])
        x = LocalManager(loc, ident_func=ident)
        self.assertListEqual(x.locals, [loc])
        self.assertIs(x.ident_func, ident)
        self.assertIs(x.locals[0].__ident_func__, ident)
        self.assertEqual(x.get_ident(), 1)

        with patch('celery.utils.threads.release_local') as release:
            x.cleanup()
            release.assert_called_with(loc)

        self.assertTrue(repr(x))
Example #2
0
    def test_init(self):
        x = LocalManager()
        assert x.locals == []
        assert x.ident_func

        def ident():
            return 1
        loc = Local()
        x = LocalManager([loc], ident_func=ident)
        assert x.locals == [loc]
        x = LocalManager(loc, ident_func=ident)
        assert x.locals == [loc]
        assert x.ident_func is ident
        assert x.locals[0].__ident_func__ is ident
        assert x.get_ident() == 1

        with patch('celery.utils.threads.release_local') as release:
            x.cleanup()
            release.assert_called_with(loc)

        assert repr(x)
Example #3
0
    def test_init(self):
        x = LocalManager()
        self.assertListEqual(x.locals, [])
        self.assertTrue(x.ident_func)

        ident = lambda: 1
        loc = Local()
        x = LocalManager([loc], ident_func=ident)
        self.assertListEqual(x.locals, [loc])
        x = LocalManager(loc, ident_func=ident)
        self.assertListEqual(x.locals, [loc])
        self.assertIs(x.ident_func, ident)
        self.assertIs(x.locals[0].__ident_func__, ident)
        self.assertEqual(x.get_ident(), 1)

        with patch('celery.utils.threads.release_local') as release:
            x.cleanup()
            release.assert_called_with(loc)

        self.assertTrue(repr(x))
Example #4
0
    def test_init(self):
        x = LocalManager()
        assert x.locals == []
        assert x.ident_func

        def ident():
            return 1
        loc = Local()
        x = LocalManager([loc], ident_func=ident)
        assert x.locals == [loc]
        x = LocalManager(loc, ident_func=ident)
        assert x.locals == [loc]
        assert x.ident_func is ident
        assert x.locals[0].__ident_func__ is ident
        assert x.get_ident() == 1

        with patch('celery.utils.threads.release_local') as release:
            x.cleanup()
            release.assert_called_with(loc)

        assert repr(x)
Example #5
0
from workflow.workflow import Workflow
from util import to_handler_function, decompress
from refine.refine import IllegalRuleCheckSum
from webui.controller.conf import settings as cnsettings

# Content Disposition Filename extraction RE.
CONTENT_DISPOSITION_FILE = re.compile('filename="([^"]+)"')

# Object returned to force recursion from source handler.
RECURSE_MARKER = {}

# pylint: disable=C0103
logger = get_task_logger(__name__)

local = Local()
local_manager = LocalManager()

FileMeta = namedtuple('FileMeta', 'path mime_by_ext mime_by_origin')
# pylint: enable=C0103


def get_dataset_dir(source_name, dataset_url):
    """Given a source name and a dataset decides which is the temporary
    dataset dir.
    """
    return os.path.join(settings.TMP_DIR, urllib.quote_plus(source_name),
                        _shorten_path_string(dataset_url))


def download_dataset(dataset):
    """Downloads dataset per source."""