Python Environment.get_environment Beispiele

Beispiel #1

Datei: test_environ.py Projekt: herberthamaral/scrapy

class EggStorageTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'project': 'mybot'}
        slot = 3
        env = self.environ.get_environment(msg, slot, '/path/to/file.egg')
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
        self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log'))
        self.assert_(env['SCRAPY_EGGFILE'].endswith('/path/to/file.egg'))
        self.failIf('SCRAPY_SETTINGS_MODULE' in env)

    def test_get_environment_without_eggfile(self):
        msg = {'project': 'newbot'}
        slot = 3
        env = self.environ.get_environment(msg, slot, None)
        self.assertEqual(env['SCRAPY_PROJECT'], 'newbot')
        self.assert_(env['SCRAPY_SQLITE_DB'].endswith('newbot.db'))
        self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log'))
        self.assertEqual(env['SCRAPY_SETTINGS_MODULE'], 'newbot.settings')
        self.failIf('SCRAPY_EGGFILE' in env)

Beispiel #2

Datei: test_environ.py Projekt: floppya/scrapy

class EnvironmentTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assertEqual(env['SCRAPY_SLOT'], '3')
        self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
        self.assertEqual(env['SCRAPY_JOB'], 'ID')
        self.assertEqual(env['SCRAPY_CONCURRENT_SPIDERS'], '1')
        self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
        self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log')))
        self.failIf('SCRAPY_SETTINGS_MODULE' in env)

Beispiel #3

class EnvironmentTest(unittest.TestCase):
    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assertEqual(env['SCRAPY_SLOT'], '3')
        self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
        self.assertEqual(env['SCRAPY_JOB'], 'ID')
        self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
        self.assert_(env['SCRAPY_LOG_FILE'].endswith(
            os.path.join('mybot', 'myspider', 'ID.log')))
        self.assert_(env['SCRAPY_FEED_URI'].endswith(
            os.path.join('mybot', 'myspider', 'ID.jl')))
        self.failIf('SCRAPY_SETTINGS_MODULE' in env)

Beispiel #4

Datei: test_environ.py Projekt: Wondermall/scrapyd

 def test_get_environment_with_logfile(self):
     config = Config(values={'items_dir': '', 'logs_dir': '.', 'logs_filename': '{project}-{spider}-{Y}{m}{d}T{H}{M}{S}'})
     msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
     slot = 3
     environ = Environment(config, initenv={})
     now = datetime.datetime.now()
     env = environ.get_environment(msg, slot)
     expected_logfilename = now.strftime("mybot-spider-%Y%m%dT%H%M%S")
     self.assert_(env['SCRAPY_LOG_FILE'], expected_logfilename)

Beispiel #5

Datei: test_environ.py Projekt: Wondermall/scrapyd

 def test_get_environment_with_no_items_dir(self):
     config = Config(values={'items_dir': '', 'logs_dir': ''})
     config.cp.add_section('settings')
     config.cp.set('settings', 'newbot', 'newbot.settings')
     msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
     slot = 3
     environ = Environment(config, initenv={})
     env = environ.get_environment(msg, slot)
     self.failUnless('SCRAPY_FEED_URI' not in env)
     self.failUnless('SCRAPY_LOG_FILE' not in env)

Beispiel #6

Datei: test_environ.py Projekt: onbillionapps/spider-server

 def test_get_environment_with_no_items_dir(self):
     config = Config(values={'items_dir': '', 'logs_dir': ''})
     config.cp.add_section('settings')
     config.cp.set('settings', 'newbot', 'newbot.settings')
     msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
     slot = 3
     environ = Environment(config, initenv={})
     env = environ.get_environment(msg, slot)
     self.failUnless('SCRAPY_FEED_URI' not in env)
     self.failUnless('SCRAPY_LOG_FILE' not in env)

Beispiel #7

Datei: test_environ.py Projekt: onbillionapps/spider-server

class EnvironmentTest(unittest.TestCase):
    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assertEqual(env['SCRAPY_SLOT'], '3')
        self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
        self.assertEqual(env['SCRAPY_JOB'], 'ID')
        self.assert_(env['SCRAPY_LOG_FILE'].endswith(
            os.path.join('mybot', 'myspider', 'ID.log')))
        if env.get('SCRAPY_FEED_URI'):  # Not compulsory
            self.assert_(env['SCRAPY_FEED_URI'].startswith('file://{}'.format(
                os.getcwd())))
            self.assert_(env['SCRAPY_FEED_URI'].endswith(
                os.path.join('mybot', 'myspider', 'ID.jl')))
        self.failIf('SCRAPY_SETTINGS_MODULE' in env)

    def test_get_environment_with_no_items_dir(self):
        config = Config(values={'items_dir': '', 'logs_dir': ''})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        environ = Environment(config, initenv={})
        env = environ.get_environment(msg, slot)
        self.failUnless('SCRAPY_FEED_URI' not in env)
        self.failUnless('SCRAPY_LOG_FILE' not in env)

    def test_get_environment_with_logfile(self):
        config = Config(
            values={
                'items_dir': '',
                'logs_dir': '.',
                'logs_filename': '{project}-{spider}-{job}-{Y}{m}{d}T{H}{M}{S}'
            })
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        environ = Environment(config, initenv={})
        now = datetime.datetime.now()
        env = environ.get_environment(msg, slot)
        expected_logfilename = now.strftime("mybot-spider-%Y%m%dT%H%M%S")
        self.assert_(env['SCRAPY_LOG_FILE'], expected_logfilename)

Beispiel #8

Datei: test_environ.py Projekt: onbillionapps/spider-server

 def test_get_environment_with_logfile(self):
     config = Config(
         values={
             'items_dir': '',
             'logs_dir': '.',
             'logs_filename': '{project}-{spider}-{job}-{Y}{m}{d}T{H}{M}{S}'
         })
     msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
     slot = 3
     environ = Environment(config, initenv={})
     now = datetime.datetime.now()
     env = environ.get_environment(msg, slot)
     expected_logfilename = now.strftime("mybot-spider-%Y%m%dT%H%M%S")
     self.assert_(env['SCRAPY_LOG_FILE'], expected_logfilename)

Beispiel #9

Datei: test_environ.py Projekt: AllenCHM/scrapyd

class EnvironmentTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assertEqual(env['SCRAPY_SLOT'], '3')
        self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
        self.assertEqual(env['SCRAPY_JOB'], 'ID')
        self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log')))
        if env.get('SCRAPY_FEED_URI'):  # Not compulsory
            self.assert_(env['SCRAPY_FEED_URI'].startswith('file://{}'.format(os.getcwd())))
            self.assert_(env['SCRAPY_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl')))
        self.failIf('SCRAPY_SETTINGS_MODULE' in env)

    def test_get_environment_with_no_items_dir(self):
        config = Config(values={'items_dir': '', 'logs_dir': ''})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        environ = Environment(config, initenv={})
        env = environ.get_environment(msg, slot)
        self.failUnless('SCRAPY_FEED_URI' not in env)
        self.failUnless('SCRAPY_LOG_FILE' not in env)

    def test_get_environment_with_logfile(self):
        config = Config(values={'items_dir': '', 'logs_dir': '.', 'logs_filename': '{project}-{spider}-{Y}{m}{d}T{H}{M}{S}'})
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        environ = Environment(config, initenv={})
        now = datetime.datetime.now()
        env = environ.get_environment(msg, slot)
        expected_logfilename = now.strftime("mybot-spider-%Y%m%dT%H%M%S")
        self.assert_(env['SCRAPY_LOG_FILE'], expected_logfilename)

Beispiel #10

Datei: test_environ.py Projekt: my8100/scrapyd-CircleCI

class EnvironmentTest(unittest.TestCase):
    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assertEqual(env['SCRAPY_SLOT'], '3')
        self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
        self.assertEqual(env['SCRAPY_JOB'], 'ID')
        self.assert_(env['SCRAPY_LOG_FILE'].endswith(
            os.path.join('mybot', 'myspider', 'ID.log')))
        if env.get('SCRAPY_FEED_URI'):  # Not compulsory
            self.assert_(env['SCRAPY_FEED_URI'].startswith('file://{}'.format(
                os.getcwd())))
            self.assert_(env['SCRAPY_FEED_URI'].endswith(
                os.path.join('mybot', 'myspider', 'ID.jl')))
        self.assertNotIn('SCRAPY_SETTINGS_MODULE', env)

    def test_get_environment_with_no_items_dir(self):
        config = Config(values={'items_dir': '', 'logs_dir': ''})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        environ = Environment(config, initenv={})
        env = environ.get_environment(msg, slot)
        self.assertNotIn('SCRAPY_FEED_URI', env)
        self.assertNotIn('SCRAPY_LOG_FILE', env)

Beispiel #11

Datei: test_environ.py Projekt: nilp0inter/scrapy

class EnvironmentTest(unittest.TestCase):
    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={"eggs_dir": d, "logs_dir": d})
        config.cp.add_section("settings")
        config.cp.set("settings", "newbot", "newbot.settings")
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {"_project": "mybot", "_spider": "myspider", "_job": "ID"}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env["SCRAPY_PROJECT"], "mybot")
        self.assertEqual(env["SCRAPY_SLOT"], "3")
        self.assertEqual(env["SCRAPY_SPIDER"], "myspider")
        self.assertEqual(env["SCRAPY_JOB"], "ID")
        self.assert_(env["SCRAPY_LOG_FILE"].endswith(os.path.join("mybot", "myspider", "ID.log")))
        self.assert_(env["SCRAPY_FEED_URI"].endswith(os.path.join("mybot", "myspider", "ID.jl")))
        self.failIf("SCRAPY_SETTINGS_MODULE" in env)

Beispiel #12

Datei: test_environ.py Projekt: jurov/scrapyd

class EnvironmentTest(unittest.TestCase):

    def setUp(self):
        d = self.mktemp()
        os.mkdir(d)
        config = Config(values={'eggs_dir': d, 'logs_dir': d})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        self.environ = Environment(config, initenv={})

    def test_interface(self):
        verifyObject(IEnvironment, self.environ)

    def test_get_environment_with_eggfile(self):
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        env = self.environ.get_environment(msg, slot)
        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
        self.assertEqual(env['SCRAPY_SLOT'], '3')
        self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
        self.assertEqual(env['SCRAPY_JOB'], 'ID')
        self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log')))
        if env.get('SCRAPY_FEED_URI'): #not compulsory
            self.assert_(env['SCRAPY_FEED_URI'].endswith(os.path.join('mybot', 'myspider', 'ID.jl')))
        self.failIf('SCRAPY_SETTINGS_MODULE' in env)

    def test_get_environment_with_no_items_dir(self):
        config = Config(values={'items_dir': '', 'logs_dir': ''})
        config.cp.add_section('settings')
        config.cp.set('settings', 'newbot', 'newbot.settings')
        msg = {'_project': 'mybot', '_spider': 'myspider', '_job': 'ID'}
        slot = 3
        environ = Environment(config, initenv={})
        env = environ.get_environment(msg, slot)
        self.failUnless('SCRAPY_FEED_URI' not in env)
        self.failUnless('SCRAPY_LOG_FILE' not in env)