def get_app(self): config = Config() scheduler_manager = SchedulerManager(config=config) scheduler_manager.init() node_manager = NodeManager(scheduler_manager) node_manager.init() secret_key = '123' with session_scope() as session: user = session.query(User).filter_by(username='******').first() user.password = encrypt_password('password', secret_key) session.add(user) session.commit() normal_user = session.query(User).filter_by( username='******').first() if not normal_user: normal_user = User() normal_user.username = '******' normal_user.is_admin = False normal_user.password = encrypt_password('passw0rd', secret_key) session.add(normal_user) session.commit() return make_app(scheduler_manager, node_manager, None, secret_key='123', enable_authentication=True)
def get_app(self): config = Config() scheduler_manager = SchedulerManager(config=config) scheduler_manager.init() node_manager = NodeManager(scheduler_manager) node_manager.init() return make_app(scheduler_manager, node_manager, None, secret_key='123')
def get_app(self): config = Config() scheduler_manager = SchedulerManager(config=config) scheduler_manager.init() node_manager = NodeManager(scheduler_manager) node_manager.init() webhook_daemon = WebhookDaemon(config, SpiderSettingLoader()) webhook_daemon.init() self.node_manager = node_manager return make_app(scheduler_manager, node_manager, webhook_daemon, secret_key='123', enable_node_registration=True)
def get_app(self): config = Config() scheduler_manager = SchedulerManager(config=config) scheduler_manager.init() node_manager = NodeManager(scheduler_manager) node_manager.init() runner_factory = RunnerFactory(config) return make_app(scheduler_manager, node_manager, None, secret_key='123', runner_factory=runner_factory)
def get_app(self): config = Config() scheduler_manager = SchedulerManager(config=config) scheduler_manager.init() node_manager = NodeManager(scheduler_manager) node_manager.init() webhook_daemon = WebhookDaemon(config, SpiderSettingLoader()) webhook_daemon.init() runner_cls = AddVersionHandlerTestProcessFail.ProcessFailProjectWorkspaceStub runner_factory = TestRunnerFactoryStub(runner_cls) return make_app(scheduler_manager, node_manager, webhook_daemon, secret_key='123', project_storage_dir='./test_data', runner_factory=runner_factory)
def get_app(self): config = Config() if self.scheduler_manager is None: self.scheduler_manager = SchedulerManager(Config()) self.scheduler_manager.init() node_manager = NodeManager(self.scheduler_manager) node_manager.init() webhook_daemon = WebhookDaemon(config, SpiderSettingLoader()) webhook_daemon.init() return make_app(self.scheduler_manager, node_manager, webhook_daemon, secret_key='123', project_storage_dir=self.project_storage_dir, runner_factory=self.runner_factory, project_manager=self.project_manager)
class AppTest(AsyncHTTPTestCase): @classmethod def setUpClass(cls): if os._exists('test.db'): os.remove('test.db') config = Config(values={'database_url': 'sqlite:///test.db'}) init_database(config) os.environ['ASYNC_TEST_TIMEOUT'] = '120' AppTest.init_project() @classmethod def init_project(self): project_name = 'test_project' version = '1.0' spiders = [ 'error_spider', 'fail_spider', 'log_spider', 'success_spider', 'warning_spider' ] ioloop = IOLoop.current() with open( os.path.join(os.path.dirname(__file__), 'test_project-1.0-py2.7.egg'), 'rb') as egg_file: def fun(): AppTest.project_manager.upload_project('test', project_name, version, egg_file) ioloop.run_sync(fun) runner_factory = TestRunnerFactoryStub() project_storage_dir = './test_data' default_project_storage_version = 2 project_manager = ProjectManager(runner_factory, project_storage_dir, default_project_storage_version) scheduler_manager = None def get_app(self): config = Config() if self.scheduler_manager is None: self.scheduler_manager = SchedulerManager(Config()) self.scheduler_manager.init() node_manager = NodeManager(self.scheduler_manager) node_manager.init() webhook_daemon = WebhookDaemon(config, SpiderSettingLoader()) webhook_daemon.init() return make_app(self.scheduler_manager, node_manager, webhook_daemon, secret_key='123', project_storage_dir=self.project_storage_dir, runner_factory=self.runner_factory, project_manager=self.project_manager) def _upload_test_project(self): post_data = {} post_data['egg'] = open( os.path.join(os.path.dirname(__file__), 'test_project-1.0-py2.7.egg'), 'rb') post_data['project'] = 'test_project' post_data['version'] = '1.0' datagen, headers = multipart_encode(post_data) databuffer = b''.join(datagen) response = self.fetch('/addversion.json', method='POST', headers=headers, body=databuffer) self.assertEqual(200, response.code) post_data['egg'].close()
def test_clear_finised_jobs(self): target = SchedulerManager() target._clear_running_jobs() jobids = [] spider_id = 1 for i in range(0, 101): job = target.add_task('weibo', 'discover') jobids.append(job.id) target.job_start(job.id, 0) job.status = 2 target.job_finished(job) jobids = jobids[-100:] target.clear_finished_jobs() session = Session() for job in session.query(HistoricalJob).filter( HistoricalJob.spider_id == spider_id): self.assertTrue(job.id in jobids) session.close()