def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
secret_key = '123'
with session_scope() as session:
user = session.query(User).filter_by(username='******').first()
user.password = encrypt_password('password', secret_key)
session.add(user)
session.commit()
normal_user = session.query(User).filter_by(
username='******').first()
if not normal_user:
normal_user = User()
normal_user.username = '******'
normal_user.is_admin = False
normal_user.password = encrypt_password('passw0rd', secret_key)
session.add(normal_user)
session.commit()
return make_app(scheduler_manager,
node_manager,
None,
secret_key='123',
enable_authentication=True)
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
return make_app(scheduler_manager,
node_manager,
None,
secret_key='123')
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
self.node_manager = node_manager
return make_app(scheduler_manager, node_manager, webhook_daemon, secret_key='123',
enable_node_registration=True)
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
runner_factory = RunnerFactory(config)
return make_app(scheduler_manager,
node_manager,
None,
secret_key='123',
runner_factory=runner_factory)
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
runner_cls = AddVersionHandlerTestProcessFail.ProcessFailProjectWorkspaceStub
runner_factory = TestRunnerFactoryStub(runner_cls)
return make_app(scheduler_manager,
node_manager,
webhook_daemon,
secret_key='123',
project_storage_dir='./test_data',
runner_factory=runner_factory)
def get_app(self):
config = Config()
if self.scheduler_manager is None:
self.scheduler_manager = SchedulerManager(Config())
self.scheduler_manager.init()
node_manager = NodeManager(self.scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
return make_app(self.scheduler_manager,
node_manager,
webhook_daemon,
secret_key='123',
project_storage_dir=self.project_storage_dir,
runner_factory=self.runner_factory,
project_manager=self.project_manager)
class AppTest(AsyncHTTPTestCase):
@classmethod
def setUpClass(cls):
if os._exists('test.db'):
os.remove('test.db')
config = Config(values={'database_url': 'sqlite:///test.db'})
init_database(config)
os.environ['ASYNC_TEST_TIMEOUT'] = '120'
AppTest.init_project()
@classmethod
def init_project(self):
project_name = 'test_project'
version = '1.0'
spiders = [
'error_spider', 'fail_spider', 'log_spider', 'success_spider',
'warning_spider'
]
ioloop = IOLoop.current()
with open(
os.path.join(os.path.dirname(__file__),
'test_project-1.0-py2.7.egg'), 'rb') as egg_file:
def fun():
AppTest.project_manager.upload_project('test', project_name,
version, egg_file)
ioloop.run_sync(fun)
runner_factory = TestRunnerFactoryStub()
project_storage_dir = './test_data'
default_project_storage_version = 2
project_manager = ProjectManager(runner_factory, project_storage_dir,
default_project_storage_version)
scheduler_manager = None
def get_app(self):
config = Config()
if self.scheduler_manager is None:
self.scheduler_manager = SchedulerManager(Config())
self.scheduler_manager.init()
node_manager = NodeManager(self.scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
return make_app(self.scheduler_manager,
node_manager,
webhook_daemon,
secret_key='123',
project_storage_dir=self.project_storage_dir,
runner_factory=self.runner_factory,
project_manager=self.project_manager)
def _upload_test_project(self):
post_data = {}
post_data['egg'] = open(
os.path.join(os.path.dirname(__file__),
'test_project-1.0-py2.7.egg'), 'rb')
post_data['project'] = 'test_project'
post_data['version'] = '1.0'
datagen, headers = multipart_encode(post_data)
databuffer = b''.join(datagen)
response = self.fetch('/addversion.json',
method='POST',
headers=headers,
body=databuffer)
self.assertEqual(200, response.code)
post_data['egg'].close()
def test_clear_finised_jobs(self):
target = SchedulerManager()
target._clear_running_jobs()
jobids = []
spider_id = 1
for i in range(0, 101):
job = target.add_task('weibo', 'discover')
jobids.append(job.id)
target.job_start(job.id, 0)
job.status = 2
target.job_finished(job)
jobids = jobids[-100:]
target.clear_finished_jobs()
session = Session()
for job in session.query(HistoricalJob).filter(
HistoricalJob.spider_id == spider_id):
self.assertTrue(job.id in jobids)
session.close()