def upgrade():
ioloop = IOLoop.current()
config = Config()
runner_factory = RunnerFactory(config)
with session_scope() as session:
for project in session.query(Project):
if project.package is not None:
continue
storage = ProjectStorage(config.get('project_storage_dir'),
project)
version, eggf = storage.get_egg()
runner = runner_factory.build(eggf)
try:
project_settings_module = ioloop.run_sync(
runner.settings_module)
spider_list = ioloop.run_sync(runner.list)
package = ProjectPackage()
package.project = project
package.type = 'scrapy'
package.settings_module = project_settings_module
package.spider_list = ','.join(spider_list)
session.add(package)
session.commit()
except (ProcessFailed, InvalidProjectEgg) as ex:
logger.error(ex)
def test_get_int_from_env(self):
target = Config()
bind_port = '1'
os.environ['SCRAPYDD_BIND_PORT'] = bind_port
self.assertEqual(int(bind_port), target.getint('bind_port'))
del os.environ['SCRAPYDD_BIND_PORT']
def test_get_raise_no_option_error(self):
target = Config()
try:
target.get('SomeNotExistOption')
self.fail('No Error caught')
except NoOptionError:
pass
v = target.get('SomeNotExistOption', 'DefaultValue')
self.assertEqual(v, 'DefaultValue')
def get_app(self):
config = Config()
if self.scheduler_manager is None:
self.scheduler_manager = SchedulerManager(Config())
self.scheduler_manager.init()
node_manager = NodeManager(self.scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
return make_app(self.scheduler_manager,
node_manager,
webhook_daemon,
secret_key='123',
project_storage_dir=self.project_storage_dir,
runner_factory=self.runner_factory,
project_manager=self.project_manager)
def setUpClass(cls):
if os._exists('test.db'):
os.remove('test.db')
config = Config(values={'database_url': 'sqlite:///test.db'})
init_database(config)
os.environ['ASYNC_TEST_TIMEOUT'] = '120'
AppTest.init_project()
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
secret_key = '123'
with session_scope() as session:
user = session.query(User).filter_by(username='******').first()
user.password = encrypt_password('password', secret_key)
session.add(user)
session.commit()
normal_user = session.query(User).filter_by(
username='******').first()
if not normal_user:
normal_user = User()
normal_user.username = '******'
normal_user.is_admin = False
normal_user.password = encrypt_password('passw0rd', secret_key)
session.add(normal_user)
session.commit()
return make_app(scheduler_manager,
node_manager,
None,
secret_key='123',
enable_authentication=True)
def test_send(self):
smtp_port = 26
smtp_server = TestingSMTPServer(smtp_port)
smtp_server.start()
smtp_from = '*****@*****.**'
config = Config(
values={
'smtp_port': str(smtp_port),
'smtp_server': 'localhost',
'smtp_user': '',
'smtp_passwd': '',
'smtp_from': smtp_from
})
target = MailSender(config)
to_address = '*****@*****.**'
subject = 'some_subject'
plain_text_content = 'some content'
try:
target.send(to_address, subject, plain_text_content)
self.assertEqual(smtp_from, smtp_server.received_mailfrom)
self.assertEqual(to_address, smtp_server.to_address)
self.assertEqual(subject, smtp_server.subject)
self.assertEqual(plain_text_content, smtp_server.body)
finally:
smtp_server.close()
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
return make_app(scheduler_manager,
node_manager,
None,
secret_key='123')
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
self.node_manager = node_manager
return make_app(scheduler_manager, node_manager, webhook_daemon, secret_key='123',
enable_node_registration=True)
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
runner_factory = RunnerFactory(config)
return make_app(scheduler_manager,
node_manager,
None,
secret_key='123',
runner_factory=runner_factory)
def get_app(self):
config = Config()
scheduler_manager = SchedulerManager(config=config)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
runner_cls = AddVersionHandlerTestProcessFail.ProcessFailProjectWorkspaceStub
runner_factory = TestRunnerFactoryStub(runner_cls)
return make_app(scheduler_manager,
node_manager,
webhook_daemon,
secret_key='123',
project_storage_dir='./test_data',
runner_factory=runner_factory)
def run(argv=None):
if argv is None:
argv = sys.argv
parser = OptionParser(prog='scrapydd server')
parser.add_option('--daemon',
action='store_true',
help='run scrapydd server in daemon mode')
parser.add_option('--pidfile',
help='pid file will be created when daemon started')
opts, args = parser.parse_args(argv)
pidfile = opts.pidfile or 'scrapydd-server.pid'
config = Config()
init_logging(config)
if opts.daemon:
print 'starting daemon.'
daemon = Daemon(pidfile=pidfile)
daemon.start()
sys.exit(0)
else:
start_server()
def start_server(argv=None):
config = Config()
logging.debug('starting server with argv : %s' % str(argv))
init_database()
bind_address = config.get('bind_address')
bind_port = config.getint('bind_port')
try:
https_port = int(
config.get('https_port')) if config.get('https_port') else None
except ValueError:
logger.warning('https_port is configured, but it is not int, %s' %
config.get('https_port'))
https_port = None
if https_port:
https_sockets = tornado.netutil.bind_sockets(https_port, bind_address)
sockets = tornado.netutil.bind_sockets(bind_port, bind_address)
# task_id is current process identifier when forked processes, start with 0
task_id = None
if not sys.platform.startswith('win'):
task_id = fork_processes(config.getint('fork_proc_count'))
else:
logger.warning(
'Windows platform does not support forking process, running in single process mode.'
)
cluster_sync_obj = None
if task_id is not None and config.get('cluster_bind_address'):
cluster_node = ClusterNode(task_id, config)
cluster_sync_obj = cluster_node.sync_obj
scheduler_manager = SchedulerManager(config=config,
syncobj=cluster_sync_obj)
scheduler_manager.init()
node_manager = NodeManager(scheduler_manager)
node_manager.init()
webhook_daemon = WebhookDaemon(config, SpiderSettingLoader())
webhook_daemon.init()
app = make_app(scheduler_manager, node_manager, webhook_daemon)
server = tornado.httpserver.HTTPServer(app)
server.add_sockets(sockets)
if https_port:
check_and_gen_ssl_keys(config)
if config.getboolean('client_validation'):
ssl_ctx = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
else:
ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
ssl_ctx.load_cert_chain(
os.path.join('keys', "%s.crt" % config.get('server_name')),
os.path.join('keys', "%s.key" % config.get('server_name')))
ssl_ctx.load_verify_locations(cafile='keys/ca.crt')
ssl_ctx.check_hostname = False
httpsserver = tornado.httpserver.HTTPServer(app, ssl_options=ssl_ctx)
httpsserver.add_sockets(https_sockets)
logger.info('starting https server on %s:%s' %
(bind_address, https_port))
ioloop = tornado.ioloop.IOLoop.current()
ioloop.start()
def setUp(self):
if os.path.exists('test.db'):
os.remove('test.db')
self.config = Config(values={'database_url': 'sqlite:///test.db'})
def setUpClass(cls):
os.environ['ASYNC_TEST_TIMEOUT'] = '500'
if os.path.exists('test.db'):
os.remove('test.db')
config = Config(values={'database_url': 'sqlite:///test.db'})
init_database(config)
def test_get(self):
target = Config()
self.assertEqual('0.0.0.0', target.get('bind_address'))
def test_getfloat(self):
target = Config()
self.assertEqual(6800, target.getfloat('bind_port'))
def test_get_from_values(self):
target = Config(values={'SomeSetting': 'Value'})
self.assertEqual('Value', target.get('SomeSetting'))
def test_get_bool_from_extrafiles(self):
target = Config(extra_sources=('tests/test.conf',))
self.assertEqual(False, target.getboolean('BoolSetting'))
self.assertEqual(True, target.getboolean('BoolSettingTrue'))
def test_get_int_from_env(self):
target = Config()
debug = 'True'
os.environ['SCRAPYDD_DEBUG'] = debug
self.assertEqual(True, target.getboolean('debug'))
del os.environ['SCRAPYDD_DEBUG']
def test_get_from_env(self):
target = Config()
database_url = 'sqlite:///test.db'
os.environ['SCRAPYDD_DATABASE_URL'] = 'sqlite:///test.db'
self.assertEqual( database_url,target.get('database_url'))
del os.environ['SCRAPYDD_DATABASE_URL']
def test_getboolean(self):
target = Config(extra_sources=('tests/test.conf', ))
self.assertEqual(False, target.getboolean('debug'))