def test_commit(self):
with session_scope() as session:
project = Project()
project.name = 'test project'
session.add(project)
# after scope closed, the data should be already commited
with session_scope() as session:
project = session.query(Project).first()
self.assertEqual('test project', project.name)
def test_get_egg_versions(self):
data_dir = 'data'
project = Project(id=1, name='test_project', storage_version=1)
target = ProjectStorage(data_dir=data_dir, project=project)
self.test_put_egg()
self.assertEqual(target.list_egg_versions(), ['1_0'])
def upload_project(self, user_id, project_name, version, eggf):
runner = self.runner_factory.build(eggf)
try:
spiders = yield runner.list()
logger.debug('spiders: %s' % spiders)
project_settings_module = yield runner.settings_module()
finally:
runner.clear()
with session_scope() as session:
project = session.query(Project).filter_by(
name=project_name).first()
if project is None:
project = Project()
project.name = project_name
project.storage_version = self.default_project_storage_version
project.version = version
session.add(project)
package = project.package
if not package:
package = ProjectPackage()
package.project = project
package.type = 'scrapy'
package.settings_module = project_settings_module
package.spider_list = ','.join(spiders)
session.add(package)
session.flush()
project_storage = ProjectStorage(self.project_storage_dir, project)
project_storage.put_egg(eggf, version)
session.refresh(project)
for spider_name in spiders:
spider = session.query(Spider).filter_by(
project_id=project.id, name=spider_name).first()
if spider is None:
spider = Spider()
spider.name = spider_name
spider.project_id = project.id
session.add(spider)
session.commit()
session.refresh(spider)
session.commit()
raise Return(project)
def test_rollback(self):
class CommitFailedError(Exception):
pass
try:
with session_scope() as session:
project = Project()
project.name = 'test project'
session.add(project)
raise CommitFailedError()
except CommitFailedError:
pass
# after an exception raised in the scope, the data should be rolled back.
with session_scope() as session:
project = session.query(Project).first()
self.assertIsNone(project)
def test_get_project_eggs_dir(self):
data_dir = '.'
project = Project(id=1, name='test_project', storage_version=2)
target = ProjectStoragePathV2(data_dir=data_dir)
expect = './projects/1/eggs'
actual = target.get_project_eggs_dir(project)
self.assertEqual(expect, actual)
def test_get_job_log_path(self):
data_dir = '.'
project = Project(id=1, name='test_project', storage_version=1)
spider = Spider(project=project, name='test_spider')
job = HistoricalJob(id=uuid4().hex, spider=spider)
target = ProjectStoragePathV2(data_dir=data_dir)
actual = target.get_job_log_path(job)
expect = './projects/1/spiders/test_spider/jobs/%s/job.log' % job.id
self.assertEqual(expect, actual)
def test_get_egg(self):
data_dir = 'data'
project = Project(id=1, name='test_project', storage_version=1)
version = None
target = ProjectStorage(data_dir=data_dir, project=project)
self.test_put_egg()
get_version, get_file = target.get_egg(version)
self.assertEqual(
open('tests/test_project-1.0-py2.7.egg', 'rb').read(),
get_file.read())
self.assertEqual('1_0', get_version)
def test_get_egg_with_none_exist_version(self):
data_dir = 'data'
project = Project(id=1, name='test_project', storage_version=1)
version = '2.0'
target = ProjectStorage(data_dir=data_dir, project=project)
self.test_put_egg()
try:
get_version, get_file = target.get_egg(version)
self.fail('Should not get non-existing version file')
except EggFileNotFound:
pass
def test_put_egg(self):
data_dir = 'data'
project = Project(id=1, name='test_project', storage_version=1)
test_project_egg = 'tests/test_project-1.0-py2.7.egg'
fegg = open(test_project_egg, 'rb')
version = '1.0'
target = ProjectStorage(data_dir=data_dir, project=project)
target.put_egg(fegg, version)
target_egg_filepath = os.path.join(
target.storage_provider.get_project_eggs_dir(project), '1_0.egg')
self.assertTrue(os.path.exists(target_egg_filepath))
self.assertTrue(cmp(test_project_egg, target_egg_filepath))
def put_job_data(self):
data_dir = 'data'
project = Project(id=1, name='test_project', storage_version=1)
spider = Spider(project=project, name='test_spider')
job = HistoricalJob(id=uuid4().hex, spider=spider)
log_stream = BytesIO(b'test log here')
items_stream = BytesIO(b'{"foo": "bar}')
target = ProjectStorage(data_dir=data_dir, project=project)
target.put_job_data(job, log_stream, items_stream)
saved_log_stream = target.get_job_log(job)
log_stream.seek(0)
self.assertEqual(log_stream.read(), saved_log_stream.read())
saved_items_stream = target.get_job_items(job)
items_stream.seek(0)
self.assertEqual(items_stream.read(), saved_items_stream.read())