class EggStorageTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
config = Config(values={'eggs_dir': d})
self.eggst = FilesystemEggStorage(config)
def test_interface(self):
verifyObject(IEggStorage, self.eggst)
def test_put_get_list_delete(self):
self.eggst.put(StringIO("egg01"), 'mybot', '01')
self.eggst.put(StringIO("egg03"), 'mybot', '03')
self.eggst.put(StringIO("egg02"), 'mybot', '02')
self.assertEqual(self.eggst.list('mybot'), ['01', '02', '03'])
self.assertEqual(self.eggst.list('mybot2'), [])
v, f = self.eggst.get('mybot')
self.assertEqual(v, "03")
self.assertEqual(f.read(), "egg03")
v, f = self.eggst.get('mybot', '02')
self.assertEqual(v, "02")
self.assertEqual(f.read(), "egg02")
self.eggst.delete('mybot', '02')
self.assertEqual(self.eggst.list('mybot'), ['01', '03'])
self.eggst.delete('mybot')
self.assertEqual(self.eggst.list('mybot'), [])
class EggStorageTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
config = Config(values={"eggs_dir": d})
self.eggst = FilesystemEggStorage(config)
def test_interface(self):
verifyObject(IEggStorage, self.eggst)
def test_put_get_list_delete(self):
self.eggst.put(StringIO("egg01"), "mybot", "01")
self.eggst.put(StringIO("egg03"), "mybot", "03")
self.eggst.put(StringIO("egg02"), "mybot", "02")
self.assertEqual(self.eggst.list("mybot"), ["01", "02", "03"])
self.assertEqual(self.eggst.list("mybot2"), [])
v, f = self.eggst.get("mybot")
self.assertEqual(v, "03")
self.assertEqual(f.read(), "egg03")
f.close()
v, f = self.eggst.get("mybot", "02")
self.assertEqual(v, "02")
self.assertEqual(f.read(), "egg02")
f.close()
self.eggst.delete("mybot", "02")
self.assertEqual(self.eggst.list("mybot"), ["01", "03"])
self.eggst.delete("mybot")
self.assertEqual(self.eggst.list("mybot"), [])
class EggStorageTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
config = Config(values={'eggs_dir': d})
self.eggst = FilesystemEggStorage(config)
def test_interface(self):
verifyObject(IEggStorage, self.eggst)
def test_put_get_list_delete(self):
self.eggst.put(StringIO("egg01"), 'mybot', '01')
self.eggst.put(StringIO("egg03"), 'mybot', '03')
self.eggst.put(StringIO("egg02"), 'mybot', '02')
self.assertEqual(self.eggst.list('mybot'), ['01', '02', '03'])
self.assertEqual(self.eggst.list('mybot2'), [])
v, f = self.eggst.get('mybot')
self.assertEqual(v, "03")
self.assertEqual(f.read(), "egg03")
f.close()
v, f = self.eggst.get('mybot', '02')
self.assertEqual(v, "02")
self.assertEqual(f.read(), "egg02")
f.close()
self.eggst.delete('mybot', '02')
self.assertEqual(self.eggst.list('mybot'), ['01', '03'])
self.eggst.delete('mybot')
self.assertEqual(self.eggst.list('mybot'), [])
class EggStorageTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
config = Config(values={'eggs_dir': d})
self.eggst = FilesystemEggStorage(config)
def test_interface(self):
verifyObject(IEggStorage, self.eggst)
def test_put_get_list_delete(self):
self.eggst.put(BytesIO(b"egg01"), 'mybot', '01')
self.eggst.put(BytesIO(b"egg03"), 'mybot', '03/ver')
self.eggst.put(BytesIO(b"egg02"), 'mybot', '02_my branch')
self.assertEqual(self.eggst.list('mybot'), [
'01',
'02_my_branch',
'03_ver'
])
self.assertEqual(self.eggst.list('mybot2'), [])
v, f = self.eggst.get('mybot')
self.assertEqual(v, "03_ver")
self.assertEqual(f.read(), b"egg03")
f.close()
v, f = self.eggst.get('mybot', '02_my branch')
self.assertEqual(v, "02_my branch")
self.assertEqual(f.read(), b"egg02")
f.close()
v, f = self.eggst.get('mybot', '02_my_branch')
self.assertEqual(v, "02_my_branch")
self.assertEqual(f.read(), b"egg02")
f.close()
self.eggst.delete('mybot', '02_my branch')
self.assertEqual(self.eggst.list('mybot'), ['01', '03_ver'])
self.eggst.delete('mybot', '03_ver')
self.assertEqual(self.eggst.list('mybot'), ['01'])
self.eggst.delete('mybot')
self.assertEqual(self.eggst.list('mybot'), [])
class EggStorageTest(unittest.TestCase):
def setUp(self):
d = self.mktemp()
config = Config(values={'eggs_dir': d})
self.eggst = FilesystemEggStorage(config)
def test_interface(self):
verifyObject(IEggStorage, self.eggst)
def test_put_get_list_delete(self):
self.eggst.put(BytesIO(get_data("scrapyd.tests", "mybot.egg")),
'mybot', '01')
self.eggst.put(BytesIO(get_data("scrapyd.tests", "mybot3.egg")),
'mybot', '03/ver')
self.eggst.put(BytesIO(get_data("scrapyd.tests", "mybot2.egg")),
'mybot', '02_my branch')
self.assertEqual(self.eggst.list('mybot'),
['01', '02_my_branch', '03_ver'])
self.assertEqual(self.eggst.list('mybot2'), [])
v, f = self.eggst.get('mybot')
self.assertEqual(v, "03_ver")
self.assertEqual(f.read(), get_data("scrapyd.tests", "mybot3.egg"))
f.close()
v, f = self.eggst.get('mybot', '02_my branch')
self.assertEqual(v, "02_my branch")
self.assertEqual(f.read(), get_data("scrapyd.tests", "mybot2.egg"))
f.close()
v, f = self.eggst.get('mybot', '02_my_branch')
self.assertEqual(v, "02_my_branch")
self.assertEqual(f.read(), get_data("scrapyd.tests", "mybot2.egg"))
f.close()
self.eggst.delete('mybot', '02_my branch')
self.assertEqual(self.eggst.list('mybot'), ['01', '03_ver'])
self.eggst.delete('mybot', '03_ver')
self.assertEqual(self.eggst.list('mybot'), ['01'])
self.eggst.delete('mybot')
self.assertEqual(self.eggst.list('mybot'), [])
class ProjectWorkspace(object):
pip = None
python = None
process = None
project_workspace_dir = None
project_check = None
temp_dir = None
def __init__(self, project_name):
project_workspace_dir = os.path.abspath(
os.path.join('workspace', project_name))
self.project_workspace_dir = project_workspace_dir
self.project_name = project_name
self.egg_storage = FilesystemEggStorage(scrapyd.config.Config())
if sys.platform.startswith('linux'):
self.pip = os.path.join(project_workspace_dir, 'bin', 'pip')
self.python = os.path.join(project_workspace_dir, 'bin', 'python')
elif sys.platform.startswith('win'):
self.pip = os.path.join(project_workspace_dir, 'Scripts',
'pip.exe')
self.python = os.path.join(project_workspace_dir, 'Scripts',
'python.exe')
else:
raise NotImplementedError('Unsupported system %s' % sys.platform)
def init(self):
'''
init project isolated workspace,
:return: future
'''
future = Future()
if os.path.exists(self.pip) and os.path.exists(self.python):
future.set_result(self)
return future
logger.debug('start creating virtualenv.')
try:
process = Popen([
'virtualenv', '--system-site-packages',
self.project_workspace_dir
],
stdout=PIPE,
stderr=PIPE)
except Exception as e:
future.set_exception(e)
return future
def check_process():
logger.debug('create virtualenv process poll.')
retcode = process.poll()
if retcode is not None:
if retcode == 0:
future.set_result(self)
else:
std_output = process.stdout.read()
err_output = process.stderr.read()
future.set_exception(
ProcessFailed('Error when init workspace virtualenv ',
std_output=std_output,
err_output=err_output))
return
IOLoop.current().call_later(1, check_process)
check_process()
return future
def find_project_requirements(self, project, egg_storage=None, eggf=None):
if eggf is None:
if egg_storage is None:
egg_storage = FilesystemEggStorage(scrapyd.config.Config())
version, eggf = egg_storage.get(project)
try:
prefix = '%s-nover-' % (project)
fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg')
logger.debug('tmp egg file saved to %s' % eggpath)
lf = os.fdopen(fd, 'wb')
eggf.seek(0)
shutil.copyfileobj(eggf, lf)
lf.close()
try:
d = pkg_resources.find_distributions(eggpath).next()
except StopIteration:
raise ValueError("Unknown or corrupt egg")
requirements = [str(x) for x in d.requires()]
return requirements
finally:
if eggpath:
os.remove(eggpath)
def test_egg(self, eggf):
future = Future()
temp_dir = tempfile.mkdtemp('scrapydd-egg-%s' % self.project_name)
self.temp_dir = temp_dir
eggf.seek(0)
egg_storage = FilesystemEggStorage(
scrapyd.config.Config({'eggs_dir': os.path.join(temp_dir,
'eggs')}))
egg_storage.put(eggf, project=self.project_name, version='1')
eggf.seek(0)
requirements = self._read_egg_requirements(eggf) + ['scrapyd']
def after_spider_list(callback_future):
logger.debug('after_spider_list')
exc = callback_future.exception()
if exc is not None:
future.set_exception(exc)
return
spider_list = callback_future.result()
#os.removedirs(temp_dir)
future.set_result(spider_list)
def after_pip_install(callback_future):
logger.debug('after_pip_install')
exc = callback_future.exception()
if exc is not None:
future.set_exception(exc)
return
self.spider_list(self.project_name,
cwd=temp_dir).add_done_callback(after_spider_list)
self.pip_install(requirements).add_done_callback(after_pip_install)
return future
def _read_egg_requirements(self, eggf):
try:
prefix = '%s-%s-' % (self.project_name, 0)
fd, eggpath = tempfile.mkstemp(prefix=prefix, suffix='.egg')
logger.debug('tmp egg file saved to %s' % eggpath)
lf = os.fdopen(fd, 'wb')
eggf.seek(0)
shutil.copyfileobj(eggf, lf)
lf.close()
try:
d = pkg_resources.find_distributions(eggpath).next()
except StopIteration:
raise ValueError("Unknown or corrupt egg")
requirements = [str(x) for x in d.requires()]
return requirements
finally:
if eggpath:
os.remove(eggpath)
def pip_install(self, requirements):
logger.debug('installing requirements: %s' % requirements)
future = Future()
try:
process = Popen([self.pip, 'install'] + requirements,
stdout=PIPE,
stderr=PIPE)
except Exception as e:
future.set_exception(e)
return future
def check_process():
logger.debug('poll')
retcode = process.poll()
if retcode is not None:
if retcode == 0:
future.set_result(self)
else:
std_out = process.stdout.read()
err_out = process.stderr.read()
future.set_exception(
ProcessFailed(std_output=std_out, err_output=err_out))
return
IOLoop.current().call_later(1, check_process)
check_process()
return future
def spider_list(self, project, cwd=None):
future = Future()
try:
env = os.environ.copy()
env['SCRAPY_PROJECT'] = project
process = Popen([self.python, '-m', 'scrapyd.runner', 'list'],
env=env,
cwd=cwd,
stdout=PIPE,
stderr=PIPE)
except Exception as e:
logger.error(e)
future.set_exception(e)
return future
def check_process():
logger.debug('poll')
retcode = process.poll()
if retcode is not None:
if retcode == 0:
future.set_result(process.stdout.read().splitlines())
else:
#future.set_exception(ProcessFailed(std_output=process.stdout.read(), err_output=process.stderr.read()))
future.set_exception(
InvalidProjectEgg(detail=process.stderr.read()))
return
IOLoop.current().call_later(1, check_process)
check_process()
return future
def clearup(self):
'''
clean up temp files.
:return:
'''
if self.temp_dir and os.path.exists(self.temp_dir):
shutil.rmtree(self.temp_dir)
def put_egg(self, eggfile, version):
eggfile.seek(0)
self.egg_storage.put(eggfile=eggfile,
project=self.project_name,
version=version)
def get_egg(self, version=None):
return self.egg_storage.get(self.project_name, version=version)
def delete_egg(self, project, version=None):
logger.info('deleting project eggs')
return self.egg_storage.delete(project, version)
def list_versions(self, project):
return self.egg_storage.list(project)
