def build_egg(project):
work_path = os.getcwd()
try:
path = os.path.abspath(merge(os.getcwd(), PROJECTS_FOLDER))
print('Path', path)
project_path = merge(path, project)
os.chdir(project_path)
settings = config(project_path, 'settings', 'default')
print('Settings', settings)
create_default_setup_py(project_path,
settings=settings,
project=project)
d = tempfile.mkdtemp(prefix="gerapy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
# 删除原文件
if find_egg(project_path):
os.remove(merge(project_path, find_egg(project_path)))
shutil.move(egg, project_path)
return merge(project_path, find_egg(project_path))
except Exception as e:
print(e.args)
finally:
os.chdir(work_path)
def build(self):
closest = closest_scrapy_cfg()
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
if not self.settings_name:
self.settings_name = 'default'
settings = get_config().get('settings', self.settings_name)
self._create_default_setup_py(settings=settings)
print 'build egg use [%s] settings' % self.settings_name
if not self.file_path:
self.file_path = tempfile.mkdtemp(prefix="scrapydeploy-")
else:
if not os.path.exists(self.file_path):
os.mkdir(self.file_path)
d = self.file_path
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
self._rename_egg_name(egg, d)
def build_egg(project: str):
'''Build egg in project root'''
click.echo('Building egg...')
closest = closest_scrapy_cfg()
if closest == '':
click.echo('No scrapy.cfg found')
exit(1)
directory = os.path.dirname(closest)
os.chdir(directory)
if not os.path.exists('setup.py'):
click.echo('No setup.py in project')
exit(1)
d = tempfile.mkdtemp(prefix='scrapydeploy-')
with open(os.path.join(d, 'stdout'), 'wb') as o, open(os.path.join(d, 'stderr'), 'wb') as e:
p = [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d]
retry_on_eintr(check_call, p, stdout=o, stderr=e)
egg = glob.glob(os.path.join(d, '*.egg'))[0]
filename = f'{project}.egg'
shutil.copyfile(egg, filename)
return f'{directory}/{filename}'
def build_egg(project):
'''
build project to egg file
:param project:
:return:
'''
work_path = os.getcwd()
try:
path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
project_path = join(path, project)
os.chdir(project_path)
settings = config(project_path, 'settings', 'default')
create_default_setup_py(project_path,
settings=settings,
project=project)
d = tempfile.mkdtemp(prefix='gerapy-')
o = open(os.path.join(d, 'stdout'), 'wb')
e = open(os.path.join(d, 'stderr'), 'wb')
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
# Delete Origin file
if find_egg(project_path):
os.remove(join(project_path, find_egg(project_path)))
shutil.move(egg, project_path)
return join(project_path, find_egg(project_path))
except Exception as e:
logger.error('error occurred %s', e.args)
finally:
os.chdir(work_path)
def build_egg(project):
print("-----build_egg-----")
work_path = os.getcwd()
try:
path = os.path.abspath(os.path.join(os.getcwd()))
project_path = os.path.join(path, project)
os.chdir(project_path)
settings = config(project_path, 'settings', 'default')
create_default_setup_py(project_path,
settings=settings,
project=project)
d = tempfile.mkdtemp(prefix="temp-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
if find_egg(project_path):
os.remove(os.path.join(project_path, find_egg(project_path)))
shutil.move(egg, project_path)
return os.path.join(project_path, find_egg(project_path))
except Exception as e:
print(e)
finally:
os.chdir(work_path)
def build_egg(project):
work_path = os.getcwd()
try:
path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
project_path = join(path, project)
os.chdir(project_path)
settings = config(project_path, 'settings', 'default')
create_default_setup_py(project_path, settings=settings, project=project)
d = tempfile.mkdtemp(prefix="gerapy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o, stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
# Delete Origin file
if find_egg(project_path):
os.remove(join(project_path, find_egg(project_path)))
shutil.move(egg, project_path)
return join(project_path, find_egg(project_path))
except Exception as e:
print(e.args)
finally:
os.chdir(work_path)
def test_template(self, tplname='crawl'):
args = ['--template=%s' % tplname] if tplname else []
spname = 'test_spider'
p = self.proc('genspider', spname, 'test.com', *args)
out = retry_on_eintr(p.stdout.read)
self.assert_("Created spider %r using template %r in module" % (spname, tplname) in out)
self.assert_(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
p = self.proc('genspider', spname, 'test.com', *args)
out = retry_on_eintr(p.stdout.read)
self.assert_("Spider %r already exists in module" % spname in out)
def test_template(self, tplname="crawl"):
args = ["--template=%s" % tplname] if tplname else []
spname = "test_spider"
p = self.proc("genspider", spname, "test.com", *args)
out = retry_on_eintr(p.stdout.read)
self.assert_("Created spider %r using template %r in module" % (spname, tplname) in out)
self.assert_(exists(join(self.proj_mod_path, "spiders", "test_spider.py")))
p = self.proc("genspider", spname, "test.com", *args)
out = retry_on_eintr(p.stdout.read)
self.assert_("Spider %r already exists in module" % spname in out)
def test_template(self, tplname='crawl'):
args = ['--template=%s' % tplname] if tplname else []
spname = 'test_spider'
p = self.proc('genspider', spname, 'test.com', *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
p = self.proc('genspider', spname, 'test.com', *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("Spider %r already exists in module" % spname, out)
def _build_egg():
closest = closest_scrapy_cfg()
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp()
f = tempfile.TemporaryFile(dir=d)
retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=f)
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def _build_egg():
closest = closest_scrapy_cfg()
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp(prefix="scrapydeploy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def _build_egg():
closest = closest_scrapy_cfg()
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp(prefix="scrapydeploy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d], stdout=o, stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def _build_egg():
closest = closest_scrapy_cfg()
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp()
f = tempfile.TemporaryFile(dir=d)
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=f)
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def build_egg(project, include_data=False):
"""
构建egg包
:param project:
:param include_data
:return:
"""
work_path = os.getcwd()
try:
path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
project_path = join(path, project)
os.chdir(project_path)
settings = config(project_path, 'settings', 'default')
if include_data:
create_data_setup_py(project_path,
settings=settings,
project=project)
else:
create_default_setup_py(project_path,
settings=settings,
project=project)
d = tempfile.mkdtemp(prefix="dt-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(
check_call,
['python', 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
# retry_on_eintr(check_call, [sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
# stdout=o, stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
# Delete Origin file
if find_egg(project_path):
os.remove(join(project_path, find_egg(project_path)))
shutil.move(egg, project_path)
return join(project_path, find_egg(project_path))
except Exception as e:
import traceback
log_common.error(">build_egg ", e)
log_common.error(">build_egg = {}", traceback.format_exc())
finally:
os.chdir(work_path)
def _build_egg():
closest = closest_scrapy_cfg()
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
scrapy_project_settings = get_config()
settings = scrapy_project_settings.get('settings', 'default')
project = scrapy_project_settings.get('deploy', 'project')
_create_default_setup_py(settings=settings, project=project)
d = 'dist'
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=sys.stdout,
stderr=sys.stderr)
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def test_startproject_template_override(self):
copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
os.mknod(join(self.tmpl_proj, 'root_template'))
assert exists(join(self.tmpl_proj, 'root_template'))
args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
p = self.proc('startproject', self.project_name, *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("New Scrapy project %r, using template directory %r, created in:" % \
(self.project_name, join(self.tmpl, 'project')), out)
assert exists(join(self.proj_path, 'root_template'))
def build_egg(project):
path = get_run_path()
path = '{path}/storage/{project}/'.format(path=path, project=project.name)
os.chdir(path)
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings, project=project.name)
d = tempfile.mkdtemp(prefix="scrapydeploy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
os.system('rm *.egg')
shutil.move(egg, path)
return find_egg(path)
def test_startproject_template_override(self):
copytree(join(scrapy.__path__[0], 'templates'), self.tmpl)
os.mknod(join(self.tmpl_proj, 'root_template'))
assert exists(join(self.tmpl_proj, 'root_template'))
args = ['--set', 'TEMPLATES_DIR=%s' % self.tmpl]
p = self.proc('startproject', self.project_name, *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("New Scrapy project %r, using template directory %r, created in:" % \
(self.project_name, join(self.tmpl, 'project')), out)
assert exists(join(self.proj_path, 'root_template'))
def test_startproject_template_override(self):
copytree(join(scrapy.__path__[0], "templates"), self.tmpl)
with open(join(self.tmpl_proj, "root_template"), "w"):
pass
assert exists(join(self.tmpl_proj, "root_template"))
args = ["--set", "TEMPLATES_DIR=%s" % self.tmpl]
p = self.proc("startproject", self.project_name, *args)
out = to_native_str(retry_on_eintr(p.stdout.read))
self.assertIn("New Scrapy project %r, using template directory" % self.project_name, out)
self.assertIn(self.tmpl_proj, out)
assert exists(join(self.proj_path, "root_template"))
