def cli(project, base_image, base_deps, add_deps, requirements):
project_dir = utils.get_project_dir()
scrapy_config = shub_utils.get_config()
if not scrapy_config.has_option('settings', project):
raise shub_exceptions.BadConfigException(
'Settings for the project is not found')
settings_module = scrapy_config.get('settings', project)
values = {
'base_image': base_image,
'system_deps': _format_system_deps(base_deps, add_deps),
'system_env': _format_system_env(settings_module),
'requirements': _format_requirements(project_dir, requirements),
}
values = {key: value if value else '' for key, value in values.items()}
source = Template(DOCKERFILE_TEMPLATE.strip())
results = source.substitute(values)
results = results.replace('\n\n', '\n')
click.echo("The following Dockerfile will be created:\n{}".format(results))
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
while True:
dockefile_path = os.path.join(project_dir, 'Dockerfile')
choice = input("Save to {}: (y/n)".format(dockefile_path)).lower()
if choice in valid:
if valid[choice]:
with open(dockefile_path, 'w') as dockerfile:
dockerfile.write(results)
click.echo('Saved.')
break
click.echo("Please respond with 'yes'('y') or 'no'(n)")
def cli(project, base_image, base_deps, add_deps, requirements):
closest_scrapy_cfg = shub_utils.closest_file('scrapy.cfg')
scrapy_config = shub_utils.get_config()
if not closest_scrapy_cfg or not scrapy_config.has_option(
'settings', project):
raise shub_exceptions.BadConfigException(
'Cannot find Scrapy project settings. Please ensure that current directory '
'contains scrapy.cfg with settings section, see example at '
'https://doc.scrapy.org/en/latest/topics/commands.html#default-structure-of-scrapy-projects'
) # NOQA
project_dir = os.path.dirname(closest_scrapy_cfg)
dockefile_path = os.path.join(project_dir, 'Dockerfile')
if os.path.exists(dockefile_path):
raise shub_exceptions.ShubException(
'Found a Dockerfile in the project directory, aborting')
settings_module = scrapy_config.get('settings', 'default')
shub_utils.create_default_setup_py(settings=settings_module)
values = {
'base_image': base_image,
'system_deps': _format_system_deps(base_deps, add_deps),
'system_env': _format_system_env(settings_module),
'requirements': _format_requirements(project_dir, requirements),
}
values = {key: value if value else '' for key, value in values.items()}
source = Template(DOCKERFILE_TEMPLATE)
results = source.substitute(values)
results = results.replace('\n\n', '\n')
with open(dockefile_path, 'w') as dockerfile:
dockerfile.write(results)
click.echo("Dockerfile is saved to {}".format(dockefile_path))
def cli(project, base_image, base_deps, add_deps, requirements):
project_dir = utils.get_project_dir()
scrapy_config = shub_utils.get_config()
if not scrapy_config.has_option('settings', project):
raise shub_exceptions.BadConfigException(
'Settings for the project is not found')
settings_module = scrapy_config.get('settings', project)
values = {
'base_image': base_image,
'system_deps': _format_system_deps(base_deps, add_deps),
'system_env': _format_system_env(settings_module),
'requirements': _format_requirements(project_dir, requirements),
}
values = {key: value if value else '' for key, value in values.items()}
source = Template(DOCKERFILE_TEMPLATE.strip())
results = source.substitute(values)
results = results.replace('\n\n', '\n')
click.echo("The following Dockerfile will be created:\n{}".format(results))
valid = {"yes": True, "y": True, "ye": True,
"no": False, "n": False}
while True:
dockefile_path = os.path.join(project_dir, 'Dockerfile')
choice = input("Save to {}: (y/n)".format(dockefile_path)).lower()
if choice in valid:
if valid[choice]:
with open(dockefile_path, 'w') as dockerfile:
dockerfile.write(results)
click.echo('Saved.')
break
click.echo("Please respond with 'yes'('y') or 'no'(n)")
def cli(project, base_image, base_deps, add_deps, requirements):
closest_scrapy_cfg = shub_utils.closest_file('scrapy.cfg')
scrapy_config = shub_utils.get_config()
if not closest_scrapy_cfg or not scrapy_config.has_option('settings', project):
raise shub_exceptions.BadConfigException(
'Cannot find Scrapy project settings. Please ensure that current directory '
'contains scrapy.cfg with settings section, see example at '
'https://doc.scrapy.org/en/latest/topics/commands.html#default-structure-of-scrapy-projects') # NOQA
project_dir = os.path.dirname(closest_scrapy_cfg)
dockefile_path = os.path.join(project_dir, 'Dockerfile')
if os.path.exists(dockefile_path):
raise shub_exceptions.ShubException('Found a Dockerfile in the project directory, aborting')
settings_module = scrapy_config.get('settings', 'default')
shub_utils.create_default_setup_py(settings=settings_module)
values = {
'base_image': base_image,
'system_deps': _format_system_deps(base_deps, add_deps),
'system_env': _format_system_env(settings_module),
'requirements': _format_requirements(project_dir, requirements),
}
values = {key: value if value else '' for key, value in values.items()}
source = Template(DOCKERFILE_TEMPLATE)
results = source.substitute(values)
results = results.replace('\n\n', '\n')
with open(dockefile_path, 'w') as dockerfile:
dockerfile.write(results)
click.echo("Dockerfile is saved to {}".format(dockefile_path))
def _build_egg():
closest = closest_file('scrapy.cfg')
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp(prefix="shub-deploy-")
run_python(['setup.py', 'clean', '-a', 'bdist_egg', '-d', d])
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def _create_setup_py_if_not_exists():
closest = closest_file('scrapy.cfg')
# create default setup.py only if scrapy.cfg is found, otherwise
# consider it as a non-scrapy/non-python project
if not closest:
warnings.warn("scrapy.cfg is not found")
return
with utils.remember_cwd():
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
def _create_setup_py_if_not_exists():
closest = closest_file('scrapy.cfg')
# create default setup.py only if scrapy.cfg is found, otherwise
# consider it as a non-scrapy/non-python project
if not closest:
warnings.warn("scrapy.cfg is not found")
return
with utils.remember_cwd():
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
def _build_egg():
closest = closest_file("scrapy.cfg")
os.chdir(os.path.dirname(closest))
if not os.path.exists("setup.py"):
settings = get_config().get("settings", "default")
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp(prefix="shub-deploy-")
with open(os.path.join(d, "stdout"), "wb") as o, open(os.path.join(d, "stderr"), "wb") as e, patch_sys_executable():
retry_on_eintr(
check_call, [sys.executable, "setup.py", "clean", "-a", "bdist_egg", "-d", d], stdout=o, stderr=e
)
egg = glob.glob(os.path.join(d, "*.egg"))[0]
return egg, d
def _build_egg():
closest = closest_file('scrapy.cfg')
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp(prefix="shub-deploy-")
o = open(os.path.join(d, "stdout"), "wb")
e = open(os.path.join(d, "stderr"), "wb")
retry_on_eintr(check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o, stderr=e)
o.close()
e.close()
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d
def _build_egg():
closest = closest_file('scrapy.cfg')
os.chdir(os.path.dirname(closest))
if not os.path.exists('setup.py'):
settings = get_config().get('settings', 'default')
_create_default_setup_py(settings=settings)
d = tempfile.mkdtemp(prefix="shub-deploy-")
with open(os.path.join(d, "stdout"), "wb") as o, \
open(os.path.join(d, "stderr"), "wb") as e, \
patch_sys_executable():
retry_on_eintr(
check_call,
[sys.executable, 'setup.py', 'clean', '-a', 'bdist_egg', '-d', d],
stdout=o,
stderr=e,
)
egg = glob.glob(os.path.join(d, '*.egg'))[0]
return egg, d