def _load_status_file(path):
""" Open status file and parse it """
data = {}
if not os.path.isfile(path):
return data
with open(path, 'r') as f:
try:
data = yaml.safe_load(f)
except yaml.YAMLError as exc:
raise shub_exceptions.BadConfigException(
"Error reading releases file:\n{}".format(exc))
if not isinstance(data, dict):
raise shub_exceptions.BadConfigException(
"Releases file has wrong format ({}).".format(data))
return data
def cli(project, base_image, base_deps, add_deps, requirements):
project_dir = utils.get_project_dir()
scrapy_config = shub_utils.get_config()
if not scrapy_config.has_option('settings', project):
raise shub_exceptions.BadConfigException(
'Settings for the project is not found')
settings_module = scrapy_config.get('settings', project)
values = {
'base_image': base_image,
'system_deps': _format_system_deps(base_deps, add_deps),
'system_env': _format_system_env(settings_module),
'requirements': _format_requirements(project_dir, requirements),
}
values = {key: value if value else '' for key, value in values.items()}
source = Template(DOCKERFILE_TEMPLATE.strip())
results = source.substitute(values)
results = results.replace('\n\n', '\n')
click.echo("The following Dockerfile will be created:\n{}".format(results))
valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
while True:
dockefile_path = os.path.join(project_dir, 'Dockerfile')
choice = input("Save to {}: (y/n)".format(dockefile_path)).lower()
if choice in valid:
if valid[choice]:
with open(dockefile_path, 'w') as dockerfile:
dockerfile.write(results)
click.echo('Saved.')
break
click.echo("Please respond with 'yes'('y') or 'no'(n)")
def cli(project, base_image, base_deps, add_deps, requirements):
closest_scrapy_cfg = shub_utils.closest_file('scrapy.cfg')
scrapy_config = shub_utils.get_config()
if not closest_scrapy_cfg or not scrapy_config.has_option(
'settings', project):
raise shub_exceptions.BadConfigException(
'Cannot find Scrapy project settings. Please ensure that current directory '
'contains scrapy.cfg with settings section, see example at '
'https://doc.scrapy.org/en/latest/topics/commands.html#default-structure-of-scrapy-projects'
) # NOQA
project_dir = os.path.dirname(closest_scrapy_cfg)
dockefile_path = os.path.join(project_dir, 'Dockerfile')
if os.path.exists(dockefile_path):
raise shub_exceptions.ShubException(
'Found a Dockerfile in the project directory, aborting')
settings_module = scrapy_config.get('settings', 'default')
shub_utils.create_default_setup_py(settings=settings_module)
values = {
'base_image': base_image,
'system_deps': _format_system_deps(base_deps, add_deps),
'system_env': _format_system_env(settings_module),
'requirements': _format_requirements(project_dir, requirements),
}
values = {key: value if value else '' for key, value in values.items()}
source = Template(DOCKERFILE_TEMPLATE)
results = source.substitute(values)
results = results.replace('\n\n', '\n')
with open(dockefile_path, 'w') as dockerfile:
dockerfile.write(results)
click.echo("Dockerfile is saved to {}".format(dockefile_path))
def get_project_dir():
""" A helper to get project root dir.
Used by init/build command to locate Dockerfile.
"""
closest = shub_utils.closest_file('scrapinghub.yml')
if not closest:
raise shub_exceptions.BadConfigException(
"Not inside a project: scrapinghub.yml not found.")
return os.path.dirname(closest)