def deprecate_debug_parameter(ctx, param, value):
if value:
print_warning(
"-d/--debug parameter is deprecated. "
"Please use -v/--verbose parameter instead.",
ShubDeprecationWarning)
return value
def inside_project():
scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
if scrapy_module is not None:
try:
import_module(scrapy_module)
except ImportError as exc:
print_warning("Cannot import scrapy settings module %s: %s"
"" % (scrapy_module, exc))
else:
return True
return bool(closest_file('scrapy.cfg'))
def inside_project():
scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
if scrapy_module is not None:
try:
import_module(scrapy_module)
except ImportError as exc:
print_warning("Cannot import scrapy settings module %s: %s"
"" % (scrapy_module, exc))
else:
return True
return bool(closest_file('scrapy.cfg'))
def load(self, stream):
"""Load Scrapinghub configuration from stream."""
# flag to mark if images/default was used in the config file
check_default_image_scope = False
try:
yaml_cfg = yaml.safe_load(stream)
if not yaml_cfg:
return
for option, shortcut in self.SHORTCUTS.items():
option_conf = getattr(self, option)
yaml_option_conf = yaml_cfg.get(option, {})
option_conf.update(yaml_option_conf)
if option == 'images' and yaml_option_conf:
print_warning(
"Images section is deprecated, please replace it with "
"global `image` setting or define `image` setting for "
"the project.\n Check for additional details in {}.".
format(CONFIG_DOCS_LINK),
category=ShubDeprecationWarning)
if 'default' in yaml_option_conf:
check_default_image_scope = True
if shortcut in yaml_cfg:
# We explicitly check yaml_option_conf and not option_conf.
# It is okay to set conflicting defaults if they are in
# different files (b/c then one of these will have
# priority)
if 'default' in yaml_option_conf:
raise BadConfigException(
"You cannot specify both '%s' and a 'default' key "
"for '%s' in the same file" % (shortcut, option))
option_conf['default'] = yaml_cfg[shortcut]
self.version = yaml_cfg.get('version', self.version)
self.requirements_file = yaml_cfg.get('requirements_file',
self.requirements_file)
self.requirements_file = yaml_cfg.get('requirements', {}).get(
'file', self.requirements_file)
self.eggs = yaml_cfg.get('requirements', {}).get('eggs', self.eggs)
except (yaml.YAMLError, AttributeError):
# AttributeError: stream is valid YAML but not dictionary-like
raise ConfigParseException
# fail if `projects` section has keys not found in `images`
if (check_default_image_scope
and not set(self.projects).issubset(set(self.images))):
raise BadConfigException(
"Found ambigious configuration: default image has global "
"scope now, but some projects were not using custom "
"images and can be broken now. Please fix your config by "
"replacing `images` section with `image` settings. Check "
"for additional details in {}".format(CONFIG_DOCS_LINK))
self._check_endpoints()
def _check_endpoints(self):
"""Check the endpoints. Send warnings if necessary."""
for endpoint, url in self.endpoints.items():
parsed = six.moves.urllib.parse.urlparse(url)
if parsed.netloc == 'staging.scrapinghub.com':
self.endpoints[endpoint] = six.moves.urllib.parse.urlunparse(
parsed._replace(netloc='app.scrapinghub.com'))
click.echo(
'WARNING: Endpoint "%s" is still using %s which has been '
'obsoleted. shub has updated it to app.scrapinghub.com '
'for this time only. Please update your configuration.' % (
endpoint,
parsed.netloc,
),
err=True)
if parsed.scheme == 'http':
print_warning('Endpoint "%s" is still using HTTP. '
'Please change it to HTTPS if possible.' %
endpoint)
def deprecate_async_parameter(ctx, param, value):
if value:
print_warning("--async parameter is deprecated.", ShubDeprecationWarning)
return value
def deprecate_async_parameter(ctx, param, value):
if value:
print_warning("--async parameter is deprecated.", ShubDeprecationWarning)
return value
def deprecate_debug_parameter(ctx, param, value):
if value:
print_warning("-d/--debug parameter is deprecated. "
"Please use -v/--verbose parameter instead.",
ShubDeprecationWarning)
return value
