def _update_settings(o, d):
# We need to convert settings to string since the S3 download handler
# doesn't work if the AWS keys are passed as unicode. Other code may also
# depend on settings being str. TODO: we should test this
for k, v in d.items():
d[to_native_str(k)] = to_native_str(v) if is_string(v) else v
o.update(d)
def _make_scrapy_args(arg, args_dict):
if not args_dict:
return []
args = []
for k, v in sorted(dict(args_dict).items()):
args += [arg, "{}={}".format(
to_native_str(k), to_native_str(v) if is_string(v) else v)]
return args
def _job_args_and_env(msg):
env = msg.get('job_env')
if not isinstance(env, dict):
env = {}
cmd = msg.get('job_cmd')
if not isinstance(cmd, list):
cmd = [str(cmd)]
return cmd, {to_native_str(k): to_native_str(v) if is_string(v) else v
for k, v in sorted(dict(env).items())}
def _make_scrapy_args(arg, args_dict):
if not args_dict:
return []
args = []
for k, v in sorted(dict(args_dict).items()):
args += [
arg, "{}={}".format(to_native_str(k),
to_native_str(v) if is_string(v) else v)
]
return args
def _job_args_and_env(msg):
env = msg.get('job_env')
if not isinstance(env, dict):
env = {}
cmd = msg.get('job_cmd')
if not isinstance(cmd, list):
cmd = [str(cmd)]
return cmd, {
to_native_str(k): to_native_str(v) if is_string(v) else v
for k, v in sorted(dict(env).items())
}
def _update_old_classpaths(settings):
"""Update user's project settings with proper class paths.
Note that the method updates only settings with dicts as values:
it's needed for proper dicts merge to avoid duplicates in paths.
For all other cases Scrapy will handle it by itself.
"""
for setting_key in settings.attributes.keys():
setting_value = settings[setting_key]
# A workaround to make it work for:
# - Scrapy==1.0.5 with dicts as values
# - Scrapy>=1.1.0 with BaseSettings as values
if hasattr(setting_value, 'copy_to_dict'):
setting_value = setting_value.copy_to_dict()
elif not isinstance(setting_value, dict):
continue
for path in setting_value.keys():
if not is_string(path):
continue
updated_path = update_classpath(path)
if updated_path != path:
order = settings[setting_key].pop(path)
settings[setting_key][updated_path] = order
def _update_old_classpaths(settings):
"""Update user's project settings with proper class paths.
Note that the method updates only settings with dicts as values:
it's needed for proper dicts merge to avoid duplicates in paths.
For all other cases Scrapy will handle it by itself.
"""
for setting_key in settings.attributes.keys():
setting_value = settings[setting_key]
# A workaround to make it work for:
# - Scrapy==1.0.5 with dicts as values
# - Scrapy>=1.1.0 with BaseSettings as values
if hasattr(setting_value, 'copy_to_dict'):
setting_value = setting_value.copy_to_dict()
elif not isinstance(setting_value, dict):
continue
for path in setting_value.keys():
if not is_string(path):
continue
updated_path = update_classpath(path)
if updated_path != path:
order = settings[setting_key].pop(path)
settings[setting_key][updated_path] = order
def set(self, name, value, priority='project'):
super(EntrypointSettings,
self).set(to_native_str(name),
to_native_str(value) if is_string(value) else value,
priority=priority)
def set(self, name, value, priority='project'):
super(EntrypointSettings, self).set(
to_native_str(name),
to_native_str(value) if is_string(value) else value,
priority=priority)