def _update_settings(o, d): # We need to convert settings to string since the S3 download handler # doesn't work if the AWS keys are passed as unicode. Other code may also # depend on settings being str. TODO: we should test this for k, v in d.items(): d[to_native_str(k)] = to_native_str(v) if is_string(v) else v o.update(d)
def _make_scrapy_args(arg, args_dict): if not args_dict: return [] args = [] for k, v in sorted(dict(args_dict).items()): args += [arg, "{}={}".format( to_native_str(k), to_native_str(v) if is_string(v) else v)] return args
def _job_args_and_env(msg): env = msg.get('job_env') if not isinstance(env, dict): env = {} cmd = msg.get('job_cmd') if not isinstance(cmd, list): cmd = [str(cmd)] return cmd, {to_native_str(k): to_native_str(v) if is_string(v) else v for k, v in sorted(dict(env).items())}
def _make_scrapy_args(arg, args_dict): if not args_dict: return [] args = [] for k, v in sorted(dict(args_dict).items()): args += [ arg, "{}={}".format(to_native_str(k), to_native_str(v) if is_string(v) else v) ] return args
def _job_args_and_env(msg): env = msg.get('job_env') if not isinstance(env, dict): env = {} cmd = msg.get('job_cmd') if not isinstance(cmd, list): cmd = [str(cmd)] return cmd, { to_native_str(k): to_native_str(v) if is_string(v) else v for k, v in sorted(dict(env).items()) }
def _update_old_classpaths(settings): """Update user's project settings with proper class paths. Note that the method updates only settings with dicts as values: it's needed for proper dicts merge to avoid duplicates in paths. For all other cases Scrapy will handle it by itself. """ for setting_key in settings.attributes.keys(): setting_value = settings[setting_key] # A workaround to make it work for: # - Scrapy==1.0.5 with dicts as values # - Scrapy>=1.1.0 with BaseSettings as values if hasattr(setting_value, 'copy_to_dict'): setting_value = setting_value.copy_to_dict() elif not isinstance(setting_value, dict): continue for path in setting_value.keys(): if not is_string(path): continue updated_path = update_classpath(path) if updated_path != path: order = settings[setting_key].pop(path) settings[setting_key][updated_path] = order
def _update_old_classpaths(settings): """Update user's project settings with proper class paths. Note that the method updates only settings with dicts as values: it's needed for proper dicts merge to avoid duplicates in paths. For all other cases Scrapy will handle it by itself. """ for setting_key in settings.attributes.keys(): setting_value = settings[setting_key] # A workaround to make it work for: # - Scrapy==1.0.5 with dicts as values # - Scrapy>=1.1.0 with BaseSettings as values if hasattr(setting_value, 'copy_to_dict'): setting_value = setting_value.copy_to_dict() elif not isinstance(setting_value, dict): continue for path in setting_value.keys(): if not is_string(path): continue updated_path = update_classpath(path) if updated_path != path: order = settings[setting_key].pop(path) settings[setting_key][updated_path] = order
def set(self, name, value, priority='project'): super(EntrypointSettings, self).set(to_native_str(name), to_native_str(value) if is_string(value) else value, priority=priority)
def set(self, name, value, priority='project'): super(EntrypointSettings, self).set( to_native_str(name), to_native_str(value) if is_string(value) else value, priority=priority)