def load_config(stream, config_globals):
"""Load a configuration file and generate importer and exporter classes.
Args:
stream: Stream containing config YAML.
config_globals: Dict to use to reference globals for code in the config.
Returns:
BulkloaderEntry
Raises:
InvalidConfiguration: If the config is invalid.
"""
builder = yaml_object.ObjectBuilder(BulkloaderEntry)
handler = yaml_builder.BuilderHandler(builder)
listener = yaml_listener.EventListener(handler)
global _global_temp_globals
_global_temp_globals = config_globals
try:
listener.Parse(stream)
finally:
_global_temp_globals = None
bulkloader_infos = handler.GetResults()
if len(bulkloader_infos) < 1:
raise bulkloader_errors.InvalidConfiguration(
'No configuration specified.')
if len(bulkloader_infos) > 1:
raise bulkloader_errors.InvalidConfiguration(
'Multiple sections in configuration.')
bulkloader_info = bulkloader_infos[0]
if not bulkloader_info.transformers:
raise bulkloader_errors.InvalidConfiguration(
'No transformers specified.')
return bulkloader_info
def create_from_options(cls, options, name):
"""Factory using an options dictionary.
Args:
options: Dictionary of options:
columns: 'from_header' or blank.
column_list: overrides columns specifically.
encoding: encoding of the file. e.g. 'utf-8' (default), 'windows-1252'.
skip_import_header_row: True to ignore the header line on import.
Defaults False, except must be True if columns=from_header.
print_export_header_row: True to print a header line on export.
Defaults to False except if columns=from_header.
import_options: Other kwargs to pass in, like "dialect".
export_options: Other kwargs to pass in, like "dialect".
name: The name of this transformer, for use in error messages.
Returns:
CsvConnector object described by the specified options.
Raises:
InvalidConfiguration: If the config is invalid.
"""
column_list = options.get('column_list', None)
columns = None
if not column_list:
columns = options.get('columns', 'from_header')
if columns != 'from_header':
raise bulkloader_errors.InvalidConfiguration(
'CSV columns must be "from_header", or a column_list '
'must be specified. (In transformer name %s.)' % name)
csv_encoding = options.get('encoding', 'utf-8')
skip_import_header_row = options.get('skip_import_header_row',
columns == 'from_header')
if columns == 'from_header' and not skip_import_header_row:
raise bulkloader_errors.InvalidConfiguration(
'When CSV columns are "from_header", the header row must always '
'be skipped. (In transformer name %s.)' % name)
print_export_header_row = options.get('print_export_header_row',
columns == 'from_header')
import_options = options.get('import_options', {})
export_options = options.get('export_options', {})
return cls(columns, column_list, skip_import_header_row,
print_export_header_row, csv_encoding, import_options,
export_options)
def create_deep_key(*path_info):
"""A method that makes multi-level Key objects.
Generates a multi-level key from multiple fields in the input dictionary.
This is typically used for keys for entities that have variable parent keys,
e.g. ones with owned relationships. It can used for both __key__ and
references.
Use create_foreign_key as a simpler way to create single-level keys.
Args:
*path_info: A list of tuples, describing (kind, property, is_id=False).
kind: The kind name.
property: The external property in the current import dictionary, or
transform.CURRENT_PROPERTY for the value passed to the transform.
is_id: If True, converts value to int and treats it as a numeric ID.
If False, the value is a string name. Default is False.
Example:
create_deep_key(('rootkind', 'rootcolumn'),
('childkind', 'childcolumn', True),
('leafkind', transform.CURRENT_PROPERTY))
Returns:
A transform function that parses the info from the current neutral
dictionary into a Key with parents as described by path_info.
"""
validated_path_info = []
for level_info in path_info:
if len(level_info) == 3:
key_is_id = level_info[2]
elif len(level_info) == 2:
key_is_id = False
else:
raise bulkloader_errors.InvalidConfiguration(
'Each list in create_deep_key must specify exactly 2 or 3 '
'parameters: (kind, property, is_id=False). You specified: %s'
% repr(path_info))
kind_name = level_info[0]
property_name = level_info[1]
validated_path_info.append((kind_name, property_name, key_is_id))
def create_deep_key_lambda(value, bulkload_state):
path = []
for kind_name, property_name, key_is_id in validated_path_info:
if property_name is CURRENT_PROPERTY:
name_or_id = value
else:
name_or_id = bulkload_state.current_dictionary[property_name]
if key_is_id:
name_or_id = int(name_or_id)
path += [kind_name, name_or_id]
return datastore.Key.from_path(*path)
return create_deep_key_lambda
def __init__(self,
template,
prolog=None,
epilog=None,
mode='text',
name=''):
"""Constructor.
Args:
template: A Python dict-interpolation string.
prolog: written before the per-record output.
epilog: written after the per-record output.
mode: one of the following, default is 'text'
text: text file mode, newlines between records.
nonewline: text file mode, no added newlines.
binary: binary file mode, no added newlines.
"""
if mode not in self.VALID_MODES:
raise bulkloader_errors.InvalidConfiguration(
'simpletext mode must be one of "%s". (In transformer name %s.)'
% ('", "'.join(self.VALID_MODES), name))
self.template = template
self.prolog = prolog
self.epilog = epilog
self.mode = mode
self.export_file_pointer = None
def create_from_options(cls, options, name):
"""Factory using an options dictionary.
Args:
options: Dictionary of options containing:
template: A Python dict-interpolation string. Required.
prolog: written before the per-record output.
epilog: written after the per-record output.
mode: one of the following, default is 'text'
text: text file mode, newlines between records.
nonewline: text file mode, no added newlines.
binary: binary file mode, no added newlines.
name: The name of this transformer, for use in error messages.
Returns:
SimpleTextConnector object described by the specified options.
Raises:
InvalidConfiguration: If the config is invalid.
"""
template = options.get('template')
if not template:
raise bulkloader_errors.InvalidConfiguration(
'simpletext must specify template. (In transformer named %s)' %
name)
prolog = options.get('prolog')
epilog = options.get('epilog')
mode = options.get('mode', 'text')
return cls(template, prolog, epilog, mode, name)
def CheckInitialized(self):
"""Check that all required (combinations) of fields are set.
Also fills in computed properties.
Raises:
InvalidConfiguration: if the config is invalid.
"""
if not self.kind and not self.model:
raise bulkloader_errors.InvalidConfiguration(
'Neither kind nor model specified for transformer.')
if self.kind and self.model:
raise bulkloader_errors.InvalidConfiguration(
'Both kind and model specified for transformer.')
if self.model:
self.kind = self.model.method.kind()
else:
if self.use_model_on_export:
raise bulkloader_errors.InvalidConfiguration(
'No model class specified but use_model_on_export is true.'
)
if not self.name:
self.name = self.kind
if not self.connector:
raise bulkloader_errors.InvalidConfiguration(
'No connector specified.')
property_names = set()
for prop in self.property_map:
if prop.property in property_names:
raise bulkloader_errors.InvalidConfiguration(
'Duplicate property specified for property %s in transform %s'
% (prop.property, self.name))
property_names.add(prop.property)
def CheckInitialized(self):
"""Check that all required (combinations) of fields are set.
Also fills in computed properties.
Raises:
InvalidConfiguration: If the config is invalid.
"""
super(PropertyEntry, self).CheckInitialized()
if not (self.external_name or self.import_template or self.export):
raise bulkloader_errors.InvalidConfiguration(
'Neither external_name nor import_template nor export specified for '
'property %s.' % self.property)
def initialize_export(self, filename, bulkload_state):
"""Initialize the output file."""
self.bulkload_state = bulkload_state
if not self.node_list:
raise bulkloader_errors.InvalidConfiguration(
'simplexml export only supports simple /root/to/node xpath_to_nodes '
'for now.')
self.output_stream = codecs.open(filename, 'wb', 'utf-8')
self.output_stream.write('<?xml version="1.0"?>\n')
self.depth = 0
for node in self.node_list:
self.output_stream.write('%s<%s>\n' % (' ' * self.depth, node))
self.depth += 1
self.indent = ' ' * self.depth
def create_from_options(cls, options, name):
"""Factory using an options dictionary.
Args:
options: Dictionary of options. Must contain:
* xpath_to_nodes: The xpath to select a record.
* style: 'element_centric' or 'attribute_centric'
name: The name of this transformer, for use in error messages.
Returns:
XmlConnector connector object described by the specified options.
Raises:
InvalidConfiguration: If the config is invalid.
"""
xpath_to_nodes = options.get('xpath_to_nodes')
if not xpath_to_nodes:
raise bulkloader_errors.InvalidConfiguration(
'simplexml must specify xpath_to_nodes. (In transformer named %s)'
% name)
if not re.match(NODE_PATH_ONLY_RE, xpath_to_nodes):
logging.warning('simplexml export only supports very simple '
'/root/to/node xpath_to_nodes for now.')
xml_style = options.get('style')
xml_style_mapping = {
'element_centric': cls.ELEMENT_CENTRIC,
'attribute_centric': cls.ATTRIBUTE_CENTRIC,
}
if xml_style not in xml_style_mapping:
raise bulkloader_errors.InvalidConfiguration(
'simplexml must specify one of these valid xml_style options: "%s". '
'You specified %s in transformer named %s.' %
('", "'.join(list(xml_style_mapping.keys())), xml_style, name))
return cls(xpath_to_nodes, xml_style_mapping[xml_style])
def create_transformer_classes(transformer_spec, config_globals, increment_id):
"""Create an importer and exporter class from a transformer spec.
Args:
transformer_spec: A bulkloader_parser.TransformerEntry.
config_globals: Dict to use to reference globals for code in the config.
increment_id: Method IncrementId(key) which will increment the
auto-allocated ids in the datastore beyond the key.id(). Can be None.
Raises:
InvalidConfig: when the config is invalid.
Returns:
Tuple, (importer class, exporter class), each which is in turn a wrapper
for the GenericImporter/GenericExporter class using a DictConvertor object
configured as per the transformer_spec.
"""
if transformer_spec.connector in CONNECTOR_FACTORIES:
connector_factory = CONNECTOR_FACTORIES[transformer_spec.connector]
elif config_globals and '.' in transformer_spec.connector:
try:
connector_factory = eval(transformer_spec.connector,
config_globals)
except (NameError, AttributeError):
raise bulkloader_errors.InvalidConfiguration(
'Invalid connector specified for name=%s. Could not evaluate %s.'
% (transformer_spec.name, transformer_spec.connector))
else:
raise bulkloader_errors.InvalidConfiguration(
'Invalid connector specified for name=%s. Must be either a built in '
'connector ("%s") or a factory method in a module imported via '
'python_preamble.' %
(transformer_spec.name, '", "'.join(CONNECTOR_FACTORIES)))
options = {}
if transformer_spec.connector_options:
options = transformer_spec.connector_options.ToDict()
try:
connector_object = connector_factory(options, transformer_spec.name)
except TypeError:
raise bulkloader_errors.InvalidConfiguration(
'Invalid connector specified for name=%s. Could not initialize %s.'
% (transformer_spec.name, transformer_spec.connector))
dict_to_model_object = DictConvertor(transformer_spec)
class ImporterClass(GenericImporter):
"""Class to pass to the bulkloader, wraps the specificed configuration."""
def __init__(self):
super(self.__class__,
self).__init__(connector_object.generate_import_record,
dict_to_model_object.dict_to_entity,
transformer_spec.name, increment_id)
importer_class = ImporterClass
class ExporterClass(GenericExporter):
"""Class to pass to the bulkloader, wraps the specificed configuration."""
def __init__(self):
super(self.__class__,
self).__init__(connector_object,
dict_to_model_object.entity_to_dict,
transformer_spec.kind,
transformer_spec.sort_key_from_entity)
exporter_class = ExporterClass
return importer_class, exporter_class
