def execute(self, config):
spreadsheet_id = config.id
gids = config.gids or []
if config.gid is not None:
gids.append(config.gid)
if not gids and not config.collection:
gids.append(0)
format_as = config.format
if (config.collection and
format_as not in GoogleSheetsPreprocessor.MAP_TYPES and
format_as not in GoogleSheetsPreprocessor.GRID_TYPES):
format_as = 'map'
gid_to_sheet, gid_to_data = GoogleSheetsPreprocessor.download(
spreadsheet_id=spreadsheet_id, gids=gids, format_as=format_as,
logger=self.pod.logger, generate_ids=config.generate_ids,
header_row_count=config.header_row_count,
header_row_index=config.header_row_index)
if config.path:
# Single sheet import.
path, key_to_update = self.parse_path(config.path)
for gid in gids:
# Preserve existing data if necessary.
gid_to_data[gid] = self._maybe_preserve_content(
new_data=gid_to_data[gid],
path=path,
key_to_update=key_to_update)
content = GoogleSheetsPreprocessor.serialize_content(
formatted_data=gid_to_data[gid], path=path,
output_style=self.config.output_style)
self.pod.write_file(path, content)
self.logger.info(
'Downloaded {} ({}) -> {}'.format(
gid_to_sheet[gid]['title'], gid, path))
else:
# Multi sheet import based on collection.
collection_path = config.collection
if not gids:
gids = gid_to_sheet.keys()
for gid in gids:
if gid_to_sheet[gid]['title'].strip().startswith(IGNORE_INITIAL):
continue
file_name = '{}.yaml'.format(
utils.slugify(gid_to_sheet[gid]['title']))
output_path = os.path.join(collection_path, file_name)
gid_to_data[gid] = self._maybe_preserve_content(
new_data=gid_to_data[gid],
path=output_path,
key_to_update=None)
# Use plain text dumper to preserve yaml constructors.
output_content = utils.dump_plain_yaml(gid_to_data[gid])
self.pod.write_file(output_path, output_content)
self.logger.info(
'Downloaded {} ({}) -> {}'.format(
gid_to_sheet[gid]['title'], gid, output_path))
def serialize_content(cls, formatted_data, path, output_style=None):
"""Serializes an object into a string as JSON, YAML, or a CSV
(default)."""
kwargs = {}
convert_to = cls.get_convert_to(path)
if convert_to == '.json':
if output_style == 'pretty':
kwargs['indent'] = 2
kwargs['separators'] = (',', ': ')
kwargs['sort_keys'] = True
return json.dumps(formatted_data, **kwargs)
elif convert_to in ('.yaml', '.yml'):
# Use plain text dumper to preserve yaml constructors.
return utils.dump_plain_yaml(formatted_data)
return formatted_data
def serialize_content(cls, formatted_data, path, output_style=None):
"""Serializes an object into a string as JSON, YAML, or a CSV
(default)."""
kwargs = {}
convert_to = cls.get_convert_to(path)
if convert_to == '.json':
if output_style == 'pretty':
kwargs['indent'] = 2
kwargs['separators'] = (',', ': ')
kwargs['sort_keys'] = True
return json.dumps(formatted_data, **kwargs)
elif convert_to in ('.yaml', '.yml'):
# Use plain text dumper to preserve yaml constructors.
return utils.dump_plain_yaml(formatted_data)
return formatted_data
def execute(self, config):
spreadsheet_id = config.id
gids = config.gids or []
if config.gid is not None:
gids.append(config.gid)
if not gids and not config.collection:
gids.append(0)
format_as = config.format
if (config.collection and
format_as not in GoogleSheetsPreprocessor.MAP_TYPES and
format_as not in GoogleSheetsPreprocessor.GRID_TYPES):
format_as = 'map'
gid_to_sheet, gid_to_data = GoogleSheetsPreprocessor.download(
spreadsheet_id=spreadsheet_id, gids=gids, format_as=format_as,
logger=self.pod.logger, generate_ids=config.generate_ids,
header_row_count=config.header_row_count,
header_row_index=config.header_row_index)
if config.path:
# Single sheet import.
path, key_to_update = self.parse_path(config.path)
for gid in gids:
if gid not in gid_to_data:
self.logger.info(
'Sheet not imported for gid {}. Skipped tab?'.format(gid))
continue
gid_to_data[gid] = self._maybe_preserve_content(
new_data=gid_to_data[gid],
path=path,
key_to_update=key_to_update,
properties=gid_to_sheet[gid])
content = GoogleSheetsPreprocessor.serialize_content(
formatted_data=gid_to_data[gid], path=path,
output_style=self.config.output_style)
self.pod.write_file(path, content)
self.logger.info(
'Downloaded {} ({}) -> {}'.format(
gid_to_sheet[gid]['title'], gid, path))
else:
# Multi sheet import based on collection.
collection_path = config.collection
if not gids:
gids = gid_to_sheet.keys()
num_saved = 0
for gid in gids:
if gid not in gid_to_data:
self.logger.info(
'Sheet not imported for gid {}. Skipped tab?'.format(gid))
continue
title = gid_to_sheet[gid]['title']
if title.strip().startswith(IGNORE_INITIAL):
continue
slug = utils.slugify(title)
file_name = '{}.yaml'.format(slug)
output_path = os.path.join(collection_path, file_name)
gid_to_data[gid] = self._maybe_preserve_content(
new_data=gid_to_data[gid],
path=output_path,
key_to_update=None,
properties=gid_to_sheet[gid])
# Use plain text dumper to preserve yaml constructors.
output_content = utils.dump_plain_yaml(gid_to_data[gid])
self.pod.write_file(output_path, output_content)
if gid_to_data[gid].get(DRAFT_KEY):
self.logger.info('Drafted tab -> {}'.format(title))
num_saved += 1
text = 'Saved {} tabs -> {}'
self.logger.info(text.format(num_saved, collection_path))
def execute(self, config):
spreadsheet_id = config.id
gids = config.gids or []
if config.gid is not None:
gids.append(config.gid)
if not gids and not config.collection:
gids.append(0)
format_as = config.format
keep_empty_values = config.keep_empty_values
if (config.collection
and format_as not in GoogleSheetsPreprocessor.MAP_TYPES
and format_as not in GoogleSheetsPreprocessor.GRID_TYPES):
format_as = 'map'
gid_to_sheet, gid_to_data = GoogleSheetsPreprocessor.download(
spreadsheet_id=spreadsheet_id,
gids=gids,
format_as=format_as,
logger=self.pod.logger,
generate_ids=config.generate_ids,
header_row_count=config.header_row_count,
header_row_index=config.header_row_index,
keep_empty_values=keep_empty_values)
if config.path:
# Single sheet import.
path, key_to_update = self.parse_path(config.path)
for gid in gids:
if gid not in gid_to_data:
self.logger.info(
'Sheet not imported for gid {}. Skipped tab?'.format(
gid))
continue
gid_to_data[gid] = self._maybe_preserve_content(
new_data=gid_to_data[gid],
path=path,
key_to_update=key_to_update,
properties=gid_to_sheet[gid])
content = GoogleSheetsPreprocessor.serialize_content(
formatted_data=gid_to_data[gid],
path=path,
output_style=self.config.output_style)
self.pod.write_file(path, content)
self.logger.info('Downloaded {} ({}) -> {}'.format(
gid_to_sheet[gid]['title'], gid, path))
else:
# Multi sheet import based on collection.
collection_path = config.collection
if not gids:
gids = list(gid_to_sheet.keys())
num_saved = 0
for gid in gids:
if gid not in gid_to_data:
self.logger.info(
'Sheet not imported for gid {}. Skipped tab?'.format(
gid))
continue
title = gid_to_sheet[gid]['title']
if title.strip().startswith(IGNORE_INITIAL):
continue
if self.pod.is_enabled(self.pod.FEATURE_OLD_SLUGIFY):
slug = utils.slugify(title)
else:
slug = slugify.slugify(title)
file_name = '{}.yaml'.format(slug)
output_path = os.path.join(collection_path, file_name)
gid_to_data[gid] = self._maybe_preserve_content(
new_data=gid_to_data[gid],
path=output_path,
key_to_update=None,
properties=gid_to_sheet[gid])
# Use plain text dumper to preserve yaml constructors.
output_content = utils.dump_plain_yaml(gid_to_data[gid])
self.pod.write_file(output_path, output_content)
if gid_to_data[gid].get(DRAFT_KEY):
self.logger.info('Drafted tab -> {}'.format(title))
num_saved += 1
text = 'Saved {} tabs -> {}'
self.logger.info(text.format(num_saved, collection_path))