def create_dump(ctx, index):
"""
Create a dump of an index. If you don't provide an ``--index`` option,
you will be prompted with a list of available index names. Dumps are
stored as a gzipped txt file in ``settings.DUMPS_DIR/<index_name>/<
timestamp>_<index-name>.gz``, and a symlink ``<index-name>_latest.gz`` is
created, pointing to the last created dump.
:param ctx: Click context, so we can issue other management commands
:param index: name of the index you want to create a dump for
"""
if not index:
available_idxs = ctx.invoke(available_indices)
if not available_idxs:
return
index = click.prompt('Name of index to dump')
if index not in available_idxs:
click.secho('"%s" is not an available index' % index, fg='red')
return
match_all = {'query': {'match_all': {}}}
total_docs = es.count(index=index).get('count')
path = _create_path(path=os.path.join(DUMPS_DIR, index))
dump_name = '%(index_name)s_%(timestamp)s.gz' % {
'index_name': index,
'timestamp': datetime.now().strftime('%Y%m%d%H%M%S')
}
new_dump = os.path.join(path, dump_name)
with gzip.open(new_dump, 'wb') as g:
with click.progressbar(es_helpers.scan(es, query=match_all, scroll='1m',
index=index),
length=total_docs) as documents:
for doc in documents:
g.write('%s\n' % json.dumps(doc))
click.secho('Generating checksum', fg='green')
checksum = _checksum_file(new_dump)
checksum_path = os.path.join(DUMPS_DIR, index, '%s.sha1' % dump_name)
with open(checksum_path, 'w') as f:
f.write(checksum)
click.secho('Created dump "%s" (checksum %s)' % (dump_name, checksum),
fg='green')
latest = os.path.join(path, '%s_latest.gz' % index)
try:
os.unlink(latest)
except OSError:
click.secho('First time creating dump, skipping unlinking',
fg='yellow')
os.symlink(new_dump, latest)
click.secho('Created symlink "%s_latest.gz" to "%s"' % (index, new_dump),
fg='green')
latest_checksum = os.path.join(os.path.dirname(checksum_path), '%s_latest.gz.sha1' % index)
try:
os.unlink(latest_checksum)
except OSError:
click.secho('First time creating dump, skipping unlinking checksum',
fg='yellow')
os.symlink(checksum_path, latest_checksum)
click.secho('Created symlink "%s_latest.gz.sha1" to "%s"' % (index, checksum_path),
fg='green')
def create_dump(ctx, index):
"""
Create a dump of an index. If you don't provide an ``--index`` option,
you will be prompted with a list of available index names. Dumps are
stored as a gzipped txt file in ``settings.DUMPS_DIR/<index_name>/<
timestamp>_<index-name>.gz``, and a symlink ``<index-name>_latest.gz`` is
created, pointing to the last created dump.
:param ctx: Click context, so we can issue other management commands
:param index: name of the index you want to create a dump for
"""
if not index:
available_idxs = ctx.invoke(available_indices)
if not available_idxs:
return
index = click.prompt('Name of index to dump')
if index not in available_idxs:
click.secho('"%s" is not an available index' % index, fg='red')
return
match_all = {'query': {'match_all': {}}}
total_docs = es.count(index=index).get('count')
path = _create_path(path=os.path.join(DUMPS_DIR, index))
dump_name = '%(index_name)s_%(timestamp)s.gz' % {
'index_name': index,
'timestamp': datetime.now().strftime('%Y%m%d%H%M%S')
}
new_dump = os.path.join(path, dump_name)
with gzip.open(new_dump, 'wb') as g:
with click.progressbar(es_helpers.scan(es, query=match_all, scroll='1m',
index=index),
length=total_docs) as documents:
for doc in documents:
g.write('%s\n' % json.dumps(doc))
click.secho('Generating checksum', fg='green')
checksum = _checksum_file(new_dump)
checksum_path = os.path.join(DUMPS_DIR, index, '%s.sha1' % dump_name)
with open(checksum_path, 'w') as f:
f.write(checksum)
click.secho('Created dump "%s" (checksum %s)' % (dump_name, checksum),
fg='green')
latest = os.path.join(path, '%s_latest.gz' % index)
try:
os.unlink(latest)
except OSError:
click.secho('First time creating dump, skipping unlinking',
fg='yellow')
os.symlink(new_dump, latest)
click.secho('Created symlink "%s_latest.gz" to "%s"' % (index, new_dump),
fg='green')
latest_checksum = os.path.join(os.path.dirname(checksum_path), '%s_latest.gz.sha1' % index)
try:
os.unlink(latest_checksum)
except OSError:
click.secho('First time creating dump, skipping unlinking checksum',
fg='yellow')
os.symlink(checksum_path, latest_checksum)
click.secho('Created symlink "%s_latest.gz.sha1" to "%s"' % (index, checksum_path),
fg='green')
