def get_db_from_db_name(db_name):
if db_name in _SQL_DBS:
return _SQL_DBS[db_name]
elif db_name == couch_config.get_db(None).dbname: # primary db
return couch_config.get_db(None)
else:
return couch_config.get_db(db_name)
def handle(self, pillow_name, **options):
fluff_configs = {config.name: config for config in get_fluff_pillow_configs()}
if pillow_name not in fluff_configs:
raise CommandError('Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format(
pillow_name, '\n\t'.join(fluff_configs)))
pillow_getter = get_pillow_by_name(pillow_name, instantiate=False)
pillow = pillow_getter(delete_filtered=True)
domains = options.get('domain') or pillow.domains
domains_not_in_pillow = set(domains) - set(pillow.domains)
if domains_not_in_pillow:
bad_domains = ', '.join(domains_not_in_pillow)
available_domains = ', '.join(pillow.domains)
raise CommandError(
"The following domains aren't for this pillow: {}.\nAvailable domains are: {}".format(
bad_domains, available_domains
))
if pillow.kafka_topic in (topics.CASE, topics.FORM):
couch_db = couch_config.get_db(None)
elif pillow.kafka_topic == topics.COMMCARE_USER:
couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB)
else:
raise CommandError('Reindexer not configured for topic: {}'.format(pillow.kafka_topic))
change_provider = CouchDomainDocTypeChangeProvider(
couch_db=couch_db,
domains=domains,
doc_types=[pillow.doc_type],
event_handler=ReindexEventHandler(pillow_name),
)
PillowChangeProviderReindexer(pillow, change_provider).reindex()
def get_db_from_db_name(db_name):
if db_name in _SQL_DBS:
return _SQL_DBS[db_name]
elif db_name == couch_config.get_db(None).dbname: # primary db
return couch_config.get_db(None)
else:
return couch_config.get_db(db_name)
def handle(self, sourcedb, targetdb, **options):
filter = options.get('filter')
continuous = options.get('continuous')
createtarget = options.get('createtarget')
server = couch_config.get_db(None).server
params = {
'continuous': continuous
}
if createtarget:
params['create_target'] = True
if filter:
params['filter'] = filter
source = _check_db_name(sourcedb)
target = _check_db_name(targetdb)
response = server.replicate(source, target, **params)
if not response['ok']:
raise CommandError(json.dumps(response))
local_id = response['_local_id']
print('\nReplication created: {}'.format(local_id))
print("\nUseful commands :")
print(" Check status : python manage.py check_couchdb_replication {}".format(local_id))
print(" Cancel : python manage.py check_couchdb_replication {} --cancel\n".format(local_id))
def handle(self, sourcedb, targetdb, **options):
filter = options.get('filter')
continuous = options.get('continuous')
createtarget = options.get('createtarget')
server = couch_config.get_db(None).server
params = {
'continuous': continuous
}
if createtarget:
params['create_target'] = True
if filter:
params['filter'] = filter
source = _check_db_name(sourcedb)
target = _check_db_name(targetdb)
response = server.replicate(source, target, **params)
if not response['ok']:
raise CommandError(json.dumps(response))
local_id = response['_local_id']
print('\nReplication created: {}'.format(local_id))
print("\nUseful commands :")
print(" Check status : python manage.py check_couchdb_replication {}".format(local_id))
print(" Cancel : python manage.py check_couchdb_replication {} --cancel\n".format(local_id))
def _lookup_id_in_database(doc_id, db_name=None):
db_result = namedtuple('db_result', 'dbname result status')
STATUSES = defaultdict(lambda: 'warning', {
'missing': 'default',
'deleted': 'danger',
})
if db_name:
db = _SQL_DBS.get(db_name, None)
if db:
dbs = [db]
else:
dbs = [couch_config.get_db(None if db_name == 'commcarehq' else db_name)]
else:
couch_dbs = couch_config.all_dbs_by_slug.values()
sql_dbs = _SQL_DBS.values()
dbs = couch_dbs + sql_dbs
db_results = []
response = {"doc_id": doc_id}
for db in dbs:
try:
doc = db.get(doc_id)
except ResourceNotFound as e:
db_results.append(db_result(db.dbname, e.msg, STATUSES[e.msg]))
else:
db_results.append(db_result(db.dbname, 'found', 'success'))
response.update({
"doc": json.dumps(doc, indent=4, sort_keys=True),
"doc_type": doc.get('doc_type', getattr(db, 'doc_type', 'Unknown')),
"dbname": db.dbname,
})
response['db_results'] = db_results
return response
def handle(self, replication_id, cancel, **options):
server = couch_config.get_db(None).server
tasks = server.active_tasks()
replication_tasks = {
task['replication_id']: task
for task in tasks if task['type'] == 'replication'
}
task = replication_tasks.get(replication_id)
if not task:
raise CommandError('Not replication task found with ID: {}'.format(
replication_id))
if cancel:
response = server.res.post('/_replicate',
payload={
'replication_id': replication_id,
'cancel': True
}).json_body
if response['ok']:
print('Replication cancelled')
else:
print(json.dumps(response, indent=4))
else:
print(json.dumps(task, indent=4))
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is ptop_reindexer_fluff %s' % self.args)
fluff_configs = {
config.name: config
for config in get_fluff_pillow_configs()
}
pillow_name = args[0]
if pillow_name not in fluff_configs:
raise CommandError(
'Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format(
pillow_name, '\n\t'.join(fluff_configs)))
pillow_getter = get_pillow_by_name(pillow_name, instantiate=False)
pillow = pillow_getter(delete_filtered=True)
if len(args) == 1:
domains = pillow.domains
else:
domains = args[1:]
domains_not_in_pillow = set(domains) - set(pillow.domains)
if domains_not_in_pillow:
bad_domains = ', '.join(domains_not_in_pillow)
available_domains = ', '.join(pillow.domains)
raise CommandError(
"The following domains aren't for this pillow: {}.\nAvailable domains are: {}"
.format(bad_domains, available_domains))
if pillow.kafka_topic in (topics.CASE, topics.FORM):
couch_db = couch_config.get_db(None)
elif pillow.kafka_topic == topics.COMMCARE_USER:
couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB)
else:
raise CommandError('Reindexer not configured for topic: {}'.format(
pillow.kafka_topic))
change_provider = CouchDomainDocTypeChangeProvider(
couch_db=couch_db,
domains=domains,
doc_types=[pillow.doc_type],
event_handler=ReindexEventHandler(pillow_name),
)
PillowChangeProviderReindexer(pillow, change_provider).reindex()
def _make_db_url(dbname, port):
if not dbname.startswith('commcarehq__'):
dbname = 'commcarehq__{}'.format(dbname)
uri = couch_config.get_db(None).server[dbname].uri
proxy_port = settings.COUCH_DATABASES['default'][
'COUCH_SERVER_ROOT'].split(':')[1]
if port != proxy_port:
uri = uri.replace(proxy_port, port)
return '{}'.format(uri)
def handle(self, replication_id, cancel, **options):
replicator = couch_config.get_db(None).server['_replicator']
replication_doc = replicator.get(replication_id)
if cancel:
response = replicator.delete_doc(replication_doc)
if response['ok']:
print('Replication cancelled')
else:
print(json.dumps(response, indent=4))
else:
print(json.dumps(replication_doc, indent=4))
def handle(self, sourcedb, targetdb, **options):
filter = options.get('filter')
continuous = options.get('continuous')
createtarget = options.get('createtarget')
couchdbport = options.get('couchdbport')
replicator_db = couch_config.get_db(None).server['_replicator']
replication_doc = {
'source': {
'url': _make_db_url(sourcedb, couchdbport)
},
'target': {
'url': _make_db_url(targetdb, couchdbport)
},
'use_checkpoints': True,
'continuous': continuous
}
if createtarget:
replication_doc['create_target'] = True
# Check for admin party
if settings.COUCH_DATABASES['default']['COUCH_USERNAME']:
auth_header = base64.b32encode(
'{COUCH_USERNAME}:{COUCH_PASSWORD}'.format(
**settings.COUCH_DATABASES['default']))
headers = {'Authorization': 'Basic {}'.format(auth_header)}
replication_doc['source']['headers'] = headers
replication_doc['target']['headers'] = headers
if filter:
replication_doc['filter'] = filter
doc_id = replicator_db.save_doc(replication_doc)['id']
replication_doc = replicator_db.get(doc_id)
print(json.dumps(replication_doc, indent=4))
print("\n\n")
print("Useful commands :")
print(
" Check status : python manage.py check_couchdb_replication {}".
format(doc_id))
print(
" Cancel : python manage.py check_couchdb_replication {} --cancel\n"
.format(doc_id))
def handle(self, doc_type, **options):
attrs = options.get('attrs', [])
db = couch_config.get_db(options.get('db', None))
blank_counts = defaultdict(lambda: 0)
max_lengths = defaultdict(lambda: 0)
print("Found {} {} docs\n".format(get_doc_count_by_type(db, doc_type),
doc_type))
docs = get_all_docs_with_doc_types(db, [doc_type])
for doc in docs:
for attr in attrs:
if doc.get(attr):
max_lengths[attr] = max(len(doc[attr]), max_lengths[attr])
else:
blank_counts[attr] += 1
for attr in attrs:
print("{} is {} blank and has max length of {}".format(
attr, 'sometimes' if blank_counts[attr] else 'never',
max_lengths[attr]))
def _lookup_id_in_couch(doc_id, db_name=None):
if db_name:
dbs = [couch_config.get_db(None if db_name == 'commcarehq' else db_name)]
else:
dbs = couch_config.all_dbs_by_slug.values()
for db in dbs:
try:
doc = db.get(doc_id)
except ResourceNotFound:
pass
else:
return {
"doc": json.dumps(doc, indent=4, sort_keys=True),
"doc_id": doc_id,
"doc_type": doc.get('doc_type', 'Unknown'),
"dbname": db.dbname,
}
return {
"doc": "NOT FOUND",
"doc_id": doc_id,
}
def get_db_from_db_name(db_name):
all_dbs = get_databases()
if db_name in all_dbs:
return all_dbs[db_name]
else:
return _CouchDb(couch_config.get_db(db_name))
def setUp(self):
super(ChangeFeedDbTest, self).setUp()
self.couch_db = couch_config.get_db(None)
self.update_seq = get_current_seq(self.couch_db)
def get_default_couch_db_change_feed_pillow(pillow_id, **kwargs):
return get_change_feed_pillow_for_db(pillow_id, couch_config.get_db(None))
def couch_db(cls):
return couch_config.get_db(cls.couch_db_slug())
def get_main_cache_invalidation_pillow(pillow_id, **kwargs):
main_db = couch_config.get_db(None)
return _get_cache_invalidation_pillow(pillow_id,
main_db,
couch_filter="hqadmin/not_case_form")