def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
try:
current_seq = int(db.info()["update_seq"])
except ValueError:
pass # we must be on bigcouch, so comparing seqs is useless
else:
if previous_export and not previous_export.is_bigcouch \
and int(previous_export.seq) > current_seq:
# something weird happened (like the couch database changing)
# better rebuild from scratch
previous_export = None
config = ExportConfiguration(db,
schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def get_docs(schema_index, previous_export=None, filter=None):
def _filter(results):
if filter is None:
return results
return [doc for doc in results if filter(doc)]
db = Database(settings.COUCH_DATABASE)
if previous_export is not None:
consumer = Consumer(db)
view_results = consumer.fetch(since=previous_export.seq)
if view_results:
try:
include_ids = set([res["id"] for res in view_results["results"]])
possible_ids = set([result['id'] for result in \
db.view("couchexport/schema_index",
key=schema_index).all()])
ids_to_use = include_ids.intersection(possible_ids)
return _filter(res["doc"] for res in \
db.all_docs(keys=list(ids_to_use), include_docs=True).all())
except Exception:
import logging
logging.exception("export failed! results: %s" % view_results)
raise
else:
# sometimes this comes back empty. I think it might be a bug
# in couchdbkit, but it's impossible to consistently reproduce.
# For now, just assume this is fine.
return []
else:
return _filter([result['doc'] for result in \
db.view("couchexport/schema_index",
key=schema_index, include_docs=True).all()])
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
try:
current_seq = int(db.info()["update_seq"])
except ValueError:
pass # we must be on bigcouch, so comparing seqs is useless
else:
if previous_export and not previous_export.is_bigcouch \
and int(previous_export.seq) > current_seq:
# something weird happened (like the couch database changing)
# better rebuild from scratch
previous_export = None
config = ExportConfiguration(db, schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def handle(self, sourcedb, destdb, doc_types, **options):
sourcedb = Database(sourcedb)
destdb = Database(destdb)
pretend = options['pretend']
if pretend:
logger.info("**** Simulated run, no data will be copied. ****")
self.copy_docs(sourcedb, destdb, pretend=pretend, doc_types=doc_types)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_doc_types %s' % self.args)
sourcedb = Database(args[0])
destdb = Database(args[1])
pretend = options['pretend']
doc_types = options['doc_types'].split(',')
if pretend:
logger.info("**** Simulated run, no data will be copied. ****")
self.copy_docs(sourcedb, destdb, pretend=pretend, doc_types=doc_types)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_domain %s' % self.args)
sourcedb = Database(args[0])
domain = args[1].strip()
simulate = options['simulate']
since = datetime.strptime(options['since'], '%Y-%m-%d').isoformat() if options['since'] else None
if options['list_types']:
self.list_types(sourcedb, domain, since)
sys.exit(0)
if simulate:
print "\nSimulated run, no data will be copied.\n"
if options['doc_types']:
doc_types = options['doc_types'].split(',')
for type in doc_types:
startkey = [x for x in [domain, type, since] if x is not None]
endkey = [x for x in [domain, type, {}] if x is not None]
self.copy_docs(sourcedb, domain, startkey, endkey, simulate, type=type, since=since)
else:
startkey = [domain]
endkey = [domain, {}]
exclude_types = DEFAULT_EXCLUDE_TYPES + options['doc_types_exclude'].split(',')
self.copy_docs(sourcedb, domain, startkey, endkey, simulate, exclude_types=exclude_types)
def handle(self, *args, **options):
db = Database(settings.BHOMA_NATIONAL_DATABASE)
results = db.view("chw/by_clinic", include_docs=True).all()
chws = [CommunityHealthWorker.wrap(res['doc']) for res in results]
map = defaultdict(lambda: 0)
def _fmt_date(dt):
return "%s-%s" % (dt.year, dt.month)
for chw in chws:
print chw.username, chw.created_on
map[_fmt_date(chw.created_on)] = map[_fmt_date(chw.created_on)] + 1
for k in sorted(map.keys()):
print "%s, %s" % (k, map[k])
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
current_seq = db.info()["update_seq"]
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(get_db(), schema_index,
previous_export=previous_export)
schema = get_schema_new(config)
if not schema:
return None
updated_checkpoint = ExportSchema(seq=current_seq, schema=schema,
index=schema_index)
updated_checkpoint.save()
return updated_checkpoint
def get_db():
"""
Get the couch database.
"""
# this is a bit of a hack, since it assumes all the models talk to the same
# db. that said a lot of our code relies on that assumption.
# this import is here because of annoying dependencies
return Database(settings.COUCH_DATABASE)
def get_db(postfix=None):
"""
Get the couch database.
"""
# this is a bit of a hack, since it assumes all the models talk to the same
# db. that said a lot of our code relies on that assumption.
# this import is here because of annoying dependencies
db_url = settings.COUCH_DATABASE
if postfix:
db_url = settings.EXTRA_COUCHDB_DATABASES[postfix]
return Database(db_url, create=True)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_domain %s' % self.args)
sourcedb = Database(args[0])
domain = args[1].strip()
simulate = options['simulate']
since = datetime.strptime(options['since'], '%Y-%m-%d').isoformat() if options['since'] else None
if options['list_types']:
self.list_types(sourcedb, domain, since)
sys.exit(0)
if simulate:
print "\nSimulated run, no data will be copied.\n"
if options['postgres_db'] and options['postgres_password']:
settings.DATABASES[options['postgres_db']]['PASSWORD'] = options['postgres_password']
self.targetdb = get_db()
domain_doc = Domain.get_by_name(domain)
if domain_doc is None:
self.copy_domain(sourcedb, domain)
if options['doc_types']:
doc_types = options['doc_types'].split(',')
for type in doc_types:
startkey = [x for x in [domain, type, since] if x is not None]
endkey = [x for x in [domain, type, {}] if x is not None]
self.copy_docs(sourcedb, domain, simulate, startkey, endkey, type=type, since=since,
postgres_db=options['postgres_db'])
elif options['id_file']:
path = options['id_file']
if not os.path.isfile(path):
print "Path '%s' does not exist or is not a file" % path
sys.exit(1)
with open(path) as input:
doc_ids = [line.rstrip('\n') for line in input]
if not doc_ids:
print "Path '%s' does not contain any document ID's" % path
sys.exit(1)
self.copy_docs(sourcedb, domain, simulate, doc_ids=doc_ids, postgres_db=options['postgres_db'])
else:
startkey = [domain]
endkey = [domain, {}]
exclude_types = DEFAULT_EXCLUDE_TYPES + options['doc_types_exclude'].split(',')
self.copy_docs(sourcedb, domain, simulate, startkey, endkey, exclude_types=exclude_types,
postgres_db=options['postgres_db'])
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_domain %s' % self.args)
sourcedb = Database(args[0])
domain = args[1].strip()
all_docs = sourcedb.view("domain/docs", startkey=[domain],
endkey=[domain, {}], reduce=False)
total = len(all_docs)
count = 0
targetdb = get_db()
print "found %s matching documents in domain: %s" % (total, domain)
for row in all_docs:
try:
count += 1
dt = DocumentTransform(sourcedb.get(row["id"]), sourcedb)
save(dt, targetdb)
print "Synced %s/%s docs (%s: %s)" % (count, total, row["key"][1], row["id"])
except Exception, e:
print "Document %s failed! Error is: %s" % (row["id"], e)
def build_latest_schema(schema_index):
"""
Build a schema, directly from the index. Also creates a saved checkpoint.
"""
from couchexport.export import ExportConfiguration
db = Database(settings.COUCH_DATABASE)
previous_export = ExportSchema.last(schema_index)
config = ExportConfiguration(db,
schema_index,
previous_export=previous_export)
schema = config.get_latest_schema()
if not schema:
return None
updated_checkpoint = config.create_new_checkpoint()
return updated_checkpoint
def get_db_for_doc_type(self, doc_type):
return Database(self.get_db_uri_for_doc_type(doc_type))
def get_db_for_class(self, klass):
return Database(self.get_db_uri_for_class(klass))
def get_db(self, postfix):
"""
Get the couch database by slug
"""
return Database(self.all_db_uris_by_slug[postfix], create=True)
def all_dbs_by_db_name(self):
return {
Database(db_uri).dbname: Database(db_uri)
for db_uri in self.all_db_uris_by_slug.values()
}
def all_dbs_by_slug(self):
return {
slug: Database(db_uri)
for slug, db_uri in self.all_db_uris_by_slug.items()
}
def __init__(self, **kwargs):
self.couchdb_url = kwargs.get('db_url', settings.COUCH_DATABASE)
self.db = Database(self.couchdb_url)
class CouchDBDocStorage(Storage):
"""
CouchDBDocStorage - a Django Storage class for CouchDB.
Uses the couchdb url for the database you want to connect to
"""
def __init__(self, **kwargs):
self.couchdb_url = kwargs.get('db_url', settings.COUCH_DATABASE)
self.db = Database(self.couchdb_url)
def _put_file(self, name, content):
doc_id, attachment_key=_split_file(name)
self.db.put_attachment(doc_id, content, name=attachment_key)
return doc_id
def get_document(self, doc_id):
return self.db.get(doc_id)
def _open(self, name, mode='rb'):
doc_id, attachment_key=_split_file(name)
couchdb_file = CouchDBAttachmentFile(doc_id, attachment_key, self, mode=mode)
return couchdb_file
def _save(self, name, content):
doc_id, attachment_key=_split_file(name)
content.open()
if hasattr(content, 'chunks'):
content_str = ''.join(chunk for chunk in content.chunks())
else:
content_str = content.read()
#name = name.replace('/', '-')
return self._put_file(name, content_str)
def exists(self, name):
doc_id, attachment_key=_split_file(name)
if self.db.doc_exist(doc_id):
return self.db.open_doc(doc_id)['_attachments'].has_key(attachment_key)
else:
return False
def size(self, name):
doc_id, attachment_key=_split_file(name)
doc = self.get_document(doc_id)
if doc:
return doc['_attachments'][attachment_key]['length']
return 0
def url(self, name):
doc_id, attachment_key=_split_file(name)
# return urljoin(self.base_url,
# os.path.join(quote_plus(self.db.name),
# quote_plus(name),
# 'content'))
return reverse('hutch.views.image_proxy', kwargs={'doc_id': doc_id, 'attachment_key': attachment_key})
def delete(self, name):
doc_id, attachment_key=_split_file(name)
try:
doc = self.get_document(doc_id)
del doc['_attachments'][attachment_key]
except Exception, ex:
print "Ex: %s" % ex
raise IOError("File not found: %s" % name)
