def setUpClass(cls):
cls.domain = 'supply-point-dbaccessors'
cls.locations = [
Location(domain=cls.domain),
Location(domain=cls.domain),
Location(domain=cls.domain),
]
Location.get_db().bulk_save(cls.locations)
cls.supply_points = [
CommCareCase(domain=cls.domain,
type='supply-point',
location_id=cls.locations[0]._id),
CommCareCase(domain=cls.domain,
type='supply-point',
location_id=cls.locations[1]._id),
CommCareCase(domain=cls.domain,
type='supply-point',
location_id=cls.locations[2]._id),
]
locations_by_id = {
location._id: location
for location in cls.locations
}
cls.location_supply_point_pairs = [
(locations_by_id[supply_point.location_id], supply_point)
for supply_point in cls.supply_points
]
CommCareCase.get_db().bulk_save(cls.supply_points)
def purge_locations(domain):
"""
Delete all location data associated with <domain>.
This means Locations, SQLLocations, LocationTypes, and anything which
has a ForeignKey relationship to SQLLocation (as of 2015-03-02, this
includes only StockStates and some custom stuff).
"""
location_ids = set([r['id'] for r in Location.get_db().view(
'locations/by_type',
reduce=False,
startkey=[domain],
endkey=[domain, {}],
).all()])
iter_bulk_delete(Location.get_db(), location_ids)
for loc in SQLLocation.objects.filter(domain=domain).iterator():
if loc.supply_point_id:
case = CommCareCase.get(loc.supply_point_id)
case.delete()
loc.delete()
db = Domain.get_db()
domain_obj = Domain.get_by_name(domain) # cached lookup is fast but stale
domain_json = db.get(domain_obj._id) # get latest raw, unwrapped doc
domain_json.pop('obsolete_location_types', None)
domain_json.pop('location_types', None)
db.save_doc(domain_json)
def handle(self, *args, **options):
with open('location_results.csv', 'wb+') as csvfile:
csv_writer = csv.writer(
csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL
)
csv_writer.writerow(['id', 'type', 'domain', 'property', 'value'])
locations = list(set(Location.get_db().view(
'locations/by_type',
reduce=False,
wrapper=lambda row: row['id'],
).all()))
problematic_domains = {}
for loc in iter_docs(Location.get_db(), locations):
loc = Location.get(loc['_id'])
if self.has_any_hardcoded_properties(loc, csv_writer):
if loc.domain in problematic_domains:
problematic_domains[loc.domain] += 1
else:
problematic_domains[loc.domain] = 1
self.stdout.write("\nDomain stats:\n")
for domain, count in problematic_domains.iteritems():
self.stdout.write("%s: %d" % (domain, count))
def handle(self, *args, **options):
with open('location_results.csv', 'wb+') as csvfile:
csv_writer = csv.writer(csvfile,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow(['id', 'type', 'domain', 'property', 'value'])
locations = list(
set(Location.get_db().view(
'locations/by_type',
reduce=False,
wrapper=lambda row: row['id'],
).all()))
problematic_domains = {}
for loc in iter_docs(Location.get_db(), locations):
loc = Location.get(loc['_id'])
if self.has_any_hardcoded_properties(loc, csv_writer):
if loc.domain in problematic_domains:
problematic_domains[loc.domain] += 1
else:
problematic_domains[loc.domain] = 1
self.stdout.write("\nDomain stats:\n")
for domain, count in problematic_domains.iteritems():
self.stdout.write("%s: %d" % (domain, count))
def handle(self, domain_name, **options):
domain = Domain.get_by_name(domain_name)
if not domain:
print(u'domain with name "{}" not found'.format(domain_name))
return
sql_location_qs = SQLLocation.objects.filter(domain=domain_name)
print('checking {} locations for issues'.format(sql_location_qs.count()))
couch_locations_to_save = []
for sql_location in sql_location_qs:
if sql_location.lineage != sql_location.couch_location.lineage:
print('would change lineage of {} from {} to {}'.format(
sql_location.name,
'-->'.join(sql_location.couch_location.lineage),
'-->'.join(sql_location.lineage),
))
sql_location.couch_location.lineage = sql_location.lineage
couch_locations_to_save.append(sql_location.couch_location.to_json())
if couch_locations_to_save:
if not options['noinput']:
confirm = raw_input(
u"""
Would you like to commit these changes? {} locations will be affected. (y/n)
""".format(len(couch_locations_to_save))
)
if confirm != 'y':
print("\n\t\taborted")
return
print(u"Committing changes")
Location.get_db().bulk_save(couch_locations_to_save)
print("Operation completed")
else:
print('no issues found')
def get_non_archived_facilities_below(location):
child_ids = (
location.sql_location.get_descendants(include_self=True)
.filter(is_archived=False, location_type__name="FACILITY")
.values_list("location_id", flat=True)
)
return [Location.wrap(doc) for doc in get_docs(Location.get_db(), child_ids)]
def handle(self, *args, **options):
try:
domain = args[0]
except IndexError:
self.stderr.write('domain required\n')
return
self.println('Migrating...')
for loc_id, case in get_supply_points_json_in_domain_by_location(domain):
loc = Location.get(loc_id)
old_code = case.get('site_code', '')
new_code = getattr(loc, 'site_code', '')
if old_code and not new_code:
loc.site_code = old_code
loc.save()
self.println('migrated %s (%s)' % (loc.name, loc.site_code))
self.println('Verifying code uniqueness...')
all_codes = Location.get_db().view('commtrack/locations_by_code',
startkey=[domain], endkey=[domain, {}])
locs_by_code = map_reduce(lambda e: [(e['key'][-1].lower(), e['id'])], data=all_codes)
for code, loc_ids in locs_by_code.iteritems():
if len(loc_ids) == 1:
continue
self.println('duplicate code [%s]' % code)
locs = Location.view('_all_docs', keys=loc_ids, include_docs=True)
for loc in locs:
self.println(' %s [%s]' % (loc.name, loc._id))
def get_wrapped_owner(owner_id):
"""
Returns the wrapped user or group object for a given ID, or None
if the id isn't a known owner type.
"""
if not owner_id:
return None
def _get_class(doc_type):
return {
'CommCareUser': CommCareUser,
'WebUser': WebUser,
'Group': Group,
'Location': Location,
}.get(doc_type)
for db in [user_db(), Location.get_db()]:
try:
owner_doc = db.get(owner_id)
except ResourceNotFound:
continue
else:
cls = _get_class(owner_doc['doc_type'])
return cls.wrap(owner_doc) if cls else None
return None
def get_wrapped_owner(owner_id):
"""
Returns the wrapped user or group object for a given ID, or None
if the id isn't a known owner type.
"""
if not owner_id:
return None
def _get_class(doc_type):
return {
'CommCareUser': CommCareUser,
'WebUser': WebUser,
'Group': Group,
'Location': Location,
}.get(doc_type)
for db in [user_db(), Location.get_db()]:
try:
owner_doc = db.get(owner_id)
except ResourceNotFound:
continue
else:
cls = _get_class(owner_doc['doc_type'])
return cls.wrap(owner_doc) if cls else None
return None
def iter_location_join_supply_point(all_location_ids, chunksize=100):
# this function was copy-paste-modified from iter_docs
database = Location.get_db()
for location_ids in chunked(all_location_ids, chunksize):
# sync supply point id
locations = [row.get('doc')
for row in get_docs(database, keys=location_ids)
if row.get('doc')
and row.get('doc')['domain'] not in EXCLUDE_DOMAINS]
supply_points = SupplyPointCase.view(
'commtrack/supply_point_by_loc',
keys=[[location['domain'], location['_id']]
for location in locations],
include_docs=True,
classes={'CommCareCase': SupplyPointCase},
).all()
supply_points_index = {}
for supply_point in supply_points:
key = (supply_point.domain, supply_point.location_id)
if key in supply_points_index:
raise Exception(
"Multiple supply points have "
"domain={!r}, location_id={!r}".format(*key))
supply_points_index[key] = supply_point
for location in locations:
yield (
location,
supply_points_index.get((location['domain'], location['_id']))
)
def bulk_delete_locs(self, loc_ids, total):
locs_to_save = []
count = 0
for loc in iter_docs(Location.get_db(), loc_ids):
loc['doc_type'] = "{}{}".format(loc['doc_type'], DELETED_SUFFIX)
loc['is_archived'] = True
locs_to_save.append(loc)
count += 1
if len(locs_to_save) > 100:
Location.get_db().bulk_save(locs_to_save)
locs_to_save = []
print "{} of {}".format(count, total)
time.sleep(5)
if locs_to_save:
Location.get_db().bulk_save(locs_to_save)
def bulk_delete_locs(self, loc_ids, total):
locs_to_save = []
count = 0
for loc in iter_docs(Location.get_db(), loc_ids):
loc['doc_type'] = "{}{}".format(loc['doc_type'], DELETED_SUFFIX)
loc['is_archived'] = True
locs_to_save.append(loc)
count += 1
if len(locs_to_save) > 100:
Location.get_db().bulk_save(locs_to_save)
locs_to_save = []
print "{} of {}".format(count, total)
time.sleep(5)
if locs_to_save:
Location.get_db().bulk_save(locs_to_save)
def get_non_archived_facilities_below(location):
child_ids = location.sql_location.get_descendants(
include_self=True).filter(is_archived=False,
location_type__name='FACILITY').values_list(
'location_id', flat=True)
return [
Location.wrap(doc) for doc in get_docs(Location.get_db(), child_ids)
]
def get_data(self):
# todo: this will probably have to paginate eventually
if self.all_relevant_forms:
sp_ids = get_relevant_supply_point_ids(
self.domain,
self.active_location,
)
form_xmlnses = [form['xmlns'] for form in self.all_relevant_forms.values()]
spoint_loc_map = {
doc['_id']: doc['location_id']
for doc in iter_docs(SupplyPointCase.get_db(), sp_ids)
}
locations = {
doc['_id']: Location.wrap(doc)
for doc in iter_docs(Location.get_db(), spoint_loc_map.values())
}
for spoint_id, loc_id in spoint_loc_map.items():
loc = locations[loc_id]
form_ids = StockReport.objects.filter(
stocktransaction__case_id=spoint_id
).exclude(
date__lte=self.start_date
).exclude(
date__gte=self.end_date
).values_list(
'form_id', flat=True
).order_by('-date').distinct() # not truly distinct due to ordering
matched = False
for form_id in form_ids:
try:
if XFormInstance.get(form_id).xmlns in form_xmlnses:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': 'reporting',
'geo': loc._geopoint,
}
matched = True
break
except ResourceNotFound:
logging.error('Stock report for location {} in {} references non-existent form {}'.format(
loc._id, loc.domain, form_id
))
if not matched:
yield {
'loc_id': loc._id,
'loc_path': loc.path,
'name': loc.name,
'type': loc.location_type,
'reporting_status': 'nonreporting',
'geo': loc._geopoint,
}
def delete_all_locations():
ids = [
doc['id'] for doc in
SupplyPointCase.get_db().view('supply_point_by_loc/view', reduce=False).all()
]
iter_bulk_delete(SupplyPointCase.get_db(), ids)
iter_bulk_delete(Location.get_db(), SQLLocation.objects.location_ids())
SQLLocation.objects.all().delete()
def location_dump(request, domain):
loc_ids = [row['id'] for row in Location.view('commtrack/locations_by_code', startkey=[domain], endkey=[domain, {}])]
resp = HttpResponse(content_type='text/csv')
resp['Content-Disposition'] = 'attachment; filename="locations_%s.csv"' % domain
w = csv.writer(resp)
w.writerow(['UUID', 'Location Type', 'SMS Code'])
for raw in iter_docs(Location.get_db(), loc_ids):
loc = Location.wrap(raw)
w.writerow([loc._id, loc.location_type, loc.site_code])
return resp
def location_dump(request, domain):
loc_ids = [row['id'] for row in Location.view('commtrack/locations_by_code', startkey=[domain], endkey=[domain, {}])]
resp = HttpResponse(content_type='text/csv')
resp['Content-Disposition'] = 'attachment; filename="locations_%s.csv"' % domain
w = csv.writer(resp)
w.writerow(['UUID', 'Location Type', 'SMS Code'])
for raw in iter_docs(Location.get_db(), loc_ids):
loc = Location.wrap(raw)
w.writerow([loc._id, loc.location_type, loc.site_code])
return resp
def handle(self, *args, **options):
relevant_ids = set([r['id'] for r in Location.get_db().view(
'locations/by_type',
reduce=False,
).all()])
total_locs = len(relevant_ids)
print 'processing {} locations'.format(total_locs)
total_saved = 0
queue = []
for i, loc in enumerate(iter_docs(Location.get_db(), relevant_ids)):
if loc['domain'] in PSI_DOMAINS:
loc['doc_type'] = 'Location-Deleted'
queue.append(loc)
if len(queue) > 500:
Location.get_db().bulk_save(queue)
total_saved += len(queue)
queue = []
print 'saved {} locations ({}/{} seen)'.format(total_saved, i, total_locs)
time.sleep(5)
if queue:
total_saved += len(queue)
Location.get_db().bulk_save(queue)
print 'successfully archived {}/{} locations'.format(total_saved, total_locs)
def handle(self, *args, **options):
self.stdout.write("Processing locations...\n")
relevant_ids = set([r['id'] for r in Location.get_db().view(
'commtrack/locations_by_code',
reduce=False,
).all()])
to_save = []
for location in iter_docs(Location.get_db(), relevant_ids):
# exclude any psi domain to make this take a realistic
# amount fo time
if (
not location.get('last_modified', False) and
'psi' not in location.get('domain', '')
):
location['last_modified'] = datetime.now().isoformat()
to_save.append(location)
if len(to_save) > 500:
Location.get_db().bulk_save(to_save)
to_save = []
if to_save:
Location.get_db().bulk_save(to_save)
def setUpClass(cls):
cls.domain = 'supply-point-dbaccessors'
cls.locations = [
Location(domain=cls.domain),
Location(domain=cls.domain),
Location(domain=cls.domain),
]
Location.get_db().bulk_save(cls.locations)
cls.supply_points = [
CommCareCase(domain=cls.domain, type='supply-point',
location_id=cls.locations[0]._id),
CommCareCase(domain=cls.domain, type='supply-point',
location_id=cls.locations[1]._id),
CommCareCase(domain=cls.domain, type='supply-point',
location_id=cls.locations[2]._id),
]
locations_by_id = {location._id: location
for location in cls.locations}
cls.location_supply_point_pairs = [
(locations_by_id[supply_point.location_id], supply_point)
for supply_point in cls.supply_points
]
CommCareCase.get_db().bulk_save(cls.supply_points)
def setUp(self):
# we have to set the fake database before any other calls
self.domain = "test-loc-parent-id"
self.evaluation_context = EvaluationContext({"domain": self.domain})
self.orig_db = Location.get_db()
self.database = FakeCouchDb()
Location.set_db(self.database)
self.parent = self._make_location(_id=uuid.uuid4().hex)
self.child = self._make_location(_id=uuid.uuid4().hex, lineage=[self.parent._id])
self.grandchild = self._make_location(_id=uuid.uuid4().hex, lineage=[self.child._id, self.parent._id])
self.expression_spec = {
"type": "location_parent_id",
"location_id_expression": {"type": "property_name", "property_name": "location_id"},
}
self.expression = ExpressionFactory.from_spec(self.expression_spec)
def lookup_by_property(domain, prop_name, val, scope, root=None):
if root and not isinstance(root, basestring):
root = root._id
index_view = "locations/prop_index_%s" % prop_name
startkey = [domain, val]
if scope == "global":
startkey.append(None)
elif scope == "descendant":
startkey.append(root)
elif scope == "child":
startkey.extend([root, 1])
else:
raise ValueError("invalid scope type")
return set(row["id"] for row in Location.get_db().view(index_view, startkey=startkey, endkey=startkey + [{}]))
def lookup_by_property(domain, prop_name, val, scope, root=None):
if root and not isinstance(root, basestring):
root = root._id
index_view = 'locations/prop_index_%s' % prop_name
startkey = [domain, val]
if scope == 'global':
startkey.append(None)
elif scope == 'descendant':
startkey.append(root)
elif scope == 'child':
startkey.extend([root, 1])
else:
raise ValueError('invalid scope type')
return set(row['id'] for row in Location.get_db().view(index_view, startkey=startkey, endkey=startkey + [{}]))
def handle(self, *args, **options):
try:
domain = args[0]
except IndexError:
self.stderr.write('domain required\n')
return
self.println('Migrating...')
supply_point_cases = CommCareCase.get_db().view(
'commtrack/supply_point_by_loc',
startkey=[domain],
endkey=[domain, {}],
include_docs=True
)
for result in supply_point_cases:
loc_id = result['key'][-1]
loc = Location.get(loc_id)
case = result['doc']
old_code = case.get('site_code', '')
new_code = getattr(loc, 'site_code', '')
if old_code and not new_code:
loc.site_code = old_code
loc.save()
self.println('migrated %s (%s)' % (loc.name, loc.site_code))
self.println('Verifying code uniqueness...')
all_codes = Location.get_db().view('commtrack/locations_by_code',
startkey=[domain], endkey=[domain, {}])
locs_by_code = map_reduce(lambda e: [(e['key'][-1].lower(), e['id'])], data=all_codes)
for code, loc_ids in locs_by_code.iteritems():
if len(loc_ids) == 1:
continue
self.println('duplicate code [%s]' % code)
locs = Location.view('_all_docs', keys=loc_ids, include_docs=True)
for loc in locs:
self.println(' %s [%s]' % (loc.name, loc._id))
def setUp(self):
# we have to set the fake database before any other calls
self.domain = 'test-loc-parent-id'
self.evaluation_context = EvaluationContext({"domain": self.domain})
self.orig_db = Location.get_db()
self.database = FakeCouchDb()
Location.set_db(self.database)
self.parent = self._make_location(_id=uuid.uuid4().hex)
self.child = self._make_location(_id=uuid.uuid4().hex,
lineage=[self.parent._id])
self.grandchild = self._make_location(
_id=uuid.uuid4().hex, lineage=[self.child._id, self.parent._id])
self.expression_spec = {
"type": "location_parent_id",
"location_id_expression": {
"type": "property_name",
"property_name": "location_id",
}
}
self.expression = ExpressionFactory.from_spec(self.expression_spec)
def lookup_by_property(domain, prop_name, val, scope, root=None):
if root and not isinstance(root, basestring):
root = root._id
if prop_name == 'site_code':
index_view = 'locations/prop_index_site_code'
else:
# this was to be backwards compatible with the api
# if this ever comes up, please take a moment to decide whether it's
# worth changing the API to raise a less nonsensical error
# (or change this function to not sound so general!)
raise ResourceNotFound('missing prop_index_%s' % prop_name)
startkey = [domain, val]
if scope == 'global':
startkey.append(None)
elif scope == 'descendant':
startkey.append(root)
elif scope == 'child':
startkey.extend([root, 1])
else:
raise ValueError('invalid scope type')
return set(row['id'] for row in Location.get_db().view(index_view, startkey=startkey, endkey=startkey + [{}]))
def lookup_by_property(domain, prop_name, val, scope, root=None):
if root and not isinstance(root, basestring):
root = root._id
if prop_name == 'site_code':
index_view = 'locations/prop_index_site_code'
else:
# this was to be backwards compatible with the api
# if this ever comes up, please take a moment to decide whether it's
# worth changing the API to raise a less nonsensical error
# (or change this function to not sound so general!)
raise ResourceNotFound('missing prop_index_%s' % prop_name)
startkey = [domain, val]
if scope == 'global':
startkey.append(None)
elif scope == 'descendant':
startkey.append(root)
elif scope == 'child':
startkey.extend([root, 1])
else:
raise ValueError('invalid scope type')
return set(row['id'] for row in Location.get_db().view(index_view, startkey=startkey, endkey=startkey + [{}]))
def handle(self, *args, **options):
self.stdout.write("Populating site codes...\n")
site_codes_by_domain = {}
relevant_ids = set([r['id'] for r in Location.get_db().view(
'locations/by_type',
reduce=False,
).all()])
to_save = []
for loc in iter_docs(Location.get_db(), relevant_ids):
if not loc['site_code']:
# triggering the safe will cause this to get populated
self.stdout.write("Updating location %s\n" % loc['name'])
if loc['domain'] not in site_codes_by_domain:
site_codes_by_domain[loc['domain']] = list(
Location.site_codes_for_domain(loc['domain'])
)
loc['site_code'] = generate_site_code(
loc['name'],
site_codes_by_domain[loc['domain']]
)
site_codes_by_domain[loc['domain']].append(loc['site_code'])
to_save.append(loc)
if len(to_save) > 500:
Location.get_db().bulk_save(to_save)
to_save = []
if to_save:
Location.get_db().bulk_save(to_save)
def tearDown(self):
self.user.delete()
SQLLocation.objects.all().delete()
delete_all_docs_by_doc_type(Location.get_db(), ['Location'])
super(LocationsTest, self).tearDown()
def _gen():
location_ids = [sp.location_id for sp in self.get_linked_supply_points()]
for doc in iter_docs(Location.get_db(), location_ids):
yield Location.wrap(doc)
def forwards(self, orm):
# hack: manually force sync Location design docs before
# we try to load from them
sync_docs.sync(location_models, verbosity=2)
properties_to_sync = [
('location_id', '_id'),
'domain',
'name',
'location_type',
'site_code',
'external_id',
'latitude',
'longitude',
'is_archived',
]
location_ids = set([r['id'] for r in Location.get_db().view(
'locations/by_name',
reduce=False,
).all()])
for location, sp in iter_location_join_supply_point(location_ids):
try:
sql_location = orm.SQLLocation.objects.get(location_id=location['_id'])
except orm.SQLLocation.DoesNotExist:
# this populates bogus mptt data because otherwise
# null constraints will blow up but do not worry, we
# rebuild this at the end
sql_location = orm.SQLLocation.objects.create(
location_id=location['_id'],
lft=0,
rght=0,
tree_id=0,
level=0
)
for prop in properties_to_sync:
if isinstance(prop, tuple):
sql_prop, couch_prop = prop
else:
sql_prop = couch_prop = prop
if couch_prop in location:
setattr(sql_location, sql_prop, location[couch_prop])
if sp:
sql_location.supply_point_id = sp._id
# sync parent connection
lineage = location.get('lineage', None)
if lineage:
try:
sql_location.parent = orm.SQLLocation.objects.get(location_id=lineage[0])
except orm.SQLLocation.DoesNotExist:
# create a placeholder for the parent if it does
# not yet exist, assuming that it will be properly
# populated with data when its turn comes up in the
# loop
sql_location.parent = orm.SQLLocation.objects.create(
location_id=lineage[0],
lft=0,
rght=0,
tree_id=0,
level=0
)
sql_location.parent.save()
sql_location.save()
# this is the important bit that rebuilds mptt tree structures
SQLLocation.objects.rebuild()
