def domains_for_user(request, selected_domain=None):
"""
Generate pulldown menu for domains.
Cache the entire string alongside the couch_user's doc_id that can get invalidated when
the user doc updates via save.
"""
domain_list = []
if selected_domain != 'public':
cached_domains = cache_core.get_cached_prop(request.couch_user.get_id, 'domain_list')
if cached_domains:
domain_list = [Domain.wrap(x) for x in cached_domains]
else:
try:
domain_list = Domain.active_for_user(request.couch_user)
cache_core.cache_doc_prop(request.couch_user.get_id, 'domain_list', [x.to_json() for x in domain_list])
except Exception:
if settings.DEBUG:
raise
else:
domain_list = Domain.active_for_user(request.user)
notify_exception(request)
domain_list = [dict(
url=reverse('domain_homepage', args=[d.name]),
name=d.long_display_name()
) for d in domain_list]
context = {
'is_public': selected_domain == 'public',
'domain_list': domain_list,
'current_domain': selected_domain,
}
template = {
style_utils.BOOTSTRAP_2: 'hqwebapp/partials/domain_list_dropdown.html',
style_utils.BOOTSTRAP_3: 'style/includes/domain_list_dropdown.html',
}[style_utils.bootstrap_version(request)]
return mark_safe(render_to_string(template, context))
def d_results(self):
hits = self.results.get('hits', {}).get('hits', [])
hits = deduplicate(hits)
d_results = []
for res in hits:
try:
domain = Domain.wrap(res['_source'])
if domain.copied_from is not None:
# this avoids putting in snapshots in the list where the
# copied_from domain has been deleted.
d_results.append(domain)
except CopiedFromDeletedException as e:
notify_exception(
self.request,
message=(
"Fetched Exchange Snapshot Error: {}. "
"The problem snapshot id: {}".format(
e.message, res['_source']['_id'])
)
)
if self.sort_by == 'newest':
pass
else:
d_results = Domain.hit_sort(d_results)
return d_results
def deployments(request, template="appstore/deployments.html"):
params, _ = parse_args_for_es(request)
params = dict([(DEPLOYMENT_MAPPING.get(p, p), params[p]) for p in params])
page = int(params.pop('page', 1))
results = es_deployments_query(params, DEPLOYMENT_FACETS)
d_results = [
Domain.wrap(res['_source']) for res in results['hits']['hits']
]
more_pages = False if len(d_results) <= page * 10 else True
facet_map = fill_mapping_with_facets(DEPLOYMENT_MAPPING, results, params)
include_unapproved = True if request.GET.get('is_approved',
"") == "false" else False
vals = {
'deployments': d_results[(page - 1) * 10:page * 10],
'page': page,
'prev_page': page - 1,
'next_page': (page + 1),
'more_pages': more_pages,
'include_unapproved': include_unapproved,
'facet_map': facet_map,
'query_str': request.META['QUERY_STRING'],
'search_url': reverse('deployments'),
'search_query': params.get('search', [""])[0]
}
return render(request, template, vals)
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info(
"Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
try:
invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end,
domain)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" % domain.name)
except CreditLineError as e:
log_accounting_error("There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e))
except InvoiceError as e:
log_accounting_error("Could not create invoice for domain %s: %s" %
(domain.name, e))
except InvoiceAlreadyCreatedError as e:
log_accounting_error("Invoice already existed for domain %s: %s" %
(domain.name, e))
except Exception as e:
log_accounting_error("Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e))
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def update_domain(doc):
domain = Domain.wrap(doc)
new_bu = name_by_map[domain.name]
if new_bu not in BUSINESS_UNITS:
print('Unknown BU: domain={}, BU={}'.format(domain.name, new_bu))
return
domain.internal.business_unit = new_bu
return DocUpdate(doc)
def update_domain(doc):
domain = Domain.wrap(doc)
new_bu = name_by_map[domain.name]
if new_bu not in BUSINESS_UNITS:
print 'Unknown BU: domain={}, BU={}'.format(domain.name, new_bu)
return
domain.internal.business_unit = new_bu
return DocUpdate(doc)
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime("%d %B %Y"),
'end': invoice_end.strftime("%d %B %Y"),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(
subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s"
% domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(
invoice_start, invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s"
% domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e)
)
except BillingContactInfoError as e:
subject = "[%s] Invoice Generation Issue" % domain.name
email_content = render_to_string(
'accounting/invoice_error_email.html', {
'project': domain.name,
'error_msg': 'BillingContactInfoError: %s' % e,
}
)
send_HTML_email(
subject, settings.BILLING_EMAIL, email_content,
email_from="Dimagi Billing Bot <%s>" % settings.DEFAULT_FROM_EMAIL
)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" % (
domain.name, e
))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e)
)
# And finally...
if not is_test:
send_bookkeeper_email()
def appstore(request, template="appstore/appstore_base.html"):
page_length = 10
include_unapproved = True if request.GET.get('is_approved',
"") == "false" else False
if include_unapproved and not request.user.is_superuser:
raise Http404()
params, _ = parse_args_for_es(request)
page = params.pop('page', 1)
page = int(page[0] if isinstance(page, list) else page)
results = es_snapshot_query(params, SNAPSHOT_FACETS)
hits = results.get('hits', {}).get('hits', [])
hits = deduplicate(hits)
d_results = [Domain.wrap(res['_source']) for res in hits]
sort_by = request.GET.get('sort_by', None)
if sort_by == 'best':
d_results = Domain.popular_sort(d_results)
elif sort_by == 'newest':
pass
else:
d_results = Domain.hit_sort(d_results)
persistent_params = {}
if sort_by:
persistent_params["sort_by"] = sort_by
if include_unapproved:
persistent_params["is_approved"] = "false"
persistent_params = urlencode(
persistent_params) # json.dumps(persistent_params)
average_ratings = list()
for result in d_results:
average_ratings.append([
result.name,
Review.get_average_rating_by_app(result.copied_from._id)
])
more_pages = False if len(d_results) <= page * page_length else True
facet_map = fill_mapping_with_facets(SNAPSHOT_MAPPING, results, params)
vals = dict(
apps=d_results[(page - 1) * page_length:page * page_length],
page=page,
prev_page=(page - 1),
next_page=(page + 1),
more_pages=more_pages,
sort_by=sort_by,
average_ratings=average_ratings,
include_unapproved=include_unapproved,
facet_map=facet_map,
facets=results.get("facets", []),
query_str=request.META['QUERY_STRING'],
search_query=params.get('search', [""])[0],
persistent_params=persistent_params,
)
return render(request, template, vals)
def copy_domain(self, sourcedb, domain):
print "Copying domain doc"
result = sourcedb.view("domain/domains", key=domain, reduce=False, include_docs=True).first()
if result and "doc" in result:
domain_doc = Domain.wrap(result["doc"])
dt = DocumentTransform(domain_doc, sourcedb)
save(dt, self.targetdb)
else:
print "Domain doc not found for domain %s." % domain
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info(
"[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime("%d %B %Y"),
'end': invoice_end.strftime("%d %B %Y"),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s" %
domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(invoice_start,
invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s" %
domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e))
except BillingContactInfoError as e:
subject = "[%s] Invoice Generation Issue" % domain.name
email_content = render_to_string(
'accounting/invoice_error_email.html', {
'project': domain.name,
'error_msg': 'BillingContactInfoError: %s' % e,
})
send_HTML_email(subject,
settings.BILLING_EMAIL,
email_content,
email_from="Dimagi Billing Bot <%s>" %
settings.DEFAULT_FROM_EMAIL)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" %
(domain.name, e))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e))
# And finally...
if not is_test:
send_bookkeeper_email()
def handle(self, *args, **options):
raise CommandError(
'copy_group_data is currently broken. '
'Ask Danny or Ethan to fix it along the lines of '
'https://github.com/dimagi/commcare-hq/pull/9180/files#diff-9d976dc051a36a028c6604581dfbce5dR95'
)
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(domain.name,
owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [
CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)
]
for case in cases:
def dump(self, output_stream):
from corehq.apps.domain.models import Domain
domain_obj = Domain.get_by_name(self.domain, strict=True)
if not domain_obj:
raise DomainDumpError("Domain not found: {}".format(self.domain))
domain_dict = _get_doc_with_attachments(Domain.get_db(), domain_obj.to_json())
domain_obj = Domain.wrap(domain_dict)
json.dump(domain_obj.to_json(), output_stream)
output_stream.write('\n')
self.stdout.write('Dumping {} Domain\n'.format(1))
return Counter({'Domain': 1})
def copy_domain(self, sourcedb, domain):
print "Copying domain doc"
result = sourcedb.view("domain/domains",
key=domain,
reduce=False,
include_docs=True).first()
if result and 'doc' in result:
domain_doc = Domain.wrap(result['doc'])
dt = DocumentTransform(domain_doc, sourcedb)
save(dt, self.targetdb)
else:
print "Domain doc not found for domain %s." % domain
def dump(self, output_stream):
from corehq.apps.domain.models import Domain
domain_obj = Domain.get_by_name(self.domain, strict=True)
if not domain_obj:
raise DomainDumpError("Domain not found: {}".format(self.domain))
domain_dict = _get_doc_with_attachments(Domain.get_db(), domain_obj.to_json())
domain_obj = Domain.wrap(domain_dict)
json.dump(domain_obj.to_json(), output_stream)
output_stream.write('\n')
self.stdout.write('Dumping {} Domain\n'.format(1))
return Counter({'Domain': 1})
def appstore(request, template="appstore/appstore_base.html"):
page_length = 10
include_unapproved = True if request.GET.get('is_approved', "") == "false" else False
if include_unapproved and not request.user.is_superuser:
raise Http404()
params, _ = parse_args_for_es(request)
page = params.pop('page', 1)
page = int(page[0] if isinstance(page, list) else page)
results = es_snapshot_query(params, SNAPSHOT_FACETS)
hits = results.get('hits', {}).get('hits', [])
hits = deduplicate(hits)
d_results = [Domain.wrap(res['_source']) for res in hits]
sort_by = request.GET.get('sort_by', None)
if sort_by == 'best':
d_results = Domain.popular_sort(d_results)
elif sort_by == 'newest':
pass
else:
d_results = Domain.hit_sort(d_results)
persistent_params = {}
if sort_by:
persistent_params["sort_by"] = sort_by
if include_unapproved:
persistent_params["is_approved"] = "false"
persistent_params = urlencode(persistent_params) # json.dumps(persistent_params)
average_ratings = list()
for result in d_results:
average_ratings.append([result.name, Review.get_average_rating_by_app(result.copied_from._id)])
more_pages = False if len(d_results) <= page*page_length else True
facet_map = fill_mapping_with_facets(SNAPSHOT_MAPPING, results, params)
vals = dict(
apps=d_results[(page-1)*page_length:page*page_length],
page=page,
prev_page=(page-1),
next_page=(page+1),
more_pages=more_pages,
sort_by=sort_by,
average_ratings=average_ratings,
include_unapproved=include_unapproved,
facet_map=facet_map,
facets=results.get("facets", []),
query_str=request.META['QUERY_STRING'],
search_query=params.get('search', [""])[0],
persistent_params=persistent_params,
)
return render(request, template, vals)
def domains_for_user(request, selected_domain=None):
"""
Generate pulldown menu for domains.
Cache the entire string alongside the couch_user's doc_id that can get invalidated when
the user doc updates via save.
"""
lst = list()
lst.append('<ul class="dropdown-menu nav-list dropdown-orange">')
new_domain_url = reverse("registration_domain")
if selected_domain == 'public':
# viewing the public domain with a different db, so the user's domains can't readily be accessed.
lst.append('<li><a href="%s">%s...</a></li>' %
(reverse("domain_select"), _("Back to My Projects")))
lst.append('<li class="divider"></li>')
else:
cached_domains = cache_core.get_cached_prop(request.couch_user.get_id,
'domain_list')
if cached_domains:
domain_list = [Domain.wrap(x) for x in cached_domains]
else:
try:
domain_list = Domain.active_for_user(request.couch_user)
cache_core.cache_doc_prop(request.couch_user.get_id,
'domain_list',
[x.to_json() for x in domain_list])
except Exception:
if settings.DEBUG:
raise
else:
domain_list = Domain.active_for_user(request.user)
notify_exception(request)
if len(domain_list) > 0:
lst.append('<li class="nav-header">%s</li>' % _('My Projects'))
for domain in domain_list:
default_url = reverse("domain_homepage", args=[domain.name])
lst.append('<li><a href="%s">%s</a></li>' %
(default_url, domain.long_display_name()))
else:
lst.append('<li class="nav-header">No Projects</li>')
lst.append('<li class="divider"></li>')
lst.append('<li><a href="%s">%s...</a></li>' %
(new_domain_url, _('New Project')))
lst.append('<li><a href="%s">%s...</a></li>' %
(reverse("appstore"), _('CommCare Exchange')))
lst.append("</ul>")
domain_list_str = "".join(lst)
return domain_list_str
def generate_invoices(based_on_date=None, check_existing=False, is_test=False):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = utils.get_previous_month_date_range(today)
logger.info("[Billing] Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
if (check_existing and
Invoice.objects.filter(
subscription__subscriber__domain=domain,
date_created__gte=today).count() != 0):
pass
elif is_test:
logger.info("[Billing] Ready to create invoice for domain %s"
% domain.name)
else:
try:
invoice_factory = DomainInvoiceFactory(
invoice_start, invoice_end, domain)
invoice_factory.create_invoices()
logger.info("[BILLING] Sent invoices for domain %s"
% domain.name)
except CreditLineError as e:
logger.error(
"[BILLING] There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e)
)
except BillingContactInfoError as e:
logger.info("BillingContactInfoError: %s" % e)
except InvoiceError as e:
logger.error(
"[BILLING] Could not create invoice for domain %s: %s" % (
domain.name, e
))
except InvoiceAlreadyCreatedError as e:
logger.error(
"[BILLING] Invoice already existed for domain %s: %s" % (
domain.name, e
))
except Exception as e:
logger.error(
"[BILLING] Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e)
)
def copy_domain(self, source_couch, domain):
print "Copying domain doc"
sourcedb = source_couch.get_db_for_class(Domain)
result = sourcedb.view("domain/domains",
key=domain,
reduce=False,
include_docs=True).first()
if result and 'doc' in result:
domain_doc = Domain.wrap(result['doc'])
dt = DocumentTransform(domain_doc._obj, sourcedb)
save(dt, self.targetdb.get_db_for_doc_type(domain_doc['doc_type']))
else:
print "Domain doc not found for domain %s." % domain
def copy_domain(self, source_couch, domain):
print "Copying domain doc"
sourcedb = source_couch.get_db_for_class(Domain)
result = sourcedb.view(
"domain/domains",
key=domain,
reduce=False,
include_docs=True
).first()
if result and 'doc' in result:
domain_doc = Domain.wrap(result['doc'])
dt = DocumentTransform(domain_doc._obj, sourcedb)
save(dt, self.targetdb.get_db_for_doc_type(domain_doc['doc_type']))
else:
print "Domain doc not found for domain %s." % domain
def appstore(request, template="appstore/appstore_base.html"):
page_length = 10
include_unapproved = True if request.GET.get('is_approved',
"") == "false" else False
if include_unapproved and not request.user.is_superuser:
raise Http404()
params, _ = parse_args_for_es(request)
params = dict([(SNAPSHOT_MAPPING.get(p, p), params[p]) for p in params])
page = params.pop('page', 1)
page = int(page[0] if isinstance(page, list) else page)
results = es_snapshot_query(params, SNAPSHOT_FACETS)
d_results = [
Domain.wrap(res['_source'])
for res in results.get('hits', {}).get('hits', [])
]
sort_by = request.GET.get('sort_by', None)
if sort_by == 'best':
d_results = Domain.popular_sort(d_results)
elif sort_by == 'newest':
pass
else:
d_results = Domain.hit_sort(d_results)
average_ratings = list()
for result in d_results:
average_ratings.append([
result.name,
Review.get_average_rating_by_app(result.copied_from._id)
])
more_pages = False if len(d_results) <= page * page_length else True
facets_sortables = generate_sortables_from_facets(
results, params, inverse_dict(SNAPSHOT_MAPPING))
vals = dict(apps=d_results[(page - 1) * page_length:page * page_length],
page=page,
prev_page=(page - 1),
next_page=(page + 1),
more_pages=more_pages,
sort_by=sort_by,
average_ratings=average_ratings,
include_unapproved=include_unapproved,
sortables=facets_sortables,
query_str=request.META['QUERY_STRING'],
search_query=params.get('search', [""])[0])
return render(request, template, vals)
def domains_for_user(request, selected_domain=None):
"""
Generate pulldown menu for domains.
Cache the entire string alongside the couch_user's doc_id that can get invalidated when
the user doc updates via save.
"""
lst = list()
lst.append('<ul class="dropdown-menu nav-list dropdown-orange">')
new_domain_url = reverse("registration_domain")
if selected_domain == 'public':
# viewing the public domain with a different db, so the user's domains can't readily be accessed.
lst.append('<li><a href="%s">%s...</a></li>' % (reverse("domain_select"), _("Back to My Projects")))
lst.append('<li class="divider"></li>')
else:
cached_domains = cache_core.get_cached_prop(request.couch_user.get_id, 'domain_list')
if cached_domains:
domain_list = [Domain.wrap(x) for x in cached_domains]
else:
try:
domain_list = Domain.active_for_user(request.couch_user)
cache_core.cache_doc_prop(request.couch_user.get_id, 'domain_list', [x.to_json() for x in domain_list])
except Exception:
if settings.DEBUG:
raise
else:
domain_list = Domain.active_for_user(request.user)
notify_exception(request)
if len(domain_list) > 0:
lst.append('<li class="nav-header">%s</li>' % _('My Projects'))
for domain in domain_list:
default_url = reverse("domain_homepage", args=[domain.name])
lst.append('<li><a href="%s">%s</a></li>' % (default_url, domain.long_display_name()))
else:
lst.append('<li class="nav-header">No Projects</li>')
lst.append('<li class="divider"></li>')
lst.append('<li><a href="%s">%s...</a></li>' % (new_domain_url, _('New Project')))
lst.append('<li><a href="%s">%s...</a></li>' % (reverse("appstore"), _('CommCare Exchange')))
lst.append("</ul>")
domain_list_str = "".join(lst)
return domain_list_str
def handle(self, dry_run=False, log_file=False, *args, **options):
if not dry_run:
confirm_force_reset = raw_input(
"Are you sure you want to apply downgrades and upgrades to "
"ALL domains? Type 'yes' to continue. \n")
if confirm_force_reset != 'yes':
return
if log_file:
orig_stdout = sys.stdout
f = file('subscription_changes.txt', 'w')
sys.stdout = f
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
logging.info("%s START" % domain.name)
print("\n")
plan_version, subscription = Subscription.get_subscribed_plan_by_domain(
domain)
if not subscription:
subscriber = Subscriber.objects.get_or_create(
domain=domain.name)[0]
print("Active subscription not found for domain %s" %
domain.name)
else:
subscriber = subscription.subscriber
if not dry_run:
print("%s => %s" % (domain.name, plan_version.plan.name))
subscriber.apply_upgrades_and_downgrades(
new_plan_version=plan_version,
verbose=True,
new_subscription=subscription,
)
else:
print("[DRY RUN] %s => %s" %
(domain.name, plan_version.plan.name))
logging.info("%s END" % domain.name)
if log_file:
sys.stdout = orig_stdout
f.close()
def handle(self, dry_run=False, log_file=False, *args, **options):
if not dry_run:
confirm_force_reset = raw_input(
"Are you sure you want to apply downgrades and upgrades to "
"ALL domains? Type 'yes' to continue. \n"
)
if confirm_force_reset != 'yes':
return
if log_file:
orig_stdout = sys.stdout
f = file('subscription_changes.txt', 'w')
sys.stdout = f
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
logging.info("%s START" % domain.name)
print ("\n")
plan_version, subscription = Subscription.get_subscribed_plan_by_domain(domain)
if not subscription:
subscriber = Subscriber.objects.get_or_create(domain=domain.name)[0]
print ("Active subscription not found for domain %s"
% domain.name)
else:
subscriber = subscription.subscriber
if not dry_run:
print ("%s => %s" %
(domain.name, plan_version.plan.name))
subscriber.apply_upgrades_and_downgrades(
new_plan_version=plan_version,
verbose=True,
new_subscription=subscription,
)
else:
print ("[DRY RUN] %s => %s" %
(domain.name, plan_version.plan.name))
logging.info("%s END" % domain.name)
if log_file:
sys.stdout = orig_stdout
f.close()
def selected_snapshots(self):
hits = self.results.get('hits', {}).get('hits', [])
hits = deduplicate(hits)
domains = []
for res in hits:
try:
domain = Domain.wrap(res['_source'])
if domain.copied_from is not None:
# this avoids putting in snapshots in the list where the
# copied_from domain has been deleted.
domains.append(domain)
except CopiedFromDeletedException as e:
notify_exception(
self.request,
message=("Fetched Exchange Snapshot Error: {}. "
"The problem snapshot id: {}".format(
e.message, res['_source']['_id'])))
return domains
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains', key=group.domain, include_docs=True,
reduce=False, limit=1).one()['doc']
)
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(
domain.name, owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
for case in cases:
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info("Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain = Domain.wrap(domain_doc)
try:
invoice_factory = DomainInvoiceFactory(
invoice_start, invoice_end, domain)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" % domain.name)
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain.name, e)
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" % (domain.name, e)
)
except InvoiceAlreadyCreatedError as e:
log_accounting_error(
"Invoice already existed for domain %s: %s" % (domain.name, e)
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain.name, e)
)
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def deployments(request, template="appstore/deployments.html"):
params, _ = parse_args_for_es(request)
params = dict([(DEPLOYMENT_MAPPING.get(p, p), params[p]) for p in params])
page = int(params.pop('page', 1))
results = es_deployments_query(params, DEPLOYMENT_FACETS)
d_results = [Domain.wrap(res['_source']) for res in results['hits']['hits']]
more_pages = False if len(d_results) <= page*10 else True
facets_sortables = generate_sortables_from_facets(results, params, inverse_dict(DEPLOYMENT_MAPPING))
include_unapproved = True if request.GET.get('is_approved', "") == "false" else False
vals = { 'deployments': d_results[(page-1)*10:page*10],
'page': page,
'prev_page': page-1,
'next_page': (page+1),
'more_pages': more_pages,
'include_unapproved': include_unapproved,
'sortables': facets_sortables,
'query_str': request.META['QUERY_STRING'],
'search_url': reverse('deployments'),
'search_query': params.get('search', [""])[0]}
return render(request, template, vals)
def selected_snapshots(self):
hits = self.results.get('hits', {}).get('hits', [])
hits = deduplicate(hits)
domains = []
for res in hits:
try:
domain = Domain.wrap(res['_source'])
if domain.copied_from is not None:
# this avoids putting in snapshots in the list where the
# copied_from domain has been deleted.
domains.append(domain)
except CopiedFromDeletedException as e:
notify_exception(
self.request,
message=(
"Fetched Exchange Snapshot Error: {}. "
"The problem snapshot id: {}".format(
six.text_type(e), res['_source']['_id'])
)
)
return domains
def appstore(request, template="appstore/appstore_base.html"):
page_length = 10
include_unapproved = True if request.GET.get('is_approved', "") == "false" else False
if include_unapproved and not request.user.is_superuser:
raise Http404()
params, _ = parse_args_for_es(request)
params = dict([(SNAPSHOT_MAPPING.get(p, p), params[p]) for p in params])
page = params.pop('page', 1)
page = int(page[0] if isinstance(page, list) else page)
results = es_snapshot_query(params, SNAPSHOT_FACETS)
d_results = [Domain.wrap(res['_source']) for res in results.get('hits', {}).get('hits', [])]
sort_by = request.GET.get('sort_by', None)
if sort_by == 'best':
d_results = Domain.popular_sort(d_results)
elif sort_by == 'newest':
pass
else:
d_results = Domain.hit_sort(d_results)
average_ratings = list()
for result in d_results:
average_ratings.append([result.name, Review.get_average_rating_by_app(result.copied_from._id)])
more_pages = False if len(d_results) <= page*page_length else True
facets_sortables = generate_sortables_from_facets(results, params, inverse_dict(SNAPSHOT_MAPPING))
vals = dict(apps=d_results[(page-1)*page_length:page*page_length],
page=page,
prev_page=(page-1),
next_page=(page+1),
more_pages=more_pages,
sort_by=sort_by,
average_ratings=average_ratings,
include_unapproved=include_unapproved,
sortables=facets_sortables,
query_str=request.META['QUERY_STRING'],
search_query = params.get('search', [""])[0])
return render(request, template, vals)
def handle(self, outfile, **options):
snapshots = Domain.get_db().view(
'domain/snapshots',
startkey=[],
endkey=[{}],
reduce=False,
include_docs=True,
).all()
snapshots = [s['doc'] for s in snapshots]
snapshots = [s for s in snapshots if s['published'] and s['is_snapshot'] and s.get('snapshot_head')]
rows = []
for s in snapshots:
domain = Domain.wrap(s)
user = None
if s['cda']['user_id']:
user = CouchUser.get_by_user_id(s['cda']['user_id'])
author = s['author']
if not author and user:
author = user.human_friendly_name
row = (
domain.copied_from,
s['title'],
author,
s['short_description'],
s['project_type'],
user.username if user else None,
datetime.strptime(s['snapshot_time'], ISO_DATETIME_FORMAT).strftime('%b %d, %Y'),
', '.join(set([lang for app in domain.applications() for lang in app.langs])),
s['license'],
)
rows.append(row)
with open(outfile, 'w', encoding='utf-8') as out:
writer = csv.writer(out, delimiter="\t")
for row in rows:
writer.writerow(row)
def deployments(request, template="appstore/deployments.html"):
params, _ = parse_args_for_es(request)
params = dict([(DEPLOYMENT_MAPPING.get(p, p), params[p]) for p in params])
page = int(params.pop("page", 1))
results = es_deployments_query(params, DEPLOYMENT_FACETS)
d_results = [Domain.wrap(res["_source"]) for res in results["hits"]["hits"]]
more_pages = False if len(d_results) <= page * 10 else True
facets_sortables = generate_sortables_from_facets(results, params, inverse_dict(DEPLOYMENT_MAPPING))
include_unapproved = True if request.GET.get("is_approved", "") == "false" else False
vals = {
"deployments": d_results[(page - 1) * 10 : page * 10],
"page": page,
"prev_page": page - 1,
"next_page": (page + 1),
"more_pages": more_pages,
"include_unapproved": include_unapproved,
"sortables": facets_sortables,
"query_str": request.META["QUERY_STRING"],
"search_url": reverse("deployments"),
"search_query": params.get("search", [""])[0],
}
return render(request, template, vals)
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info("Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain_obj = Domain.wrap(domain_doc)
if not domain_obj.is_active:
continue
try:
invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end, domain_obj)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" % domain_obj.name)
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
all_customer_billing_accounts = BillingAccount.objects.filter(is_customer_billing_account=True)
for account in all_customer_billing_accounts:
try:
if account.invoicing_plan == InvoicingPlan.QUARTERLY:
customer_invoice_start = invoice_start - relativedelta(months=2)
elif account.invoicing_plan == InvoicingPlan.YEARLY:
customer_invoice_start = invoice_start - relativedelta(months=11)
else:
customer_invoice_start = invoice_start
invoice_factory = CustomerAccountInvoiceFactory(
account=account,
date_start=customer_invoice_start,
date_end=invoice_end
)
invoice_factory.create_invoice()
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError("Usage is copy_group_data %s" % self.args)
sourcedb = Database(args[0])
group_id = args[1]
print "getting group"
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print "getting domain"
domain = Domain.wrap(
sourcedb.view("domain/domains", key=group.domain, include_docs=True, reduce=False, limit=1).one()["doc"]
)
domain.save(force_update=True)
print "getting cases"
cases = sourcedb.view(
"hqcase/by_owner",
keys=[[group.domain, group_id, False], [group.domain, group_id, True]],
wrapper=lambda row: CommCareCase.wrap(row["doc"]),
reduce=False,
include_docs=True,
).all()
self.lenient_bulk_save(CommCareCase, cases)
print "compiling xform_ids"
xform_ids = set()
for case in cases:
xform_ids.update(case.xform_ids)
print "getting xforms"
user_ids = set(group.users)
CHUNK_SIZE = 100
def form_wrapper(row):
doc = row["doc"]
doc.pop("_attachments", None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(keys=list(subset), include_docs=True, wrapper=form_wrapper).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print "getting users"
def wrap_user(row):
doc = row["doc"]
try:
return CouchUser.wrap_correctly(doc)
except Exception as e:
logging.exception("trouble with user %s" % doc["_id"])
return None
users = sourcedb.all_docs(keys=list(user_ids), include_docs=True, wrapper=wrap_user).all()
for user in users:
# if we use bulk save, django user doesn't get sync'd
user.save(force_update=True)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains', key=group.domain, include_docs=True,
reduce=False, limit=1).one()['doc']
)
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
def keys_for_owner(domain, owner_id):
return [
[domain, owner_id, False],
[domain, owner_id, True],
]
def get_case_ids(owners):
keys = list(itertools.chain(*[keys_for_owner(domain.name, owner_id) for owner_id in owners]))
results = sourcedb.view(
'hqcase/by_owner',
keys=keys,
reduce=False,
include_docs=False,
)
return [res['id'] for res in results]
print 'getting case ids'
case_ids = get_case_ids(owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
for case in cases:
xform_ids.update(case.xform_ids)
self.lenient_bulk_save(CommCareCase, cases)
if not exclude_user_owned:
# also grab submissions that may not have included any case data
for user_id in group.users:
xform_ids.update(res['id'] for res in sourcedb.view(
'reports_forms/all_forms',
startkey=['submission user', domain.name, user_id],
endkey=['submission user', domain.name, user_id, {}],
reduce=False
))
print 'copying %s xforms' % len(xform_ids)
user_ids = set(group.users)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(
keys=list(subset),
include_docs=True,
wrapper=form_wrapper,
).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print 'copying %s users' % len(user_ids)
def wrap_user(row):
try:
doc = row['doc']
except KeyError:
logging.exception('trouble with user result %r' % row)
return None
try:
return CouchUser.wrap_correctly(doc)
except Exception:
logging.exception('trouble with user %s' % doc['_id'])
return None
users = sourcedb.all_docs(
keys=list(user_ids),
include_docs=True,
wrapper=wrap_user,
).all()
role_ids = set([])
for user in filter(lambda u: u is not None, users):
# if we use bulk save, django user doesn't get sync'd
if user.get_domain_membership(domain.name).role_id:
role_ids.add(user.domain_membership.role_id)
user.save(force_update=True)
print 'copying %s roles' % len(role_ids)
for i, subset in enumerate(chunked(role_ids, CHUNK_SIZE)):
roles = [UserRole.wrap(role['doc']) for role in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(UserRole, roles)
if options['include_sync_logs']:
print 'copying sync logs'
for user_id in user_ids:
log_ids = [res['id'] for res in sourcedb.view("phone/sync_logs_by_user",
startkey=[user_id, {}],
endkey=[user_id],
descending=True,
reduce=False,
include_docs=True
)]
print 'user: %s, logs: %s' % (user_id, len(log_ids))
for i, subset in enumerate(chunked(log_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
logs = [SyncLog.wrap(log['doc']) for log in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(SyncLog, logs)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
include_user_owned = options["include_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
domain.save(force_update=True)
owners = [group_id]
if include_user_owned:
owners.extend(group.users)
def keys_for_owner(domain, owner_id):
return [
[domain, owner_id, False],
[domain, owner_id, True],
]
def get_case_ids(owners):
keys = list(
itertools.chain(*[
keys_for_owner(domain.name, owner_id)
for owner_id in owners
]))
results = sourcedb.view(
'hqcase/by_owner',
keys=keys,
reduce=False,
include_docs=False,
)
return [res['id'] for res in results]
CHUNK_SIZE = 100
print 'getting case ids'
case_ids = get_case_ids(owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [
CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)
]
for case in cases:
def d_results(self):
d_results = [
Domain.wrap(res['_source']) for res in self.results['hits']['hits']
]
return d_results
def handle(self, *args, **options):
raise CommandError(
'copy_group_data is currently broken. '
'Ask Danny or Ethan to fix it along the lines of '
'https://github.com/dimagi/commcare-hq/pull/9180/files#diff-9d976dc051a36a028c6604581dfbce5dR95'
)
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains', key=group.domain, include_docs=True,
reduce=False, limit=1).one()['doc']
)
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(
domain.name, owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
for case in cases:
xform_ids.update(case.xform_ids)
self.lenient_bulk_save(CommCareCase, cases)
if not exclude_user_owned:
# also grab submissions that may not have included any case data
for user_id in group.users:
xform_ids.update(res['id'] for res in sourcedb.view(
'all_forms/view',
startkey=['submission user', domain.name, user_id],
endkey=['submission user', domain.name, user_id, {}],
reduce=False
))
print 'copying %s xforms' % len(xform_ids)
user_ids = set(group.users)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
doc.pop('external_blobs', None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(
keys=list(subset),
include_docs=True,
wrapper=form_wrapper,
).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print 'copying %s users' % len(user_ids)
def wrap_user(row):
try:
doc = row['doc']
except KeyError:
logging.exception('trouble with user result %r' % row)
return None
try:
return CouchUser.wrap_correctly(doc)
except Exception:
logging.exception('trouble with user %s' % doc['_id'])
return None
users = sourcedb.all_docs(
keys=list(user_ids),
include_docs=True,
wrapper=wrap_user,
).all()
role_ids = set([])
for user in filter(lambda u: u is not None, users):
# if we use bulk save, django user doesn't get sync'd
domain_membership = user.get_domain_membership(domain.name)
if domain_membership and domain_membership.role_id:
role_ids.add(user.domain_membership.role_id)
user.save(force_update=True)
print 'copying %s roles' % len(role_ids)
for i, subset in enumerate(chunked(role_ids, CHUNK_SIZE)):
roles = [UserRole.wrap(role['doc']) for role in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(UserRole, roles)
if options['include_sync_logs']:
print 'copying sync logs'
for user_id in user_ids:
log_ids = [res['id'] for res in sourcedb.view("phone/sync_logs_by_user",
startkey=[user_id, {}],
endkey=[user_id],
descending=True,
reduce=False,
include_docs=True
)]
print 'user: %s, logs: %s' % (user_id, len(log_ids))
for i, subset in enumerate(chunked(log_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
logs = [SyncLog.wrap(log['doc']) for log in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(SyncLog, logs)
def d_results(self):
d_results = [Domain.wrap(res['_source']) for res in self.results['hits']['hits']]
return d_results
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
domain.save(force_update=True)
print 'getting cases'
cases = sourcedb.view(
'hqcase/by_owner',
keys=[
[group.domain, group_id, False],
[group.domain, group_id, True],
],
wrapper=lambda row: CommCareCase.wrap(row['doc']),
reduce=False,
include_docs=True).all()
self.lenient_bulk_save(CommCareCase, cases)
print 'compiling xform_ids'
xform_ids = set()
for case in cases:
xform_ids.update(case.xform_ids)
print 'getting xforms'
user_ids = set(group.users)
CHUNK_SIZE = 100
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(
keys=list(subset),
include_docs=True,
wrapper=form_wrapper,
).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print 'getting users'
def wrap_user(row):
doc = row['doc']
try:
return CouchUser.wrap_correctly(doc)
except Exception as e:
logging.exception('trouble with user %s' % doc['_id'])
return None
users = sourcedb.all_docs(keys=list(user_ids),
include_docs=True,
wrapper=wrap_user).all()
for user in users:
# if we use bulk save, django user doesn't get sync'd
user.save(force_update=True)
def generate_invoices(based_on_date=None):
"""
Generates all invoices for the past month.
"""
today = based_on_date or datetime.date.today()
invoice_start, invoice_end = get_previous_month_date_range(today)
log_accounting_info(
"Starting up invoices for %(start)s - %(end)s" % {
'start': invoice_start.strftime(USER_DATE_FORMAT),
'end': invoice_end.strftime(USER_DATE_FORMAT),
})
all_domain_ids = [d['id'] for d in Domain.get_all(include_docs=False)]
for domain_doc in iter_docs(Domain.get_db(), all_domain_ids):
domain_obj = Domain.wrap(domain_doc)
if not domain_obj.is_active:
continue
try:
invoice_factory = DomainInvoiceFactory(invoice_start, invoice_end,
domain_obj)
invoice_factory.create_invoices()
log_accounting_info("Sent invoices for domain %s" %
domain_obj.name)
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" %
(domain_obj.name, e),
show_stack_trace=True,
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
all_customer_billing_accounts = BillingAccount.objects.filter(
is_customer_billing_account=True)
for account in all_customer_billing_accounts:
try:
if account.invoicing_plan == InvoicingPlan.QUARTERLY:
customer_invoice_start = invoice_start - relativedelta(
months=2)
elif account.invoicing_plan == InvoicingPlan.YEARLY:
customer_invoice_start = invoice_start - relativedelta(
months=11)
else:
customer_invoice_start = invoice_start
invoice_factory = CustomerAccountInvoiceFactory(
account=account,
date_start=customer_invoice_start,
date_end=invoice_end)
invoice_factory.create_invoice()
except CreditLineError as e:
log_accounting_error(
"There was an error utilizing credits for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
except InvoiceError as e:
log_accounting_error(
"Could not create invoice for domain %s: %s" %
(domain_obj.name, e),
show_stack_trace=True,
)
except Exception as e:
log_accounting_error(
"Error occurred while creating invoice for "
"domain %s: %s" % (domain_obj.name, e),
show_stack_trace=True,
)
if not settings.UNIT_TESTING:
_invoicing_complete_soft_assert(False, "Invoicing is complete!")
def update_domain(doc):
Domain.wrap(doc).internal.business_unit = updates[doc['_id']]
return DocUpdate(doc)
def handle(self, **options):
for domain_doc in self._get_domains_without_last_modified_date():
print("Updating domain {}".format(domain_doc['name']))
domain = Domain.wrap(domain_doc)
domain.save()
def appstore(request, template="appstore/appstore_base.html"):
page_length = 10
include_unapproved = True if request.GET.get('is_approved', "") == "false" else False
if include_unapproved and not request.user.is_superuser:
raise Http404()
params, _ = parse_args_for_es(request)
page = params.pop('page', 1)
page = int(page[0] if isinstance(page, list) else page)
results = es_snapshot_query(params, SNAPSHOT_FACETS)
hits = results.get('hits', {}).get('hits', [])
hits = deduplicate(hits)
d_results = []
for res in hits:
try:
domain = Domain.wrap(res['_source'])
if domain.copied_from is not None:
# this avoids putting in snapshots in the list where the
# copied_from domain has been deleted.
d_results.append(domain)
except CopiedFromDeletedException as e:
notify_exception(
request,
message=(
"Fetched Exchange Snapshot Error: {}. "
"The problem snapshot id: {}".format(
e.message, res['_source']['_id'])
)
)
starter_apps = request.GET.get('is_starter_app', None)
sort_by = request.GET.get('sort_by', None)
if sort_by == 'newest':
pass
else:
d_results = Domain.hit_sort(d_results)
persistent_params = {}
if sort_by:
persistent_params["sort_by"] = sort_by
if include_unapproved:
persistent_params["is_approved"] = "false"
persistent_params = urlencode(persistent_params) # json.dumps(persistent_params)
more_pages = False if len(d_results) <= page * page_length else True
facet_map = fill_mapping_with_facets(SNAPSHOT_MAPPING, results, params)
vals = dict(
apps=d_results[(page - 1) * page_length:page * page_length],
page=page,
prev_page=(page - 1),
next_page=(page + 1),
more_pages=more_pages,
sort_by=sort_by,
show_starter_apps=starter_apps,
include_unapproved=include_unapproved,
facet_map=facet_map,
facets=results.get("facets", []),
query_str=request.META['QUERY_STRING'],
search_query=params.get('search', [""])[0],
persistent_params=persistent_params,
)
return render(request, template, vals)
