def get_payload_properties(sector):
if sector not in SECTORS:
raise ValueError('sector argument should be one of {}'.format(
",".join(SECTORS)))
properties = {}
spec = load_api_spec()
for param in spec.api_params:
if param.payload_object:
properties[param.api_param_name] = jsonobject.ObjectProperty(
to_function(param.payload_object),
required=param.required_,
exclude_if_none=param.exclude_if_none,
)
else:
properties[param.api_param_name] = jsonobject.StringProperty(
choices=param.get_by_sector('choices', sector),
required=param.required_,
exclude_if_none=param.exclude_if_none,
validators=([
to_function(param.validator)(sector,
param.validator_values)
] if param.validator else []),
)
return properties
def handle(self, django_app, class_name, chunk_size, **options):
self.class_name = class_name
self.django_app = django_app
self.models_path = f"corehq.apps.{self.django_app}.models.{self.class_name}"
self.couch_class = to_function(self.models_path)
while not self.couch_class:
self.models_path = input(f"Could not find {self.models_path}, please enter path: ")
self.couch_class = to_function(self.models_path)
self.class_name = self.models_path.split(".")[-1]
docs = DocsIterator(self.couch_class, chunk_size)
print("Found {} {} docs\n".format(len(docs), self.class_name))
print("CTRL+C to stop evaluating documents.")
try:
for doc in with_progress_bar(docs, oneline="concise"):
self.evaluate_doc(doc)
except KeyboardInterrupt:
pass
self.standardize_max_lengths()
self.correlate_with_couch_schema(self.couch_class)
models_file = self.models_path[:-(len(self.class_name) + 1)].replace(".", os.path.sep) + ".py"
sql_model, couch_model_additions = self.generate_models_changes()
print(f"################# edit {models_file} #################")
print(sql_model)
print(f"\n################# update {self.class_name} #################")
print(couch_model_additions)
command_file = "populate_" + self.class_name.lower() + ".py"
command_file = os.path.join("corehq", "apps", self.django_app, "management", "commands", command_file)
command_content = self.generate_management_command()
print(f"\n################# add {command_file} #################")
print(command_content)
def handle(self, django_app, class_name, **options):
self.class_name = class_name
self.django_app = django_app
self.models_path = f"corehq.apps.{self.django_app}.models.{self.class_name}"
self.couch_class = to_function(self.models_path)
while not self.couch_class:
self.models_path = input(
f"Could not find {self.models_path}, please enter path: ")
self.couch_class = to_function(self.models_path)
self.class_name = self.models_path.split(".")[-1]
doc_ids = get_doc_ids_by_class(self.couch_class)
print("Found {} {} docs\n".format(len(doc_ids), self.class_name))
for doc in iter_docs(self.couch_class.get_db(), doc_ids):
self.evaluate_doc(doc)
self.standardize_max_lengths()
models_file = self.models_path[:-(len(self.class_name) + 1)].replace(
".", os.path.sep) + ".py"
models_content = self.generate_models_changes()
print(f"################# edit {models_file} #################")
print(models_content)
command_file = "populate_" + self.class_name.lower() + ".py"
command_file = os.path.join("corehq", "apps", self.django_app,
"management", "commands", command_file)
command_content = self.generate_management_command()
print(f"################# add {command_file} #################")
print(command_content)
def handle(self, *args, **options):
domainname, subsection, reportname, reportclass, config_raw = args
domain = Domain.get_by_name(domainname)
if not domain:
self.stderr.write('no domain [%s]\n' % domainname)
return
try:
to_function(reportclass)
except:
self.stderr.write('cannot find class [%s]\n' % reportclass)
return
try:
config = json.loads(
config_raw) if config_raw != '-' else json.load(sys.stdin)
except (TypeError, ValueError):
self.stderr.write('json config not valid\n')
return
try:
section = dict((s.section_title, s)
for s in domain.dynamic_reports)[subsection]
except KeyError:
section = DynamicReportSet(section_title=subsection)
domain.dynamic_reports.append(section)
self.stdout.write('creating subsection [%s]\n' % subsection)
try:
report = dict((r.name, r) for r in section.reports)[reportname]
if not options['update']:
self.stderr.write('report [%s] exists! use -u to update.\n' %
reportname)
return
except KeyError:
report = DynamicReportConfig(name=reportname)
section.reports.append(report)
self.stdout.write('creating report [%s]\n' % reportname)
report.report = reportclass
report.kwargs = config
report.previewers_only = not bool(options['all'])
if options['execute']:
domain.save()
self.stdout.write('done; report visible to %s\n' %
('all' if options['all'] else 'previewers only'))
else:
self.stdout.write('dry run only. no changes saves (use -x)\n')
def dispatch(self, request, *args, **kwargs):
self.request = request
form_type = kwargs.get('form_type')
action = kwargs.get('action', 'new')
item_id = kwargs.get("item_id")
try:
form_class = to_function("%s.%s" % (self.base_loc, form_type))
except Exception:
form_class = None
if not inspect.isclass(form_class):
return HttpResponseBadRequest("'%s' should be a class name in %s" %
(form_type, self.base_loc))
if self.is_form_class_valid(form_class):
try:
form_manager = self.form_request_manager(
request,
form_class,
self.template_name,
doc_id=item_id,
delete=bool(action == 'delete'))
return HttpResponse(form_manager.json_response)
except CRUDActionError as e:
return HttpResponseBadRequest(e)
else:
return HttpResponseBadRequest("Not a valid form class.")
def __init__(self):
self._generator_funcs = []
if hasattr(settings, "FIXTURE_GENERATORS"):
for func_path in settings.FIXTURE_GENERATORS:
func = to_function(func_path)
if func:
self._generator_funcs.append(func)
def get_max_level_function():
if settings.LOGISTICS_MAX_REPORT_LEVEL_FUNCTION:
return to_function(settings.LOGISTICS_MAX_REPORT_LEVEL_FUNCTION)
elif settings.LOGISTICS_MAX_REPORT_LEVEL_FACTOR:
return check_max_levels
else:
return None
def default_resource_class():
from django.conf import settings
if hasattr(settings, 'COUCHDBKIT_RESOURCE_CLASS'):
return to_function(settings.COUCHDBKIT_RESOURCE_CLASS,
failhard=True)
else:
return couchdbkit.resource.CouchdbResource
def _make_dynamic_report(report_config, keyprefix):
"""create a report class the descends from a generic report class but has specific parameters set"""
# a unique key to distinguish this particular configuration of the generic report
report_key = keyprefix + [report_config.report, report_config.name]
slug = hashlib.sha1(':'.join(report_key)).hexdigest()[:12]
kwargs = dict(report_config.kwargs)
kwargs.update({
'name': report_config.name,
'slug': slug,
})
if report_config.previewers_only:
# note this is a classmethod that will be injected into the dynamic class below
@classmethod
def show_in_navigation(cls, domain=None, project=None, user=None):
return user and user.is_previewer()
kwargs['show_in_navigation'] = show_in_navigation
try:
metaclass = to_function(report_config.report, failhard=True)
except Exception:
logging.error('dynamic report config for [%s] is invalid' %
report_config.report)
return None
# dynamically create a report class
return type('DynamicReport%s' % slug, (metaclass, ), kwargs)
def _get_data_legacyreport(self, params, filters):
Report = to_function(params['report'])
assert issubclass(
Report, GenericTabularReport
), '[%s] must be a GenericTabularReport!' % params['report']
# TODO it would be nice to indicate to the report that it was being used in a map context, (so
# that it could add a geo column) but it does not seem like reports can be arbitrarily
# parameterized in this way
report = Report(request=self.request,
domain=self.domain,
**params.get('report_params', {}))
def _headers(e, root=[]):
if hasattr(e, '__iter__'):
if hasattr(e, 'html'):
root = list(root) + [unicode(e.html)]
for sub in e:
for k in _headers(sub, root):
yield k
else:
yield root + [unicode(e.html)]
headers = ['::'.join(k) for k in _headers(report.headers)]
for row in report.rows:
yield dict(zip(headers, row))
def dispatch(self, request, domain, indicator_type=None, *args, **kwargs):
self.domain = domain
try:
self.indicator_class = to_function(
"%s.%s" % (self.indicator_loc, indicator_type))
except AttributeError:
return HttpBadRequest("%s.%s does not exist" %
(self.indicator_loc, indicator_type))
status = {}
if request.method == 'POST':
form = BulkCopyIndicatorsForm(data=request.POST,
domain=self.domain,
couch_user=request.couch_user,
indicator_class=self.indicator_class)
if form.is_valid():
status = form.copy_indicators()
else:
form = BulkCopyIndicatorsForm(domain=self.domain,
couch_user=request.couch_user,
indicator_class=self.indicator_class)
return render(
request, self.template_name, {
"form": form,
"status": status,
"domain": self.domain,
"indicator_type": self.indicator_class.__name__,
"indicator_name": self.indicator_class.get_nice_name(),
})
def dispatch(self, request, domain, indicator_type=None, *args, **kwargs):
self.domain = domain
try:
self.indicator_class = to_function("%s.%s" % (self.indicator_loc, indicator_type))
except AttributeError:
return HttpBadRequest("%s.%s does not exist" % (self.indicator_loc, indicator_type))
status = {}
if request.method == 'POST':
form = BulkCopyIndicatorsForm(data=request.POST, domain=self.domain,
couch_user=request.couch_user, indicator_class=self.indicator_class)
if form.is_valid():
status = form.copy_indicators()
else:
form = BulkCopyIndicatorsForm(domain=self.domain,
couch_user=request.couch_user, indicator_class=self.indicator_class)
return render(request, self.template_name, {
"form": form,
"status": status,
"domain": self.domain,
"indicator_type": self.indicator_class.__name__,
"indicator_name": self.indicator_class.get_nice_name(),
})
def recipient(self):
if self.recipient_type == self.RECIPIENT_TYPE_SELF:
return self.case
elif self.recipient_type == self.RECIPIENT_TYPE_CASE_OWNER:
return self.case_owner
elif self.recipient_type == self.RECIPIENT_TYPE_LAST_SUBMITTING_USER:
if self.case and self.case.modified_by:
return CouchUser.get_by_user_id(self.case.modified_by, domain=self.domain)
return None
elif self.recipient_type == self.RECIPIENT_TYPE_PARENT_CASE:
if self.case:
return self.case.parent
return None
elif self.recipient_type == self.RECIPIENT_TYPE_ALL_CHILD_CASES:
if self.case:
return list(self.case.get_subcases(index_identifier=DEFAULT_PARENT_IDENTIFIER))
return None
elif self.recipient_type == self.RECIPIENT_TYPE_CUSTOM:
custom_function = to_function(
settings.AVAILABLE_CUSTOM_SCHEDULING_RECIPIENTS[self.recipient_id][0]
)
return custom_function(self)
else:
return super(CaseScheduleInstanceMixin, self).recipient
def get_correct_wrap(cls, docid):
data = cls.get_db().get(docid)
try:
correct_class = to_function("hqbilling.models.%s" % data['doc_type'])
except Exception:
correct_class = cls
return correct_class.get(docid)
def is_editable_datespan(field):
if isinstance(field, six.string_types):
soft_assert_type_text(field)
field_fn = to_function(field) if isinstance(
field, six.string_types) else field
return issubclass(field_fn,
DatespanFilter) and field_fn.is_editable
def __init__(self):
self._generator_providers = {}
if hasattr(settings, "FIXTURE_GENERATORS"):
for group, func_paths in settings.FIXTURE_GENERATORS.items():
self._generator_providers[group] = filter(None, [
to_function(func_path, failhard=True) for func_path in func_paths
])
def process_incoming(msg, delay=True):
v = VerifiedNumber.by_phone(msg.phone_number, include_pending=True)
if v is not None and v.verified:
msg.couch_recipient_doc_type = v.owner_doc_type
msg.couch_recipient = v.owner_id
msg.domain = v.domain
msg.save()
if msg.domain_scope:
# only process messages for phones known to be associated with this domain
if v is None or v.domain != msg.domain_scope:
raise DomainScopeValidationError(
'Attempted to simulate incoming sms from phone number not ' \
'verified with this domain'
)
create_billable_for_sms(msg, msg.backend_api, delay=delay)
if v is not None and v.verified:
for h in settings.SMS_HANDLERS:
try:
handler = to_function(h)
except:
logging.exception('error loading sms handler: %s' % h)
continue
try:
was_handled = handler(v, msg.text, msg=msg)
except Exception, e:
logging.exception('unhandled error in sms handler %s for message [%s]: %s' % (h, msg._id, e))
was_handled = False
if was_handled:
break
def get_current(cls,
namespace,
domain,
slug,
version=None,
wrap=True,
**kwargs):
couch_key = cls._generate_couch_key(namespace=namespace,
domain=domain,
slug=slug,
version=version,
reverse=True,
**kwargs)
results = cache_core.cached_view(cls.get_db(),
cls.indicator_list_view(),
cache_expire=60 * 60 * 6,
reduce=False,
include_docs=False,
descending=True,
**couch_key)
doc = results[0] if results else None
if wrap and doc:
try:
doc_class = to_function(
doc.get('value',
"%s.%s" % (cls._class_path, cls.__name__)))
doc_instance = doc_class.get(doc.get('id'))
return doc_instance
except Exception as e:
logging.error(
"No matching documents found for indicator %s: %s" %
(slug, e))
return None
return doc
def process_incoming(msg, delay=True):
v = VerifiedNumber.by_phone(msg.phone_number, include_pending=True)
if v is not None and v.verified:
msg.couch_recipient_doc_type = v.owner_doc_type
msg.couch_recipient = v.owner_id
msg.domain = v.domain
msg.save()
if msg.domain_scope:
# only process messages for phones known to be associated with this domain
if v is None or v.domain != msg.domain_scope:
raise DomainScopeValidationError(
'Attempted to simulate incoming sms from phone number not ' \
'verified with this domain'
)
if v is not None and v.verified:
for h in settings.SMS_HANDLERS:
try:
handler = to_function(h)
except:
notify_exception(None,
message=('error loading sms handler: %s' % h))
continue
try:
was_handled = handler(v, msg.text, msg=msg)
except Exception, e:
log_sms_exception(msg)
was_handled = False
if was_handled:
break
def __init__(self):
self._generator_providers = {}
if hasattr(settings, "FIXTURE_GENERATORS"):
for group, func_paths in settings.FIXTURE_GENERATORS.items():
self._generator_providers[group] = filter(None, [
to_function(func_path) for func_path in func_paths
])
def report_navigation_list(cls, context):
request = context.get('request')
report_nav = list()
dispatcher = cls()
args, kwargs = dispatcher.args_kwargs_from_context(context)
reports = dispatcher.get_reports(request, *args, **kwargs)
current_slug = kwargs.get('report_slug')
for key, models in reports.items():
section = list()
section_header = '<li class="nav-header">%s</li>' % escape(key)
for model in models:
if not dispatcher.permissions_check(model, request, *args, **kwargs):
continue
report = to_function(model)
if report.show_in_navigation(request, *args, **kwargs):
selected_report = bool(report.slug == current_slug)
section.append("""<li class="%(css_class)s"><a href="%(link)s" title="%(link_title)s">
%(icon)s%(title)s
</a></li>""" % dict(
css_class="active" if selected_report else "",
link=report.get_url(*args),
link_title=report.description or "",
icon='<i class="icon%s %s"></i> ' % ("-white" if selected_report else "", report.icon) if report.icon else "",
title=report.name
))
if section:
report_nav.append(section_header)
report_nav.extend(section)
return "\n".join(report_nav)
def get_backend_by_class_name(class_name):
backends = dict([(d.split('.')[-1], d)
for d in settings.SMS_LOADED_BACKENDS])
backend_path = backends.get(class_name)
if backend_path is not None:
return to_function(backend_path)
return None
def to_python(self, value):
if not value:
return SerializableFunction()
try:
return SerializableFunction.loads(value)
except ValueError:
return SerializableFunction(to_function(value))
def get_max_level_function():
if settings.LOGISTICS_MAX_REPORT_LEVEL_FUNCTION:
return to_function(settings.LOGISTICS_MAX_REPORT_LEVEL_FUNCTION)
elif settings.LOGISTICS_MAX_REPORT_LEVEL_FACTOR:
return check_max_levels
else:
return None
def make_dynamic_report(report_config, keyprefix):
"""create a report class the descends from a generic report class but has specific parameters set"""
# a unique key to distinguish this particular configuration of the generic report
report_key = keyprefix + [report_config.report, report_config.name]
slug = hashlib.sha1(':'.join(report_key)).hexdigest()[:12]
kwargs = dict(report_config.kwargs)
kwargs.update({
'name': report_config.name,
'slug': slug,
})
if report_config.previewers_only:
# note this is a classmethod that will be injected into the dynamic class below
@classmethod
def show_in_navigation(cls, domain=None, project=None, user=None):
return user and user.is_previewer()
kwargs['show_in_navigation'] = show_in_navigation
try:
metaclass = to_function(report_config.report, failhard=True)
except StandardError:
logging.error('dynamic report config for [%s] is invalid' % report_config.report)
return None
# dynamically create a report class
return type('DynamicReport%s' % slug, (metaclass,), kwargs)
def __init__(self):
self._generator_funcs = []
if hasattr(settings, "FIXTURE_GENERATORS"):
for func_path in settings.FIXTURE_GENERATORS:
func = to_function(func_path)
if func:
self._generator_funcs.append(func)
def to_python(self, value):
if not value:
return SerializableFunction()
try:
return SerializableFunction.loads(value)
except ValueError:
return SerializableFunction(to_function(value))
def update_cases(self):
sector = get_sector(self._person_case)
case_updates = []
for prop, value in six.iteritems(self.request_json):
try:
param = self.api_spec.get_param(prop, sector)
except KeyError:
raise NinetyNineDotsException(
"{} is not a valid parameter to update".format(prop))
if not param.direction & DIRECTION_INBOUND:
raise NinetyNineDotsException(
"{} is not a valid parameter to update".format(prop))
case_type = param.get_by_sector('case_type', sector)
case_id = self.case_types_to_cases[case_type].case_id
if param.setter:
update = to_function(param.setter)(param, value, sector)
else:
update = {param.get_by_sector('case_property', sector): value}
case_updates.append((case_id, update, False))
return bulk_update_cases(
self.domain,
case_updates,
"{}.{}".format(self.__module__, self.__class__.__name__),
)
def get_current(cls, namespace, domain, slug, version=None, wrap=True, **kwargs):
couch_key = cls._generate_couch_key(
namespace=namespace,
domain=domain,
slug=slug,
version=version,
reverse=True,
**kwargs
)
results = cache_core.cached_view(cls.get_db(), cls.indicator_list_view(),
cache_expire=60*60*6,
reduce=False,
include_docs=False,
descending=True,
**couch_key
)
doc = results[0] if results else None
if wrap and doc:
try:
doc_class = to_function(doc.get('value', "%s.%s" % (cls._class_path, cls.__name__)))
doc_instance = doc_class.get(doc.get('id'))
return doc_instance
except Exception as e:
logging.error("No matching documents found for indicator %s: %s" % (slug, e))
return None
return doc
def handle(self, *args, **options):
domainname, subsection, reportname, reportclass, config_raw = args
domain = Domain.get_by_name(domainname)
if not domain:
self.stderr.write('no domain [%s]\n' % domainname)
return
try:
to_function(reportclass)
except:
self.stderr.write('cannot find class [%s]\n' % reportclass)
return
try:
config = json.loads(config_raw) if config_raw != '-' else json.load(sys.stdin)
except (TypeError, ValueError):
self.stderr.write('json config not valid\n')
return
try:
section = dict((s.section_title, s) for s in domain.dynamic_reports)[subsection]
except KeyError:
section = DynamicReportSet(section_title=subsection)
domain.dynamic_reports.append(section)
self.stdout.write('creating subsection [%s]\n' % subsection)
try:
report = dict((r.name, r) for r in section.reports)[reportname]
if not options['update']:
self.stderr.write('report [%s] exists! use -u to update.\n' % reportname)
return
except KeyError:
report = DynamicReportConfig(name=reportname)
section.reports.append(report)
self.stdout.write('creating report [%s]\n' % reportname)
report.report = reportclass
report.kwargs = config
report.previewers_only = not bool(options['all'])
if options['execute']:
domain.save()
self.stdout.write('done; report visible to %s\n' %
('all' if options['all'] else 'previewers only'))
else:
self.stdout.write('dry run only. no changes saves (use -x)\n')
def get_backend(backend_name):
if not backend_name:
backend = SqlBackend()
else:
backend_class_name = backends()[backend_name]
backend_class = to_function(backend_class_name, failhard=True)
backend = backend_class()
return backend
def get_warehouse_runner():
"""
Get the configured runner, bsed on the WAREHOUSE_RUNNER setting, or
the default demo runner if none found.
"""
classpath = settings.WAREHOUSE_RUNNER if hasattr(settings, "WAREHOUSE_RUNNER") \
else "warehouse.runner.DemoWarehouseRunner"
return to_function(classpath, failhard=True)()
def loads(cls, data):
def object_hook(d):
if d.get('type') == 'SerializedFunction':
return cls.loads(d['dump'])
else:
return d
try:
functions = json.loads(data, object_hook=object_hook)
except Exception:
# then it's just a simple path
return cls(to_function(data))
self = cls()
for o in functions:
f, kwargs = o['function'], o['kwargs']
f = to_function(f)
self.add(f, **kwargs)
return self
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is ptop_fast_reindex_fluff %s' % self.args)
self.domain = args[0]
self.pillow_class = to_function(args[1])
super(Command, self).handle(*args, **options)
def patch(resource_class):
if isinstance(resource_class, basestring):
resource_class = to_function(resource_class,
failhard=True)
# monkey patch the couchdbkit database to use our new custom
# resource whenever it's not explicitly specified
couchdbkit.client.Server = PatchedServer
couchdbkit.resource.CouchdbResource = resource_class
def _convert_transform(serializable_transform):
transform_fn = to_function(serializable_transform.dumps_simple())
if not transform_fn:
return None
for slug, fn in list(TRANSFORM_FUNCTIONS.iteritems()) + list(DEID_TRANSFORM_FUNCTIONS.iteritems()):
if fn == transform_fn:
return slug
return None
def _convert_transform(serializable_transform):
transform_fn = to_function(serializable_transform.dumps_simple())
if not transform_fn:
return None
for slug, fn in list(TRANSFORM_FUNCTIONS.iteritems()) + list(DEID_TRANSFORM_FUNCTIONS.iteritems()):
if fn == transform_fn:
return slug
return None
def _all(cls):
for path in settings.STATIC_DATA_SOURCES:
yield cls.wrap(_read_file(path)), path
for provider_path in settings.STATIC_DATA_SOURCE_PROVIDERS:
provider_fn = to_function(provider_path, failhard=True)
for wrapped, path in provider_fn():
yield wrapped, path
def get_report(self):
Report = to_function(self.data_source["report"])
assert issubclass(Report, GenericTabularReport), (
"[%s] must be a GenericTabularReport!" % self.data_source["report"]
)
report = Report(request=self.request, domain=self.domain, **self.data_source.get("report_params", {}))
return report
def loads(cls, data):
def object_hook(d):
if d.get('type') == 'SerializedFunction':
return cls.loads(d['dump'])
else:
return d
try:
functions = json.loads(data, object_hook=object_hook)
except Exception:
# then it's just a simple path
return cls(to_function(data))
self = cls()
for o in functions:
f, kwargs = o['function'], o['kwargs']
f = to_function(f)
self.add(f, **kwargs)
return self
def load_from_view(self):
"""
Loads entire view, saves to file, set pillowtop checkpoint
"""
xform_handlers = []
for full_str in getattr(settings, 'XFORM_PILLOW_HANDLERS', []):
func = to_function(full_str)
xform_handlers.append(func())
dynamic_domains = [x.domain for x in xform_handlers]
print "loaded full xform domains: %s" % dynamic_domains
def full_view_iter():
for domain in dynamic_domains:
print "XForm View iter for domain: %s" % domain
start_seq = 0
startkey = [domain]
endkey = [domain, {}]
view_chunk = self.db.view(self.view_name,
startkey=startkey,
endkey=endkey,
reduce=False,
limit=self.chunk_size *
self.chunk_size,
skip=start_seq)
while len(view_chunk) > 0:
for item in view_chunk:
yield item
start_seq += self.chunk_size * self.chunk_size
view_chunk = self.db.view(self.view_name,
startkey=startkey,
endkey=endkey,
reduce=False,
limit=CHUNK_SIZE *
self.chunk_size,
skip=start_seq)
# Set pillowtop checkpoint for doc_class
# though this might cause some superfluous reindexes of docs,
# we're going to set the checkpoint BEFORE we start our operation so that any changes
# that happen to cases while we're doing our reindexing would not get skipped once we
# finish.
current_db_seq = self.pillow.couch_db.info()['update_seq']
self.pillow.set_checkpoint({'seq': current_db_seq})
#Write sequence file to disk
with open(self.get_seq_filename(), 'w') as fout:
fout.write(str(current_db_seq))
#load entire view to disk
print "Getting full view list: %s" % datetime.utcnow().isoformat()
with open(self.get_dump_filename(), 'w') as fout:
fout.write("[")
fout.write(','.join(
simplejson.dumps(row) for row in full_view_iter()))
fout.write("]")
print "View and sequence written to disk: %s" % datetime.utcnow(
).isoformat()
def incoming(phone_number, text, backend_api, timestamp=None, domain_scope=None, delay=True):
"""
entry point for incoming sms
phone_number - originating phone number
text - message content
backend_api - backend ID of receiving sms backend
timestamp - message received timestamp; defaults to now (UTC)
domain_scope - if present, only messages from phone numbers that can be
definitively linked to this domain will be processed; others will be
dropped (useful to provide security when simulating incoming sms)
"""
phone_number = clean_phone_number(phone_number)
v = VerifiedNumber.by_phone(phone_number, include_pending=True)
if domain_scope:
# only process messages for phones known to be associated with this domain
if v is None or v.domain != domain_scope:
raise RuntimeError("attempted to simulate incoming sms from phone number not verified with this domain")
# Log message in message log
msg = SMSLog(
phone_number=phone_number,
direction=INCOMING,
date=timestamp or datetime.utcnow(),
text=text,
backend_api=backend_api,
)
if v is not None and v.verified:
msg.couch_recipient_doc_type = v.owner_doc_type
msg.couch_recipient = v.owner_id
msg.domain = v.domain
msg.save()
create_billable_for_sms(msg, backend_api, delay=delay)
if v is not None and v.verified:
for h in settings.SMS_HANDLERS:
try:
handler = to_function(h)
except:
logging.exception("error loading sms handler: %s" % h)
continue
try:
was_handled = handler(v, text)
except:
logging.exception("unhandled error in sms handler %s for message [%s]" % (h, text))
was_handled = False
if was_handled:
break
else:
if not process_sms_registration(msg):
import verify
verify.process_verification(phone_number, text)
return msg
def handle(self, *args, **options):
if self.doc_class is None and self.doc_class_str is None:
#in the case where we want to make this a generic anyclass user
if len(args) != 1:
print "\tEnter a doc class!\n"
sys.exit(1)
self.doc_class_str = args[0].split('.')[-1]
self.doc_class = to_function(args[0])
self.finish_handle()
def _all(cls):
for path in settings.STATIC_DATA_SOURCES:
with open(path) as f:
yield cls.wrap(json.load(f)), path
for provider_path in settings.STATIC_DATA_SOURCE_PROVIDERS:
provider_fn = to_function(provider_path, failhard=True)
for wrapped, path in provider_fn():
yield wrapped, path
def disable_extensions(ext_point):
if isinstance(ext_point, str):
ext_point = to_function(ext_point)
extensions = ext_point.extensions
ext_point.extensions = []
try:
yield
finally:
ext_point.extensions = extensions
def process_incoming(msg):
v = PhoneNumber.by_phone(msg.phone_number, include_pending=True)
if v is not None and v.verified:
msg.couch_recipient_doc_type = v.owner_doc_type
msg.couch_recipient = v.owner_id
msg.domain = v.domain
msg.location_id = get_location_id_by_verified_number(v)
msg.save()
if msg.domain_scope:
# only process messages for phones known to be associated with this domain
if v is None or v.domain != msg.domain_scope:
raise DomainScopeValidationError(
'Attempted to simulate incoming sms from phone number not ' \
'verified with this domain'
)
can_receive_sms = PhoneBlacklist.can_receive_sms(msg.phone_number)
opt_in_keywords, opt_out_keywords = get_opt_keywords(msg)
domain = v.domain if v else None
if is_opt_message(msg.text, opt_out_keywords) and can_receive_sms:
if PhoneBlacklist.opt_out_sms(msg.phone_number, domain=domain):
metadata = MessageMetadata(ignore_opt_out=True)
text = get_message(MSG_OPTED_OUT, v, context=(opt_in_keywords[0],))
if v:
send_sms_to_verified_number(v, text, metadata=metadata)
else:
send_sms(msg.domain, None, msg.phone_number, text, metadata=metadata)
elif is_opt_message(msg.text, opt_in_keywords) and not can_receive_sms:
if PhoneBlacklist.opt_in_sms(msg.phone_number, domain=domain):
text = get_message(MSG_OPTED_IN, v, context=(opt_out_keywords[0],))
if v:
send_sms_to_verified_number(v, text)
else:
send_sms(msg.domain, None, msg.phone_number, text)
elif v is not None and v.verified:
if (
domain_has_privilege(msg.domain, privileges.INBOUND_SMS) and
is_contact_active(v.domain, v.owner_doc_type, v.owner_id)
):
for h in settings.SMS_HANDLERS:
try:
handler = to_function(h)
except:
notify_exception(None, message=('error loading sms handler: %s' % h))
continue
try:
was_handled = handler(v, msg.text, msg=msg)
except Exception, e:
log_sms_exception(msg)
was_handled = False
if was_handled:
break
def _all(cls):
for path in settings.STATIC_DATA_SOURCES:
with open(path) as f:
yield cls.wrap(json.load(f)), path
for provider_path in settings.STATIC_DATA_SOURCE_PROVIDERS:
provider_fn = to_function(provider_path, failhard=True)
for wrapped, path in provider_fn():
yield wrapped, path
def hq_fixtures(user, version, last_sync):
if hasattr(user, "_hq_user") and user._hq_user is not None:
user = user._hq_user
if isinstance(user, CommCareUser):
for func_path in settings.HQ_FIXTURE_GENERATORS:
func = to_function(func_path)
if func:
for fixture in func(user, version, last_sync):
yield fixture
def get_correct_wrap(cls, docid):
try:
adm_doc = get_db().get(docid)
adm_class = adm_doc.get('doc_type')
adm = to_function('corehq.apps.adm.models.%s' % adm_class)
return adm.wrap(adm_doc)
except Exception:
pass
return None
def get_reports(cls, domain=None):
reports = dict((rep, cls.get_report(rep)) for rep in SCHEDULABLE_REPORTS)
if domain is not None:
for heading, models in settings.CUSTOM_REPORT_MAP.get(domain, {}).items():
for model in models:
klass = to_function(model)
slug = "custom-" + klass.slug
reports[slug] = CustomReportSchedule(klass)
return reports
def get_warehouse_runner():
"""
Get the configured runner, bsed on the WAREHOUSE_RUNNER setting, or
the default demo runner if none found.
"""
classpath = (
settings.WAREHOUSE_RUNNER if hasattr(settings, "WAREHOUSE_RUNNER") else "warehouse.runner.DemoWarehouseRunner"
)
return to_function(classpath, failhard=True)()
def handle(self, *args, **options):
if self.doc_class is None and self.doc_class_str is None:
#in the case where we want to make this a generic anyclass user
if len(args) != 1:
print "\tEnter a doc class!\n"
sys.exit(1)
self.doc_class_str = args[0].split('.')[-1]
self.doc_class = to_function(args[0])
self.finish_handle()
def get_correct_wrap(cls, docid):
try:
adm_doc = get_db().get(docid)
adm_class = adm_doc.get('doc_type')
adm = to_function('corehq.apps.adm.models.%s' % adm_class)
return adm.wrap(adm_doc)
except Exception:
pass
return None
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is ptop_fast_reindex_fluff %s' %
self.args)
self.domain = args[0]
self.pillow_class = to_function(args[1])
super(Command, self).handle(*args, **options)
def get_possible_reports(domain):
reports = []
report_map = []
report_map.extend(settings.PROJECT_REPORT_MAP.items())
report_map.extend(settings.CUSTOM_REPORT_MAP.get(domain, {}).items())
for heading, models in report_map:
for model in models:
reports.append({'path': model, 'name': to_function(model).name})
return reports
def hq_fixtures(user, version, last_sync):
if hasattr(user, "_hq_user") and user._hq_user is not None:
user = user._hq_user
if isinstance(user, CommCareUser):
for func_path in settings.HQ_FIXTURE_GENERATORS:
func = to_function(func_path)
if func:
for fixture in func(user, version, last_sync):
yield fixture
def get_available_backends(index_by_api_id=False):
result = {}
for backend_class in settings.SMS_LOADED_BACKENDS:
klass = to_function(backend_class)
if index_by_api_id:
api_id = klass.get_api_id()
result[api_id] = klass
else:
result[klass.__name__] = klass
return result
def filter_classes(self):
filters = []
fields = self.fields
for field in fields or []:
if isinstance(field, basestring):
klass = to_function(field, failhard=True)
else:
klass = field
filters.append(klass(self.request, self.domain, self.timezone))
return filters
def __init__(self, **kwargs):
super(FullXFormPillow, self).__init__(**kwargs)
#Pillow Handlers are custom processing classes that can add new mapping definitions
# beyond the default/core mapping types found in self.default_xform_mapping
#it also provides for more custom transform prior to transmission
for full_str in getattr(settings, 'XFORM_PILLOW_HANDLERS', []):
func = to_function(full_str)
self.xform_handlers.append(func())
self.handler_domain_map = dict((x.domain, x) for x in self.xform_handlers)
