def obj_update(self, bundle, **kwargs):
bundle.obj = Repeater.get(kwargs['pk'])
assert bundle.obj.domain == kwargs['domain']
bundle = self._update(bundle)
assert bundle.obj.domain == kwargs['domain']
bundle.obj.save()
return bundle
def obj_update(self, bundle, **kwargs):
bundle.obj = Repeater.get(kwargs['pk'])
assert bundle.obj.domain == kwargs['domain']
bundle = self._update(bundle)
assert bundle.obj.domain == kwargs['domain']
bundle.obj.save()
return bundle
def send_repeater_payloads(repeater_id, payload_ids, email_id):
from corehq.motech.repeaters.models import Repeater, RepeatRecord
repeater = Repeater.get(repeater_id)
repeater_type = repeater.doc_type
payloads = dict()
headers = ['note']
result_file_name = "bulk-payloads-%s-%s-%s.csv" % (
repeater.doc_type, repeater.get_id,
datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S"))
def get_payload(payload_id):
dummy_repeat_record = RepeatRecord(
domain=repeater.domain,
next_check=datetime.datetime.utcnow(),
repeater_id=repeater.get_id,
repeater_type=repeater_type,
payload_id=payload_id,
)
payload = repeater.get_payload(dummy_repeat_record)
if isinstance(payload, dict):
return payload
else:
return json.loads(payload)
def populate_payloads(headers):
for payload_id in payload_ids:
try:
payload = get_payload(payload_id)
payloads[payload_id] = payload
headers = list(set(headers + payload.keys()))
except Exception as e:
payloads[payload_id] = {
'note': 'Could not generate payload, %s' % str(e)
}
return headers
def create_result_file():
_, temp_file_path = tempfile.mkstemp()
with open(temp_file_path, 'w') as csvfile:
headers.append('payload_id')
writer = csv.DictWriter(csvfile, fieldnames=headers)
writer.writeheader()
for payload_id, payload in payloads.items():
row = payload
row['payload_id'] = payload_id
writer.writerow(row)
return temp_file_path
def email_result(download_url):
send_HTML_email(
'Bulk Payload generated for %s' % repeater_type, email_id,
'This email is to just let you know that there is a '
'download waiting for you at %s. It will expire in 24 hours' %
download_url)
headers = populate_payloads(headers)
temp_file_path = create_result_file()
download_url = ExposeBlobDownload().get_link(temp_file_path,
result_file_name, Format.CSV)
email_result(download_url)
def create_repeat_records_on_dest_repeater(self, source_repeater_id,
dest_repeater_id, state):
dest_repeater = Repeater.get(dest_repeater_id)
retriggered = set()
records = iter_repeat_records_by_domain(domain,
repeater_id=source_repeater_id,
state=state)
record_count = get_repeat_record_count(domain,
repeater_id=source_repeater_id,
state=state)
accessor = CaseAccessors(domain)
print("Iterating over records and adding new record for them")
for record in with_progress_bar(records, length=record_count):
if record.payload_id in retriggered:
self.record_failure(record.get_id,
record.payload_id,
error_message="Already triggered")
continue
try:
episode = accessor.get_case(record.payload_id)
episode_case_properties = episode.dynamic_case_properties()
if (episode_case_properties.get('nikshay_registered', 'false')
== 'false' and episode_case_properties.get(
'private_nikshay_registered', 'false') == 'false'
and not episode_case_properties.get('nikshay_id')
and episode_case_properties.get('episode_type')
== 'confirmed_tb'
and is_valid_episode_submission(episode)):
new_record = RepeatRecord(
domain=domain,
next_check=datetime.utcnow(),
repeater_id=dest_repeater_id,
repeater_type=dest_repeater.doc_type,
payload_id=record.payload_id,
)
if not self.dry_run:
new_record.save()
retriggered.add(record.payload_id)
self.add_row(
record,
episode_case_properties.get('migration_created_case'),
new_record.get_id)
else:
self.record_failure(record.get_id,
record.payload_id,
error_message="Not to be re-triggered")
except Exception as e:
self.record_failure(record.get_id,
record.payload_id,
error_message="{error}: {message}".format(
error=e.__name__, message=e.message))
def handle(self, domain, repeater_id, *args, **options):
self.domain = domain
self.repeater_id = repeater_id
repeater = Repeater.get(repeater_id)
print("Looking up repeat records for '{}'".format(
repeater.friendly_name))
redundant_records = []
records_by_payload_id = defaultdict(list)
records = iter_repeat_records_by_domain(domain,
repeater_id=repeater_id,
state=RECORD_CANCELLED_STATE)
total_records = 0
for record in records:
total_records += 1
most_recent_success = self.most_recent_success.get(
record.payload_id)
if most_recent_success and record.last_checked < most_recent_success:
# another record with this payload has succeeded after this record failed
redundant_records.append(record)
else:
records_by_payload_id[record.payload_id].append(record)
unique_payloads = len(records_by_payload_id)
redundant_payloads = len(redundant_records)
print(
"There are {total} total cancelled records, {redundant} with payloads which "
"have since succeeded, and {unique} unsent unique payload ids.".
format(total=total_records,
redundant=redundant_payloads,
unique=unique_payloads))
print("Delete {} duplicate records?".format(total_records -
unique_payloads))
if not input("(y/n)") == 'y':
print("Aborting")
return
redundant_log = self.delete_already_successful_records(
redundant_records)
duplicates_log = self.resolve_duplicates(records_by_payload_id)
filename = "cancelled_{}_records-{}.csv".format(
repeater.__class__.__name__,
datetime.datetime.utcnow().isoformat())
print("Writing log of changes to {}".format(filename))
with open(filename, 'w') as f:
writer = csv.writer(f)
writer.writerow(('RepeatRecord ID', 'Payload ID', 'Failure Reason',
'Deleted?', 'Reason'))
writer.writerows(redundant_log)
writer.writerows(duplicates_log)
def add_repeater_form(self):
if self.request.method == 'POST':
return self.repeater_form_class(self.request.POST,
domain=self.domain,
repeater_class=self.repeater_class)
else:
repeater_id = self.kwargs['repeater_id']
repeater = Repeater.get(repeater_id)
return self.repeater_form_class(
domain=self.domain,
repeater_class=self.repeater_class,
data=repeater.to_json(),
submit_btn_text=_("Update Repeater"),
)
def add_repeater_form(self):
if self.request.method == 'POST':
return self.repeater_form_class(self.request.POST,
domain=self.domain,
repeater_class=self.repeater_class)
else:
repeater_id = self.kwargs['repeater_id']
repeater = Repeater.get(repeater_id)
data = repeater.to_json()
data['password'] = PASSWORD_PLACEHOLDER
return self.repeater_form_class(
domain=self.domain,
repeater_class=self.repeater_class,
data=data,
submit_btn_text=_("Update Repeater"),
)
def add_repeater_form(self):
if self.request.method == 'POST':
return self.repeater_form_class(
self.request.POST,
domain=self.domain,
repeater_class=self.repeater_class
)
else:
repeater_id = self.kwargs['repeater_id']
repeater = Repeater.get(repeater_id)
data = repeater.to_json()
data['password'] = PASSWORD_PLACEHOLDER
return self.repeater_form_class(
domain=self.domain,
repeater_class=self.repeater_class,
data=data,
submit_btn_text=_("Update Repeater"),
)
def handle(self, domain, repeater_id, *args, **options):
self.domain = domain
self.repeater_id = repeater_id
repeater = Repeater.get(repeater_id)
print("Looking up repeat records for '{}'".format(repeater.friendly_name))
redundant_records = []
records_by_payload_id = defaultdict(list)
records = iter_repeat_records_by_domain(domain, repeater_id=repeater_id, state=RECORD_CANCELLED_STATE)
total_records = 0
for record in records:
total_records += 1
most_recent_success = self.most_recent_success.get(record.payload_id)
if most_recent_success and record.last_checked < most_recent_success:
# another record with this payload has succeeded after this record failed
redundant_records.append(record)
else:
records_by_payload_id[record.payload_id].append(record)
unique_payloads = len(records_by_payload_id)
redundant_payloads = len(redundant_records)
print ("There are {total} total cancelled records, {redundant} with payloads which "
"have since succeeded, and {unique} unsent unique payload ids."
.format(total=total_records,
redundant=redundant_payloads,
unique=unique_payloads))
print("Delete {} duplicate records?".format(total_records - unique_payloads))
if not input("(y/n)") == 'y':
print("Aborting")
return
redundant_log = self.delete_already_successful_records(redundant_records)
duplicates_log = self.resolve_duplicates(records_by_payload_id)
filename = "cancelled_{}_records-{}.csv".format(
repeater.__class__.__name__,
datetime.datetime.utcnow().isoformat())
print("Writing log of changes to {}".format(filename))
with open(filename, 'w', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(('RepeatRecord ID', 'Payload ID', 'Failure Reason', 'Deleted?', 'Reason'))
writer.writerows(redundant_log)
writer.writerows(duplicates_log)
def resume_repeater(request, domain, repeater_id):
rep = Repeater.get(repeater_id)
rep.resume()
messages.success(request, "Forwarding resumed!")
return HttpResponseRedirect(
reverse(DomainForwardingOptionsView.urlname, args=[domain]))
def initialize_repeater(self):
return Repeater.get(self.kwargs['repeater_id'])
def initialize_repeater(self):
return Repeater.get(self.kwargs['repeater_id'])
def resume_repeater(request, domain, repeater_id):
rep = Repeater.get(repeater_id)
rep.resume()
messages.success(request, "Forwarding resumed!")
return HttpResponseRedirect(reverse(DomainForwardingOptionsView.urlname, args=[domain]))
def handle(self, domain, repeater_id, *args, **options):
sleep_time = options.get('sleep_time')
include_regexps = options.get('include_regexps')
exclude_regexps = options.get('exclude_regexps')
verbose = options.get('verbose')
action = options.get('action')
success_message = options.get('success_message')
response_status = options.get('response_status')
repeater = Repeater.get(repeater_id)
print("Looking up repeat records for '{}'".format(
repeater.friendly_name))
def meets_filter(record):
if exclude_regexps: # Match none of the exclude expressions
if record.failure_reason:
if any(
re.search(exclude_regex, record.failure_reason)
for exclude_regex in exclude_regexps):
return False
if include_regexps: # Match any of the include expressions
if not record.failure_reason:
return False
return any(
re.search(include_regex, record.failure_reason)
for include_regex in include_regexps)
return True # No filter applied
records = list(
filter(
meets_filter,
iter_repeat_records_by_domain(domain,
repeater_id=repeater_id,
state=RECORD_CANCELLED_STATE)))
if verbose:
for record in records:
print(record.payload_id, record.failure_reason)
total_records = len(records)
print("Found {} matching records. {} them?".format(
total_records, action))
if not input("(y/n)") == 'y':
print("Aborting")
return
filename = "{}_{}_records-{}.csv".format(
action, repeater.__class__.__name__,
datetime.datetime.utcnow().strftime('%Y-%m-%d_%H.%M.%S'))
with open(filename, 'w', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(('record_id', 'payload_id', 'state', 'message'))
for i, record in enumerate(records):
try:
if action == 'retrigger':
if record.next_check is None:
record.next_check = datetime.datetime.utcnow()
record.fire(force_send=True)
elif action == 'succeed':
self._succeed_record(record, success_message,
response_status)
except Exception as e:
print("{}/{}: {} {}".format(i + 1, total_records,
'EXCEPTION', repr(e)))
writer.writerow(
(record._id, record.payload_id, record.state, repr(e)))
else:
print("{}/{}: {}, {}".format(i + 1, total_records,
record.state,
record.attempts[-1].message))
writer.writerow(
(record._id, record.payload_id, record.state,
record.attempts[-1].message))
if sleep_time:
time.sleep(float(sleep_time))
print("Wrote log of changes to {}".format(filename))
def handle(self, file_path, *args, **options):
with open(file_path, 'r') as file:
doc_ids = [line.strip() for line in file.readlines()]
repeater_type = options['repeater_type']
repeater_id = options['repeater_id']
repeater_name_re = None
if options['repeater_name']:
repeater_name_re = re.compile(options['repeater_name'])
if repeater_id:
repeater = Repeater.get(repeater_id)
if repeater_type and repeater_type != repeater.doc_type:
raise CommandError(f"Repeater type does not match: {repeater_type} != {repeater.doc_type}")
if repeater_name_re and not repeater_name_re.match(repeater.name):
raise CommandError(f"Repeater name does not match: {repeater.name}")
def _get_repeaters(doc):
assert doc.domain == repeater.domain
return [repeater]
else:
by_domain = {}
def _get_repeaters(doc):
if doc.domain not in by_domain:
repeater_class = get_all_repeater_types()[repeater_type] if repeater_type else None
repeaters = get_repeaters_for_type_in_domain(doc.domain, repeater_class)
if repeater_name_re:
repeaters = [r for r in repeaters if repeater_name_re.match(r.name)]
if not repeaters:
logger.info(f"No repeaters matched for domain '{doc.domain}'")
by_domain[doc.domain] = repeaters
return by_domain[doc.domain]
def doc_iterator(doc_ids):
try:
yield from bulk_accessor(doc_ids)
except Exception:
logger.exception("Unable to fetch bulk docs, falling back to individual fetches")
for doc_id in doc_ids:
try:
yield single_accessor(doc_id)
except Exception:
logger.exception(f"Unable to fetch doc '{doc_id}'")
forms = XFormInstance.objects
cases = CommCareCase.objects
bulk_accessor = forms.get_forms if options['doc_type'] == 'form' else cases.get_cases
single_accessor = forms.get_form if options['doc_type'] == 'form' else cases.get_case
for doc_ids in chunked(with_progress_bar(doc_ids), 100):
for doc in doc_iterator(list(doc_ids)):
try:
repeaters = _get_repeaters(doc)
except Exception:
logger.exception(f"Unable to fetch repeaters for doc '{doc.get_id}'")
continue
for repeater in repeaters:
try:
repeater.register(doc)
except Exception:
logger.exception(f"Unable to create records for doc '{doc.get_id}'")
def handle(self, domain, repeater_id, *args, **options):
sleep_time = options.get('sleep_time')
include_regexps = options.get('include_regexps')
exclude_regexps = options.get('exclude_regexps')
verbose = options.get('verbose')
action = options.get('action')
success_message = options.get('success_message')
response_status = options.get('response_status')
repeater = Repeater.get(repeater_id)
print("Looking up repeat records for '{}'".format(repeater.friendly_name))
def meets_filter(record):
if exclude_regexps: # Match none of the exclude expressions
if record.failure_reason:
if any(re.search(exclude_regex, record.failure_reason)
for exclude_regex in exclude_regexps):
return False
if include_regexps: # Match any of the include expressions
if not record.failure_reason:
return False
return any(re.search(include_regex, record.failure_reason)
for include_regex in include_regexps)
return True # No filter applied
records = list(filter(
meets_filter,
iter_repeat_records_by_domain(domain, repeater_id=repeater_id, state=RECORD_CANCELLED_STATE)
))
if verbose:
for record in records:
print(record.payload_id, record.failure_reason)
total_records = len(records)
print("Found {} matching records. {} them?".format(total_records, action))
if not input("(y/n)") == 'y':
print("Aborting")
return
filename = "{}_{}_records-{}.csv".format(
action,
repeater.__class__.__name__,
datetime.datetime.utcnow().strftime('%Y-%m-%d_%H.%M.%S'))
with open(filename, 'w', encoding='utf-8') as f:
writer = csv.writer(f)
writer.writerow(('record_id', 'payload_id', 'state', 'message'))
for i, record in enumerate(records):
try:
if action == 'retrigger':
if record.next_check is None:
record.next_check = datetime.datetime.utcnow()
record.fire(force_send=True)
elif action == 'succeed':
self._succeed_record(record, success_message, response_status)
except Exception as e:
print("{}/{}: {} {}".format(i + 1, total_records, 'EXCEPTION', repr(e)))
writer.writerow((record._id, record.payload_id, record.state, repr(e)))
else:
print("{}/{}: {}, {}".format(i + 1, total_records, record.state, record.attempts[-1].message))
writer.writerow((record._id, record.payload_id, record.state, record.attempts[-1].message))
if sleep_time:
time.sleep(float(sleep_time))
print("Wrote log of changes to {}".format(filename))
def send_repeater_payloads(repeater_id, payload_ids, email_id):
from corehq.motech.repeaters.models import Repeater, RepeatRecord
repeater = Repeater.get(repeater_id)
repeater_type = repeater.doc_type
payloads = dict()
headers = ['note']
result_file_name = "bulk-payloads-%s-%s-%s.csv" % (
repeater.doc_type, repeater.get_id,
datetime.datetime.utcnow().strftime("%Y-%m-%d--%H-%M-%S")
)
def get_payload(payload_id):
dummy_repeat_record = RepeatRecord(
domain=repeater.domain,
next_check=datetime.datetime.utcnow(),
repeater_id=repeater.get_id,
repeater_type=repeater_type,
payload_id=payload_id,
)
payload = repeater.get_payload(dummy_repeat_record)
if isinstance(payload, dict):
return payload
else:
return json.loads(payload)
def populate_payloads(headers):
for payload_id in payload_ids:
try:
payload = get_payload(payload_id)
payloads[payload_id] = payload
headers = list(set(headers + list(payload)))
except Exception as e:
payloads[payload_id] = {'note': 'Could not generate payload, %s' % str(e)}
return headers
def create_result_file():
_, temp_file_path = tempfile.mkstemp()
with open(temp_file_path, 'w') as csvfile:
headers.append('payload_id')
writer = csv.DictWriter(csvfile, fieldnames=headers)
writer.writeheader()
for payload_id, payload in payloads.items():
row = payload
row['payload_id'] = payload_id
writer.writerow(row)
return temp_file_path
def email_result(download_url):
send_HTML_email('Bulk Payload generated for %s' % repeater_type,
email_id,
'This email is to just let you know that there is a '
'download waiting for you at %s. It will expire in 24 hours' % download_url)
headers = populate_payloads(headers)
temp_file_path = create_result_file()
download_url = expose_zipped_blob_download(
temp_file_path,
result_file_name,
Format.CSV,
repeater.domain,
)
email_result(download_url)
