def simple_post_with_cached_timeout(data,
url,
expiry=60 * 60,
force_send=False,
*args,
**kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5('{0} timeout {1}'.format(__name__, url)).hexdigest()
cache_value = cache.get(key)
if cache_value and not force_send:
raise RequestConnectionError(cache_value)
try:
resp = simple_post(data, url, *args, **kwargs)
except (Timeout, ConnectionError) as e:
cache.set(key, e.message, expiry)
raise RequestConnectionError(e.message)
if not 200 <= resp.status_code < 300:
message = u'Status Code {}: {}. {}'.format(
resp.status_code, resp.reason, getattr(resp, 'content', None))
cache.set(key, message, expiry)
return resp
def process_xform(self, form_id, new_domain, new_owner, level):
if form_id not in self.forms_processing:
self.forms_processing[form_id] = self.new_id()
else:
return
print "{}=== xform: {} -> {}".format(' ' * (level+1), form_id, self.forms_processing[form_id])
instance = XFormInstance.get(form_id)
xml = instance.get_xml()
referenced_case_ids = set(re.findall(r'case_id="([\w-]*)"', xml))
for ref_case_id in referenced_case_ids:
if ref_case_id not in self.cases_processing:
self.duplicate_case(ref_case_id, new_domain, new_owner, level=level+1)
xml = xml.replace(ref_case_id, self.cases_processing[ref_case_id])
referenced_user_ids = set(re.findall(r'user_id="([\w-]*)"', xml))
for ref_form_id in referenced_user_ids:
xml = xml.replace(ref_form_id, new_owner)
instance_ids = set(re.findall(r'instanceID>([\w-]*)</', xml))
for inst_id in instance_ids:
xml = xml.replace(inst_id, self.forms_processing[form_id])
resp = simple_post(xml, self.submit_url, content_type='text/xml')
if not resp.status in [200, 201]:
raise Exception(resp.read())
# with open("{}/{}.xml".format(OUT, self.forms_processing[form_id]), "w") as form:
# form.write(xml)
self.processed_docs.append(('form', form_id, self.forms_processing[form_id]))
def test_repeater(request, domain):
url = request.POST["url"]
form = FormRepeaterForm({"url": url})
if form.is_valid():
url = form.cleaned_data["url"]
# now we fake a post
fake_post = "<?xml version='1.0' ?><data id='test'><TestString>Test post from CommCareHQ on %s</TestString></data>" \
% (datetime.datetime.utcnow())
try:
resp = simple_post(fake_post, url)
if 200 <= resp.status < 300:
return HttpResponse(
json.dumps({
"success": True,
"response": resp.read(),
"status": resp.status
}))
else:
return HttpResponse(
json.dumps({
"success": False,
"response": resp.read(),
"status": resp.status
}))
except Exception, e:
errors = str(e)
return HttpResponse(json.dumps({"success": False, "response": errors}))
def test_repeater(request, domain):
url = request.POST["url"]
repeater_type = request.POST['repeater_type']
format = request.POST.get('format', None)
repeater_class = get_all_repeater_types()[repeater_type]
auth_type = request.POST.get('auth_type')
form = GenericRepeaterForm({
"url": url,
"format": format
},
domain=domain,
repeater_class=repeater_class)
if form.is_valid():
url = form.cleaned_data["url"]
format = format or RegisterGenerator.default_format_by_repeater(
repeater_class)
generator_class = RegisterGenerator.generator_class_by_repeater_format(
repeater_class, format)
generator = generator_class(repeater_class())
fake_post = generator.get_test_payload(domain)
headers = generator.get_headers()
username = request.POST.get('username')
password = request.POST.get('password')
if auth_type == BASIC_AUTH:
auth = HTTPBasicAuth(username, password)
elif auth_type == DIGEST_AUTH:
auth = HTTPDigestAuth(username, password)
else:
auth = None
try:
resp = simple_post(fake_post, url, headers=headers, auth=auth)
if 200 <= resp.status_code < 300:
return HttpResponse(
json.dumps({
"success": True,
"response": resp.content,
"status": resp.status_code
}))
else:
return HttpResponse(
json.dumps({
"success": False,
"response": resp.content,
"status": resp.status_code
}))
except Exception as e:
errors = str(e)
return HttpResponse(json.dumps({"success": False, "response": errors}))
else:
return HttpResponse(
json.dumps({
"success": False,
"response": "Please enter a valid url."
}))
def send_request(self, repeat_record, payload):
headers = self.get_headers(repeat_record)
auth = self.get_auth()
url = self.get_url(repeat_record)
return simple_post(payload,
url,
headers=headers,
timeout=POST_TIMEOUT,
auth=auth)
def simple_post_with_cached_timeout(data, url, expiry=60 * 60, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5("{0} timeout {1}".format(__name__, url)).hexdigest()
if cache.get(key) == "timeout":
raise socket.timeout("recently timed out, not retrying")
try:
return simple_post(data, url, *args, **kwargs)
except socket.timeout:
cache.set(key, "timeout", expiry)
raise
def simple_post_with_cached_timeout(data, url, expiry=60*60, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5(
'{0} timeout {1}'.format(__name__, url)
).hexdigest()
if cache.get(key) == 'timeout':
raise socket.timeout('recently timed out, not retrying')
try:
return simple_post(data, url, *args, **kwargs)
except socket.timeout:
cache.set(key, 'timeout', expiry)
raise
def simple_post_with_cached_timeout(data, url, expiry=60 * 60, force_send=False, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5(
'{0} timeout {1}'.format(__name__, url)
).hexdigest()
cache_value = cache.get(key)
if cache_value and not force_send:
raise RequestConnectionError(cache_value)
try:
resp = simple_post(data, url, *args, **kwargs)
except (Timeout, ConnectionError), e:
cache.set(key, e.message, expiry)
raise RequestConnectionError(e.message)
def simple_post_with_cached_timeout(data, url, expiry=60 * 60, force_send=False, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5(
'{0} timeout {1}'.format(__name__, url)
).hexdigest()
cache_value = cache.get(key)
if cache_value and not force_send:
raise RequestConnectionError(cache_value)
try:
resp = simple_post(data, url, *args, **kwargs)
except (Timeout, ConnectionError), e:
cache.set(key, e.message, expiry)
raise RequestConnectionError(e.message)
def test_repeater(request, domain):
url = request.POST["url"]
repeater_type = request.POST['repeater_type']
format = request.POST.get('format', None)
repeater_class = get_all_repeater_types()[repeater_type]
auth_type = request.POST.get('auth_type')
form = GenericRepeaterForm(
{"url": url, "format": format},
domain=domain,
repeater_class=repeater_class
)
if form.is_valid():
url = form.cleaned_data["url"]
format = format or RegisterGenerator.default_format_by_repeater(repeater_class)
generator_class = RegisterGenerator.generator_class_by_repeater_format(repeater_class, format)
generator = generator_class(repeater_class())
fake_post = generator.get_test_payload(domain)
headers = generator.get_headers()
username = request.POST.get('username')
password = request.POST.get('password')
verify = not request.POST.get('skip_cert_verify') == 'true'
if auth_type == BASIC_AUTH:
auth = HTTPBasicAuth(username, password)
elif auth_type == DIGEST_AUTH:
auth = HTTPDigestAuth(username, password)
else:
auth = None
try:
resp = simple_post(fake_post, url, headers=headers, auth=auth, verify=verify)
if 200 <= resp.status_code < 300:
return HttpResponse(json.dumps({"success": True,
"response": resp.content,
"status": resp.status_code}))
else:
return HttpResponse(json.dumps({"success": False,
"response": resp.content,
"status": resp.status_code}))
except Exception as e:
errors = str(e)
return HttpResponse(json.dumps({"success": False, "response": errors}))
else:
return HttpResponse(json.dumps({"success": False, "response": "Please enter a valid url."}))
def fire_for_record(self, repeat_record):
headers = self.get_headers(repeat_record)
auth = self.get_auth()
payload = self.get_payload(repeat_record)
url = self.get_url(repeat_record)
try:
response = simple_post(payload,
url,
headers=headers,
timeout=POST_TIMEOUT,
auth=auth)
except (Timeout, ConnectionError) as error:
log_repeater_timeout_in_datadog(self.domain)
return self.handle_response(RequestConnectionError(error),
repeat_record)
except Exception as e:
return self.handle_response(e, repeat_record)
else:
return self.handle_response(response, repeat_record)
def simple_post_with_cached_timeout(data, url, expiry=60 * 60, force_send=False, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5("{0} timeout {1}".format(__name__, url)).hexdigest()
cache_value = cache.get(key)
if cache_value == "timeout" and not force_send:
raise socket.timeout("recently timed out, not retrying")
elif cache_value == "error" and not force_send:
raise socket.timeout("recently errored, not retrying")
try:
resp = simple_post(data, url, *args, **kwargs)
except socket.timeout:
cache.set(key, "timeout", expiry)
raise
if not 200 <= resp.status_code < 300:
cache.set(key, "error", expiry)
return resp
def simple_post_with_cached_timeout(data, url, expiry=60 * 60, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5(
'{0} timeout {1}'.format(__name__, url)
).hexdigest()
cache_value = cache.get(key)
if cache_value == 'timeout':
raise socket.timeout('recently timed out, not retrying')
elif cache_value == 'error':
raise socket.timeout('recently errored, not retrying')
try:
resp = simple_post(data, url, *args, **kwargs)
except socket.timeout:
cache.set(key, 'timeout', expiry)
raise
if not 200 <= resp.status < 300:
cache.set(key, 'error', expiry)
return resp
def simple_post_with_cached_timeout(data, url, expiry=60 * 60, *args, **kwargs):
# no control characters (e.g. '/') in keys
key = hashlib.md5(
'{0} timeout {1}'.format(__name__, url)
).hexdigest()
cache_value = cache.get(key)
if cache_value == 'timeout':
raise socket.timeout('recently timed out, not retrying')
elif cache_value == 'error':
raise socket.timeout('recently errored, not retrying')
try:
resp = simple_post(data, url, *args, **kwargs)
except socket.timeout:
cache.set(key, 'timeout', expiry)
raise
if not 200 <= resp.status < 300:
cache.set(key, 'error', expiry)
return resp
def test_repeater(request, domain):
url = request.POST["url"]
form = FormRepeaterForm({"url": url})
if form.is_valid():
url = form.cleaned_data["url"]
# now we fake a post
fake_post = "<?xml version='1.0' ?><data id='test'><TestString>Test post from CommCareHQ on %s</TestString></data>" \
% (datetime.datetime.utcnow())
try:
resp = simple_post(fake_post, url)
if 200 <= resp.status < 300:
return HttpResponse(json.dumps({"success": True,
"response": resp.read(),
"status": resp.status}))
else:
return HttpResponse(json.dumps({"success": False,
"response": resp.read(),
"status": resp.status}))
except Exception, e:
errors = str(e)
return HttpResponse(json.dumps({"success": False, "response": errors}))
def send_request(self, repeat_record, payload):
headers = self.get_headers(repeat_record)
auth = self.get_auth()
url = self.get_url(repeat_record)
return simple_post(payload, url, headers=headers, timeout=POST_TIMEOUT, auth=auth, verify=self.verify)
