def is_global_dup(doc_id, title, content, pub_date):
data = generate_dedup_data(doc_id, title, content, pub_date)
resp = upload(GLOBAL_DUP_API, data)
if resp is None or resp.status != 200:
_LOGGER.error("[Dedup] Unexcepted result from dedup service. (url: %s data %s status %d)" % (GLOBAL_DUP_API, data, resp.status if resp else 0))
return False
try:
data = simplejson.loads(resp.body)
if isinstance(data, dict) and 'dupId' in data:
return data['dupId']
except simplejson.JSONDecodeError, err:
_LOGGER.error("[Dedup] Failed to load response from dedup service. (url: %s data %s body %s)" % (GLOBAL_DUP_API, data, resp.body))
_LOGGER.error("[Dedup] err: %s traceback %s" % (err, traceback.format_exc()))
return False
def is_users_dup(user_ids, doc_id):
if user_ids is None or doc_id is None:
return True
data = {
'did': doc_id,
'uids': user_ids,
}
resp = upload(USERS_DUP_API, data)
if resp is None or resp.status != 200:
_LOGGER.error("[Dedup] Unexcepted result from dedup service. (url: %s status %d)" % (USERS_DUP_API, resp.status if resp else 0))
return None
try:
data = simplejson.loads(resp.body)
if isinstance(data, dict):
return data
except simplejson.JSONDecodeError, err:
_LOGGER.error("[Dedup] Failed to load response from dedup service. (url: %s body %s)" % (USERS_DUP_API, resp.body))
_LOGGER.error("[Dedup] err: %s traceback %s" % (err, traceback.format_exc()))
return None
def is_global_dup(doc_id, title, content, pub_date):
data = generate_dedup_data(doc_id, title, content, pub_date)
resp = upload(GLOBAL_DUP_API, data)
if resp is None or resp.status != 200:
_LOGGER.error(
"[Dedup] Unexcepted result from dedup service. (url: %s data %s status %d)"
% (GLOBAL_DUP_API, data, resp.status if resp else 0))
return False
try:
data = simplejson.loads(resp.body)
if isinstance(data, dict) and 'dupId' in data:
return data['dupId']
except simplejson.JSONDecodeError, err:
_LOGGER.error(
"[Dedup] Failed to load response from dedup service. (url: %s data %s body %s)"
% (GLOBAL_DUP_API, data, resp.body))
_LOGGER.error("[Dedup] err: %s traceback %s" %
(err, traceback.format_exc()))
return False
def is_users_dup(user_ids, doc_id):
if user_ids is None or doc_id is None:
return True
data = {
'did': doc_id,
'uids': user_ids,
}
resp = upload(USERS_DUP_API, data)
if resp is None or resp.status != 200:
_LOGGER.error(
"[Dedup] Unexcepted result from dedup service. (url: %s status %d)"
% (USERS_DUP_API, resp.status if resp else 0))
return None
try:
data = simplejson.loads(resp.body)
if isinstance(data, dict):
return data
except simplejson.JSONDecodeError, err:
_LOGGER.error(
"[Dedup] Failed to load response from dedup service. (url: %s body %s)"
% (USERS_DUP_API, resp.body))
_LOGGER.error("[Dedup] err: %s traceback %s" %
(err, traceback.format_exc()))
return None