def mongo_connect(uri, conn_timeout=None, **kwargs):
conn_timeout_mills = (conn_timeout or CONN_TIMEOUT) * 1000
kwargs = kwargs or {}
kwargs["connectTimeoutMS"] = conn_timeout_mills
kwargs["socketTimeoutMS"] = SOCKET_TIMEOUT * 1000
# default connection timeout and convert to mills
uri_wrapper = parse_mongo_uri(uri)
try:
dbname = uri_wrapper.database
if not dbname:
if uri.endswith("/"):
uri += "admin"
else:
uri += "/admin"
# add serverSelectionTimeoutMS for pymongo 3.2
if pymongo.get_version_string().startswith("3.2"):
kwargs["serverSelectionTimeoutMS"] = conn_timeout_mills
kwargs["maxPoolSize"] = 1
mongo_client = _mongo_client(uri, **kwargs)
return mongo_client
except Exception, e:
if is_connection_exception(e):
raise ConnectionError(uri_wrapper.masked_uri, cause=e)
elif "authentication failed" in safe_stringify(e):
raise AuthenticationFailedError(uri_wrapper.masked_uri, cause=e)
else:
raise
def mongo_connect(uri, conn_timeout=None, **kwargs):
conn_timeout_mills = (conn_timeout or CONN_TIMEOUT) * 1000
kwargs = kwargs or {}
kwargs["connectTimeoutMS"] = conn_timeout_mills
kwargs["socketTimeoutMS"] = SOCKET_TIMEOUT * 1000
# default connection timeout and convert to mills
uri_wrapper = parse_mongo_uri(uri)
try:
dbname = uri_wrapper.database
if not dbname:
if uri.endswith("/"):
uri += "admin"
else:
uri += "/admin"
# add serverSelectionTimeoutMS for pymongo 3.2
if pymongo.get_version_string().startswith("3.2"):
kwargs["serverSelectionTimeoutMS"] = conn_timeout_mills
kwargs["maxPoolSize"] = 1
mongo_client = _mongo_client(uri, **kwargs)
return mongo_client
except Exception, e:
if is_connection_exception(e):
raise ConnectionError(uri_wrapper.masked_uri, cause=e)
elif "authentication failed" in safe_stringify(e):
raise AuthenticationFailedError(uri_wrapper.masked_uri, cause=e)
else:
raise
def worker_fail(self, exception, trace=None):
if isinstance(exception, MBSError):
log_msg = exception.message
else:
log_msg = "Unexpected error. Please contact admin"
details = safe_stringify(exception)
task = self._task
self.get_task_collection().update_task(
task,
event_type=EventType.ERROR,
message=log_msg,
details=details,
error_code=to_mbs_error_code(exception))
# update retry info
set_task_retry_info(task, exception)
self.worker_finished(State.FAILED)
# send a notification only if the task is not reschedulable
# if there is an event queue configured then do not notify (because it should be handled by the backup
# event listener)
if not get_mbs().event_queue and task.exceeded_max_tries():
get_mbs().notifications.notify_on_task_failure(
task, exception, trace)
def raise_dump_error(returncode, error_log_line, last_namespace=None):
error_log_line = utils.safe_stringify(error_log_line)
# encode error log line
if (("Failed: error creating bson file" in error_log_line and
"no such file or directory" in error_log_line) or
"contains a path separator" in error_log_line):
error_type = BadCollectionNameError
elif "10334" in error_log_line:
if "BSONObj size: 0 (0x00000000)" in error_log_line:
error_type = CorruptionError
else:
error_type = InvalidBSONObjSizeError
elif "Document is corrupted" in error_log_line:
error_type = CorruptionError
elif "13106" in error_log_line and "extent_manager.cpp" in error_log_line:
error_type = CorruptionError
elif "assertion src/mongo/db/storage/mmap_v1/btree/key.cpp:443" in error_log_line:
error_type = CorruptionError
elif "error reading collection: bad offset" in error_log_line:
error_type = CorruptionError
elif "13338" in error_log_line:
error_type = CappedCursorOverrunError
elif "13280" in error_log_line:
error_type = InvalidDBNameError
elif "10320" in error_log_line:
error_type = BadTypeError
elif "Cannot connect" in error_log_line:
error_type = MongoctlConnectionError
elif "cursor didn't exist on server" in error_log_line:
error_type = CursorDoesNotExistError
elif "16465" in error_log_line:
error_type = ExhaustReceiveError
elif ("SocketException" in error_log_line or
"socket error" in error_log_line or
"transport error" in error_log_line or
"no reachable servers" in error_log_line or
"error connecting to db server" in error_log_line):
error_type = DumpConnectivityError
elif (("DBClientCursor" in error_log_line and "failed" in error_log_line) or
"invalid cursor" in error_log_line or
"Closed explicitly" in error_log_line):
error_type = DBClientCursorFailError
elif "index out of range" in error_log_line:
error_type = IndexOutOfRangeDumpError
elif "error reading collection" in error_log_line:
error_type = CollectionReadError
elif "oplog overflow" in error_log_line:
error_type = OplogOverflowError
elif "mongoctl error" in error_log_line and "Unable to find a compatible 'mongodump'" in error_log_line:
error_type = NoCompatibleMongodumpExeFoundError
elif returncode == 245: # segmentation fault
error_type = MongodumpSegmentationFaultError
# Generic retriable errors
elif is_retriable_dump_error(returncode, error_log_line):
error_type = RetriableDumpError
else:
error_type = DumpError
raise error_type(returncode, error_log_line, last_namespace=last_namespace)
def raise_if_not_ec2_retriable(exception):
# retry on boto request limit and other ec2 errors
msg = utils.safe_stringify(exception)
if ((isinstance(exception, BotoServerError) and
exception.status == 503) or "ConcurrentTagAccess" in msg):
logger.warn("Caught a retriable exception: %s" % exception)
else:
raise_if_not_retriable(exception)
def is_connection_exception(exception):
if isinstance(exception, ConnectionFailure):
return True
else:
msg = utils.safe_stringify(exception)
return ("timed out" in msg or "refused" in msg or "reset" in msg or
"Broken pipe" in msg or "closed" in msg or "IncompleteRead" in msg or
"Connection aborted" in msg)
def is_connection_exception(exception):
if isinstance(exception, ConnectionFailure):
return True
else:
msg = utils.safe_stringify(exception)
return ("timed out" in msg or "refused" in msg or "reset" in msg or
"Broken pipe" in msg or "closed" in msg or "IncompleteRead" in msg or
"Connection aborted" in msg)
def raise_if_not_ec2_retriable(exception):
log_ec2_limit_error(exception)
# retry on boto request limit and other ec2 errors
msg = utils.safe_stringify(exception)
if ((isinstance(exception, BotoServerError) and
exception.status == 503) or "ConcurrentTagAccess" in msg):
logger.warn("Caught a retriable exception: %s" % exception)
else:
raise_if_not_retriable(exception)
def set_account_temp_url_key(storage_url, auth_token, temp_key):
headers = {
"X-Auth-Token": auth_token,
"X-Account-Meta-Temp-Url-Key": temp_key
}
try:
return fetch_url(storage_url, headers=headers, method="POST")
except Exception, e:
if "204" in safe_stringify(e):
pass
else:
raise
def raise_dump_error(returncode, error_log_line, last_namespace=None):
error_log_line = utils.safe_stringify(error_log_line)
# encode error log line
if (("Failed: error creating bson file" in error_log_line and
"no such file or directory" in error_log_line) or
"contains a path separator" in error_log_line):
error_type = BadCollectionNameError
elif "10334" in error_log_line:
if "BSONObj size: 0 (0x00000000)" in error_log_line:
error_type = CorruptionError
else:
error_type = InvalidBSONObjSizeError
elif "13338" in error_log_line:
error_type = CappedCursorOverrunError
elif "13280" in error_log_line:
error_type = InvalidDBNameError
elif "10320" in error_log_line:
error_type = BadTypeError
elif "Cannot connect" in error_log_line:
error_type = MongoctlConnectionError
elif "cursor didn't exist on server" in error_log_line:
error_type = CursorDoesNotExistError
elif "16465" in error_log_line:
error_type = ExhaustReceiveError
elif ("SocketException" in error_log_line or
"socket error" in error_log_line or
"transport error" in error_log_line or
"no reachable servers" in error_log_line or
"error connecting to db server" in error_log_line):
error_type = DumpConnectivityError
elif (("DBClientCursor" in error_log_line and "failed" in error_log_line) or
"invalid cursor" in error_log_line or
"Closed explicitly" in error_log_line):
error_type = DBClientCursorFailError
elif "index out of range" in error_log_line:
error_type = IndexOutOfRangeDumpError
elif "error reading collection" in error_log_line:
error_type = CollectionReadError
elif "oplog overflow" in error_log_line:
error_type = OplogOverflowError
# Generic retriable errors
elif is_retriable_dump_error(returncode, error_log_line):
error_type = RetriableDumpError
else:
error_type = DumpError
raise error_type(returncode, error_log_line, last_namespace=last_namespace)
def to_document(self, display_only=False):
doc = {
"_type": self.full_type_name,
"message": self.message
}
if self.cause:
if isinstance(self.cause, MBSError):
doc["cause"] = self.cause.to_document(display_only=display_only)
else:
doc["cause"] = {
"causeType": utils.object_full_type_name(self.cause),
"message": utils.safe_stringify(self.cause)
}
return doc
def set_account_temp_url_key(storage_url, auth_token, temp_key):
headers = {
"X-Auth-Token": auth_token,
"X-Account-Meta-Temp-Url-Key": temp_key
}
try:
return fetch_url(storage_url, headers=headers, method="POST")
except Exception, e:
if "204" in safe_stringify(e):
pass
else:
raise
def to_document(self, display_only=False):
doc = {
"_type": self.full_type_name,
"message": self.message
}
if self.cause:
if isinstance(self.cause, MBSError):
doc["cause"] = self.cause.to_document(display_only=display_only)
else:
doc["cause"] = {
"causeType": utils.object_full_type_name(self.cause),
"message": utils.safe_stringify(self.cause)
}
return doc
def worker_fail(self, exception, trace=None):
if isinstance(exception, MBSError):
log_msg = exception.message
else:
log_msg = "Unexpected error. Please contact admin"
details = safe_stringify(exception)
task = self._task
self.get_task_collection().update_task(
task, event_type=EventType.ERROR,
message=log_msg, details=details, error_code=to_mbs_error_code(exception))
# update retry info
set_task_retry_info(task, exception)
self.worker_finished(State.FAILED)
# send a notification only if the task is not reschedulable
# if there is an event queue configured then do not notify (because it should be handled by the backup
# event listener)
if not get_mbs().event_queue and task.exceeded_max_tries():
get_mbs().notifications.notify_on_task_failure(task, exception, trace)
def log_ec2_limit_error(exception):
"""
Logs more details if the exception is RequestLimitExceeded. Not implemented in the best way/right place but
should work just fine
:param exception:
:return:
"""
if isinstance(exception, BotoServerError) and "RequestLimitExceeded" in utils.safe_stringify(exception):
stack_str = traceback.format_exc()
if "add_tag" in stack_str:
op_name = "CreateTag"
elif "create_snapshot" in stack_str:
op_name = "CreateSnapshot"
elif "delete_snapshot" in stack_str:
op_name = "DeleteSnapshot"
elif "get_all_snapshots" in stack_str:
op_name = "DescribeSnapshots"
elif "get_all_volumes" in stack_str:
op_name = "DescribeVolumes"
else:
op_name = "UNKNOWN"
logger.info("EC2_THROTTLE: Got a RequestLimitExceeded on op '%s', Error body: %s, Trace: %s" %
(op_name, exception.body, stack_str))
def log_ec2_limit_error(exception):
"""
Logs more details if the exception is RequestLimitExceeded. Not implemented in the best way/right place but
should work just fine
:param exception:
:return:
"""
if isinstance(exception, BotoServerError) and "RequestLimitExceeded" in utils.safe_stringify(exception):
stack_str = traceback.format_exc()
if "add_tag" in stack_str:
op_name = "CreateTag"
elif "create_snapshot" in stack_str:
op_name = "CreateSnapshot"
elif "delete_snapshot" in stack_str:
op_name = "DeleteSnapshot"
elif "get_all_snapshots" in stack_str:
op_name = "DescribeSnapshots"
elif "get_all_volumes" in stack_str:
op_name = "DescribeVolumes"
else:
op_name = "UNKNOWN"
logger.info("EC2_THROTTLE: Got a RequestLimitExceeded on op '%s', Error body: %s, Trace: %s" %
(op_name, exception.body, stack_str))
def to_mbs_error_code(error):
if isinstance(error, MBSError):
return error
else:
return MBSErrorWrapper(msg=utils.safe_stringify(error), cause=error)
def to_mbs_error_code(error):
if isinstance(error, MBSError):
return error
else:
return MBSErrorWrapper(msg=utils.safe_stringify(error), cause=error)