def mongo_connect(uri, conn_timeout=None, **kwargs):
    conn_timeout_mills = (conn_timeout or CONN_TIMEOUT) * 1000

    kwargs = kwargs or {}
    kwargs["connectTimeoutMS"] = conn_timeout_mills
    kwargs["socketTimeoutMS"] = SOCKET_TIMEOUT * 1000
    # default connection timeout and convert to mills

    uri_wrapper = parse_mongo_uri(uri)

    try:
        dbname = uri_wrapper.database
        if not dbname:
            if uri.endswith("/"):
                uri += "admin"
            else:
                uri += "/admin"

        # add serverSelectionTimeoutMS for pymongo 3.2
        if pymongo.get_version_string().startswith("3.2"):
            kwargs["serverSelectionTimeoutMS"] = conn_timeout_mills

        kwargs["maxPoolSize"] = 1

        mongo_client = _mongo_client(uri, **kwargs)

        return mongo_client

    except Exception, e:
        if is_connection_exception(e):
            raise ConnectionError(uri_wrapper.masked_uri, cause=e)
        elif "authentication failed" in safe_stringify(e):
            raise AuthenticationFailedError(uri_wrapper.masked_uri, cause=e)
        else:
            raise
def mongo_connect(uri, conn_timeout=None, **kwargs):
    conn_timeout_mills = (conn_timeout or CONN_TIMEOUT) * 1000

    kwargs = kwargs or {}
    kwargs["connectTimeoutMS"] = conn_timeout_mills
    kwargs["socketTimeoutMS"] = SOCKET_TIMEOUT * 1000
    # default connection timeout and convert to mills

    uri_wrapper = parse_mongo_uri(uri)

    try:
        dbname = uri_wrapper.database
        if not dbname:
            if uri.endswith("/"):
                uri += "admin"
            else:
                uri += "/admin"

        # add serverSelectionTimeoutMS for pymongo 3.2
        if pymongo.get_version_string().startswith("3.2"):
            kwargs["serverSelectionTimeoutMS"] = conn_timeout_mills

        kwargs["maxPoolSize"] = 1

        mongo_client = _mongo_client(uri, **kwargs)

        return mongo_client

    except Exception, e:
        if is_connection_exception(e):
            raise ConnectionError(uri_wrapper.masked_uri, cause=e)
        elif "authentication failed" in safe_stringify(e):
            raise AuthenticationFailedError(uri_wrapper.masked_uri, cause=e)
        else:
            raise
    def worker_fail(self, exception, trace=None):
        if isinstance(exception, MBSError):
            log_msg = exception.message
        else:
            log_msg = "Unexpected error. Please contact admin"

        details = safe_stringify(exception)
        task = self._task

        self.get_task_collection().update_task(
            task,
            event_type=EventType.ERROR,
            message=log_msg,
            details=details,
            error_code=to_mbs_error_code(exception))

        # update retry info
        set_task_retry_info(task, exception)

        self.worker_finished(State.FAILED)

        # send a notification only if the task is not reschedulable
        # if there is an event queue configured then do not notify (because it should be handled by the backup
        # event listener)
        if not get_mbs().event_queue and task.exceeded_max_tries():
            get_mbs().notifications.notify_on_task_failure(
                task, exception, trace)
Exemple #4
0
def raise_dump_error(returncode, error_log_line, last_namespace=None):
    error_log_line = utils.safe_stringify(error_log_line)
    # encode error log line
    if (("Failed: error creating bson file" in error_log_line and
                 "no such file or directory" in error_log_line) or
        "contains a path separator" in error_log_line):
        error_type = BadCollectionNameError
    elif "10334" in error_log_line:
        if "BSONObj size: 0 (0x00000000)" in error_log_line:
            error_type = CorruptionError
        else:
            error_type = InvalidBSONObjSizeError
    elif "Document is corrupted" in error_log_line:
        error_type = CorruptionError
    elif "13106" in error_log_line and "extent_manager.cpp" in error_log_line:
        error_type = CorruptionError
    elif "assertion src/mongo/db/storage/mmap_v1/btree/key.cpp:443" in error_log_line:
        error_type = CorruptionError
    elif "error reading collection: bad offset" in error_log_line:
        error_type = CorruptionError
    elif "13338" in error_log_line:
        error_type = CappedCursorOverrunError
    elif "13280" in error_log_line:
        error_type = InvalidDBNameError
    elif "10320" in error_log_line:
        error_type = BadTypeError
    elif "Cannot connect" in error_log_line:
        error_type = MongoctlConnectionError
    elif "cursor didn't exist on server" in error_log_line:
        error_type = CursorDoesNotExistError
    elif "16465" in error_log_line:
        error_type = ExhaustReceiveError
    elif ("SocketException" in error_log_line or
          "socket error" in error_log_line or
          "transport error" in error_log_line or
          "no reachable servers" in error_log_line or
          "error connecting to db server" in error_log_line):
        error_type = DumpConnectivityError
    elif (("DBClientCursor" in error_log_line and "failed" in error_log_line) or
          "invalid cursor" in error_log_line or
          "Closed explicitly" in error_log_line):
        error_type = DBClientCursorFailError
    elif "index out of range" in error_log_line:
        error_type = IndexOutOfRangeDumpError
    elif "error reading collection" in error_log_line:
        error_type = CollectionReadError
    elif "oplog overflow" in error_log_line:
        error_type = OplogOverflowError
    elif "mongoctl error" in error_log_line and "Unable to find a compatible 'mongodump'" in error_log_line:
        error_type = NoCompatibleMongodumpExeFoundError
    elif returncode == 245: # segmentation fault
        error_type = MongodumpSegmentationFaultError
    # Generic retriable errors
    elif is_retriable_dump_error(returncode, error_log_line):
        error_type = RetriableDumpError
    else:
        error_type = DumpError

    raise error_type(returncode, error_log_line, last_namespace=last_namespace)
Exemple #5
0
def raise_if_not_ec2_retriable(exception):
    # retry on boto request limit and other ec2 errors
    msg = utils.safe_stringify(exception)
    if ((isinstance(exception, BotoServerError) and
         exception.status == 503) or "ConcurrentTagAccess" in msg):
        logger.warn("Caught a retriable exception: %s" % exception)
    else:
        raise_if_not_retriable(exception)
Exemple #6
0
def is_connection_exception(exception):
    if isinstance(exception, ConnectionFailure):
        return True
    else:
        msg = utils.safe_stringify(exception)
        return ("timed out" in msg or "refused" in msg or "reset" in msg or
                "Broken pipe" in msg or "closed" in msg or "IncompleteRead" in msg or
                "Connection aborted" in msg)
Exemple #7
0
def is_connection_exception(exception):
    if isinstance(exception, ConnectionFailure):
        return True
    else:
        msg = utils.safe_stringify(exception)
        return ("timed out" in msg or "refused" in msg or "reset" in msg or
                "Broken pipe" in msg or "closed" in msg or "IncompleteRead" in msg or
                "Connection aborted" in msg)
Exemple #8
0
def raise_if_not_ec2_retriable(exception):
    log_ec2_limit_error(exception)
    # retry on boto request limit and other ec2 errors
    msg = utils.safe_stringify(exception)
    if ((isinstance(exception, BotoServerError) and
         exception.status == 503) or "ConcurrentTagAccess" in msg):
        logger.warn("Caught a retriable exception: %s" % exception)
    else:
        raise_if_not_retriable(exception)
Exemple #9
0
def set_account_temp_url_key(storage_url, auth_token, temp_key):

    headers = {
        "X-Auth-Token": auth_token,
        "X-Account-Meta-Temp-Url-Key": temp_key
    }

    try:
        return fetch_url(storage_url, headers=headers, method="POST")
    except Exception, e:
        if "204" in safe_stringify(e):
            pass
        else:
            raise
Exemple #10
0
def raise_dump_error(returncode, error_log_line, last_namespace=None):
    error_log_line = utils.safe_stringify(error_log_line)
    # encode error log line
    if (("Failed: error creating bson file" in error_log_line and
                 "no such file or directory" in error_log_line) or
        "contains a path separator" in error_log_line):
        error_type = BadCollectionNameError
    elif "10334" in error_log_line:
        if "BSONObj size: 0 (0x00000000)" in error_log_line:
            error_type = CorruptionError
        else:
            error_type = InvalidBSONObjSizeError
    elif "13338" in error_log_line:
        error_type = CappedCursorOverrunError
    elif "13280" in error_log_line:
        error_type = InvalidDBNameError
    elif "10320" in error_log_line:
        error_type = BadTypeError
    elif "Cannot connect" in error_log_line:
        error_type = MongoctlConnectionError
    elif "cursor didn't exist on server" in error_log_line:
        error_type = CursorDoesNotExistError
    elif "16465" in error_log_line:
        error_type = ExhaustReceiveError
    elif ("SocketException" in error_log_line or
          "socket error" in error_log_line or
          "transport error" in error_log_line or
          "no reachable servers" in error_log_line or
          "error connecting to db server" in error_log_line):
        error_type = DumpConnectivityError
    elif (("DBClientCursor" in error_log_line and "failed" in error_log_line) or
          "invalid cursor" in error_log_line or
          "Closed explicitly" in error_log_line):
        error_type = DBClientCursorFailError
    elif "index out of range" in error_log_line:
        error_type = IndexOutOfRangeDumpError
    elif "error reading collection" in error_log_line:
        error_type = CollectionReadError
    elif "oplog overflow" in error_log_line:
        error_type = OplogOverflowError

    # Generic retriable errors
    elif is_retriable_dump_error(returncode, error_log_line):
        error_type = RetriableDumpError
    else:
        error_type = DumpError

    raise error_type(returncode, error_log_line, last_namespace=last_namespace)
Exemple #11
0
    def to_document(self, display_only=False):
        doc = {
            "_type": self.full_type_name,
            "message": self.message
        }

        if self.cause:
            if isinstance(self.cause, MBSError):
                doc["cause"] = self.cause.to_document(display_only=display_only)
            else:
                doc["cause"] = {
                    "causeType": utils.object_full_type_name(self.cause),
                    "message": utils.safe_stringify(self.cause)
                }

        return doc
def set_account_temp_url_key(storage_url,  auth_token, temp_key):


    headers = {
        "X-Auth-Token": auth_token,
        "X-Account-Meta-Temp-Url-Key": temp_key

    }

    try:
        return fetch_url(storage_url, headers=headers, method="POST")
    except Exception, e:
        if "204" in safe_stringify(e):
            pass
        else:
            raise
Exemple #13
0
    def to_document(self, display_only=False):
        doc = {
            "_type": self.full_type_name,
            "message": self.message
        }

        if self.cause:
            if isinstance(self.cause, MBSError):
                doc["cause"] = self.cause.to_document(display_only=display_only)
            else:
                doc["cause"] = {
                    "causeType": utils.object_full_type_name(self.cause),
                    "message": utils.safe_stringify(self.cause)
                }

        return doc
    def worker_fail(self, exception, trace=None):
        if isinstance(exception, MBSError):
            log_msg = exception.message
        else:
            log_msg = "Unexpected error. Please contact admin"

        details = safe_stringify(exception)
        task = self._task

        self.get_task_collection().update_task(
            task, event_type=EventType.ERROR,
            message=log_msg, details=details, error_code=to_mbs_error_code(exception))

        # update retry info
        set_task_retry_info(task, exception)

        self.worker_finished(State.FAILED)

        # send a notification only if the task is not reschedulable
        # if there is an event queue configured then do not notify (because it should be handled by the backup
        # event listener)
        if not get_mbs().event_queue and task.exceeded_max_tries():
            get_mbs().notifications.notify_on_task_failure(task, exception, trace)
Exemple #15
0
def log_ec2_limit_error(exception):
    """
    Logs more details if the exception is RequestLimitExceeded. Not implemented in the best way/right place but
    should work just fine
    :param exception:
    :return:
    """
    if isinstance(exception, BotoServerError) and "RequestLimitExceeded" in utils.safe_stringify(exception):
        stack_str = traceback.format_exc()

        if "add_tag" in stack_str:
            op_name = "CreateTag"
        elif "create_snapshot" in stack_str:
            op_name = "CreateSnapshot"
        elif "delete_snapshot" in stack_str:
            op_name = "DeleteSnapshot"
        elif "get_all_snapshots" in stack_str:
            op_name = "DescribeSnapshots"
        elif "get_all_volumes" in stack_str:
            op_name = "DescribeVolumes"
        else:
            op_name = "UNKNOWN"
        logger.info("EC2_THROTTLE: Got a RequestLimitExceeded on op '%s', Error body: %s, Trace: %s" %
                    (op_name, exception.body, stack_str))
def log_ec2_limit_error(exception):
    """
    Logs more details if the exception is RequestLimitExceeded. Not implemented in the best way/right place but
    should work just fine
    :param exception:
    :return:
    """
    if isinstance(exception, BotoServerError) and "RequestLimitExceeded" in utils.safe_stringify(exception):
        stack_str = traceback.format_exc()

        if "add_tag" in stack_str:
            op_name = "CreateTag"
        elif "create_snapshot" in stack_str:
            op_name = "CreateSnapshot"
        elif "delete_snapshot" in stack_str:
            op_name = "DeleteSnapshot"
        elif "get_all_snapshots" in stack_str:
            op_name = "DescribeSnapshots"
        elif "get_all_volumes" in stack_str:
            op_name = "DescribeVolumes"
        else:
            op_name = "UNKNOWN"
        logger.info("EC2_THROTTLE: Got a RequestLimitExceeded on op '%s', Error body: %s, Trace: %s" %
                    (op_name, exception.body, stack_str))
Exemple #17
0
def to_mbs_error_code(error):
    if isinstance(error, MBSError):
        return error
    else:
        return MBSErrorWrapper(msg=utils.safe_stringify(error), cause=error)
Exemple #18
0
def to_mbs_error_code(error):
    if isinstance(error, MBSError):
        return error
    else:
        return MBSErrorWrapper(msg=utils.safe_stringify(error), cause=error)