def consume_design( opts, sink_spec: str, sink_map, source_bucket, source_map, source_design: Union[str, bytes]) -> couchbaseConstants.PUMP_ERROR: if not source_design: return 0 try: sd = json.loads(source_design) except ValueError as e: return f'error: could not parse source design; exception: {e!s}' if not sd: return 0 if (not sink_map['buckets'] or len(sink_map['buckets']) != 1 or not sink_map['buckets'][0] or not sink_map['buckets'][0]['name']): return "error: design sink incorrect sink_map bucket" spec_parts = pump.parse_spec(opts, sink_spec, 8091) if not spec_parts: return "error: design sink no spec_parts: " + sink_spec sink_bucket = sink_map['buckets'][0] sink_nodes = pump.filter_bucket_nodes(sink_bucket, spec_parts) or \ sink_bucket['nodes'] if not sink_nodes: return "error: design sink nodes missing" couch_api_base = sink_nodes[0].get('couchApiBase') if not couch_api_base: return f'error: cannot restore bucket design on a couchbase cluster that does not support couch API;' \ f' the couchbase cluster may be an older, pre-2.0 version; please check your cluster URL: {sink_spec}' host, port, user, pswd, path = \ pump.parse_spec(opts, couch_api_base, 8092) if user is None: user = spec_parts[2] # Default to the main REST user/pwsd. pswd = spec_parts[3] if opts.username_dest is not None and opts.password_dest is not None: user = opts.username_dest user = opts.password_dest if type(sd) is dict: id = sd.get('_id', None) if id: str_source = _to_string(source_design) err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=str_source, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' else: stmts = sd.get('statements', []) hostname = f'http://{spec_parts[0]}:{spec_parts[1]!s}' cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error( f'N1QL query {stmt["statement"]} failed due to {errors}' ) if result and 'errors' in result: for error in result['errors']: logging.error( f'N1QL query {stmt["statement"]} failed due to error `{error["msg"]}`' ) except ServiceNotAvailableException as e: logging.error( "Failed to restore indexes, cluster does not contain a query node" ) elif type(sd) is list: for row in sd: logging.debug(f'design_doc row: {row!s}') doc = row.get('doc', None) if not doc: stmt = row.get('statement', None) if not stmt: return f'error: missing design doc or index statement in row: {row}' # publish index return 0 if 'json' in doc and 'meta' in doc: js = doc['json'] id = doc['meta'].get('id', None) if not id: return f'error: missing id for design doc: {row}' else: # Handle design-doc from 2.0DP4. js = doc if '_rev' in js: del js['_rev'] id = row.get('id', None) if not id: return f'error: missing id for row: {row}' js_doc = json.dumps(js) if id.startswith(CBSink.DDOC_HEAD): id = CBSink.DDOC_HEAD + urllib.parse.quote( id[len(CBSink.DDOC_HEAD):], '') else: id = urllib.parse.quote(id, '') logging.debug(f'design_doc: {js_doc}') logging.debug(f'design_doc id: {id} at: {path}/{id}') try: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=js_doc, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' except Exception as e: return f'error: design sink exception: {e}; couch_api_base: {couch_api_base}' logging.debug(f'design_doc created at: {path}/{id}') return 0
err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, path + "/" + id, method='PUT', body=source_design, reason="consume_design") if conn: conn.close() if err: return ("error: could not restore design doc id: %s" + "; response: %s; err: %s") % (id, response, err) else: stmts = sd.get('statements', []) hostname = 'http://' + spec_parts[0] + ':' + str(spec_parts[1]) cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error('N1QL query %s failed due to %s' % (stmt['statement'], errors)) if result and 'errors' in result: for error in result['errors']: logging.error( 'N1QL query %s failed due to error `%s`' % (stmt['statement'], error['msg'])) except ServiceNotAvailableException, e: logging.error( "Failed to restore indexes, cluster does not contain a" + " query node") elif type(sd) is list: for row in sd:
err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, path + "/" + id, method='PUT', body=source_design, reason="consume_design") if conn: conn.close() if err: return ("error: could not restore design doc id: %s" + "; response: %s; err: %s") % (id, response, err) else: stmts = sd.get('statements', []) hostname = 'http://' + spec_parts[0] + ':' + str(spec_parts[1]) cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error('N1QL query %s failed due to %s' % (stmt['statement'], errors)) if result and 'errors' in result: for error in result['errors']: logging.error('N1QL query %s failed due to error `%s`' % (stmt['statement'], error['msg'])) except ServiceNotAvailableException, e: logging.error("Failed to restore indexes, cluster does not contain a" + " query node") elif type(sd) is list: for row in sd: logging.debug("design_doc row: " + str(row)) doc = row.get('doc', None) if not doc:
def consume_design(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: sd = json.loads(source_design) except ValueError as e: return "error: could not parse source design; exception: %s" % (e) if not sd: return 0 if (not sink_map['buckets'] or len(sink_map['buckets']) != 1 or not sink_map['buckets'][0] or not sink_map['buckets'][0]['name']): return "error: design sink incorrect sink_map bucket" spec_parts = pump.parse_spec(opts, sink_spec, 8091) if not spec_parts: return "error: design sink no spec_parts: " + sink_spec sink_bucket = sink_map['buckets'][0] sink_nodes = pump.filter_bucket_nodes(sink_bucket, spec_parts) or \ sink_bucket['nodes'] if not sink_nodes: return "error: design sink nodes missing" couch_api_base = sink_nodes[0].get('couchApiBase') if not couch_api_base: return "error: cannot restore bucket design" \ " on a couchbase cluster that does not support couch API;" \ " the couchbase cluster may be an older, pre-2.0 version;" \ " please check your cluster URL: " + sink_spec host, port, user, pswd, path = \ pump.parse_spec(opts, couch_api_base, 8092) if user is None: user = spec_parts[2] # Default to the main REST user/pwsd. pswd = spec_parts[3] if opts.username_dest is not None and opts.password_dest is not None: user = opts.username_dest user = opts.password_dest if type(sd) is dict: id = sd.get('_id', None) if id: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, path + "/" + id, method='PUT', body=source_design, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return ("error: could not restore design doc id: %s" + "; response: %s; err: %s") % (id, response, err) else: stmts = sd.get('statements', []) hostname = 'http://' + spec_parts[0] + ':' + str(spec_parts[1]) cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error('N1QL query %s failed due to %s' % (stmt['statement'], errors)) if result and 'errors' in result: for error in result['errors']: logging.error('N1QL query %s failed due to error `%s`' % (stmt['statement'], error['msg'])) except ServiceNotAvailableException as e: logging.error("Failed to restore indexes, cluster does not contain a" + " query node") elif type(sd) is list: for row in sd: logging.debug("design_doc row: " + str(row)) doc = row.get('doc', None) if not doc: stmt = row.get('statement', None) if not stmt: return "error: missing design doc or index statement in row: %s" % (row) else: #publish index return 0 if 'json' in doc and 'meta' in doc: js = doc['json'] id = doc['meta'].get('id', None) if not id: return "error: missing id for design doc: %s" % (row) else: # Handle design-doc from 2.0DP4. js = doc if '_rev' in js: del js['_rev'] id = row.get('id', None) if not id: return "error: missing id for row: %s" % (row) js_doc = json.dumps(js) if id.startswith(CBSink.DDOC_HEAD): id = CBSink.DDOC_HEAD + urllib.parse.quote(id[len(CBSink.DDOC_HEAD):], '') else: id = urllib.parse.quote(id, '') logging.debug("design_doc: " + js_doc) logging.debug("design_doc id: " + id + " at: " + path + "/" + id) try: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, path + "/" + id, method='PUT', body=js_doc, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return ("error: could not restore design doc id: %s" + "; response: %s; err: %s") % (id, response, err) except Exception as e: return ("error: design sink exception: %s" + "; couch_api_base: %s") % (e, couch_api_base) logging.debug("design_doc created at: " + path + "/" + id) return 0
def consume_design(opts, sink_spec: str, sink_map, source_bucket, source_map, source_design: Union[str, bytes]) -> couchbaseConstants.PUMP_ERROR: if not source_design: return 0 try: sd = json.loads(source_design) except ValueError as e: return f'error: could not parse source design; exception: {e!s}' if not sd: return 0 if (not sink_map['buckets'] or len(sink_map['buckets']) != 1 or not sink_map['buckets'][0] or not sink_map['buckets'][0]['name']): return "error: design sink incorrect sink_map bucket" spec_parts = pump.parse_spec(opts, sink_spec, 8091) if not spec_parts: return "error: design sink no spec_parts: " + sink_spec sink_bucket = sink_map['buckets'][0] sink_nodes = pump.filter_bucket_nodes(sink_bucket, spec_parts) or \ sink_bucket['nodes'] if not sink_nodes: return "error: design sink nodes missing" couch_api_base = sink_nodes[0].get('couchApiBase') if not couch_api_base: return f'error: cannot restore bucket design on a couchbase cluster that does not support couch API;' \ f' the couchbase cluster may be an older, pre-2.0 version; please check your cluster URL: {sink_spec}' host, port, user, pswd, path = \ pump.parse_spec(opts, couch_api_base, 8092) if user is None: user = spec_parts[2] # Default to the main REST user/pwsd. pswd = spec_parts[3] if opts.username_dest is not None and opts.password_dest is not None: user = opts.username_dest user = opts.password_dest if type(sd) is dict: id = sd.get('_id', None) if id: str_source = _to_string(source_design) err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=str_source, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' else: stmts = sd.get('statements', []) hostname = f'http://{spec_parts[0]}:{spec_parts[1]!s}' cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error(f'N1QL query {stmt["statement"]} failed due to {errors}') if result and 'errors' in result: for error in result['errors']: logging.error(f'N1QL query {stmt["statement"]} failed due to error `{error["msg"]}`') except ServiceNotAvailableException as e: logging.error("Failed to restore indexes, cluster does not contain a query node") elif type(sd) is list: for row in sd: logging.debug(f'design_doc row: {row!s}') doc = row.get('doc', None) if not doc: stmt = row.get('statement', None) if not stmt: return f'error: missing design doc or index statement in row: {row}' else: #publish index return 0 if 'json' in doc and 'meta' in doc: js = doc['json'] id = doc['meta'].get('id', None) if not id: return f'error: missing id for design doc: {row}' else: # Handle design-doc from 2.0DP4. js = doc if '_rev' in js: del js['_rev'] id = row.get('id', None) if not id: return f'error: missing id for row: {row}' js_doc = json.dumps(js) if id.startswith(CBSink.DDOC_HEAD): id = CBSink.DDOC_HEAD + urllib.parse.quote(id[len(CBSink.DDOC_HEAD):], '') else: id = urllib.parse.quote(id, '') logging.debug(f'design_doc: {js_doc}') logging.debug(f'design_doc id: {id} at: {path}/{id}') try: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=js_doc, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' except Exception as e: return f'error: design sink exception: {e}; couch_api_base: {couch_api_base}' logging.debug(f'design_doc created at: {path}/{id}') return 0