def main(min_replica=1, max_replica=2, aws_region='vir', runtime_env='dev', dry_run=True, debug=True): ''' Main function ''' service_name = 'sample_application' namespace = 'alexrhino' if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") if int(max_replica) >= int(min_replica) and int(min_replica) > 0: cm = ClusterManager(runtime_env, aws_region) my_service = SampleApplication('sample_application', 'alexrhino') manifest_spec = my_service.load_general_manifest( 'hpa', runtime_env, aws_region) manifest_spec['spec']['maxReplicas'] = max_replica manifest_spec['spec']['minReplicas'] = min_replica cm.deploy_manifest(manifest_spec, dry_run) else: logging.error('Wrong replica range, from %s to %s, please check', min_replica, max_replica)
def validate_manifests(aws_region, runtime_env, debug=True): set_log_level(debug) runtime_envs = config.get_runtime_envs(runtime_env) for env in runtime_envs: for region in config.get_regions(env, aws_region): cm = ClusterManager(runtime_env=env, aws_region=region) result = cm.export('manifests')
def main(service_name='sample_application', aws_region='tyo', runtime_env='test', dry_run=True, namespace='alexrhino', debug=True, extra_vars='', manifest_type='configmap'): ''' Main function ''' if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") cluster_controller = ClusterManager(runtime_env, aws_region) general_service = MicroService(service_name, namespace) if manifest_type == 'configmap': manifests = general_service.get_configmap_manifests( runtime_env, aws_region) else: manifest = general_service.load_general_manifest( manifest_type, runtime_env, aws_region, extra_vars) manifests = [manifest] for manifest in manifests: cluster_controller.deploy_manifest(manifest, dry_run)
def consume_index( opts, sink_spec: str, sink_map, source_bucket, source_map, source_design: Union[str, bytes]) -> couchbaseConstants.PUMP_ERROR: if not source_design: return 0 try: sd = json.loads(source_design) if not sd: return 0 except ValueError as e: return f'error: could not parse source design; exception: {e!s}' try: sink_bucket = sink_map['buckets'][0] username = opts.username password = opts.password if opts.username_dest is not None and opts.password_dest is not None: username = opts.username_dest password = opts.password_dest rest = ClusterManager(sink_spec, username, password, opts.ssl, opts.no_ssl_verify, opts.cacert, False) _, errors = rest.restore_index_metadata(sink_bucket['name'], sd) return errors except ServiceNotAvailableException as e: return "No index service in cluster, skipping restore of indexes"
def consume_index(opts, sink_spec: str, sink_map, source_bucket, source_map, source_design: Union[str, bytes]) -> couchbaseConstants.PUMP_ERROR: if not source_design: return 0 try: sd = json.loads(source_design) if not sd: return 0 except ValueError as e: return f'error: could not parse source design; exception: {e!s}' try: sink_bucket = sink_map['buckets'][0] username = opts.username password = opts.password if opts.username_dest is not None and opts.password_dest is not None: username = opts.username_dest password = opts.password_dest rest = ClusterManager(sink_spec, username, password, opts.ssl, opts.no_ssl_verify, opts.cacert, False) _, errors = rest.restore_index_metadata(sink_bucket['name'], sd) return errors except ServiceNotAvailableException as e: return "No index service in cluster, skipping restore of indexes"
def main(image_tag, aws_region='vir', runtime_env='dev', dry_run=True, debug=True): ''' Main function ''' service_name = 'sample_application' namespace = 'alexrhino' if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") cm = ClusterManager(runtime_env, aws_region) my_service = SampleApplication('sample_application', 'alexrhino') image_url = my_service.get_image_url(image_tag) extra_vars = {'image_url': image_url} manifest_spec = my_service.load_general_manifest('deploy', runtime_env, aws_region, extra_vars=extra_vars) if not dry_run: my_service.update_s3_image_url(runtime_env, aws_region, image_tag) cm.deploy_manifest(manifest_spec, dry_run)
def rest_couchbase( opts, spec: str, check_sink_credential: bool = False ) -> Tuple[couchbaseConstants.PUMP_ERROR, Optional[Dict[str, Any]]]: spec = spec.replace('couchbase://', 'http://') username = opts.username password = opts.password if check_sink_credential and opts.username_dest is not None and opts.password_dest is not None: username = opts.username_dest password = opts.password_dest rest = ClusterManager(spec, username, password, opts.ssl, False, None, False) result, errors = rest.list_buckets(True) if errors: return errors[0], None buckets = [] for bucket in result: if bucket["bucketType"] in ["membase", "couchbase", "ephemeral"]: buckets.append(bucket) return 0, { 'spec': spec, 'buckets': buckets, 'spec_parts': parse_spec(opts, spec, 8091) }
def provide_fts_index(opts, source_spec, source_bucket, source_map): try: rest = ClusterManager(source_spec, opts.username, opts.password, opts.ssl, False, None, False) result, errors = rest.get_fts_index_metadata(source_bucket['name']) if errors: return errors, None return 0, json.dumps(result) except ServiceNotAvailableException, e: return 0, None
def test_topology_progress(self): client = ClusterManager('http://localhost:8091', 'u', 'p') tasks = TaskGetter([ { 'type': 'rebalance', 'status': 'running', 'progress': 0.5, 'recommendedRefreshPeriod': 0.1, 'detailedProgress': {} }, { 'type': 'rebalance', 'status': 'running', 'progress': 0.6, 'recommendedRefreshPeriod': 0.1, 'detailedProgress': {} }, { 'type': 'rebalance', 'status': 'running', 'progress': 0.7, 'recommendedRefreshPeriod': 0.1, 'detailedProgress': {} }, { 'type': 'rebalance', 'status': 'notRunning', 'statusIsStale': True }, { 'type': 'rebalance', 'status': 'notRunning', 'masterRequestTimedOut': True }, { 'type': 'rebalance', 'status': 'running', 'progress': 0.9, 'recommendedRefreshPeriod': 0.1, 'detailedProgress': {} }, { 'type': 'rebalance', 'status': 'notRunning' }, ]) client.get_tasks = tasks.get_task bar = TopologyProgressBar(client, 'rebalance', True) err = bar.show() self.assertIsNone(err) self.assertEqual(tasks.calls, 7)
def provide_fts_alias(opts, source_spec: str, source_bucket: Dict[str, Any], source_map) -> \ Tuple[couchbaseConstants.PUMP_ERROR, Optional[str]]: try: rest = ClusterManager(source_spec, opts.username, opts.password, opts.ssl, opts.no_ssl_verify, opts.cacert, False) result, errors = rest.get_fts_index_alias() if errors: return errors, None return 0, json.dumps(result) except ServiceNotAvailableException: return 0, None
def provide_fts_alias(opts, source_spec: str, source_bucket: Dict[str, Any], source_map) -> \ Tuple[couchbaseConstants.PUMP_ERROR, Optional[str]]: try: rest = ClusterManager(source_spec, opts.username, opts.password, opts.ssl, opts.no_ssl_verify, opts.cacert, False) result, errors = rest.get_fts_index_alias() if errors: return errors, None return 0, json.dumps(result) except ServiceNotAvailableException as e: return 0, None
def provide_index(opts, source_spec, source_bucket, source_map): try: rest = ClusterManager(source_spec, opts.username, opts.password, opts.ssl, opts.no_ssl_verify, opts.cacert, False) result, errors = rest.get_index_metadata(source_bucket['name']) if errors: return errors, None return 0, json.dumps(result["result"]) except ServiceNotAvailableException as e: return 0, None
def main(aws_region='fra', runtime_env='test', namespace='alexrhino', debug=True, container='sample_application-service-log', tail_lines=1): ''' Main function ''' if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") cm = ClusterManager(runtime_env, aws_region) cm.get_logs(namespace, container, tail_lines)
def rest_couchbase(opts, spec): spec = spec.replace('couchbase://', 'http://') spec_parts = parse_spec(opts, spec, 8091) rest = ClusterManager(spec, opts.username, opts.password, opts.ssl, False, None, False) result, errors = rest.list_buckets(True) if errors: return errors[0], None buckets = [] for bucket in result: if bucket["bucketType"] in ["membase", "couchbase", "ephemeral"]: buckets.append(bucket) return 0, {'spec': spec, 'buckets': buckets, 'spec_parts': parse_spec(opts, spec, 8091)}
def main(pod_name, container, aws_region='vir', runtime_env='dev', namespace='alexrhino', command='date', debug=False): ''' Main function ''' if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") cm = ClusterManager(runtime_env, aws_region) cm.exec_command(namespace, pod_name, container, command)
def main(aws_region='vir', runtime_env='dev', debug=False): ''' Main function ''' service_name = 'sample_application' namespace = 'alexrhino' if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") cm = ClusterManager(runtime_env, aws_region) sample_application_service = SampleApplication('sample_application', 'alexrhino') manifest_spec = sample_application_service.load_general_manifest( 'deploy', runtime_env, aws_region) cm.list_images(manifest_spec)
def main(aws_region='vir', runtime_env='dev', debug=False): ''' Main function ''' service_name='sample_application' if runtime_env == 'dev': namespace='alexrhino' else: namespace='alexrhino-' + runtime_env if debug: logging.basicConfig(level=logging.DEBUG) logging.info("DEBUG ON") else: logging.basicConfig(level=logging.INFO) logging.info("DEBUG OFF") cm = ClusterManager(runtime_env, aws_region) cm.list_pods(namespace)
def consume_fts_index(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: index_defs = json.loads(source_design) if not index_defs: return 0 except ValueError as e: return "error: could not parse fts index definitions; exception: %s" % (e) try: username = opts.username password = opts.password if opts.username_dest is not None and opts.password_dest is not None: username = opts.username_dest password = opts.password_dest rest = ClusterManager(sink_spec, username, password, opts.ssl, opts.no_ssl_verify, opts.cacert, False) _, errors = rest.restore_fts_index_metadata(index_defs) return errors except ServiceNotAvailableException as e: return "No fts service in cluster, skipping restore of indexes"
def run_contest_remotely(self, hosts, resume_folder=None, first=True): self.prepare_dirs() if resume_folder is not None: contest_folder = os.path.split(self.tmp_dir)[1] resume_folder = os.path.join(resume_folder, contest_folder) shutil.rmtree(self.tmp_logs_dir) shutil.copytree(os.path.join(resume_folder, "logs-run"), self.tmp_logs_dir) shutil.rmtree(self.tmp_replays_dir) shutil.copytree(os.path.join(resume_folder, "replays-run"), self.tmp_replays_dir) jobs = self.resume_contest_jobs() else: jobs = self.run_contest_jobs() # This is the core package to be transferable to each host core_req_file = TransferableFile( local_path=os.path.join(TMP_DIR, CORE_CONTEST_TEAM_ZIP_FILE), remote_path=os.path.join("/tmp", CORE_CONTEST_TEAM_ZIP_FILE), ) # create cluster with hosts and jobs and run it by starting it, and then analyze output results # results will contain all outputs from every game played if first: cm = ClusterManager(hosts, jobs, [core_req_file]) else: # subsequent contests don't need to transfer the files again cm = ClusterManager(hosts, jobs, None) # sys.exit(0) results = cm.start() print( "========================= GAMES FINISHED - NEXT ANALYSING OUTPUT OF GAMES ========================= " ) self._analyse_all_outputs(results) self._calculate_team_stats()
def deploy_all_regions(runtime_env, namespace, service_name, manifest_type, dry_run=True, debug=True): ''' Main function ''' set_log_level(debug) all_changed = [] runtime_envs = config.get_runtime_envs(runtime_env) if service_name == 'sample_application': my_service = SampleApplication(service_name, namespace) else: my_service = MicroService(service_name, namespace) for env in runtime_envs: for region in config.get_regions(env, 'all'): cm = ClusterManager(env, region) if config.has_manifest(env, region, namespace, service_name, manifest_type): manifest_spec = my_service.load_general_manifest( manifest_type, runtime_env, region) cm.deploy_manifest(manifest_spec, dry_run)
def consume_design( opts, sink_spec: str, sink_map, source_bucket, source_map, source_design: Union[str, bytes]) -> couchbaseConstants.PUMP_ERROR: if not source_design: return 0 try: sd = json.loads(source_design) except ValueError as e: return f'error: could not parse source design; exception: {e!s}' if not sd: return 0 if (not sink_map['buckets'] or len(sink_map['buckets']) != 1 or not sink_map['buckets'][0] or not sink_map['buckets'][0]['name']): return "error: design sink incorrect sink_map bucket" spec_parts = pump.parse_spec(opts, sink_spec, 8091) if not spec_parts: return "error: design sink no spec_parts: " + sink_spec sink_bucket = sink_map['buckets'][0] sink_nodes = pump.filter_bucket_nodes(sink_bucket, spec_parts) or \ sink_bucket['nodes'] if not sink_nodes: return "error: design sink nodes missing" couch_api_base = sink_nodes[0].get('couchApiBase') if not couch_api_base: return f'error: cannot restore bucket design on a couchbase cluster that does not support couch API;' \ f' the couchbase cluster may be an older, pre-2.0 version; please check your cluster URL: {sink_spec}' host, port, user, pswd, path = \ pump.parse_spec(opts, couch_api_base, 8092) if user is None: user = spec_parts[2] # Default to the main REST user/pwsd. pswd = spec_parts[3] if opts.username_dest is not None and opts.password_dest is not None: user = opts.username_dest user = opts.password_dest if type(sd) is dict: id = sd.get('_id', None) if id: str_source = _to_string(source_design) err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=str_source, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' else: stmts = sd.get('statements', []) hostname = f'http://{spec_parts[0]}:{spec_parts[1]!s}' cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error( f'N1QL query {stmt["statement"]} failed due to {errors}' ) if result and 'errors' in result: for error in result['errors']: logging.error( f'N1QL query {stmt["statement"]} failed due to error `{error["msg"]}`' ) except ServiceNotAvailableException as e: logging.error( "Failed to restore indexes, cluster does not contain a query node" ) elif type(sd) is list: for row in sd: logging.debug(f'design_doc row: {row!s}') doc = row.get('doc', None) if not doc: stmt = row.get('statement', None) if not stmt: return f'error: missing design doc or index statement in row: {row}' # publish index return 0 if 'json' in doc and 'meta' in doc: js = doc['json'] id = doc['meta'].get('id', None) if not id: return f'error: missing id for design doc: {row}' else: # Handle design-doc from 2.0DP4. js = doc if '_rev' in js: del js['_rev'] id = row.get('id', None) if not id: return f'error: missing id for row: {row}' js_doc = json.dumps(js) if id.startswith(CBSink.DDOC_HEAD): id = CBSink.DDOC_HEAD + urllib.parse.quote( id[len(CBSink.DDOC_HEAD):], '') else: id = urllib.parse.quote(id, '') logging.debug(f'design_doc: {js_doc}') logging.debug(f'design_doc id: {id} at: {path}/{id}') try: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=js_doc, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' except Exception as e: return f'error: design sink exception: {e}; couch_api_base: {couch_api_base}' logging.debug(f'design_doc created at: {path}/{id}') return 0
def consume_design(opts, sink_spec: str, sink_map, source_bucket, source_map, source_design: Union[str, bytes]) -> couchbaseConstants.PUMP_ERROR: if not source_design: return 0 try: sd = json.loads(source_design) except ValueError as e: return f'error: could not parse source design; exception: {e!s}' if not sd: return 0 if (not sink_map['buckets'] or len(sink_map['buckets']) != 1 or not sink_map['buckets'][0] or not sink_map['buckets'][0]['name']): return "error: design sink incorrect sink_map bucket" spec_parts = pump.parse_spec(opts, sink_spec, 8091) if not spec_parts: return "error: design sink no spec_parts: " + sink_spec sink_bucket = sink_map['buckets'][0] sink_nodes = pump.filter_bucket_nodes(sink_bucket, spec_parts) or \ sink_bucket['nodes'] if not sink_nodes: return "error: design sink nodes missing" couch_api_base = sink_nodes[0].get('couchApiBase') if not couch_api_base: return f'error: cannot restore bucket design on a couchbase cluster that does not support couch API;' \ f' the couchbase cluster may be an older, pre-2.0 version; please check your cluster URL: {sink_spec}' host, port, user, pswd, path = \ pump.parse_spec(opts, couch_api_base, 8092) if user is None: user = spec_parts[2] # Default to the main REST user/pwsd. pswd = spec_parts[3] if opts.username_dest is not None and opts.password_dest is not None: user = opts.username_dest user = opts.password_dest if type(sd) is dict: id = sd.get('_id', None) if id: str_source = _to_string(source_design) err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=str_source, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' else: stmts = sd.get('statements', []) hostname = f'http://{spec_parts[0]}:{spec_parts[1]!s}' cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error(f'N1QL query {stmt["statement"]} failed due to {errors}') if result and 'errors' in result: for error in result['errors']: logging.error(f'N1QL query {stmt["statement"]} failed due to error `{error["msg"]}`') except ServiceNotAvailableException as e: logging.error("Failed to restore indexes, cluster does not contain a query node") elif type(sd) is list: for row in sd: logging.debug(f'design_doc row: {row!s}') doc = row.get('doc', None) if not doc: stmt = row.get('statement', None) if not stmt: return f'error: missing design doc or index statement in row: {row}' else: #publish index return 0 if 'json' in doc and 'meta' in doc: js = doc['json'] id = doc['meta'].get('id', None) if not id: return f'error: missing id for design doc: {row}' else: # Handle design-doc from 2.0DP4. js = doc if '_rev' in js: del js['_rev'] id = row.get('id', None) if not id: return f'error: missing id for row: {row}' js_doc = json.dumps(js) if id.startswith(CBSink.DDOC_HEAD): id = CBSink.DDOC_HEAD + urllib.parse.quote(id[len(CBSink.DDOC_HEAD):], '') else: id = urllib.parse.quote(id, '') logging.debug(f'design_doc: {js_doc}') logging.debug(f'design_doc id: {id} at: {path}/{id}') try: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, f'{path}/{id}', method='PUT', body=js_doc, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return f'error: could not restore design doc id: {id}; response: {response}; err: {err}' except Exception as e: return f'error: design sink exception: {e}; couch_api_base: {couch_api_base}' logging.debug(f'design_doc created at: {path}/{id}') return 0
return rv @staticmethod def consume_fts_index(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: index_defs = json.loads(source_design) if not index_defs: return 0 except ValueError, e: return "error: could not parse fts index definitions; exception: %s" % (e) try: rest = ClusterManager(sink_spec, opts.username, opts.password, opts.ssl, False, None, False) _, errors = rest.restore_fts_index_metadata(index_defs) return errors except ServiceNotAvailableException, e: return "No fts service in cluster, skipping restore of indexes" @staticmethod def consume_index(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: sd = json.loads(source_design) if not sd: return 0 except ValueError, e:
jobs.append( Job(commands=commands, required_files=required_files, return_files=return_files, id=job_id)) return jobs def create_hosts(workers_settings): hosts = [] for worker in workers_settings: host = Host( no_cpu=worker['no_cpu'], hostname=worker['hostname'], username=worker['username'], password=worker['password'], key_filename=worker['private_key_file'] ) hosts.append(host) return hosts if __name__ == "__main__": args = load_settings() settings = load_json(args.config_file) jobs = create_jobs(settings) workers_settings = load_json("workers.json") hosts = create_hosts(args.workers_file) cm = ClusterManager(hosts, jobs) results = cm.start()
def consume_design(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: sd = json.loads(source_design) except ValueError as e: return "error: could not parse source design; exception: %s" % (e) if not sd: return 0 if (not sink_map['buckets'] or len(sink_map['buckets']) != 1 or not sink_map['buckets'][0] or not sink_map['buckets'][0]['name']): return "error: design sink incorrect sink_map bucket" spec_parts = pump.parse_spec(opts, sink_spec, 8091) if not spec_parts: return "error: design sink no spec_parts: " + sink_spec sink_bucket = sink_map['buckets'][0] sink_nodes = pump.filter_bucket_nodes(sink_bucket, spec_parts) or \ sink_bucket['nodes'] if not sink_nodes: return "error: design sink nodes missing" couch_api_base = sink_nodes[0].get('couchApiBase') if not couch_api_base: return "error: cannot restore bucket design" \ " on a couchbase cluster that does not support couch API;" \ " the couchbase cluster may be an older, pre-2.0 version;" \ " please check your cluster URL: " + sink_spec host, port, user, pswd, path = \ pump.parse_spec(opts, couch_api_base, 8092) if user is None: user = spec_parts[2] # Default to the main REST user/pwsd. pswd = spec_parts[3] if opts.username_dest is not None and opts.password_dest is not None: user = opts.username_dest user = opts.password_dest if type(sd) is dict: id = sd.get('_id', None) if id: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, path + "/" + id, method='PUT', body=source_design, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return ("error: could not restore design doc id: %s" + "; response: %s; err: %s") % (id, response, err) else: stmts = sd.get('statements', []) hostname = 'http://' + spec_parts[0] + ':' + str(spec_parts[1]) cm = ClusterManager(hostname, user, pswd, opts.ssl) try: for stmt in stmts: result, errors = cm.n1ql_query(stmt['statement'], stmt.get('args', None)) if errors: logging.error('N1QL query %s failed due to %s' % (stmt['statement'], errors)) if result and 'errors' in result: for error in result['errors']: logging.error('N1QL query %s failed due to error `%s`' % (stmt['statement'], error['msg'])) except ServiceNotAvailableException as e: logging.error("Failed to restore indexes, cluster does not contain a" + " query node") elif type(sd) is list: for row in sd: logging.debug("design_doc row: " + str(row)) doc = row.get('doc', None) if not doc: stmt = row.get('statement', None) if not stmt: return "error: missing design doc or index statement in row: %s" % (row) else: #publish index return 0 if 'json' in doc and 'meta' in doc: js = doc['json'] id = doc['meta'].get('id', None) if not id: return "error: missing id for design doc: %s" % (row) else: # Handle design-doc from 2.0DP4. js = doc if '_rev' in js: del js['_rev'] id = row.get('id', None) if not id: return "error: missing id for row: %s" % (row) js_doc = json.dumps(js) if id.startswith(CBSink.DDOC_HEAD): id = CBSink.DDOC_HEAD + urllib.parse.quote(id[len(CBSink.DDOC_HEAD):], '') else: id = urllib.parse.quote(id, '') logging.debug("design_doc: " + js_doc) logging.debug("design_doc id: " + id + " at: " + path + "/" + id) try: err, conn, response = \ pump.rest_request(host, int(port), user, pswd, opts.ssl, path + "/" + id, method='PUT', body=js_doc, reason="consume_design", verify=opts.no_ssl_verify, ca_cert=opts.cacert) if conn: conn.close() if err: return ("error: could not restore design doc id: %s" + "; response: %s; err: %s") % (id, response, err) except Exception as e: return ("error: design sink exception: %s" + "; couch_api_base: %s") % (e, couch_api_base) logging.debug("design_doc created at: " + path + "/" + id) return 0
@staticmethod def consume_fts_index(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: index_defs = json.loads(source_design) if not index_defs: return 0 except ValueError, e: return "error: could not parse fts index definitions; exception: %s" % ( e) try: rest = ClusterManager(sink_spec, opts.username, opts.password, opts.ssl, False, None, False) _, errors = rest.restore_fts_index_metadata(index_defs) return errors except ServiceNotAvailableException, e: return "No fts service in cluster, skipping restore of indexes" @staticmethod def consume_index(opts, sink_spec, sink_map, source_bucket, source_map, source_design): if not source_design: return 0 try: sd = json.loads(source_design) if not sd: return 0
def run(self, is_local="false"): if is_local == "true": self.run_local() return start_time = datetime.now() end_time = datetime.now() time_diff = end_time - start_time total_seconds = time_diff.total_seconds() slacker.do_slack("Dataproc Cluster Status", self.job_name, [{ "title": "Cluster Status", "value": "Intiating" }], 0) dpm = ClusterManager(self.project_config, self.cluster_config) dpm.create_cluster() dpm.wait_for_cluster_creation() slacker.do_slack("Dataproc Cluster Status", self.job_name, [{ "title": "Cluster Status", "value": "Created" }], 0) cm = CodeManager(self.job_config.get('job_name')) job_config_gcs = cm.upload_files_to_gcs() job_config_gcs.update(self.job_config) job_id = dpm.submit_pyspark_job(job_config_gcs) slacker.do_slack("Dataproc Job Status", self.job_name, [{ "title": "Job Status", "value": "Submitted and Running" }], 0) dpm.wait_for_job(job_id) slacker.do_slack("Dataproc Job Status", self.job_name, [{ "title": "Job Status", "value": "Completed" }], 0) dpm.delete_cluster() slacker.do_slack("Dataproc Cluster Status", self.job_name, [{ "title": "Cluster Status", "value": "Deleting" }], 0)
from color_manager import ColorRequestManager from cluster_manager import ClusterManager from dotenv import load_dotenv import itertools import os load_dotenv() mongo_uri = os.getenv("MONGO") app = Flask(__name__) CORS(app) app.config["MONGO_URI"] = mongo_uri mongo = PyMongo(app) db = mongo.cx.music # get to db via MongoClient clusterer = ClusterManager() NEXT_ID = 0 def generate_album(doc): album = dict() global NEXT_ID album["id"] = NEXT_ID NEXT_ID += 1 album["name"] = doc["album"] album["artist"] = doc["artist"] album["year"] = doc["year"] album["genres"] = doc["genres"] album["spotifyUrl"] = doc["spotify_url"]