# distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from eggroll.core.conf_keys import SessionConfKeys, TransferConfKeys, \ ClusterManagerConfKeys, NodeManagerConfKeys from eggroll.core.constants import DeployModes from eggroll.core.constants import ProcessorTypes, ProcessorStatus from eggroll.core.constants import StoreTypes from eggroll.core.meta_model import ErStore, ErStoreLocator, ErEndpoint, \ ErProcessor from eggroll.core.session import ErSession ER_STORE1 = ErStore(store_locator=ErStoreLocator( store_type=StoreTypes.ROLLPAIR_LEVELDB, namespace="namespace", name="name")) def get_debug_test_context(is_standalone=False, manager_port=4670, egg_port=20001, transfer_port=20002, session_id='testing'): manager_port = manager_port egg_ports = [egg_port] egg_transfer_ports = [transfer_port] self_server_node_id = 2 options = {} if is_standalone:
def map_values(_tagged_key, is_standalone, roll_site_header): if is_standalone: dst_name = _tagged_key store_type = rp.get_store_type() else: dst_name = DELIM.join([ _tagged_key, self.dst_host, str(self.dst_port), obj_type ]) store_type = StoreTypes.ROLLPAIR_ROLLSITE if is_standalone: status_rp = self.ctx.rp_ctx.load( namespace, STATUS_TABLE_NAME + DELIM + self.roll_site_session_id, options=_options) status_rp.disable_gc() if isinstance(obj, RollPair): status_rp.put(_tagged_key, (obj_type.encode("utf-8"), rp.get_name(), rp.get_namespace())) else: status_rp.put( _tagged_key, (obj_type.encode("utf-8"), dst_name, namespace)) else: store = rp.get_store() store_locator = store._store_locator new_store_locator = ErStoreLocator( store_type=store_type, namespace=namespace, name=dst_name, total_partitions=store_locator._total_partitions, partitioner=store_locator._partitioner, serdes=store_locator._serdes) # TODO:0: move options from job to store when database modification finished options = { "roll_site_header": roll_site_header, "proxy_endpoint": self.ctx.proxy_endpoint, "obj_type": obj_type } if isinstance(obj, RollPair): roll_site_header._options[ 'total_partitions'] = obj.get_store( )._store_locator._total_partitions L.info( f"RollSite.push: pushing {roll_site_header}, type: RollPair, count: {obj.count()}" ) else: L.info( f"RollSite.push: pushing {roll_site_header}, type: object" ) rp.map_values( lambda v: v, output=ErStore(store_locator=new_store_locator), options=options) L.info( f"RollSite.push: push {roll_site_header} done. type:{type(obj)}" ) return _tagged_key