# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from eggroll.core.conf_keys import SessionConfKeys, TransferConfKeys, \
ClusterManagerConfKeys, NodeManagerConfKeys
from eggroll.core.constants import DeployModes
from eggroll.core.constants import ProcessorTypes, ProcessorStatus
from eggroll.core.constants import StoreTypes
from eggroll.core.meta_model import ErStore, ErStoreLocator, ErEndpoint, \
ErProcessor
from eggroll.core.session import ErSession
ER_STORE1 = ErStore(store_locator=ErStoreLocator(
store_type=StoreTypes.ROLLPAIR_LEVELDB, namespace="namespace",
name="name"))
def get_debug_test_context(is_standalone=False,
manager_port=4670,
egg_port=20001,
transfer_port=20002,
session_id='testing'):
manager_port = manager_port
egg_ports = [egg_port]
egg_transfer_ports = [transfer_port]
self_server_node_id = 2
options = {}
if is_standalone:
def map_values(_tagged_key, is_standalone, roll_site_header):
if is_standalone:
dst_name = _tagged_key
store_type = rp.get_store_type()
else:
dst_name = DELIM.join([
_tagged_key, self.dst_host,
str(self.dst_port), obj_type
])
store_type = StoreTypes.ROLLPAIR_ROLLSITE
if is_standalone:
status_rp = self.ctx.rp_ctx.load(
namespace,
STATUS_TABLE_NAME + DELIM + self.roll_site_session_id,
options=_options)
status_rp.disable_gc()
if isinstance(obj, RollPair):
status_rp.put(_tagged_key,
(obj_type.encode("utf-8"), rp.get_name(),
rp.get_namespace()))
else:
status_rp.put(
_tagged_key,
(obj_type.encode("utf-8"), dst_name, namespace))
else:
store = rp.get_store()
store_locator = store._store_locator
new_store_locator = ErStoreLocator(
store_type=store_type,
namespace=namespace,
name=dst_name,
total_partitions=store_locator._total_partitions,
partitioner=store_locator._partitioner,
serdes=store_locator._serdes)
# TODO:0: move options from job to store when database modification finished
options = {
"roll_site_header": roll_site_header,
"proxy_endpoint": self.ctx.proxy_endpoint,
"obj_type": obj_type
}
if isinstance(obj, RollPair):
roll_site_header._options[
'total_partitions'] = obj.get_store(
)._store_locator._total_partitions
L.info(
f"RollSite.push: pushing {roll_site_header}, type: RollPair, count: {obj.count()}"
)
else:
L.info(
f"RollSite.push: pushing {roll_site_header}, type: object"
)
rp.map_values(
lambda v: v,
output=ErStore(store_locator=new_store_locator),
options=options)
L.info(
f"RollSite.push: push {roll_site_header} done. type:{type(obj)}"
)
return _tagged_key
