def initialize(setupObjectToUse=None, useLocalEvaluator=True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def main(parsedArguments):
Runtime.initialize()
testMissingMapFile()
testMissingDatFiles()
testMissingIndexFiles()
testCorruptMapFile()
testCorruptDatFiles()
testCorruptIndexFiles()
def main(parsedArguments):
Runtime.initialize()
testMissingMapFile()
testMissingDatFiles()
testMissingIndexFiles()
testCorruptMapFile()
testCorruptDatFiles()
testCorruptIndexFiles()
def initialize(setupObjectToUse = None, useLocalEvaluator = True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def main(argv): t0 = time.time() Runtime.initialize(Setup.defaultSetup()) runtime = Runtime.getMainRuntime() handler = runtime.getInterpreterTraceHandler() handler.replayTracesFromFile(argv[1]) print "took ", time.time() - t0
def main(argv):
t0 = time.time()
Runtime.initialize(Setup.defaultSetup())
runtime = Runtime.getMainRuntime()
handler = runtime.getInterpreterTraceHandler()
handler.replayTracesFromFile(argv[1])
print "took ", time.time() - t0
def main(parsedArguments):
Runtime.initialize()
tester = AxiomTester(
parsedArguments.numRandVals,
parsedArguments.numRelaxations,
parsedArguments.maxForRelax,
parsedArguments.maxForRand,
parsedArguments.testAxiomsPath,
parsedArguments.seed
)
tester.test_axiom_consistency()
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory,
s3InterfaceFactory=None,
objectStore=None):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.s3InterfaceFactory = s3InterfaceFactory
self.objectStore = objectStore
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc
self.eventHandler = eventHandler
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcmalloc:
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
#If the "s3InterfaceFactory" is not in-memory, we use real out of process python.
#it would be better if this were more explicit
outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess)
self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
self.datasetLoadService = None
if self.s3InterfaceFactory:
externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
callbackScheduler
)
self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
self.s3InterfaceFactory,
self.objectStore,
self.vdm,
externalDatasetChannel.makeQueuelike(callbackScheduler)
)
self.cumulusWorker.startComputations()
if self.datasetLoadService:
self.datasetLoadService.startService()
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride
)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()
def __init__(self,
callbackScheduler,
sharedStateViewFactory,
computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(
sharedStateViewFactory.createView()
)
logging.info("attached shared state view.")
simultaneously(
initSynchronizer,
initValueGateway
)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(
self.graph,
self.computedValueGateway,
self.synchronizer
)
def __init__(self, callbackScheduler, sharedStateViewFactory, computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(sharedStateViewFactory.createView())
logging.info("attached shared state view.")
simultaneously(initSynchronizer, initValueGateway)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(self.graph, self.computedValueGateway, self.synchronizer)
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
rebuildAxiomSearchFunction
Generate appropriate axiom build functions after we've built the project successfully.
"""
import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime
Runtime.initialize(Setup.defaultSetup())
axioms = Runtime.getMainRuntime().getAxioms()
implvalCode, jovtCode = axioms.getCppWrapperCode()
uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"), "w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
print >> f, implvalCode
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"), "w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
print >> f, jovtCode
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
rebuildAxiomSearchFunction
Generate appropriate axiom build functions after we've built the project successfully.
"""
import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime
Runtime.initialize(Setup.defaultSetup())
axioms = Runtime.getMainRuntime().getAxioms()
implvalCode, jovtCode = axioms.getCppWrapperCode()
uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"),
"w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
print >> f, implvalCode
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"),
"w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
def __init__(self, ownAddress, channelListener, channelFactory,
eventHandler, callbackScheduler, diagnosticsDir, config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4())))
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set(
) # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler, self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler, self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None ()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval, 1024 * 1024)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride, checkpointPolicy,
ExecutionContext.createContextConfiguration(), diagnosticsDir
or ""), self.vdm, self.offlineCache, eventHandler)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()