threadCount=threadCount,
returnSimulation=True,
ioTaskThreadOverride=8,
useInMemoryCache=False,
timeout=30,
objectStore=self.createObjectStore(s3),
pageSizeOverride=pageSizeOverride)
result = simulation.executeExternalIoTask(
CumulusNative.ExternalIoTask.WriteCharBigvecToS3(
setupResults.asResult.result.getVectorBigvecGuid(),
CumulusNative.S3KeyAndCredentials(self.getTestDataBucket(),
keyGuid, "", "", "")),
timeout=60)
self.assertTrue(result.isSuccess(), result)
assert s3().getKeySize(self.getTestDataBucket(),
keyGuid) == bytecount
finally:
try:
s3().deleteKey(self.getTestDataBucket(), keyGuid)
except:
logging.warn("Failed to cleanup the test key: %s",
traceback.format_exc())
if __name__ == "__main__":
import ufora.config.Mainline as Mainline
Mainline.UnitTestMainline([FORA, Runtime])
def runTests():
if args.timeout is not None:
UnitTestCommon.startTimeoutThread(args.timeout[0])
try:
return executeTests(args)
except:
import traceback
logging.error("executeTests() threw an exception: \n%s",
traceback.format_exc())
return 1
if args.repeat:
while True:
runTests()
else:
return runTests()
if __name__ == "__main__":
#parse args, return zero and exit if help string was printed
parser = UnitTestArgumentParser()
generalArgumentsGroup = parser.add_argument_group(
title="General Arguments",
description="Arguments that affect the system as a whole")
Setup.addDefaultArguments(generalArgumentsGroup)
Mainline.addNoseVerbosityArgument(parser)
Mainline.UserFacingMainline(main, sys.argv, [], parser=parser)
f,
maxNToSearch=20,
timeoutInSec=20.0)
def test_vector_dot_product_large(self):
self.vector_dot_product(1000000.0)
def test_vector_dot_product_small(self):
self.vector_dot_product(1000.0)
def vector_dot_product(self, dimension):
with self.ufora.remotely:
a = np.arange(dimension)
b = np.arange(dimension)
def f(n):
with self.ufora.remotely:
for _ in xrange(n):
np.dot(a, b)
PerformanceTestReporter.testThroughput(
"pyfora.numpy.vector_dot_product_%d" % dimension,
f,
maxNToSearch=20,
timeoutInSec=20.0)
if __name__ == '__main__':
import ufora.config.Mainline as Mainline
Mainline.UnitTestMainline([])
dataLen = struct.unpack('I', sys.stdin.read(struct.calcsize('I')))[0]
data = sys.stdin.read(dataLen)
connectionData = pickle.loads(data)
maxFD = os.sysconf("SC_OPEN_MAX")
for fd in range(3, maxFD):
if fd != connectionData['socketFd']:
try:
os.close(fd)
except:
pass
handler = BackendGatewayRequestHandler(
connectionData['socketFd'],
connectionData['sharedStateAddress']
)
handler.handle()
finally:
sys.stderr.write(traceback.format_exc())
sys.stderr.write("closing connection handler\n")
sys.stderr.flush()
return 0
if __name__ == "__main__":
Mainline.UserFacingMainline(main, sys.argv, [FORA])
CumulusNative.replayCumulusWorkerEventStream(
events, parsedArguments.validation)
else:
CumulusNative.replayCumulusWorkerEventStreamFromFile(
parsedArguments.file, parsedArguments.validation)
return 0
def createParser():
desc = """Utility for validating a stream of LocalSchedulerEvent objects in a test failure. """
parser = Setup.defaultParser(description=desc)
parser.add_argument('file', help='names of file to be read')
parser.add_argument('-n',
'--no_validation',
dest='validation',
action='store_false',
default=True,
required=False,
help="don't validate the response stream")
return parser
if __name__ == "__main__":
Mainline.UserFacingMainline(main,
sys.argv,
modulesToInitialize=[],
parser=createParser())
if __name__ == "__main__":
callbackSchedulerFactory = CallbackScheduler.createSimpleCallbackSchedulerFactory(
)
callbackScheduler = callbackSchedulerFactory.createScheduler(
"sharedStateScheduler", 1)
parser = argparse.ArgumentParser()
parser.add_argument('--cacheDir')
parsed, remaining = parser.parse_known_args(sys.argv[1:])
def startSharedState(args):
if parsed.cacheDir:
logging.info("Shared state cache directory: %s", parsed.cacheDir)
Setup.config().sharedStateCache = parsed.cacheDir
service = SharedStateService.SharedStateService(callbackScheduler)
service.startService()
service.blockUntilListening()
def handleSigTerm(signum, frame):
logging.info("SIGTERM received - shutting down")
service.stopService()
signal.signal(signal.SIGTERM, handleSigTerm)
signal.pause()
logging.info("process exiting")
Mainline.UserFacingMainline(startSharedState, [sys.argv[0]] + remaining)
import signal
import sys
import ufora.config.Mainline as Mainline
import ufora.test.ClusterSimulation as ClusterSimulation
def main(args):
print "Starting cluster simulation..."
simulator = ClusterSimulation.Simulator.createGlobalSimulator()
simulator.startService()
def signal_handler(sig, _):
signal_name = '(unknown)'
if sig == signal.SIGINT:
signal_name = 'SIGINT'
elif sig == signal.SIGTERM:
signal_name = 'SIGTERM'
print 'Received ', signal_name, 'signal. Exiting.'
simulator.stopService()
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
print "Press Ctrl+C to exit."
signal.pause()
if __name__ == "__main__":
Mainline.UserFacingMainline(main, sys.argv, modulesToInitialize=[])
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import unittest
import pyfora
import ufora.config.Setup as Setup
import ufora.FORA.CUDA.test.GpuPerformanceTestCases as GpuPerformanceTestCases
import ufora.FORA.CUDA.test.GpuTestUtil as GpuTestUtil
import ufora.FORA.python.FORA as FORA
class GpuPythonPerformanceTests(
unittest.TestCase,
GpuTestUtil.GpuTestUtil,
GpuPerformanceTestCases.GpuPerformanceTestCases
):
pass
if __name__ == '__main__':
import ufora.config.Mainline as Mainline
Mainline.UnitTestMainline([FORA])
args.lang,args.native,args.py,args.scripts,args.node = True,True,True,True,False
def runTests():
if args.timeout is not None:
UnitTestCommon.startTimeoutThread(args.timeout[0])
try:
return executeTests(args)
except:
import traceback
logging.error("executeTests() threw an exception: \n%s", traceback.format_exc())
return 1
if args.repeat:
while True:
runTests()
else:
return runTests()
if __name__ == "__main__":
#parse args, return zero and exit if help string was printed
parser = UnitTestArgumentParser()
generalArgumentsGroup = parser.add_argument_group(
title="General Arguments",
description="Arguments that affect the system as a whole")
Setup.addDefaultArguments(generalArgumentsGroup)
Mainline.addNoseVerbosityArgument(parser)
Mainline.UserFacingMainline(main, sys.argv, [], parser=parser)
testMessage, {
"DEBUG":
testMessage if loggingLevel <= logging.DEBUG else None,
"INFO":
testMessage if loggingLevel <= logging.INFO else None,
"WARN":
testMessage if loggingLevel <= logging.WARN else None,
"ERROR":
testMessage if loggingLevel <= logging.ERROR else None,
"CRITICAL":
testMessage
if loggingLevel <= logging.CRITICAL else None
})
except:
traceback.print_exc()
raise
pass
def test_logging_basic(self):
for level in levels:
loggingLevel = getattr(logging, level)
NativeLogging.setLogLevel(loggingLevel)
for testLevel in levels:
self.assertEqual(
NativeLogging.shouldLog(getattr(logging, testLevel)), True
if loggingLevel <= getattr(logging, testLevel) else False)
if __name__ == '__main__':
Mainline.UnitTestMainline([], disableLogCapture=True)
for t in threads:
t.start()
for t in threads:
t.join()
expectedDict = {}
for ix in range(threadcount):
expectedDict[ix] = True
self.assertEqual(expectedDict, worked)
def testWithTcp(self):
self.stressMultipleSharedStateReadWrites(True, 200)
def testWithRelay(self):
self.stressMultipleSharedStateReadWrites(False, 200)
@classmethod
def tearDownClass(cls):
cls.simulator.stopService()
if __name__ == '__main__':
import ufora.config.Mainline as Mainline
import ufora.config.LoginConfiguration as LoginConfiguration
Mainline.UnitTestMainline(
loginConfiguration=LoginConfiguration.LoginConfiguration(
"test_admin", "asdfasdf", True, {}))
# should appear as a dependent module
def removeFirstLine(string):
return string[string.find('\n') + 1:]
def testInMem():
val = subprocess.Popen(
['fora -e "1+2"'],
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
#make sure we pass the script-setup environment variables down
env=dict(os.environ)).communicate()
if removeFirstLine(val[0]).lstrip().rstrip() == '3':
return True
print 'testInMem failed:', val
return False
def main(parsedArguments):
if testInMem():
return 0
return 1
if __name__ == "__main__":
Mainline.UserFacingMainline(main, sys.argv)
for k in keyspaces:
storageForKeyspace = storage.storageForKeyspace(k, 0)
data = storageForKeyspace.readKeyValueMap()
entries = []
for key, val in data.iteritems():
entries.append(
NativeJson.Json({
'key': tuple(key[ix] for ix in range(len(key))),
'val': val
}))
toWrite.append({
'keyspace': (k.type, k.name, k.dimension),
'values': entries
})
print >> outputStream, NativeJson.Json(toWrite)
def main(parsedArgs):
sharedStateToHumanReadableFormat(parsedArgs.path, sys.stdout)
if __name__ == "__main__":
parser = Setup.defaultParser(
description="Dump the contents of SharedState to standard out")
parser.add_argument('path', help='path to the shared-state directory')
Mainline.UserFacingMainline(main, sys.argv, parser=parser)
FORA.eval(
"fun(){ let f = fun() { throw 1 }; try { f() } catch(...) { throw 2 } }"
)), FORANative.symbol_Call)))
context.compute()
self.assertTrue(context.getFinishedResult().isException())
def stringAllocShouldFailFun(self, ct):
return FORA.extractImplValContainer(
FORA.eval("""fun()
{
let s = "*";
let i = 0;
while (i < 100000)
{
s = s + "%s" + s + "%s";
i = i + 1;
}
}
""" % (" ", " " * ct)))
def test_large_string_alloc_fails_and_raises_foravalue_error(self):
for ix in range(10):
val = ForaValue.FORAValue(self.stringAllocShouldFailFun(ix))
self.assertRaises(ForaValue.FORAFailure, val)
if __name__ == "__main__":
import ufora.config.Mainline as Mainline
Mainline.UnitTestMainline([ExecutionContext, Evaluator])
