def validateOpfJsonValue(value, opfJsonSchemaFilename):
""" Validate a python object against an OPF json schema file
target: target python object to validate (typically a dictionary)
opfJsonSchemaFilename: OPF json schema filename containing the json schema
object. (e.g., opfTaskControlSchema.json)
Returns: nothing
Raises: jsonhelpers.ValidationError when value fails json validation
"""
# Create a path by joining the filename with our local json schema root
jsonSchemaPath = os.path.join(os.path.dirname(__file__), "jsonschema",
opfJsonSchemaFilename)
# Validate
jsonhelpers.validate(value, schemaPath=jsonSchemaPath)
return
def validateOpfJsonValue(value, opfJsonSchemaFilename):
""" Validate a python object against an OPF json schema file
target: target python object to validate (typically a dictionary)
opfJsonSchemaFilename: OPF json schema filename containing the json schema
object. (e.g., opfTaskControlSchema.json)
Returns: nothing
Raises: jsonhelpers.ValidationError when value fails json validation
"""
# Create a path by joining the filename with our local json schema root
jsonSchemaPath = os.path.join(os.path.dirname(__file__),
"jsonschema",
opfJsonSchemaFilename)
# Validate
jsonhelpers.validate(value, schemaPath=jsonSchemaPath)
return
def _runExperimentImpl(options, model=None):
"""Creates and runs the experiment
Args:
options: namedtuple ParseCommandLineOptionsResult
model: For testing: may pass in an existing OPF Model instance
to use instead of creating a new one.
Returns: referece to OPFExperiment instance that was constructed (this
is provided to aid with debugging) or None, if none was
created.
"""
jsonhelpers.validate(options.privateOptions, schemaDict=g_parsedPrivateCommandLineOptionsSchema)
# Load the experiment's description.py module
experimentDir = options.experimentDir
descriptionPyModule = opfhelpers.loadExperimentDescriptionScriptFromDir(experimentDir)
expIface = opfhelpers.getExperimentDescriptionInterfaceFromModule(descriptionPyModule)
# Handle "list checkpoints" request
if options.privateOptions["listAvailableCheckpoints"]:
_printAvailableCheckpoints(experimentDir)
return None
# Load experiment tasks
experimentTasks = expIface.getModelControl().get("tasks", [])
# If the tasks list is empty, and this is a nupic environment description
# file being run from the OPF, convert it to a simple OPF description file.
if len(experimentTasks) == 0 and expIface.getModelControl()["environment"] == OpfEnvironment.Nupic:
expIface.convertNupicEnvToOPF()
experimentTasks = expIface.getModelControl().get("tasks", [])
# Handle listTasks
if options.privateOptions["listTasks"]:
print "Available tasks:"
for label in [t["taskLabel"] for t in experimentTasks]:
print "\t", label
return None
# Construct the experiment instance
if options.privateOptions["runCheckpointName"]:
assert model is None
checkpointName = options.privateOptions["runCheckpointName"]
model = ModelFactory.loadFromCheckpoint(savedModelDir=_getModelCheckpointDir(experimentDir, checkpointName))
elif model is not None:
print "Skipping creation of OPFExperiment instance: caller provided his own"
else:
modelDescription = expIface.getModelDescription()
model = ModelFactory.create(modelDescription)
# Handle "create model" request
if options.privateOptions["createCheckpointName"]:
checkpointName = options.privateOptions["createCheckpointName"]
_saveModel(model=model, experimentDir=experimentDir, checkpointLabel=checkpointName)
return model
# Build the task list
# Default task execution index list is in the natural list order of the tasks
taskIndexList = range(len(experimentTasks))
customTaskExecutionLabelsList = options.privateOptions["taskLabels"]
if customTaskExecutionLabelsList:
taskLabelsList = [t["taskLabel"] for t in experimentTasks]
taskLabelsSet = set(taskLabelsList)
customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)
assert customTaskExecutionLabelsSet.issubset(taskLabelsSet), (
"Some custom-provided task execution labels don't correspond "
"to actual task labels: mismatched labels: %r; actual task "
"labels: %r."
) % (customTaskExecutionLabelsSet - taskLabelsSet, customTaskExecutionLabelsList)
taskIndexList = [taskLabelsList.index(label) for label in customTaskExecutionLabelsList]
print "#### Executing custom task list: %r" % [taskLabelsList[i] for i in taskIndexList]
# Run all experiment tasks
for taskIndex in taskIndexList:
task = experimentTasks[taskIndex]
# Create a task runner and run it!
taskRunner = _TaskRunner(model=model, task=task, cmdOptions=options)
taskRunner.run()
del taskRunner
if options.privateOptions["checkpointModel"]:
_saveModel(model=model, experimentDir=experimentDir, checkpointLabel=task["taskLabel"])
return model
def __init__(self, streamDef, bookmark=None, saveOutput=False,
isBlocking=True, maxTimeout=0, eofOnTimeout=False):
""" Base class constructor, performs common initialization
Parameters:
----------------------------------------------------------------
streamDef: The stream definition, potentially containing multiple sources
(not supported yet). See
/nupic/frameworks/opf/jsonschema/stream_def.json for the format
of this dict
bookmark: Bookmark to start reading from. This overrides the first_record
field of the streamDef if provided.
saveOutput: If true, save the output to a csv file in a temp directory.
The path to the generated file can be found in the log
output.
isBlocking: should read operation block *forever* if the next row of data
is not available, but the stream is not marked as 'completed'
yet?
maxTimeout: if isBlocking is False, max seconds to wait for more data before
timing out; ignored when isBlocking is True.
eofOnTimeout: If True and we get a read timeout (isBlocking must be False
to get read timeouts), assume we've reached the end of the
input and produce the last aggregated record, if one can be
completed.
"""
# Call superclass constructor
super(StreamReader, self).__init__()
loggerPrefix = 'com.numenta.nupic.data.StreamReader'
self._logger = logging.getLogger(loggerPrefix)
jsonhelpers.validate(streamDef,
schemaPath=pkg_resources.resource_filename(
jsonschema.__name__, "stream_def.json"))
assert len(streamDef['streams']) == 1, "Only 1 source stream is supported"
# Save constructor args
sourceDict = streamDef['streams'][0]
self._recordCount = 0
self._eofOnTimeout = eofOnTimeout
self._logger.debug('Reading stream with the def: %s', sourceDict)
# Dictionary to store record statistics (min and max of scalars for now)
self._stats = None
# ---------------------------------------------------------------------
# Get the stream definition params
# Limiting window of the stream. It would not return any records until
# 'first_record' ID is read (or very first with the ID above that). The
# stream will return EOS once it reads record with ID 'last_record' or
# above (NOTE: the name 'lastRecord' is misleading because it is NOT
# inclusive).
firstRecordIdx = sourceDict.get('first_record', None)
self._sourceLastRecordIdx = sourceDict.get('last_record', None)
# If a bookmark was given, then override first_record from the stream
# definition.
if bookmark is not None:
firstRecordIdx = None
# Column names must be provided in the streamdef json
# Special case is ['*'], meaning all available names from the record stream
self._streamFieldNames = sourceDict.get('columns', None)
if self._streamFieldNames != None and self._streamFieldNames[0] == '*':
self._needFieldsFiltering = False
else:
self._needFieldsFiltering = True
# Types must be specified in streamdef json, or in case of the
# file_recod_stream types could be implicit from the file
streamFieldTypes = sourceDict.get('types', None)
self._logger.debug('Types from the def: %s', streamFieldTypes)
# Validate that all types are valid
if streamFieldTypes is not None:
for dataType in streamFieldTypes:
assert FieldMetaType.isValid(dataType)
# Reset, sequence and time fields might be provided by streamdef json
streamResetFieldName = streamDef.get('resetField', None)
streamTimeFieldName = streamDef.get('timeField', None)
streamSequenceFieldName = streamDef.get('sequenceIdField', None)
self._logger.debug('r, t, s fields: %s, %s, %s', streamResetFieldName,
streamTimeFieldName,
streamSequenceFieldName)
# =======================================================================
# Open up the underlying record store
dataUrl = sourceDict.get('source', None)
assert dataUrl is not None
self._recordStore = self._openStream(dataUrl, isBlocking, maxTimeout,
bookmark, firstRecordIdx)
assert self._recordStore is not None
# =======================================================================
# Prepare the data structures we need for returning just the fields
# the caller wants from each record
recordStoreFields = self._recordStore.getFields()
self._recordStoreFieldNames = self._recordStore.getFieldNames()
if not self._needFieldsFiltering:
self._streamFieldNames = self._recordStoreFieldNames
# Build up the field definitions for each field. This is a list of tuples
# of (name, type, special)
self._streamFields = []
for dstIdx, name in enumerate(self._streamFieldNames):
if name not in self._recordStoreFieldNames:
raise RuntimeError("The column '%s' from the stream definition "
"is not present in the underlying stream which has the following "
"columns: %s" % (name, self._recordStoreFieldNames))
fieldIdx = self._recordStoreFieldNames.index(name)
fieldType = recordStoreFields[fieldIdx].type
fieldSpecial = recordStoreFields[fieldIdx].special
# If the types or specials were defined in the stream definition,
# then override what was found in the record store
if streamFieldTypes is not None:
fieldType = streamFieldTypes[dstIdx]
if streamResetFieldName is not None and streamResetFieldName == name:
fieldSpecial = FieldMetaSpecial.reset
if streamTimeFieldName is not None and streamTimeFieldName == name:
fieldSpecial = FieldMetaSpecial.timestamp
if (streamSequenceFieldName is not None and
streamSequenceFieldName == name):
fieldSpecial = FieldMetaSpecial.sequence
self._streamFields.append(FieldMetaInfo(name, fieldType, fieldSpecial))
# ========================================================================
# Create the aggregator which will handle aggregation of records before
# returning them.
self._aggregator = Aggregator(
aggregationInfo=streamDef.get('aggregation', None),
inputFields=recordStoreFields,
timeFieldName=streamDef.get('timeField', None),
sequenceIdFieldName=streamDef.get('sequenceIdField', None),
resetFieldName=streamDef.get('resetField', None))
# We rely on the aggregator to tell us the bookmark of the last raw input
# that contributed to the aggregated record
self._aggBookmark = None
# Compute the aggregation period in terms of months and seconds
if 'aggregation' in streamDef:
self._aggMonthsAndSeconds = nupic.support.aggregationToMonthsSeconds(
streamDef.get('aggregation'))
else:
self._aggMonthsAndSeconds = None
# ========================================================================
# Are we saving the generated output to a csv?
if saveOutput:
tmpDir = tempfile.mkdtemp()
outFilename = os.path.join(tmpDir, "generated_output.csv")
self._logger.info("StreamReader: Saving generated records to: '%s'" %
outFilename)
self._writer = FileRecordStream(streamID=outFilename,
write=True,
fields=self._streamFields)
else:
self._writer = None
def _runExperimentImpl(options, model=None):
"""Creates and runs the experiment
Args:
options: namedtuple ParseCommandLineOptionsResult
model: For testing: may pass in an existing OPF Model instance
to use instead of creating a new one.
Returns: reference to OPFExperiment instance that was constructed (this
is provided to aid with debugging) or None, if none was
created.
"""
jsonhelpers.validate(options.privateOptions,
schemaDict=g_parsedPrivateCommandLineOptionsSchema)
# Load the experiment's description.py module
experimentDir = options.experimentDir
descriptionPyModule = opf_helpers.loadExperimentDescriptionScriptFromDir(
experimentDir)
expIface = opf_helpers.getExperimentDescriptionInterfaceFromModule(
descriptionPyModule)
# Handle "list checkpoints" request
if options.privateOptions['listAvailableCheckpoints']:
_printAvailableCheckpoints(experimentDir)
return None
# Load experiment tasks
experimentTasks = expIface.getModelControl().get('tasks', [])
# If the tasks list is empty, and this is a nupic environment description
# file being run from the OPF, convert it to a simple OPF description file.
if (len(experimentTasks) == 0 and expIface.getModelControl()['environment']
== OpfEnvironment.Nupic):
expIface.convertNupicEnvToOPF()
experimentTasks = expIface.getModelControl().get('tasks', [])
# Ensures all the source locations are either absolute paths or relative to
# the nupic.datafiles package_data location.
expIface.normalizeStreamSources()
# Extract option
newSerialization = options.privateOptions['newSerialization']
# Handle listTasks
if options.privateOptions['listTasks']:
print "Available tasks:"
for label in [t['taskLabel'] for t in experimentTasks]:
print "\t", label
return None
# Construct the experiment instance
if options.privateOptions['runCheckpointName']:
assert model is None
checkpointName = options.privateOptions['runCheckpointName']
model = ModelFactory.loadFromCheckpoint(
savedModelDir=_getModelCheckpointDir(experimentDir,
checkpointName),
newSerialization=newSerialization)
elif model is not None:
print "Skipping creation of OPFExperiment instance: caller provided his own"
else:
modelDescription = expIface.getModelDescription()
model = ModelFactory.create(modelDescription)
# Handle "create model" request
if options.privateOptions['createCheckpointName']:
checkpointName = options.privateOptions['createCheckpointName']
_saveModel(model=model,
experimentDir=experimentDir,
checkpointLabel=checkpointName,
newSerialization=newSerialization)
return model
# Build the task list
# Default task execution index list is in the natural list order of the tasks
taskIndexList = range(len(experimentTasks))
customTaskExecutionLabelsList = options.privateOptions['taskLabels']
if customTaskExecutionLabelsList:
taskLabelsList = [t['taskLabel'] for t in experimentTasks]
taskLabelsSet = set(taskLabelsList)
customTaskExecutionLabelsSet = set(customTaskExecutionLabelsList)
assert customTaskExecutionLabelsSet.issubset(taskLabelsSet), \
("Some custom-provided task execution labels don't correspond "
"to actual task labels: mismatched labels: %r; actual task "
"labels: %r.") % (customTaskExecutionLabelsSet - taskLabelsSet,
customTaskExecutionLabelsList)
taskIndexList = [
taskLabelsList.index(label)
for label in customTaskExecutionLabelsList
]
print "#### Executing custom task list: %r" % [
taskLabelsList[i] for i in taskIndexList
]
# Run all experiment tasks
for taskIndex in taskIndexList:
task = experimentTasks[taskIndex]
# Create a task runner and run it!
taskRunner = _TaskRunner(model=model, task=task, cmdOptions=options)
taskRunner.run()
del taskRunner
if options.privateOptions['checkpointModel']:
_saveModel(model=model,
experimentDir=experimentDir,
checkpointLabel=task['taskLabel'],
newSerialization=newSerialization)
return model
def run(self):
""" Run this worker.
Parameters:
----------------------------------------------------------------------
retval: jobID of the job we ran. This is used by unit test code
when calling this working using the --params command
line option (which tells this worker to insert the job
itself).
"""
# Easier access to options
options = self._options
# ---------------------------------------------------------------------
# Connect to the jobs database
self.logger.info("Connecting to the jobs database")
cjDAO = ClientJobsDAO.get()
# Get our worker ID
self._workerID = cjDAO.getConnectionID()
if options.clearModels:
cjDAO.modelsClearAll()
# -------------------------------------------------------------------------
# if params were specified on the command line, insert a new job using
# them.
if options.params is not None:
options.jobID = cjDAO.jobInsert(client='hwTest', cmdLine="echo 'test mode'",
params=options.params, alreadyRunning=True,
minimumWorkers=1, maximumWorkers=1,
jobType = cjDAO.JOB_TYPE_HS)
if options.workerID is not None:
wID = options.workerID
else:
wID = self._workerID
buildID = Configuration.get('nupic.software.buildNumber', 'N/A')
logPrefix = '<BUILDID=%s, WORKER=HW, WRKID=%s, JOBID=%s> ' % \
(buildID, wID, options.jobID)
ExtendedLogger.setLogPrefix(logPrefix)
# ---------------------------------------------------------------------
# Get the search parameters
# If asked to reset the job status, do that now
if options.resetJobStatus:
cjDAO.jobSetFields(options.jobID,
fields={'workerCompletionReason': ClientJobsDAO.CMPL_REASON_SUCCESS,
'cancel': False,
#'engWorkerState': None
},
useConnectionID=False,
ignoreUnchanged=True)
jobInfo = cjDAO.jobInfo(options.jobID)
self.logger.info("Job info retrieved: %s" % (str(clippedObj(jobInfo))))
# ---------------------------------------------------------------------
# Instantiate the swarm object, which will handle the logic of
# which models to create when we need more to evaluate.
jobParams = json.loads(jobInfo.params)
# Validate job params
jsonSchemaPath = os.path.join(os.path.dirname(__file__),
"jsonschema",
"jobParamsSchema.json")
jsonhelpers.validate(jobParams, schemaPath=jsonSchemaPath)
hsVersion = jobParams.get('hsVersion', None)
if hsVersion == 'v2':
self._hs = SwarmV2(searchParams=jobParams, workerID=self._workerID,
cjDAO=cjDAO, jobID=options.jobID, logLevel=options.logLevel)
else:
raise RuntimeError("Invalid swarm implementation (%s) specified" \
% (hsVersion))
# =====================================================================
# The main loop.
try:
exit = False
numModelsTotal = 0
print >>sys.stderr, "reporter:status:Evaluating first model..."
while not exit:
# ------------------------------------------------------------------
# Choose a model to evaluate
batchSize = 10 # How many to try at a time.
modelIDToRun = None
while modelIDToRun is None:
if options.modelID is None:
# -----------------------------------------------------------------
# Get the latest results on all running models and send them to
# the swarm implementation
# This calls cjDAO.modelsGetUpdateCounters(), compares the
# updateCounters with what we have cached, fetches the results for the
# changed and new models, and sends those to the swarm
# implementation's self._hs.recordModelProgress() method.
self._processUpdatedModels(cjDAO)
# --------------------------------------------------------------------
# Create a new batch of models
(exit, newModels) = self._hs.createModels(numModels = batchSize)
if exit:
break
# No more models left to create, just loop. The _hs is waiting for
# all remaining running models to complete, and may pick up on an
# orphan if it detects one.
if len(newModels) == 0:
continue
# Try and insert one that we will run
for (modelParams, modelParamsHash, particleHash) in newModels:
jsonModelParams = json.dumps(modelParams)
(modelID, ours) = cjDAO.modelInsertAndStart(options.jobID,
jsonModelParams, modelParamsHash, particleHash)
# Some other worker is already running it, tell the swarm object
# so that it doesn't try and insert it again
if not ours:
mParamsAndHash = cjDAO.modelsGetParams([modelID])[0]
mResult = cjDAO.modelsGetResultAndStatus([modelID])[0]
results = mResult.results
if results is not None:
results = json.loads(results)
modelParams = json.loads(mParamsAndHash.params)
particleHash = cjDAO.modelsGetFields(modelID,
['engParticleHash'])[0]
particleInst = "%s.%s" % (
modelParams['particleState']['id'],
modelParams['particleState']['genIdx'])
self.logger.info("Adding model %d to our internal DB " \
"because modelInsertAndStart() failed to insert it: " \
"paramsHash=%s, particleHash=%s, particleId='%s'", modelID,
mParamsAndHash.engParamsHash.encode('hex'),
particleHash.encode('hex'), particleInst)
self._hs.recordModelProgress(modelID = modelID,
modelParams = modelParams,
modelParamsHash = mParamsAndHash.engParamsHash,
results = results,
completed = (mResult.status == cjDAO.STATUS_COMPLETED),
completionReason = mResult.completionReason,
matured = mResult.engMatured,
numRecords = mResult.numRecords)
else:
modelIDToRun = modelID
break
else:
# A specific modelID was passed on the command line
modelIDToRun = int(options.modelID)
mParamsAndHash = cjDAO.modelsGetParams([modelIDToRun])[0]
modelParams = json.loads(mParamsAndHash.params)
modelParamsHash = mParamsAndHash.engParamsHash
# Make us the worker
cjDAO.modelSetFields(modelIDToRun,
dict(engWorkerConnId=self._workerID))
if False:
# Change the hash and params of the old entry so that we can
# create a new model with the same params
for attempt in range(1000):
paramsHash = hashlib.md5("OrphanParams.%d.%d" % (modelIDToRun,
attempt)).digest()
particleHash = hashlib.md5("OrphanParticle.%d.%d" % (modelIDToRun,
attempt)).digest()
try:
cjDAO.modelSetFields(modelIDToRun,
dict(engParamsHash=paramsHash,
engParticleHash=particleHash))
success = True
except:
success = False
if success:
break
if not success:
raise RuntimeError("Unexpected failure to change paramsHash and "
"particleHash of orphaned model")
(modelIDToRun, ours) = cjDAO.modelInsertAndStart(options.jobID,
mParamsAndHash.params, modelParamsHash)
# ^^^ end while modelIDToRun ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# ---------------------------------------------------------------
# We have a model, evaluate it now
# All done?
if exit:
break
# Run the model now
self.logger.info("RUNNING MODEL GID=%d, paramsHash=%s, params=%s",
modelIDToRun, modelParamsHash.encode('hex'), modelParams)
# ---------------------------------------------------------------------
# Construct model checkpoint GUID for this model:
# jobParams['persistentJobGUID'] contains the client's (e.g., API Server)
# persistent, globally-unique model identifier, which is what we need;
persistentJobGUID = jobParams['persistentJobGUID']
assert persistentJobGUID, "persistentJobGUID: %r" % (persistentJobGUID,)
modelCheckpointGUID = jobInfo.client + "_" + persistentJobGUID + (
'_' + str(modelIDToRun))
self._hs.runModel(modelID=modelIDToRun, jobID = options.jobID,
modelParams=modelParams, modelParamsHash=modelParamsHash,
jobsDAO=cjDAO, modelCheckpointGUID=modelCheckpointGUID)
# TODO: don't increment for orphaned models
numModelsTotal += 1
self.logger.info("COMPLETED MODEL GID=%d; EVALUATED %d MODELs",
modelIDToRun, numModelsTotal)
print >>sys.stderr, "reporter:status:Evaluated %d models..." % \
(numModelsTotal)
print >>sys.stderr, "reporter:counter:swarmWorker,numModels,1"
if options.modelID is not None:
exit = True
# ^^^ end while not exit
finally:
# Provide swarm instance an opportunity to clean up temporary files
self._hs.close()
self.logger.info("FINISHED. Evaluated %d models." % (numModelsTotal))
print >>sys.stderr, "reporter:status:Finished, evaluated %d models" % (numModelsTotal)
return options.jobID
def __init__(self,
streamDef,
bookmark=None,
saveOutput=False,
isBlocking=True,
maxTimeout=0,
eofOnTimeout=False):
""" Base class constructor, performs common initialization
Parameters:
----------------------------------------------------------------
streamDef: The stream definition, potentially containing multiple sources
(not supported yet). See
/nupic/frameworks/opf/jsonschema/stream_def.json for the format
of this dict
bookmark: Bookmark to start reading from. This overrides the first_record
field of the streamDef if provided.
saveOutput: If true, save the output to a csv file in a temp directory.
The path to the generated file can be found in the log
output.
isBlocking: should read operation block *forever* if the next row of data
is not available, but the stream is not marked as 'completed'
yet?
maxTimeout: if isBlocking is False, max seconds to wait for more data before
timing out; ignored when isBlocking is True.
eofOnTimeout: If True and we get a read timeout (isBlocking must be False
to get read timeouts), assume we've reached the end of the
input and produce the last aggregated record, if one can be
completed.
"""
# Call superclass constructor
super(StreamReader, self).__init__()
loggerPrefix = 'com.numenta.nupic.data.StreamReader'
self._logger = logging.getLogger(loggerPrefix)
jsonhelpers.validate(streamDef,
schemaPath=pkg_resources.resource_filename(
jsonschema.__name__, "stream_def.json"))
assert len(
streamDef['streams']) == 1, "Only 1 source stream is supported"
# Save constructor args
sourceDict = streamDef['streams'][0]
self._recordCount = 0
self._eofOnTimeout = eofOnTimeout
self._logger.debug('Reading stream with the def: %s', sourceDict)
# Dictionary to store record statistics (min and max of scalars for now)
self._stats = None
# ---------------------------------------------------------------------
# Get the stream definition params
# Limiting window of the stream. It would not return any records until
# 'first_record' ID is read (or very first with the ID above that). The
# stream will return EOS once it reads record with ID 'last_record' or
# above (NOTE: the name 'lastRecord' is misleading because it is NOT
# inclusive).
firstRecordIdx = sourceDict.get('first_record', None)
self._sourceLastRecordIdx = sourceDict.get('last_record', None)
# If a bookmark was given, then override first_record from the stream
# definition.
if bookmark is not None:
firstRecordIdx = None
# Column names must be provided in the streamdef json
# Special case is ['*'], meaning all available names from the record stream
self._streamFieldNames = sourceDict.get('columns', None)
if self._streamFieldNames != None and self._streamFieldNames[0] == '*':
self._needFieldsFiltering = False
else:
self._needFieldsFiltering = True
# Types must be specified in streamdef json, or in case of the
# file_recod_stream types could be implicit from the file
streamFieldTypes = sourceDict.get('types', None)
self._logger.debug('Types from the def: %s', streamFieldTypes)
# Validate that all types are valid
if streamFieldTypes is not None:
for dataType in streamFieldTypes:
assert FieldMetaType.isValid(dataType)
# Reset, sequence and time fields might be provided by streamdef json
streamResetFieldName = streamDef.get('resetField', None)
streamTimeFieldName = streamDef.get('timeField', None)
streamSequenceFieldName = streamDef.get('sequenceIdField', None)
self._logger.debug('r, t, s fields: %s, %s, %s', streamResetFieldName,
streamTimeFieldName, streamSequenceFieldName)
# =======================================================================
# Open up the underlying record store
dataUrl = sourceDict.get('source', None)
assert dataUrl is not None
self._recordStore = self._openStream(dataUrl, isBlocking, maxTimeout,
bookmark, firstRecordIdx)
assert self._recordStore is not None
# =======================================================================
# Prepare the data structures we need for returning just the fields
# the caller wants from each record
recordStoreFields = self._recordStore.getFields()
self._recordStoreFieldNames = self._recordStore.getFieldNames()
if not self._needFieldsFiltering:
self._streamFieldNames = self._recordStoreFieldNames
# Build up the field definitions for each field. This is a list of tuples
# of (name, type, special)
self._streamFields = []
for dstIdx, name in enumerate(self._streamFieldNames):
if name not in self._recordStoreFieldNames:
raise RuntimeError(
"The column '%s' from the stream definition "
"is not present in the underlying stream which has the following "
"columns: %s" % (name, self._recordStoreFieldNames))
fieldIdx = self._recordStoreFieldNames.index(name)
fieldType = recordStoreFields[fieldIdx].type
fieldSpecial = recordStoreFields[fieldIdx].special
# If the types or specials were defined in the stream definition,
# then override what was found in the record store
if streamFieldTypes is not None:
fieldType = streamFieldTypes[dstIdx]
if streamResetFieldName is not None and streamResetFieldName == name:
fieldSpecial = FieldMetaSpecial.reset
if streamTimeFieldName is not None and streamTimeFieldName == name:
fieldSpecial = FieldMetaSpecial.timestamp
if (streamSequenceFieldName is not None
and streamSequenceFieldName == name):
fieldSpecial = FieldMetaSpecial.sequence
self._streamFields.append(
FieldMetaInfo(name, fieldType, fieldSpecial))
# ========================================================================
# Create the aggregator which will handle aggregation of records before
# returning them.
self._aggregator = Aggregator(
aggregationInfo=streamDef.get('aggregation', None),
inputFields=recordStoreFields,
timeFieldName=streamDef.get('timeField', None),
sequenceIdFieldName=streamDef.get('sequenceIdField', None),
resetFieldName=streamDef.get('resetField', None))
# We rely on the aggregator to tell us the bookmark of the last raw input
# that contributed to the aggregated record
self._aggBookmark = None
# Compute the aggregation period in terms of months and seconds
if 'aggregation' in streamDef:
self._aggMonthsAndSeconds = nupic.support.aggregationToMonthsSeconds(
streamDef.get('aggregation'))
else:
self._aggMonthsAndSeconds = None
# ========================================================================
# Are we saving the generated output to a csv?
if saveOutput:
tmpDir = tempfile.mkdtemp()
outFilename = os.path.join(tmpDir, "generated_output.csv")
self._logger.info(
"StreamReader: Saving generated records to: '%s'" %
outFilename)
self._writer = FileRecordStream(streamID=outFilename,
write=True,
fields=self._streamFields)
else:
self._writer = None
def run(self):
""" Run this worker.
Parameters:
----------------------------------------------------------------------
retval: jobID of the job we ran. This is used by unit test code
when calling this working using the --params command
line option (which tells this worker to insert the job
itself).
"""
# Easier access to options
options = self._options
# ---------------------------------------------------------------------
# Connect to the jobs database
self.logger.info("Connecting to the jobs database")
cjDAO = ClientJobsDAO.get()
# Get our worker ID
self._workerID = cjDAO.getConnectionID()
if options.clearModels:
cjDAO.modelsClearAll()
# -------------------------------------------------------------------------
# if params were specified on the command line, insert a new job using
# them.
if options.params is not None:
options.jobID = cjDAO.jobInsert(client='hwTest',
cmdLine="echo 'test mode'",
params=options.params,
alreadyRunning=True,
minimumWorkers=1,
maximumWorkers=1,
jobType=cjDAO.JOB_TYPE_HS)
if options.workerID is not None:
wID = options.workerID
else:
wID = self._workerID
buildID = Configuration.get('nupic.software.buildNumber', 'N/A')
logPrefix = '<BUILDID=%s, WORKER=HW, WRKID=%s, JOBID=%s> ' % \
(buildID, wID, options.jobID)
ExtendedLogger.setLogPrefix(logPrefix)
# ---------------------------------------------------------------------
# Get the search parameters
# If asked to reset the job status, do that now
if options.resetJobStatus:
cjDAO.jobSetFields(
options.jobID,
fields={
'workerCompletionReason':
ClientJobsDAO.CMPL_REASON_SUCCESS,
'cancel': False,
#'engWorkerState': None
},
useConnectionID=False,
ignoreUnchanged=True)
jobInfo = cjDAO.jobInfo(options.jobID)
self.logger.info("Job info retrieved: %s" % (str(clippedObj(jobInfo))))
# ---------------------------------------------------------------------
# Instantiate the swarm object, which will handle the logic of
# which models to create when we need more to evaluate.
jobParams = json.loads(jobInfo.params)
# Validate job params
jsonSchemaPath = os.path.join(os.path.dirname(__file__), "jsonschema",
"jobParamsSchema.json")
jsonhelpers.validate(jobParams, schemaPath=jsonSchemaPath)
hsVersion = jobParams.get('hsVersion', None)
if hsVersion == 'v2':
self._hs = SwarmV2(searchParams=jobParams,
workerID=self._workerID,
cjDAO=cjDAO,
jobID=options.jobID,
logLevel=options.logLevel)
else:
raise RuntimeError("Invalid swarm implementation (%s) specified" \
% (hsVersion))
# =====================================================================
# The main loop.
try:
exit = False
numModelsTotal = 0
print >> sys.stderr, "reporter:status:Evaluating first model..."
while not exit:
# ------------------------------------------------------------------
# Choose a model to evaluate
batchSize = 10 # How many to try at a time.
modelIDToRun = None
while modelIDToRun is None:
if options.modelID is None:
# -----------------------------------------------------------------
# Get the latest results on all running models and send them to
# the swarm implementation
# This calls cjDAO.modelsGetUpdateCounters(), compares the
# updateCounters with what we have cached, fetches the results for the
# changed and new models, and sends those to the swarm
# implementation's self._hs.recordModelProgress() method.
self._processUpdatedModels(cjDAO)
# --------------------------------------------------------------------
# Create a new batch of models
(exit, newModels) = self._hs.createModels(
numModels=batchSize)
if exit:
break
# No more models left to create, just loop. The _hs is waiting for
# all remaining running models to complete, and may pick up on an
# orphan if it detects one.
if len(newModels) == 0:
continue
# Try and insert one that we will run
for (modelParams, modelParamsHash,
particleHash) in newModels:
jsonModelParams = json.dumps(modelParams)
(modelID, ours) = cjDAO.modelInsertAndStart(
options.jobID, jsonModelParams,
modelParamsHash, particleHash)
# Some other worker is already running it, tell the swarm object
# so that it doesn't try and insert it again
if not ours:
mParamsAndHash = cjDAO.modelsGetParams(
[modelID])[0]
mResult = cjDAO.modelsGetResultAndStatus(
[modelID])[0]
results = mResult.results
if results is not None:
results = json.loads(results)
modelParams = json.loads(mParamsAndHash.params)
particleHash = cjDAO.modelsGetFields(
modelID, ['engParticleHash'])[0]
particleInst = "%s.%s" % (
modelParams['particleState']['id'],
modelParams['particleState']['genIdx'])
self.logger.info("Adding model %d to our internal DB " \
"because modelInsertAndStart() failed to insert it: " \
"paramsHash=%s, particleHash=%s, particleId='%s'", modelID,
mParamsAndHash.engParamsHash.encode('hex'),
particleHash.encode('hex'), particleInst)
self._hs.recordModelProgress(
modelID=modelID,
modelParams=modelParams,
modelParamsHash=mParamsAndHash.
engParamsHash,
results=results,
completed=(mResult.status ==
cjDAO.STATUS_COMPLETED),
completionReason=mResult.completionReason,
matured=mResult.engMatured,
numRecords=mResult.numRecords)
else:
modelIDToRun = modelID
break
else:
# A specific modelID was passed on the command line
modelIDToRun = int(options.modelID)
mParamsAndHash = cjDAO.modelsGetParams([modelIDToRun
])[0]
modelParams = json.loads(mParamsAndHash.params)
modelParamsHash = mParamsAndHash.engParamsHash
# Make us the worker
cjDAO.modelSetFields(
modelIDToRun, dict(engWorkerConnId=self._workerID))
if False:
# Change the hash and params of the old entry so that we can
# create a new model with the same params
for attempt in range(1000):
paramsHash = hashlib.md5(
"OrphanParams.%d.%d" %
(modelIDToRun, attempt)).digest()
particleHash = hashlib.md5(
"OrphanParticle.%d.%d" %
(modelIDToRun, attempt)).digest()
try:
cjDAO.modelSetFields(
modelIDToRun,
dict(engParamsHash=paramsHash,
engParticleHash=particleHash))
success = True
except:
success = False
if success:
break
if not success:
raise RuntimeError(
"Unexpected failure to change paramsHash and "
"particleHash of orphaned model")
(modelIDToRun, ours) = cjDAO.modelInsertAndStart(
options.jobID, mParamsAndHash.params,
modelParamsHash)
# ^^^ end while modelIDToRun ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# ---------------------------------------------------------------
# We have a model, evaluate it now
# All done?
if exit:
break
# Run the model now
self.logger.info(
"RUNNING MODEL GID=%d, paramsHash=%s, params=%s",
modelIDToRun, modelParamsHash.encode('hex'), modelParams)
# ---------------------------------------------------------------------
# Construct model checkpoint GUID for this model:
# jobParams['persistentJobGUID'] contains the client's (e.g., API Server)
# persistent, globally-unique model identifier, which is what we need;
persistentJobGUID = jobParams['persistentJobGUID']
assert persistentJobGUID, "persistentJobGUID: %r" % (
persistentJobGUID, )
modelCheckpointGUID = jobInfo.client + "_" + persistentJobGUID + (
'_' + str(modelIDToRun))
self._hs.runModel(modelID=modelIDToRun,
jobID=options.jobID,
modelParams=modelParams,
modelParamsHash=modelParamsHash,
jobsDAO=cjDAO,
modelCheckpointGUID=modelCheckpointGUID)
# TODO: don't increment for orphaned models
numModelsTotal += 1
self.logger.info("COMPLETED MODEL GID=%d; EVALUATED %d MODELs",
modelIDToRun, numModelsTotal)
print >>sys.stderr, "reporter:status:Evaluated %d models..." % \
(numModelsTotal)
print >> sys.stderr, "reporter:counter:swarmWorker,numModels,1"
if options.modelID is not None:
exit = True
# ^^^ end while not exit
finally:
# Provide swarm instance an opportunity to clean up temporary files
self._hs.close()
self.logger.info("FINISHED. Evaluated %d models." % (numModelsTotal))
print >> sys.stderr, "reporter:status:Finished, evaluated %d models" % (
numModelsTotal)
return options.jobID
def __init__(self, streamDef, bookmark=None, saveOutput=False,
isBlocking=True, maxTimeout=0, eofOnTimeout=False):
""" Base class constructor, performs common initialization
Parameters:
----------------------------------------------------------------
streamDef: The stream definition, potentially containing multiple sources
(not supported yet). See
/nupic/frameworks/opf/jsonschema/stream_def.json for the format
of this dict
bookmark: Bookmark to start reading from. This overrides the first_record
field of the streamDef if provided.
saveOutput: If true, save the output to a csv file in a temp directory.
The path to the generated file can be found in the log
output.
isBlocking: should read operation block *forever* if the next row of data
is not available, but the stream is not marked as 'completed'
yet?
maxTimeout: if isBlocking is False, max seconds to wait for more data before
timing out; ignored when isBlocking is True.
eofOnTimeout: If True and we get a read timeout (isBlocking must be False
to get read timeouts), assume we've reached the end of the
input and produce the last aggregated record, if one can be
completed.
"""
# Call superclass constructor
super(StreamReader, self).__init__()
loggerPrefix = 'com.numenta.nupic.data.StreamReader'
self._logger = logging.getLogger(loggerPrefix)
jsonhelpers.validate(streamDef,
schemaPath=resource_filename(
jsonschema.__name__, "stream_def.json"))
assert len(streamDef['streams']) == 1, "Only 1 source stream is supported"
# Save constructor args
sourceDict = streamDef['streams'][0]
self._recordCount = 0
self._eofOnTimeout = eofOnTimeout
self._logger.debug('Reading stream with the def: %s', sourceDict)
# Dictionary to store record statistics (min and max of scalars for now)
self._stats = None
# ---------------------------------------------------------------------
# Get the stream definition params
# Limiting window of the stream. It would not return any records until
# 'first_record' ID is read (or very first with the ID above that). The
# stream will return EOS once it reads record with ID 'last_record' or
# above (NOTE: the name 'lastRecord' is misleading because it is NOT
# inclusive).
firstRecordIdx = sourceDict.get('first_record', None)
self._sourceLastRecordIdx = sourceDict.get('last_record', None)
# If a bookmark was given, then override first_record from the stream
# definition.
if bookmark is not None:
firstRecordIdx = None
# Column names must be provided in the streamdef json
# Special case is ['*'], meaning all available names from the record stream
self._streamFieldNames = sourceDict.get('columns', None)
if self._streamFieldNames != None and self._streamFieldNames[0] == '*':
self._needFieldsFiltering = False
else:
self._needFieldsFiltering = True
# Types must be specified in streamdef json, or in case of the
# file_recod_stream types could be implicit from the file
streamFieldTypes = sourceDict.get('types', None)
self._logger.debug('Types from the def: %s', streamFieldTypes)
# Validate that all types are valid
if streamFieldTypes != None:
for dataType in streamFieldTypes:
assert(dataType in TYPES)
# Reset, sequence and time fields might be provided by streamdef json
streamResetFieldName = streamDef.get('resetField', None)
streamTimeFieldName = streamDef.get('timeField', None)
streamSequenceFieldName = streamDef.get('sequenceIdField', None)
self._logger.debug('r, t, s fields: %s, %s, %s', streamResetFieldName,
streamTimeFieldName,
streamSequenceFieldName)
# ================================
