def swarm(inputFile):
swarmWorkDir()
permutations_runner.runWithConfig(
SWARM_DESCRIPTION,
{"maxWorkers", 1, "overwrite", True},
outputLable= "rec_center",
outDir
)
pass
def swarm(inputFile):
swarmWorkDir = os.path.abspath( "swarm")
if not os.path.exists( swarmWorkDir):
os.mkdir( swarmWorkDir)
permutations_runner.runWithConfig(
SWARM_DESCRIPTION,
{"maxWorkers": 1, "overwrite": True},
outputLabel = "rec_center",
outDir =swarmWorkDir,
permWorkDir=swarmWorkDir
)
def swarmForBestModelParams(swarmConfig, filePath, maxWorkers=4):
name = os.path.splitext(os.path.basename(filePath))[0]
outputLabel = name
permWorkDir = os.path.abspath('swarm')
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
#print "swarmConfig: " + str(swarmConfig)
#print
#print "swarmConfig['streamDef']['streams']['source']: " + str(swarmConfig['streamDef']['streams'][0]['source'])
#print
#set filesource to arg0
swarmConfig ['streamDef']['streams'][0]['source'] = "file://" + filePath
#print
#print "swarmConfig: " + str(swarmConfig)
#print
modelParams = permutations_runner.runWithConfig(
swarmConfig,
{"maxWorkers": maxWorkers, "overwrite": True},
outputLabel=outputLabel,
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=0
)
modelParamsFile = writeModelParamsToFile(modelParams, name)
return modelParamsFile
def swarm(output_dir):
'''
Run swarming.
Args:
output_dir: a directory name for swarm results.
'''
swarm_dir = os.path.abspath(output_dir)
if not os.path.exists(swarm_dir):
os.mkdir(swarm_dir)
f = open(os.path.join(swarm_dir, '__init__.py'), 'w')
f.close()
model = permutations_runner.runWithConfig(SWARM_DESCRIPTION, {
"maxWorkers": 4,
"overwrite": True
},
outputLabel="sine",
outDir=swarm_dir,
permWorkDir=swarm_dir)
f = open(os.path.join(swarm_dir, "model_0", '__init__.py'), 'w')
f.close()
return model
def swarmForBestModelParams(swarmConfig, areaDirPath, areaId, maxWorkers=4):
"""
Trigger the swarm passing the configuration file to the permutation runner in NuPIC.
Also trigger the writing of the resultant model params file
:param swarmConfig:
:param areaDirPath:
:param areaId:
:param maxWorkers:
:return:
"""
outputLabel = "model_params"
# Define where the work will take place (Should be in the area1 folder, in a tmp_work folder?)
# Need the path to the area1 folder to do this...
permuWorkDir = os.path.abspath(areaDirPath + "tmpWork")
outputDir = os.path.abspath(areaDirPath)
createDirIfNotExisting(permuWorkDir)
createDirIfNotExisting(outputDir)
log.info("Starting permutations_runner for area %s" % areaId)
modelParams = permutations_runner.runWithConfig(swarmConfig, {
"maxWorkers": maxWorkers,
"overwrite": True
},
outputLabel=outputLabel,
outDir=outputDir,
permWorkDir=permuWorkDir)
log.info("Writing swarm results to model params file for area %s" % areaId)
modelParamsFilePath = writeModelParamsToFile(modelParams, outputDir)
return modelParamsFilePath
def swarm_over_data(attr):
permWorkDir = os.path.abspath("swarm")
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
SWARM_CONFIG = {
"includedFields": [
{"fieldName": "timestamp", "fieldType": "datetime"},
{"fieldName": attr["name"], "fieldType": attr["type"], "maxValue": attr["max"], "minValue": attr["min"]},
],
"streamDef": {
"info": "lol",
"version": 1,
"streams": [{"info": "reddit_lol", "source": "file://reddit_lol.csv", "columns": ["*"]}],
},
"inferenceType": "TemporalAnomaly",
"inferenceArgs": {"predictionSteps": [1], "predictedField": attr["name"]},
"iterationCount": -1,
"swarmSize": "medium",
}
modelParams = permutations_runner.runWithConfig(
SWARM_CONFIG,
{"maxWorkers": 8, "overwrite": True},
outputLabel="num",
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=1,
)
modelParamsFile = writeModelParamsToFile(modelParams, attr["name"])
def swarm_over_data(input_file):
SWARM_DEF = "search_def.json"
SWARM_CONFIG = {
"includedFields": [{
"fieldName": "pitch",
"fieldType": "float",
"maxValue": 12.0,
"minValue": 0.0
}],
"streamDef": {
"info":
"eggs",
"version":
1,
"streams": [{
"info": input_file,
"source": "file://" + input_file,
"columns": ["*"]
}]
},
"inferenceType":
"TemporalAnomaly",
"inferenceArgs": {
"predictionSteps": [1],
"predictedField": "pitch"
},
"swarmSize":
"medium"
}
return permutations_runner.runWithConfig(SWARM_CONFIG, {
'maxWorkers': 8,
'overwrite': True
})
def swarm_over_data(input_file):
SWARM_DEF = "search_def.json"
SWARM_CONFIG = {
"includedFields": [
{
"fieldName": "pitch",
"fieldType": "float",
"maxValue": 12.0,
"minValue": 0.0
}
],
"streamDef": {
"info": "eggs",
"version": 1,
"streams": [
{
"info": input_file,
"source": "file://"+input_file,
"columns": [
"*"
]
}
]
},
"inferenceType": "TemporalAnomaly",
"inferenceArgs": {
"predictionSteps": [
1
],
"predictedField": "pitch"
},
"swarmSize": "medium"
}
return permutations_runner.runWithConfig(SWARM_CONFIG,
{'maxWorkers': 8, 'overwrite': True})
def swarm_over_data():
cur_dir = os.getcwd()
dis = distination = os.path.join(cur_dir, 'disk')
outdir = distination = os.path.join(cur_dir, 'disk/disk_model_store')
return permutations_runner.runWithConfig(SWARM_CONFIG, {'overwrite': True},
outDir=outdir,
permWorkDir=dis)
def runSwarm(sampleFileName, workers = 4):
label = getSampleName(sampleFileName)
modelParams = permutations_runner.runWithConfig(
description.getDescription(sampleFileName), {"maxWorkers": workers, "overwrite": True},
outDir = label + MODEL_PARAMS_DIR_POSTFIX,
outputLabel = label,
verbosity=1
)
return modelParams
def swarm_over_data(SWARM_CONFIG):
cur_dir = os.getcwd()
dis = os.path.join(cur_dir, 'cpu')
outdir = os.path.join(cur_dir, 'cpu/cpu_model_store')
return permutations_runner.runWithConfig(SWARM_CONFIG, {
'maxWorkers': 8,
'overwrite': True
},
outDir=outdir,
permWorkDir=dis)
def swarm_over_data():
cur_dir = os.getcwd()
dist = os.path.join(cur_dir, 'mem')
print 'dist', dist
outdir = os.path.join(cur_dir, 'mem/mem_model_store')
return permutations_runner.runWithConfig(SWARM_CONFIG, {
'maxWorkers': 2,
'overwrite': True
},
outDir=outdir,
permWorkDir=dist)
def swarm_over_data():
swarmWorkDir = os.path.abspath("swarm")
if not os.path.exists(swarmWorkDir):
os.mkdir(swarmWorkDir)
modelParams = permutations_runner.runWithConfig(
description,
{'maxWorkers': 4, "overwrite": True},
outDir= swarmWorkDir,
permWorkDir=swarmWorkDir
)
writeModelParams(modelParams)
def swarm(inputFile):
swarmWorkDir = os.path.abspath("swarm")
if not os.path.exists(swarmWorkDir):
os.mkdir(swarmWorkDir)
modelParams = permutations_runner.runWithConfig(SWARM_DESCRIPTION, {
"maxWorkers": 4,
"overwrite": True
},
outputLabel="rec center",
outDir=swarmWorkDir,
permWorkDir=swarmWorkDir)
writeModelParams(modelParams)
def swarm_over_data():
permWorkDir = os.path.abspath('swarm')
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
modelParams = permutations_runner.runWithConfig(SWARM_CONFIG,
{'maxWorkers': 8, 'overwrite': True},
outputLabel='num',
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=1
)
modelParamsFile = writeModelParamsToFile(modelParams, 'num')
def run_swarm_ng(working_dir, sensors_dir, sensor, store, swarmconfig):
store_dir = sensors_dir + sensor + '/stores/' + store + '/'
swarms_dir = store_dir + '/swarms/'
swarm_dir = next_file_name(prefix='swarm', suffix='', path=swarms_dir)
path, swarm = os.path.split(swarm_dir)
swarm_dir = swarm_dir + '/'
egg_file = swarm_dir + "eggs.csv"
store_file = store_dir + 'out.csv'
swarmconfig['streamDef']['streams'][0]['source'] = 'file://' + egg_file
print swarmconfig['streamDef']['streams'][0]['source']
if not os.path.exists(swarms_dir):
os.makedirs(swarms_dir)
if not os.path.exists(swarm_dir):
os.makedirs(swarm_dir)
fieldNames = map(lambda x: x['fieldName'], swarmconfig['includedFields'])
fieldTypes = map(lambda x: x['fieldType'], swarmconfig['includedFields'])
prepare_swarm(field_names=fieldNames,
field_types=fieldTypes,
egg_file=egg_file,
store_file=store_file)
#swarm_config = {} # complete swarm config here
#nupic_config = permutations_runner.runWithJsonFile(os.getcwd() + "/nupic/search_def.json",{'maxWorkers': 6, 'overwrite': True}, "test", os.getcwd() + "/working")
nupic_config = permutations_runner.runWithConfig(swarmconfig, {
'maxWorkers': 6,
'overwrite': True
},
outDir=swarm_dir,
outputLabel='',
permWorkDir=swarm_dir,
verbosity=1)
#dump the model params generated by this swarm in 2 different file formats
pickle_file = open(swarm_dir + 'model_params.pkl', 'w')
pickle.dump(nupic_config, pickle_file, pickle.HIGHEST_PROTOCOL)
pickle_file.close()
json_file = open(swarm_dir + 'model_params.json', 'w')
json.dump(nupic_config, json_file)
json_file.close()
#dump the swarmconfig used for this swarm in 2 different file formats
pickle_file = open(swarm_dir + 'swarm_config.pkl', 'w')
pickle.dump(swarmconfig, pickle_file, pickle.HIGHEST_PROTOCOL)
pickle_file.close()
json_file = open(swarm_dir + 'swarm_config.json', 'w')
json.dump(swarmconfig, json_file)
json_file.close()
return swarm
def swarm_over_data():
permWorkDir = os.path.abspath("swarm")
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
modelParams = permutations_runner.runWithConfig(
SWARM_CONFIG,
{"maxWorkers": 8, "overwrite": True},
outputLabel="exercise",
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=1,
)
modelParamsFile = writeModelParamsToFile(modelParams, "exercise")
def swarm(inputFile):
swarmWorkDir = os.path.abspath("swarmHTM")
if not os.path.exists(swarmWorkDir):
os.mkdir(swarmWorkDir),
modelParams = permutations_runner.runWithConfig(fieldX, {
"maxWorker": 4,
"overwrite": True
},
outputLabel="output",
outDir=swarmWorkDir,
permWorkDir=swarmWorkDir)
modelParamsName = os.path.splitext(inputFile)[0]
writeModelParams(modelParams, modelParamsName)
def swarm():
swarmWorkDir = os.path.abspath("swarm")
if not os.path.exists(swarmWorkDir):
os.mkdir(swarmWorkDir)
print swarmWorkDir
modelParams = permutations_runner.runWithConfig(
SWARM_DESCRIPTION,
{"maxWorkers": 4, "overwrite": True},
outputLabel="rec_center",
outDir=swarmWorkDir,
permWorkDir=swarmWorkDir
)
writeModelParams(modelParams)
def swarm():
swarmWorkDir = os.path.abspath("swarm_results")
if not os.path.exists(swarmWorkDir):
os.mkdir(swarmWorkDir)
modelParams = permutations_runner.runWithConfig(
SWARM_DESCRIPTION,
{"maxWorkers": swarm_setup.MAX_WORKERS, "overwrite": True},
outputLabel="cla_model",
outDir=swarmWorkDir,
permWorkDir=swarmWorkDir,
verbosity=0)
saveModelParams(modelParams)
def run_swarm_ng(working_dir,sensors_dir, sensor, store, swarmconfig):
store_dir = sensors_dir + sensor + '/stores/' + store +'/'
swarms_dir = store_dir + '/swarms/'
swarm_dir = next_file_name(prefix='swarm',suffix='',path=swarms_dir)
path,swarm = os.path.split(swarm_dir)
swarm_dir = swarm_dir + '/'
egg_file = swarm_dir + "eggs.csv"
store_file = store_dir + 'out.csv'
swarmconfig['streamDef']['streams'][0]['source'] = 'file://' + egg_file
print swarmconfig['streamDef']['streams'][0]['source']
if not os.path.exists(swarms_dir):
os.makedirs(swarms_dir)
if not os.path.exists(swarm_dir):
os.makedirs(swarm_dir)
fieldNames = map(lambda x: x['fieldName'],swarmconfig['includedFields'])
fieldTypes = map(lambda x: x['fieldType'],swarmconfig['includedFields'])
prepare_swarm(field_names=fieldNames, field_types= fieldTypes, egg_file=egg_file, store_file = store_file)
#swarm_config = {} # complete swarm config here
#nupic_config = permutations_runner.runWithJsonFile(os.getcwd() + "/nupic/search_def.json",{'maxWorkers': 6, 'overwrite': True}, "test", os.getcwd() + "/working")
nupic_config = permutations_runner.runWithConfig(swarmconfig,
{'maxWorkers': 6,
'overwrite': True
},
outDir= swarm_dir,
outputLabel='',
permWorkDir= swarm_dir,
verbosity=1)
#dump the model params generated by this swarm in 2 different file formats
pickle_file = open(swarm_dir + 'model_params.pkl', 'w')
pickle.dump(nupic_config,pickle_file,pickle.HIGHEST_PROTOCOL)
pickle_file.close()
json_file = open(swarm_dir +'model_params.json', 'w')
json.dump(nupic_config, json_file)
json_file.close()
#dump the swarmconfig used for this swarm in 2 different file formats
pickle_file = open(swarm_dir + 'swarm_config.pkl', 'w')
pickle.dump(swarmconfig,pickle_file,pickle.HIGHEST_PROTOCOL)
pickle_file.close()
json_file = open(swarm_dir+ 'swarm_config.json', 'w')
json.dump(swarmconfig, json_file)
json_file.close()
return swarm
def swarmForBestModelParams(swarmConfig, name, maxWorkers=4):
outputLabel = name
permWorkDir = os.path.abspath('swarm')
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
modelParams = permutations_runner.runWithConfig(
swarmConfig,
{"maxWorkers": maxWorkers, "overwrite": True},
outputLabel=outputLabel,
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=0
)
modelParamsFile = writeModelParamsToFile(modelParams, name)
return modelParamsFile
def swarmForBestModelParams(swarmConfig, name, maxWorkers=4):
outputLabel = name
permWorkDir = os.path.abspath('swarm')
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
modelParams = permutations_runner.runWithConfig(swarmConfig, {
"maxWorkers": maxWorkers,
"overwrite": True
},
outputLabel=outputLabel,
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=0)
modelParamsFile = writeModelParamsToFile(modelParams, name)
return modelParamsFile
def swarm_for_best_model_params(swarm_config, name, max_workers=4):
output_label = name
perm_work_dir = os.path.abspath('swarm')
if not os.path.exists(perm_work_dir):
os.mkdir(perm_work_dir)
model_params = permutations_runner.runWithConfig(swarm_config, {
"maxWorkers": max_workers,
"overwrite": True
},
outputLabel=output_label,
outDir=perm_work_dir,
permWorkDir=perm_work_dir,
verbosity=0)
model_params_file = write_model_params_to_file(model_params, name)
return model_params_file
def swarm_for_best_model_params(swarm_config, name, max_workers=4):
output_label = name
perm_work_dir = os.path.abspath('swarm')
if not os.path.exists(perm_work_dir):
os.mkdir(perm_work_dir)
model_params = permutations_runner.runWithConfig(
swarm_config,
{"maxWorkers": max_workers, "overwrite": True},
outputLabel=output_label,
outDir=perm_work_dir,
permWorkDir=perm_work_dir,
verbosity=0
)
model_params_file = write_model_params_to_file(model_params, name)
return model_params_file
def _swarmForBestModelParams(swarmConfig, name, workingDir, maxWorkers=8):
outputLabel = name
permWorkDir = os.path.join(workingDir, "swarm")
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
modelParams = permutations_runner.runWithConfig(
swarmConfig,
{"maxWorkers": maxWorkers, "overwrite": True},
outputLabel=outputLabel,
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=0
)
_writeModelParamsToFile(modelParams, workingDir)
return modelParams
def __swarm(self):
"""
Starts a swarm and writes a generated files to swarm/ directory for the business.
@return modelParam:(object) Models Parameters object.
"""
# Create directory for swarm details
swarmWorkDir = self.__createSwarmWorkDir()
modelParams = permutations_runner.runWithConfig(
self._swarmDescriptionObject,
{"maxWorkers": 2, "overwrite": True},
outputLabel=self._swarmName,
outDir=swarmWorkDir,
permWorkDir=swarmWorkDir
)
# Write the model parameters to swarm directory.
self.__writeModelParams(modelParams)
return modelParams
def _runSwarm(self):
if self.swarmDescType == 'template':
pred_field = self.swarm_desc["inferenceArgs"]["predictedField"]
self.model_params = permutations_runner.runWithConfig(
self.swarm_desc, {
"maxWorkers": self.max_workers,
"overwrite": True
},
outputLabel=pred_field,
outDir=self.field_output_fpath,
permWorkDir=self.output_fpath,
verbosity=0)
elif self.swarmDescType == 'permScript':
swarm_fpath = os.path.join(self.field_output_fpath,
self.swarm_desc)
import_name = os.path.join(self.field_output_fpath, 'description')
#import_name = os.path.join(os.path.dirname(self.swarm_desc), 'description')
import_name = import_name.replace('/', '.')
try:
print import_name
imported_desc = importlib.import_module(import_name).control
except ImportError as err:
print("Import error: {}".format(err))
pred_field = imported_desc["inferenceArgs"]["predictedField"]
print pred_field
self.model_params = permutations_runner.runWithPermutationsScript(
swarm_fpath, {
"maxWorkers": self.max_workers,
"overwrite": True
},
outputLabel=pred_field,
permWorkDir=self.field_output_fpath)
self._params_to_file(pred_field)
return self.model_params_file
def swarm_for_best_model_params(swarm_config, name, max_workers):
output_label = name
perm_work_dir = os.path.abspath('swarm')
if not os.path.exists(perm_work_dir):
os.mkdir(perm_work_dir)
if verbose:
print "Using %i swarm workers." % max_workers
print "\n** STARTING SWARM **\n\n"
model_params = permutations_runner.runWithConfig(
swarm_config,
{"maxWorkers": max_workers, "overwrite": True},
outputLabel=output_label,
outDir=perm_work_dir,
permWorkDir=perm_work_dir,
verbosity=0
)
model_params_file = write_model_params_to_file(model_params, name)
return model_params_file
def swarmParams(swarmConfig, intersection):
outputLabel = intersection
permWorkDir = os.path.abspath('swarm')
if not os.path.exists(permWorkDir):
os.mkdir(permWorkDir)
import multiprocessing as mp
# use 3/4 of your CPUs
maxWorkers = 3 * mp.cpu_count() / 2
print "Running swarm!"
modelParams = permutations_runner.runWithConfig(
swarmConfig,
{"maxWorkers": maxWorkers, "overwrite": True},
outputLabel=outputLabel,
outDir=permWorkDir,
permWorkDir=permWorkDir,
verbosity=2
)
modelParamsFile = writeModelParams(modelParams, intersection)
return modelParamsFile
def swarm_for_best_model_params(swarm_config, name, max_workers):
output_label = name
perm_work_dir = os.path.abspath('swarm')
if not os.path.exists(perm_work_dir):
os.mkdir(perm_work_dir)
if verbose:
print "Using %i swarm workers." % max_workers
print "\n** STARTING SWARM **\n\n"
model_params = permutations_runner.runWithConfig(swarm_config, {
"maxWorkers": max_workers,
"overwrite": True
},
outputLabel=output_label,
outDir=perm_work_dir,
permWorkDir=perm_work_dir,
verbosity=0)
model_params_file = write_model_params_to_file(model_params, name)
return model_params_file
def swarm(cwd, input_file, swarm_description, model_name=None,
write_model=False, max_workers=4):
swarm_work_dir = os.path.abspath('swarm')
if not os.path.exists(swarm_work_dir):
os.mkdir(swarm_work_dir)
stream = swarm_description['streamDef']['streams'][0]
full_path = os.path.join(cwd, input_file)
stream['source'] = 'file://{}'.format(full_path)
label = swarm_description['streamDef']['info']
model_params = permutations_runner.runWithConfig(
swarm_description,
{'maxWorkers': max_workers,
'overwrite': True},
outputLabel=label,
outDir=swarm_work_dir,
permWorkDir=swarm_work_dir
)
if write_model:
write_model_params(cwd, model_params, model_name)
return model_params
def run_swarms():
'''Runs every swarm found in the directory structure found above using a local, scratch
get the data directories, screen out git and other junk files'''
dirs=[]
for ddir in os.listdir(BASE_DIR):
if os.path.isdir(os.path.join(BASE_DIR, ddir)) and ddir[0].isdigit():
dirs.append(ddir)
#for each of the swarm description directories, dig out the description files ...
swarm_files=[]
for d in dirs:
data_id = d.split('_')[0]
swarm_files=os.listdir(os.path.join(BASE_DIR, d, data_id+'_swarm'))
#All files in this dir with the right tag (naming convention) are swarm descriptors
for source_file in swarm_files:
if SWARM_DESC_TAG in source_file:
#set up the scratch dir
swarm_dir=tempfile.mkdtemp()
sys.path.append(swarm_dir)
#... make a scratch copy of the file and import the description dict
swarm_file_name=os.path.join(swarm_dir, SWARM_DESC_TAG)
shutil.copyfile(os.path.join(BASE_DIR, d, ODIR_SUBDIR.format(data_id), source_file), swarm_file_name )
##os.chdir(scatch) not thread safe
from swarm_description import SWARM_DESCTIPTION
#...and run the built in runWithConfig() method from permutaions_runner
name = source_file.split('_swarm')[0] #this will be something like '666_energy_only'
param_fn = PARAMS_FILE.format(name)
print('running swarm for {} started {}'.
format(source_file.split('_swarm')[0],
datetime.now().strftime('%m-%d-%y %H:%M')),
end='' )
if DEBUG:
model_params = 'test params'
with open(os.path.join(swarm_dir, param_fn), 'w') as f:
f.write (model_params)
else:
model_params = permutations_runner.runWithConfig(
SWARM_DESCTIPTION,
{"maxWorkers": MAX_WORKERS, "overwrite": True},
outputLabel=name,
outDir=swarm_dir,
permWorkDir=swarm_dir,
verbosity=0
)
print(' ... finished at {}'.format(datetime.now().strftime('%m-%d-%y %H:%M')))
#dump parameters to the local scratch dir
writeModelParamsToFile(model_params, param_fn)
#finally, copy it back to the original data file directory
shutil.copyfile(os.path.join(swarm_dir, param_fn),
os.path.join(BASE_DIR, d, ODIR_SUBDIR.format(data_id), PARAMS_FILE.format(name))
)
shutil.rmtree(swarm_dir)
def swarm_over_data(swarm_config):
permutations_runner.runWithConfig(swarm_config, {'maxWorkers': 1, 'overwrite': True})
shutil.copyfile("model_0/model_params.py", "model_params.py")
def swarm_over_data():
model_params = permutations_runner.runWithConfig(swarm_config, {
'maxWorkers': 2,
'overwrite': True
})
shutil.copyfile("model_0/model_params.py", "model_params.py")
def swarm_over_data():
return permutations_runner.runWithConfig(SWARM_CONFIG, {
'maxWorkers': 4,
'overwrite': True
})
#!/usr/bin/env python
from nupic.swarming import permutations_runner
import os
import json
import logging
logging.basicConfig()
if __name__ == "__main__":
with open('swarm_config.json', 'r') as conf:
swarm_config = json.load(conf)
model_params = permutations_runner.runWithConfig(swarm_config, {
'maxWorkers': 1,
'overwrite': True
},
verbosity=3)
exit()
model_params = permutations_runner.runWithPermutationsScript(
'permutations.py',
{
'maxWorkers': 1,
'overwrite': True,
# 'expDescConfig': 'description.py',
'verbosityCount': 0
},
'output',
'./model_0')
print(json.dumps(model_params))
def swarm_over_data():
return permutations_runner.runWithConfig(SWARM_CONFIG, {"maxWorkers": 8, "overwrite": True})
def swarm_over_data():
return permutations_runner.runWithConfig(SWARM_CONFIG,
{'maxWorkers': 8, 'overwrite': True})
