def swarm(inputFile): swarmWorkDir() permutations_runner.runWithConfig( SWARM_DESCRIPTION, {"maxWorkers", 1, "overwrite", True}, outputLable= "rec_center", outDir ) pass
def swarm(inputFile): swarmWorkDir = os.path.abspath( "swarm") if not os.path.exists( swarmWorkDir): os.mkdir( swarmWorkDir) permutations_runner.runWithConfig( SWARM_DESCRIPTION, {"maxWorkers": 1, "overwrite": True}, outputLabel = "rec_center", outDir =swarmWorkDir, permWorkDir=swarmWorkDir )
def swarmForBestModelParams(swarmConfig, filePath, maxWorkers=4): name = os.path.splitext(os.path.basename(filePath))[0] outputLabel = name permWorkDir = os.path.abspath('swarm') if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) #print "swarmConfig: " + str(swarmConfig) #print #print "swarmConfig['streamDef']['streams']['source']: " + str(swarmConfig['streamDef']['streams'][0]['source']) #print #set filesource to arg0 swarmConfig ['streamDef']['streams'][0]['source'] = "file://" + filePath #print #print "swarmConfig: " + str(swarmConfig) #print modelParams = permutations_runner.runWithConfig( swarmConfig, {"maxWorkers": maxWorkers, "overwrite": True}, outputLabel=outputLabel, outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=0 ) modelParamsFile = writeModelParamsToFile(modelParams, name) return modelParamsFile
def swarm(output_dir): ''' Run swarming. Args: output_dir: a directory name for swarm results. ''' swarm_dir = os.path.abspath(output_dir) if not os.path.exists(swarm_dir): os.mkdir(swarm_dir) f = open(os.path.join(swarm_dir, '__init__.py'), 'w') f.close() model = permutations_runner.runWithConfig(SWARM_DESCRIPTION, { "maxWorkers": 4, "overwrite": True }, outputLabel="sine", outDir=swarm_dir, permWorkDir=swarm_dir) f = open(os.path.join(swarm_dir, "model_0", '__init__.py'), 'w') f.close() return model
def swarmForBestModelParams(swarmConfig, areaDirPath, areaId, maxWorkers=4): """ Trigger the swarm passing the configuration file to the permutation runner in NuPIC. Also trigger the writing of the resultant model params file :param swarmConfig: :param areaDirPath: :param areaId: :param maxWorkers: :return: """ outputLabel = "model_params" # Define where the work will take place (Should be in the area1 folder, in a tmp_work folder?) # Need the path to the area1 folder to do this... permuWorkDir = os.path.abspath(areaDirPath + "tmpWork") outputDir = os.path.abspath(areaDirPath) createDirIfNotExisting(permuWorkDir) createDirIfNotExisting(outputDir) log.info("Starting permutations_runner for area %s" % areaId) modelParams = permutations_runner.runWithConfig(swarmConfig, { "maxWorkers": maxWorkers, "overwrite": True }, outputLabel=outputLabel, outDir=outputDir, permWorkDir=permuWorkDir) log.info("Writing swarm results to model params file for area %s" % areaId) modelParamsFilePath = writeModelParamsToFile(modelParams, outputDir) return modelParamsFilePath
def swarm_over_data(attr): permWorkDir = os.path.abspath("swarm") if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) SWARM_CONFIG = { "includedFields": [ {"fieldName": "timestamp", "fieldType": "datetime"}, {"fieldName": attr["name"], "fieldType": attr["type"], "maxValue": attr["max"], "minValue": attr["min"]}, ], "streamDef": { "info": "lol", "version": 1, "streams": [{"info": "reddit_lol", "source": "file://reddit_lol.csv", "columns": ["*"]}], }, "inferenceType": "TemporalAnomaly", "inferenceArgs": {"predictionSteps": [1], "predictedField": attr["name"]}, "iterationCount": -1, "swarmSize": "medium", } modelParams = permutations_runner.runWithConfig( SWARM_CONFIG, {"maxWorkers": 8, "overwrite": True}, outputLabel="num", outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=1, ) modelParamsFile = writeModelParamsToFile(modelParams, attr["name"])
def swarm_over_data(input_file): SWARM_DEF = "search_def.json" SWARM_CONFIG = { "includedFields": [{ "fieldName": "pitch", "fieldType": "float", "maxValue": 12.0, "minValue": 0.0 }], "streamDef": { "info": "eggs", "version": 1, "streams": [{ "info": input_file, "source": "file://" + input_file, "columns": ["*"] }] }, "inferenceType": "TemporalAnomaly", "inferenceArgs": { "predictionSteps": [1], "predictedField": "pitch" }, "swarmSize": "medium" } return permutations_runner.runWithConfig(SWARM_CONFIG, { 'maxWorkers': 8, 'overwrite': True })
def swarm_over_data(input_file): SWARM_DEF = "search_def.json" SWARM_CONFIG = { "includedFields": [ { "fieldName": "pitch", "fieldType": "float", "maxValue": 12.0, "minValue": 0.0 } ], "streamDef": { "info": "eggs", "version": 1, "streams": [ { "info": input_file, "source": "file://"+input_file, "columns": [ "*" ] } ] }, "inferenceType": "TemporalAnomaly", "inferenceArgs": { "predictionSteps": [ 1 ], "predictedField": "pitch" }, "swarmSize": "medium" } return permutations_runner.runWithConfig(SWARM_CONFIG, {'maxWorkers': 8, 'overwrite': True})
def swarm_over_data(): cur_dir = os.getcwd() dis = distination = os.path.join(cur_dir, 'disk') outdir = distination = os.path.join(cur_dir, 'disk/disk_model_store') return permutations_runner.runWithConfig(SWARM_CONFIG, {'overwrite': True}, outDir=outdir, permWorkDir=dis)
def runSwarm(sampleFileName, workers = 4): label = getSampleName(sampleFileName) modelParams = permutations_runner.runWithConfig( description.getDescription(sampleFileName), {"maxWorkers": workers, "overwrite": True}, outDir = label + MODEL_PARAMS_DIR_POSTFIX, outputLabel = label, verbosity=1 ) return modelParams
def swarm_over_data(SWARM_CONFIG): cur_dir = os.getcwd() dis = os.path.join(cur_dir, 'cpu') outdir = os.path.join(cur_dir, 'cpu/cpu_model_store') return permutations_runner.runWithConfig(SWARM_CONFIG, { 'maxWorkers': 8, 'overwrite': True }, outDir=outdir, permWorkDir=dis)
def swarm_over_data(): cur_dir = os.getcwd() dist = os.path.join(cur_dir, 'mem') print 'dist', dist outdir = os.path.join(cur_dir, 'mem/mem_model_store') return permutations_runner.runWithConfig(SWARM_CONFIG, { 'maxWorkers': 2, 'overwrite': True }, outDir=outdir, permWorkDir=dist)
def swarm_over_data(): swarmWorkDir = os.path.abspath("swarm") if not os.path.exists(swarmWorkDir): os.mkdir(swarmWorkDir) modelParams = permutations_runner.runWithConfig( description, {'maxWorkers': 4, "overwrite": True}, outDir= swarmWorkDir, permWorkDir=swarmWorkDir ) writeModelParams(modelParams)
def swarm(inputFile): swarmWorkDir = os.path.abspath("swarm") if not os.path.exists(swarmWorkDir): os.mkdir(swarmWorkDir) modelParams = permutations_runner.runWithConfig(SWARM_DESCRIPTION, { "maxWorkers": 4, "overwrite": True }, outputLabel="rec center", outDir=swarmWorkDir, permWorkDir=swarmWorkDir) writeModelParams(modelParams)
def swarm_over_data(): permWorkDir = os.path.abspath('swarm') if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) modelParams = permutations_runner.runWithConfig(SWARM_CONFIG, {'maxWorkers': 8, 'overwrite': True}, outputLabel='num', outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=1 ) modelParamsFile = writeModelParamsToFile(modelParams, 'num')
def run_swarm_ng(working_dir, sensors_dir, sensor, store, swarmconfig): store_dir = sensors_dir + sensor + '/stores/' + store + '/' swarms_dir = store_dir + '/swarms/' swarm_dir = next_file_name(prefix='swarm', suffix='', path=swarms_dir) path, swarm = os.path.split(swarm_dir) swarm_dir = swarm_dir + '/' egg_file = swarm_dir + "eggs.csv" store_file = store_dir + 'out.csv' swarmconfig['streamDef']['streams'][0]['source'] = 'file://' + egg_file print swarmconfig['streamDef']['streams'][0]['source'] if not os.path.exists(swarms_dir): os.makedirs(swarms_dir) if not os.path.exists(swarm_dir): os.makedirs(swarm_dir) fieldNames = map(lambda x: x['fieldName'], swarmconfig['includedFields']) fieldTypes = map(lambda x: x['fieldType'], swarmconfig['includedFields']) prepare_swarm(field_names=fieldNames, field_types=fieldTypes, egg_file=egg_file, store_file=store_file) #swarm_config = {} # complete swarm config here #nupic_config = permutations_runner.runWithJsonFile(os.getcwd() + "/nupic/search_def.json",{'maxWorkers': 6, 'overwrite': True}, "test", os.getcwd() + "/working") nupic_config = permutations_runner.runWithConfig(swarmconfig, { 'maxWorkers': 6, 'overwrite': True }, outDir=swarm_dir, outputLabel='', permWorkDir=swarm_dir, verbosity=1) #dump the model params generated by this swarm in 2 different file formats pickle_file = open(swarm_dir + 'model_params.pkl', 'w') pickle.dump(nupic_config, pickle_file, pickle.HIGHEST_PROTOCOL) pickle_file.close() json_file = open(swarm_dir + 'model_params.json', 'w') json.dump(nupic_config, json_file) json_file.close() #dump the swarmconfig used for this swarm in 2 different file formats pickle_file = open(swarm_dir + 'swarm_config.pkl', 'w') pickle.dump(swarmconfig, pickle_file, pickle.HIGHEST_PROTOCOL) pickle_file.close() json_file = open(swarm_dir + 'swarm_config.json', 'w') json.dump(swarmconfig, json_file) json_file.close() return swarm
def swarm_over_data(): permWorkDir = os.path.abspath("swarm") if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) modelParams = permutations_runner.runWithConfig( SWARM_CONFIG, {"maxWorkers": 8, "overwrite": True}, outputLabel="exercise", outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=1, ) modelParamsFile = writeModelParamsToFile(modelParams, "exercise")
def swarm(inputFile): swarmWorkDir = os.path.abspath("swarmHTM") if not os.path.exists(swarmWorkDir): os.mkdir(swarmWorkDir), modelParams = permutations_runner.runWithConfig(fieldX, { "maxWorker": 4, "overwrite": True }, outputLabel="output", outDir=swarmWorkDir, permWorkDir=swarmWorkDir) modelParamsName = os.path.splitext(inputFile)[0] writeModelParams(modelParams, modelParamsName)
def swarm(): swarmWorkDir = os.path.abspath("swarm") if not os.path.exists(swarmWorkDir): os.mkdir(swarmWorkDir) print swarmWorkDir modelParams = permutations_runner.runWithConfig( SWARM_DESCRIPTION, {"maxWorkers": 4, "overwrite": True}, outputLabel="rec_center", outDir=swarmWorkDir, permWorkDir=swarmWorkDir ) writeModelParams(modelParams)
def swarm(): swarmWorkDir = os.path.abspath("swarm_results") if not os.path.exists(swarmWorkDir): os.mkdir(swarmWorkDir) modelParams = permutations_runner.runWithConfig( SWARM_DESCRIPTION, {"maxWorkers": swarm_setup.MAX_WORKERS, "overwrite": True}, outputLabel="cla_model", outDir=swarmWorkDir, permWorkDir=swarmWorkDir, verbosity=0) saveModelParams(modelParams)
def run_swarm_ng(working_dir,sensors_dir, sensor, store, swarmconfig): store_dir = sensors_dir + sensor + '/stores/' + store +'/' swarms_dir = store_dir + '/swarms/' swarm_dir = next_file_name(prefix='swarm',suffix='',path=swarms_dir) path,swarm = os.path.split(swarm_dir) swarm_dir = swarm_dir + '/' egg_file = swarm_dir + "eggs.csv" store_file = store_dir + 'out.csv' swarmconfig['streamDef']['streams'][0]['source'] = 'file://' + egg_file print swarmconfig['streamDef']['streams'][0]['source'] if not os.path.exists(swarms_dir): os.makedirs(swarms_dir) if not os.path.exists(swarm_dir): os.makedirs(swarm_dir) fieldNames = map(lambda x: x['fieldName'],swarmconfig['includedFields']) fieldTypes = map(lambda x: x['fieldType'],swarmconfig['includedFields']) prepare_swarm(field_names=fieldNames, field_types= fieldTypes, egg_file=egg_file, store_file = store_file) #swarm_config = {} # complete swarm config here #nupic_config = permutations_runner.runWithJsonFile(os.getcwd() + "/nupic/search_def.json",{'maxWorkers': 6, 'overwrite': True}, "test", os.getcwd() + "/working") nupic_config = permutations_runner.runWithConfig(swarmconfig, {'maxWorkers': 6, 'overwrite': True }, outDir= swarm_dir, outputLabel='', permWorkDir= swarm_dir, verbosity=1) #dump the model params generated by this swarm in 2 different file formats pickle_file = open(swarm_dir + 'model_params.pkl', 'w') pickle.dump(nupic_config,pickle_file,pickle.HIGHEST_PROTOCOL) pickle_file.close() json_file = open(swarm_dir +'model_params.json', 'w') json.dump(nupic_config, json_file) json_file.close() #dump the swarmconfig used for this swarm in 2 different file formats pickle_file = open(swarm_dir + 'swarm_config.pkl', 'w') pickle.dump(swarmconfig,pickle_file,pickle.HIGHEST_PROTOCOL) pickle_file.close() json_file = open(swarm_dir+ 'swarm_config.json', 'w') json.dump(swarmconfig, json_file) json_file.close() return swarm
def swarmForBestModelParams(swarmConfig, name, maxWorkers=4): outputLabel = name permWorkDir = os.path.abspath('swarm') if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) modelParams = permutations_runner.runWithConfig( swarmConfig, {"maxWorkers": maxWorkers, "overwrite": True}, outputLabel=outputLabel, outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=0 ) modelParamsFile = writeModelParamsToFile(modelParams, name) return modelParamsFile
def swarmForBestModelParams(swarmConfig, name, maxWorkers=4): outputLabel = name permWorkDir = os.path.abspath('swarm') if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) modelParams = permutations_runner.runWithConfig(swarmConfig, { "maxWorkers": maxWorkers, "overwrite": True }, outputLabel=outputLabel, outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=0) modelParamsFile = writeModelParamsToFile(modelParams, name) return modelParamsFile
def swarm_for_best_model_params(swarm_config, name, max_workers=4): output_label = name perm_work_dir = os.path.abspath('swarm') if not os.path.exists(perm_work_dir): os.mkdir(perm_work_dir) model_params = permutations_runner.runWithConfig(swarm_config, { "maxWorkers": max_workers, "overwrite": True }, outputLabel=output_label, outDir=perm_work_dir, permWorkDir=perm_work_dir, verbosity=0) model_params_file = write_model_params_to_file(model_params, name) return model_params_file
def swarm_for_best_model_params(swarm_config, name, max_workers=4): output_label = name perm_work_dir = os.path.abspath('swarm') if not os.path.exists(perm_work_dir): os.mkdir(perm_work_dir) model_params = permutations_runner.runWithConfig( swarm_config, {"maxWorkers": max_workers, "overwrite": True}, outputLabel=output_label, outDir=perm_work_dir, permWorkDir=perm_work_dir, verbosity=0 ) model_params_file = write_model_params_to_file(model_params, name) return model_params_file
def _swarmForBestModelParams(swarmConfig, name, workingDir, maxWorkers=8): outputLabel = name permWorkDir = os.path.join(workingDir, "swarm") if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) modelParams = permutations_runner.runWithConfig( swarmConfig, {"maxWorkers": maxWorkers, "overwrite": True}, outputLabel=outputLabel, outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=0 ) _writeModelParamsToFile(modelParams, workingDir) return modelParams
def __swarm(self): """ Starts a swarm and writes a generated files to swarm/ directory for the business. @return modelParam:(object) Models Parameters object. """ # Create directory for swarm details swarmWorkDir = self.__createSwarmWorkDir() modelParams = permutations_runner.runWithConfig( self._swarmDescriptionObject, {"maxWorkers": 2, "overwrite": True}, outputLabel=self._swarmName, outDir=swarmWorkDir, permWorkDir=swarmWorkDir ) # Write the model parameters to swarm directory. self.__writeModelParams(modelParams) return modelParams
def _runSwarm(self): if self.swarmDescType == 'template': pred_field = self.swarm_desc["inferenceArgs"]["predictedField"] self.model_params = permutations_runner.runWithConfig( self.swarm_desc, { "maxWorkers": self.max_workers, "overwrite": True }, outputLabel=pred_field, outDir=self.field_output_fpath, permWorkDir=self.output_fpath, verbosity=0) elif self.swarmDescType == 'permScript': swarm_fpath = os.path.join(self.field_output_fpath, self.swarm_desc) import_name = os.path.join(self.field_output_fpath, 'description') #import_name = os.path.join(os.path.dirname(self.swarm_desc), 'description') import_name = import_name.replace('/', '.') try: print import_name imported_desc = importlib.import_module(import_name).control except ImportError as err: print("Import error: {}".format(err)) pred_field = imported_desc["inferenceArgs"]["predictedField"] print pred_field self.model_params = permutations_runner.runWithPermutationsScript( swarm_fpath, { "maxWorkers": self.max_workers, "overwrite": True }, outputLabel=pred_field, permWorkDir=self.field_output_fpath) self._params_to_file(pred_field) return self.model_params_file
def swarm_for_best_model_params(swarm_config, name, max_workers): output_label = name perm_work_dir = os.path.abspath('swarm') if not os.path.exists(perm_work_dir): os.mkdir(perm_work_dir) if verbose: print "Using %i swarm workers." % max_workers print "\n** STARTING SWARM **\n\n" model_params = permutations_runner.runWithConfig( swarm_config, {"maxWorkers": max_workers, "overwrite": True}, outputLabel=output_label, outDir=perm_work_dir, permWorkDir=perm_work_dir, verbosity=0 ) model_params_file = write_model_params_to_file(model_params, name) return model_params_file
def swarmParams(swarmConfig, intersection): outputLabel = intersection permWorkDir = os.path.abspath('swarm') if not os.path.exists(permWorkDir): os.mkdir(permWorkDir) import multiprocessing as mp # use 3/4 of your CPUs maxWorkers = 3 * mp.cpu_count() / 2 print "Running swarm!" modelParams = permutations_runner.runWithConfig( swarmConfig, {"maxWorkers": maxWorkers, "overwrite": True}, outputLabel=outputLabel, outDir=permWorkDir, permWorkDir=permWorkDir, verbosity=2 ) modelParamsFile = writeModelParams(modelParams, intersection) return modelParamsFile
def swarm_for_best_model_params(swarm_config, name, max_workers): output_label = name perm_work_dir = os.path.abspath('swarm') if not os.path.exists(perm_work_dir): os.mkdir(perm_work_dir) if verbose: print "Using %i swarm workers." % max_workers print "\n** STARTING SWARM **\n\n" model_params = permutations_runner.runWithConfig(swarm_config, { "maxWorkers": max_workers, "overwrite": True }, outputLabel=output_label, outDir=perm_work_dir, permWorkDir=perm_work_dir, verbosity=0) model_params_file = write_model_params_to_file(model_params, name) return model_params_file
def swarm(cwd, input_file, swarm_description, model_name=None, write_model=False, max_workers=4): swarm_work_dir = os.path.abspath('swarm') if not os.path.exists(swarm_work_dir): os.mkdir(swarm_work_dir) stream = swarm_description['streamDef']['streams'][0] full_path = os.path.join(cwd, input_file) stream['source'] = 'file://{}'.format(full_path) label = swarm_description['streamDef']['info'] model_params = permutations_runner.runWithConfig( swarm_description, {'maxWorkers': max_workers, 'overwrite': True}, outputLabel=label, outDir=swarm_work_dir, permWorkDir=swarm_work_dir ) if write_model: write_model_params(cwd, model_params, model_name) return model_params
def run_swarms(): '''Runs every swarm found in the directory structure found above using a local, scratch get the data directories, screen out git and other junk files''' dirs=[] for ddir in os.listdir(BASE_DIR): if os.path.isdir(os.path.join(BASE_DIR, ddir)) and ddir[0].isdigit(): dirs.append(ddir) #for each of the swarm description directories, dig out the description files ... swarm_files=[] for d in dirs: data_id = d.split('_')[0] swarm_files=os.listdir(os.path.join(BASE_DIR, d, data_id+'_swarm')) #All files in this dir with the right tag (naming convention) are swarm descriptors for source_file in swarm_files: if SWARM_DESC_TAG in source_file: #set up the scratch dir swarm_dir=tempfile.mkdtemp() sys.path.append(swarm_dir) #... make a scratch copy of the file and import the description dict swarm_file_name=os.path.join(swarm_dir, SWARM_DESC_TAG) shutil.copyfile(os.path.join(BASE_DIR, d, ODIR_SUBDIR.format(data_id), source_file), swarm_file_name ) ##os.chdir(scatch) not thread safe from swarm_description import SWARM_DESCTIPTION #...and run the built in runWithConfig() method from permutaions_runner name = source_file.split('_swarm')[0] #this will be something like '666_energy_only' param_fn = PARAMS_FILE.format(name) print('running swarm for {} started {}'. format(source_file.split('_swarm')[0], datetime.now().strftime('%m-%d-%y %H:%M')), end='' ) if DEBUG: model_params = 'test params' with open(os.path.join(swarm_dir, param_fn), 'w') as f: f.write (model_params) else: model_params = permutations_runner.runWithConfig( SWARM_DESCTIPTION, {"maxWorkers": MAX_WORKERS, "overwrite": True}, outputLabel=name, outDir=swarm_dir, permWorkDir=swarm_dir, verbosity=0 ) print(' ... finished at {}'.format(datetime.now().strftime('%m-%d-%y %H:%M'))) #dump parameters to the local scratch dir writeModelParamsToFile(model_params, param_fn) #finally, copy it back to the original data file directory shutil.copyfile(os.path.join(swarm_dir, param_fn), os.path.join(BASE_DIR, d, ODIR_SUBDIR.format(data_id), PARAMS_FILE.format(name)) ) shutil.rmtree(swarm_dir)
def swarm_over_data(swarm_config): permutations_runner.runWithConfig(swarm_config, {'maxWorkers': 1, 'overwrite': True}) shutil.copyfile("model_0/model_params.py", "model_params.py")
def swarm_over_data(): model_params = permutations_runner.runWithConfig(swarm_config, { 'maxWorkers': 2, 'overwrite': True }) shutil.copyfile("model_0/model_params.py", "model_params.py")
def swarm_over_data(): return permutations_runner.runWithConfig(SWARM_CONFIG, { 'maxWorkers': 4, 'overwrite': True })
#!/usr/bin/env python from nupic.swarming import permutations_runner import os import json import logging logging.basicConfig() if __name__ == "__main__": with open('swarm_config.json', 'r') as conf: swarm_config = json.load(conf) model_params = permutations_runner.runWithConfig(swarm_config, { 'maxWorkers': 1, 'overwrite': True }, verbosity=3) exit() model_params = permutations_runner.runWithPermutationsScript( 'permutations.py', { 'maxWorkers': 1, 'overwrite': True, # 'expDescConfig': 'description.py', 'verbosityCount': 0 }, 'output', './model_0') print(json.dumps(model_params))
def swarm_over_data(): return permutations_runner.runWithConfig(SWARM_CONFIG, {"maxWorkers": 8, "overwrite": True})
def swarm_over_data(): return permutations_runner.runWithConfig(SWARM_CONFIG, {'maxWorkers': 8, 'overwrite': True})