def on_get(self, req, resp, id): topology_id = self.validated_params['id'] try: topology = MongodbFactory.get_topology(topology_id) if topology is None: return ResponsesHandler.handle_404(resp, 'Topology not found') text = LogViewer.get_log(topology[TopologyConst.FIELD_NAME]) return ResponsesHandler.handle_200(resp, { 'id': topology_id, 'log': text }) except TopologyInvalidId as Ex: logger.error(Ex.message) return ResponsesHandler.handle_404(resp, Ex.message) except MissingMandatoryFields as Ex: logger.error(Ex.message) return ResponsesHandler.handle_409(resp) except Error as Ex: logger.critical(Ex.message) return ResponsesHandler.handle_500(resp) except Exception as Ex: logger.critical(Ex.message) return ResponsesHandler.handle_500(resp)
def run_command(command,ignore_errors=False,log_output=True,callback=None,startupinfo=None,creationflags=0,timeout=120): stdout="" stderr="" if command != None: logger.debug('run_command: %s' % command) logger.debug(' called from %s:%d' % inspect.stack()[1][1:3]) ssh_process=subprocess.Popen(command,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True,universal_newlines=True,startupinfo=startupinfo,creationflags=creationflags) #(stdout,stderr) = ssh_process.communicate(command) t=threading.Timer(timeout,ssh_process.kill) t.start() (stdout,stderr) = ssh_process.communicate() t.cancel() ssh_process.wait() if log_output: logger.debug('command stdout: %s' % repr(stdout)) logger.debug('command stderr: %s' % repr(stderr)) if not ignore_errors and len(stderr) > 0: error_message = 'Error running command: "%s" at line %d' % (command, inspect.stack()[1][2]) logger.error('Nonempty stderr and ignore_errors == False; exiting the launcher with error dialog: ' + error_message) if (callback != None): callback(error_message) return stdout, stderr
def plotResults(history): try: plt.plot(history.history['accuracy'], label='accuracy') plt.plot(history.history['val_accuracy'], label='val_accuracy') plt.xlabel('Epoch') plt.ylabel('Accuracy') # plt.ylim([0.5, 1]) plt.legend(loc='lower right') plt.show() except KeyError as ke: logger.error("Caught key error: {}".format(ke.__str__()))
def run(self): logger.debug("loadkeyThread: started") try: logger.debug("loadkeyThread: Trying to open the key file") with open(self.keydistObject.keyModel.sshpaths.sshKeyPath,'r'): pass event = KeyDist.sshKeyDistEvent(KeyDist.EVT_KEYDIST_LOADKEY,self.keydistObject) except Exception as e: logger.error("loadkeyThread: Failed to open the key file %s" % str(e)) self.keydistObject.cancel("Failed to open the key file %s" % str(e)) return if (not self.stopped()): logger.debug("loadkeyThread: generating LOADKEY event from loadkeyThread") wx.PostEvent(self.keydistObject.notifywindow.GetEventHandler(),event)
def run_command(command, ignore_errors=False, log_output=True, callback=None, startupinfo=None, creationflags=0, timeout=120): stdout = "" stderr = "" if command != None: logger.debug('run_command: %s' % command) logger.debug(' called from %s:%d' % inspect.stack()[1][1:3]) ssh_process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, universal_newlines=True, startupinfo=startupinfo, creationflags=creationflags) #(stdout,stderr) = ssh_process.communicate(command) t = threading.Timer(timeout, ssh_process.kill) t.start() (stdout, stderr) = ssh_process.communicate() t.cancel() ssh_process.wait() if log_output: logger.debug('command stdout: %s' % stdout) logger.debug('command stderr: %s' % stderr) if not ignore_errors and len(stderr) > 0: error_message = 'Error running command: "%s" at line %d' % ( command, inspect.stack()[1][2]) logger.error( 'Nonempty stderr and ignore_errors == False; exiting the launcher with error dialog: ' + error_message) if (callback != None): callback(error_message) return stdout, stderr
def run(self): logger.debug("loadkeyThread: started") try: logger.debug("loadkeyThread: Trying to open the key file") with open(self.keydistObject.keyModel.sshpaths.sshKeyPath, 'r'): pass event = KeyDist.sshKeyDistEvent(KeyDist.EVT_KEYDIST_LOADKEY, self.keydistObject) except Exception as e: logger.error("loadkeyThread: Failed to open the key file %s" % str(e)) self.keydistObject.cancel("Failed to open the key file %s" % str(e)) return if (not self.stopped()): logger.debug( "loadkeyThread: generating LOADKEY event from loadkeyThread" ) wx.PostEvent(self.keydistObject.notifywindow.GetEventHandler(), event)
def on_delete(self, req, res, id, version_id): module_id = self.validated_params['id'] module_version_id = self.validated_params['version_id'] compiler = None try: module = MongodbFactory.get_module(module_id) module_version = MongodbFactory.get_module_version( module_version_id) if not module or not module_version: return ResponsesHandler.handle_404(res, 'Module not found') m_name = module[ModuleConst.FIELD_NAME] m_version = module_version[ModuleVersionConst.FIELD_VERSION_CODE] compiler = Compiler.delete_module(m_name, m_version) except ModuleVersionInvalidId as Ex: logger.error(Ex.message) return ResponsesHandler.handle_404(res, msg=Ex.message) except MissingMandatoryFields as Ex: logger.error(Ex.message) return ResponsesHandler.handle_409(res) except Error as Ex: logger.error(Ex.message) return ResponsesHandler.handle_400(res) except Exception as e: logger.critical(e.message) return ResponsesHandler.handle_500(res) finally: if compiler: compiler.remove() return ResponsesHandler.handle_200(res)
def __init__(self, host='localhost', port=27017, db_name=None, user=None, pwd=None): if not db_name: raise MongodbInvalidDatabase() self.client = MongoClient('mongodb://' + host + ':' + str(port)) if user and pwd: try: is_logged = self.client[db_name].authenticate(user, pwd) except OperationFailure as e: logger.error(e.message) is_logged = False if not is_logged: raise MongodbInvalidCredentials() self.db = self.client[db_name]
def on_post(self, req, resp, id): topology_id = self.validated_params['id'] try: topology = MongodbFactory.get_topology(topology_id) if not topology: return ResponsesHandler.handle_404(resp, 'Topology not found') info = StormUI.kill_topology_by_name( topology[TopologyConst.FIELD_NAME]) if info['status'] == 'success': MongodbFactory.update_stopped_state(topology_id) return ResponsesHandler.handle_200( resp, { 'id': topology_id, 'status': TopologyConst.STATUS_STOPPED }) else: return ResponsesHandler.handle_409(resp, { 'id': topology_id, 'status': info['status'] }) except TopologyInvalidId as Ex: logger.error(Ex.message) return ResponsesHandler.handle_404(resp) except MissingMandatoryFields as Ex: logger.error(Ex.message) return ResponsesHandler.handle_409(resp) except TopologyNotInCluster as Ex: MongodbFactory.update_stopped_state(topology_id) logger.error(Ex.message) return ResponsesHandler.handle_409(resp) except Error as Ex: logger.error(Ex.message) return ResponsesHandler.handle_500(resp) except Exception as Ex: logger.critical(Ex.message) return ResponsesHandler.handle_500(resp)
def on_post(self, req, resp, id): topology_id = self.validated_params['id'] builder = None try: topology = MongodbFactory.get_topology(topology_id) if not topology: return ResponsesHandler.handle_404(resp, 'Topology not found') topology_name = topology[TopologyConst.FIELD_NAME] try: StormUI.kill_topology_by_name(topology_name) except TopologyNotInCluster: pass logger.debug('Creating topology project...') builder = TopologyBuilder(topology) logger.debug('Building topology jar...') builder.build_jar_with_dependencies() logger.debug('Uploading topology to storm...') result = StormProcess.launch_topology_cmd( topology_name, os.path.join(builder.base_path, 'target', builder.jar_name)) logger.debug(result.stdout) MongodbFactory.update_running_state(topology_id) return ResponsesHandler.handle_200( resp, { 'id': topology_id, 'status': TopologyConst.STATUS_RUNNING }) except TopologyInvalidId as Ex: logger.error(Ex.message) return ResponsesHandler.handle_404(resp) except CommandException as Ex: logger.error(Ex.message) logger.debug(Ex.log()) return ResponsesHandler.handle_500(resp) except Error as Ex: logger.error(Ex.message) return ResponsesHandler.handle_500(resp) except Exception as e: logger.critical(e.message) return ResponsesHandler.handle_500(resp) finally: if builder is not None: builder.remove()
def on_post(self, req, res, id, version_id): module_id = self.validated_params['id'] module_version_id = self.validated_params['version_id'] compiler = None try: module = MongodbFactory.get_module(module_id) module_version = MongodbFactory.get_module_version( module_version_id) if not module_version or not module: return ResponsesHandler.handle_404(res, 'Module not found') if not module_version[ModuleVersionConst.FIELD_SOURCE_CODE]: raise ModuleException('The module\'s ' + ModuleVersionConst.FIELD_SOURCE_CODE + ' is mandatory.') m_name = module[ModuleConst.FIELD_NAME] m_type = module.get(ModuleConst.FIELD_TYPE) if not m_type and not module['container']: raise ModuleException('We can not found the module\'s type') elif not m_type: m_type = module['container'][ModuleConst.FIELD_TYPE] m_lang = module[ModuleConst.FIELD_LANG].lower() m_code = module_version[ModuleVersionConst.FIELD_SOURCE_CODE] m_version = str( module_version[ModuleVersionConst.FIELD_VERSION_CODE]) m_libraries = module_version.get( ModuleVersionConst.FIELD_LIBRARIES) if m_libraries is not None: dependencies = list( map(Compiler.build_mvn_dependency_from_url, [ l[ModuleVersionConst.FIELD_LIBRARY_URL] for l in m_libraries ])) else: dependencies = list() compiler = Compiler(m_name, m_version, m_type, m_lang, m_code, dependencies=dependencies) MongodbFactory.update_status_builder_module( module_version_id, ModuleVersionConst.BUILD_STATUS_SUCCESS) MongodbFactory.update_module_last_modify(module_id) compiler.install_jar() except ModuleInvalidId as Ex: logger.error(Ex.message) return ResponsesHandler.handle_404(res, msg=Ex.message) except ModuleVersionInvalidId as Ex: logger.error(Ex.message) return ResponsesHandler.handle_404(res, msg=Ex.message) except MissingMandatoryFields as Ex: logger.error(Ex.message) return ResponsesHandler.handle_409(res, Ex.message) except ModuleException as Ex: logger.error(Ex.message) return ResponsesHandler.handle_803(res, data=Ex.message) except ModuleInvalidLanguage as Ex: logger.error(Ex.message) return ResponsesHandler.handle_803(res, data=Ex.message) except ModuleErrorCompilingException as Ex: logger.error(Ex.message) MongodbFactory.update_status_builder_module( module_version_id, ModuleVersionConst.BUILD_STATUS_FAILURE) MongodbFactory.update_module_last_modify(module_id) return ResponsesHandler.handle_804(res, trace=Ex.message) except CommandException as Ex: logger.error(Ex.log()) return ResponsesHandler.handle_500(res) except Error as Ex: logger.error(Ex.message) return ResponsesHandler.handle_500(res) except Exception as e: logger.critical(e.message) return ResponsesHandler.handle_500(res) finally: if compiler: compiler.remove() return ResponsesHandler.handle_200(res)
help='neural network model used for classification, \navailable models: %(choices)s') parser.add_argument('method', choices=['rus', 'ros', 'tpl', 'ds', 'fl', 'csdnn', 'mfse', 'crlahm'], help='method to handle imbalance problem, \navailbale techniques: %(choices)s') if __name__ == '__main__': args = parser.parse_args() factory = SimpleFactory() handleImbObj = factory.createHandleImbObj(args.method) neuralNetworkModel = factory.createNeuralNetworkModel(args.model) if isinstance(neuralNetworkModel, CNN): (train_data, train_labels), (test_data, test_labels) = datasets.cifar10.load_data() # CIFAR-10 concatenatedData = np.concatenate((train_data, test_data)) elif isinstance(neuralNetworkModel, RNN): (train_data, train_labels), (test_data, test_labels) = datasets.imdb.load_data(num_words=20000) # IMBD - REVIEWS concatenatedData = np.concatenate((train_data, test_data)) concatenatedData = tf.keras.preprocessing.sequence.pad_sequences(concatenatedData, maxlen=80) try: data, labels = concatenatedData, np.concatenate((train_labels, test_labels)) # data, labels = DataProcessor.reduceDatasetSize(concatenatedData, np.concatenate((train_labels, test_labels)), 2) data, labels = DataImbalancer.imbalanceImageDataset(data, labels, args.imbalance_ratio, [0]) except RuntimeError as re: logger.error('Imbalance error {}'.format(re.__str__())) history = handleImbObj.handleImbalanceProblem(data, labels, neuralNetworkModel, epochs=10) ResultPlotter.plotResults(history)
def log(self): logger.info(self.stdout) if self.stderr: logger.error(self.stderr)