def on_get(self, req, resp, id):
topology_id = self.validated_params['id']
try:
topology = MongodbFactory.get_topology(topology_id)
if topology is None:
return ResponsesHandler.handle_404(resp, 'Topology not found')
text = LogViewer.get_log(topology[TopologyConst.FIELD_NAME])
return ResponsesHandler.handle_200(resp, {
'id': topology_id,
'log': text
})
except TopologyInvalidId as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_404(resp, Ex.message)
except MissingMandatoryFields as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_409(resp)
except Error as Ex:
logger.critical(Ex.message)
return ResponsesHandler.handle_500(resp)
except Exception as Ex:
logger.critical(Ex.message)
return ResponsesHandler.handle_500(resp)
def run_command(command,ignore_errors=False,log_output=True,callback=None,startupinfo=None,creationflags=0,timeout=120):
stdout=""
stderr=""
if command != None:
logger.debug('run_command: %s' % command)
logger.debug(' called from %s:%d' % inspect.stack()[1][1:3])
ssh_process=subprocess.Popen(command,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True,universal_newlines=True,startupinfo=startupinfo,creationflags=creationflags)
#(stdout,stderr) = ssh_process.communicate(command)
t=threading.Timer(timeout,ssh_process.kill)
t.start()
(stdout,stderr) = ssh_process.communicate()
t.cancel()
ssh_process.wait()
if log_output:
logger.debug('command stdout: %s' % repr(stdout))
logger.debug('command stderr: %s' % repr(stderr))
if not ignore_errors and len(stderr) > 0:
error_message = 'Error running command: "%s" at line %d' % (command, inspect.stack()[1][2])
logger.error('Nonempty stderr and ignore_errors == False; exiting the launcher with error dialog: ' + error_message)
if (callback != None):
callback(error_message)
return stdout, stderr
def plotResults(history):
try:
plt.plot(history.history['accuracy'], label='accuracy')
plt.plot(history.history['val_accuracy'], label='val_accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
# plt.ylim([0.5, 1])
plt.legend(loc='lower right')
plt.show()
except KeyError as ke:
logger.error("Caught key error: {}".format(ke.__str__()))
def run(self):
logger.debug("loadkeyThread: started")
try:
logger.debug("loadkeyThread: Trying to open the key file")
with open(self.keydistObject.keyModel.sshpaths.sshKeyPath,'r'): pass
event = KeyDist.sshKeyDistEvent(KeyDist.EVT_KEYDIST_LOADKEY,self.keydistObject)
except Exception as e:
logger.error("loadkeyThread: Failed to open the key file %s" % str(e))
self.keydistObject.cancel("Failed to open the key file %s" % str(e))
return
if (not self.stopped()):
logger.debug("loadkeyThread: generating LOADKEY event from loadkeyThread")
wx.PostEvent(self.keydistObject.notifywindow.GetEventHandler(),event)
def run_command(command,
ignore_errors=False,
log_output=True,
callback=None,
startupinfo=None,
creationflags=0,
timeout=120):
stdout = ""
stderr = ""
if command != None:
logger.debug('run_command: %s' % command)
logger.debug(' called from %s:%d' % inspect.stack()[1][1:3])
ssh_process = subprocess.Popen(command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
universal_newlines=True,
startupinfo=startupinfo,
creationflags=creationflags)
#(stdout,stderr) = ssh_process.communicate(command)
t = threading.Timer(timeout, ssh_process.kill)
t.start()
(stdout, stderr) = ssh_process.communicate()
t.cancel()
ssh_process.wait()
if log_output:
logger.debug('command stdout: %s' % stdout)
logger.debug('command stderr: %s' % stderr)
if not ignore_errors and len(stderr) > 0:
error_message = 'Error running command: "%s" at line %d' % (
command, inspect.stack()[1][2])
logger.error(
'Nonempty stderr and ignore_errors == False; exiting the launcher with error dialog: '
+ error_message)
if (callback != None):
callback(error_message)
return stdout, stderr
def run(self):
logger.debug("loadkeyThread: started")
try:
logger.debug("loadkeyThread: Trying to open the key file")
with open(self.keydistObject.keyModel.sshpaths.sshKeyPath,
'r'):
pass
event = KeyDist.sshKeyDistEvent(KeyDist.EVT_KEYDIST_LOADKEY,
self.keydistObject)
except Exception as e:
logger.error("loadkeyThread: Failed to open the key file %s" %
str(e))
self.keydistObject.cancel("Failed to open the key file %s" %
str(e))
return
if (not self.stopped()):
logger.debug(
"loadkeyThread: generating LOADKEY event from loadkeyThread"
)
wx.PostEvent(self.keydistObject.notifywindow.GetEventHandler(),
event)
def on_delete(self, req, res, id, version_id):
module_id = self.validated_params['id']
module_version_id = self.validated_params['version_id']
compiler = None
try:
module = MongodbFactory.get_module(module_id)
module_version = MongodbFactory.get_module_version(
module_version_id)
if not module or not module_version:
return ResponsesHandler.handle_404(res, 'Module not found')
m_name = module[ModuleConst.FIELD_NAME]
m_version = module_version[ModuleVersionConst.FIELD_VERSION_CODE]
compiler = Compiler.delete_module(m_name, m_version)
except ModuleVersionInvalidId as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_404(res, msg=Ex.message)
except MissingMandatoryFields as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_409(res)
except Error as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_400(res)
except Exception as e:
logger.critical(e.message)
return ResponsesHandler.handle_500(res)
finally:
if compiler:
compiler.remove()
return ResponsesHandler.handle_200(res)
def __init__(self,
host='localhost',
port=27017,
db_name=None,
user=None,
pwd=None):
if not db_name:
raise MongodbInvalidDatabase()
self.client = MongoClient('mongodb://' + host + ':' + str(port))
if user and pwd:
try:
is_logged = self.client[db_name].authenticate(user, pwd)
except OperationFailure as e:
logger.error(e.message)
is_logged = False
if not is_logged:
raise MongodbInvalidCredentials()
self.db = self.client[db_name]
def on_post(self, req, resp, id):
topology_id = self.validated_params['id']
try:
topology = MongodbFactory.get_topology(topology_id)
if not topology:
return ResponsesHandler.handle_404(resp, 'Topology not found')
info = StormUI.kill_topology_by_name(
topology[TopologyConst.FIELD_NAME])
if info['status'] == 'success':
MongodbFactory.update_stopped_state(topology_id)
return ResponsesHandler.handle_200(
resp, {
'id': topology_id,
'status': TopologyConst.STATUS_STOPPED
})
else:
return ResponsesHandler.handle_409(resp, {
'id': topology_id,
'status': info['status']
})
except TopologyInvalidId as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_404(resp)
except MissingMandatoryFields as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_409(resp)
except TopologyNotInCluster as Ex:
MongodbFactory.update_stopped_state(topology_id)
logger.error(Ex.message)
return ResponsesHandler.handle_409(resp)
except Error as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_500(resp)
except Exception as Ex:
logger.critical(Ex.message)
return ResponsesHandler.handle_500(resp)
def on_post(self, req, resp, id):
topology_id = self.validated_params['id']
builder = None
try:
topology = MongodbFactory.get_topology(topology_id)
if not topology:
return ResponsesHandler.handle_404(resp, 'Topology not found')
topology_name = topology[TopologyConst.FIELD_NAME]
try:
StormUI.kill_topology_by_name(topology_name)
except TopologyNotInCluster:
pass
logger.debug('Creating topology project...')
builder = TopologyBuilder(topology)
logger.debug('Building topology jar...')
builder.build_jar_with_dependencies()
logger.debug('Uploading topology to storm...')
result = StormProcess.launch_topology_cmd(
topology_name,
os.path.join(builder.base_path, 'target', builder.jar_name))
logger.debug(result.stdout)
MongodbFactory.update_running_state(topology_id)
return ResponsesHandler.handle_200(
resp, {
'id': topology_id,
'status': TopologyConst.STATUS_RUNNING
})
except TopologyInvalidId as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_404(resp)
except CommandException as Ex:
logger.error(Ex.message)
logger.debug(Ex.log())
return ResponsesHandler.handle_500(resp)
except Error as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_500(resp)
except Exception as e:
logger.critical(e.message)
return ResponsesHandler.handle_500(resp)
finally:
if builder is not None:
builder.remove()
def on_post(self, req, res, id, version_id):
module_id = self.validated_params['id']
module_version_id = self.validated_params['version_id']
compiler = None
try:
module = MongodbFactory.get_module(module_id)
module_version = MongodbFactory.get_module_version(
module_version_id)
if not module_version or not module:
return ResponsesHandler.handle_404(res, 'Module not found')
if not module_version[ModuleVersionConst.FIELD_SOURCE_CODE]:
raise ModuleException('The module\'s ' +
ModuleVersionConst.FIELD_SOURCE_CODE +
' is mandatory.')
m_name = module[ModuleConst.FIELD_NAME]
m_type = module.get(ModuleConst.FIELD_TYPE)
if not m_type and not module['container']:
raise ModuleException('We can not found the module\'s type')
elif not m_type:
m_type = module['container'][ModuleConst.FIELD_TYPE]
m_lang = module[ModuleConst.FIELD_LANG].lower()
m_code = module_version[ModuleVersionConst.FIELD_SOURCE_CODE]
m_version = str(
module_version[ModuleVersionConst.FIELD_VERSION_CODE])
m_libraries = module_version.get(
ModuleVersionConst.FIELD_LIBRARIES)
if m_libraries is not None:
dependencies = list(
map(Compiler.build_mvn_dependency_from_url, [
l[ModuleVersionConst.FIELD_LIBRARY_URL]
for l in m_libraries
]))
else:
dependencies = list()
compiler = Compiler(m_name,
m_version,
m_type,
m_lang,
m_code,
dependencies=dependencies)
MongodbFactory.update_status_builder_module(
module_version_id, ModuleVersionConst.BUILD_STATUS_SUCCESS)
MongodbFactory.update_module_last_modify(module_id)
compiler.install_jar()
except ModuleInvalidId as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_404(res, msg=Ex.message)
except ModuleVersionInvalidId as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_404(res, msg=Ex.message)
except MissingMandatoryFields as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_409(res, Ex.message)
except ModuleException as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_803(res, data=Ex.message)
except ModuleInvalidLanguage as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_803(res, data=Ex.message)
except ModuleErrorCompilingException as Ex:
logger.error(Ex.message)
MongodbFactory.update_status_builder_module(
module_version_id, ModuleVersionConst.BUILD_STATUS_FAILURE)
MongodbFactory.update_module_last_modify(module_id)
return ResponsesHandler.handle_804(res, trace=Ex.message)
except CommandException as Ex:
logger.error(Ex.log())
return ResponsesHandler.handle_500(res)
except Error as Ex:
logger.error(Ex.message)
return ResponsesHandler.handle_500(res)
except Exception as e:
logger.critical(e.message)
return ResponsesHandler.handle_500(res)
finally:
if compiler:
compiler.remove()
return ResponsesHandler.handle_200(res)
help='neural network model used for classification, \navailable models: %(choices)s')
parser.add_argument('method',
choices=['rus', 'ros', 'tpl', 'ds', 'fl', 'csdnn', 'mfse', 'crlahm'],
help='method to handle imbalance problem, \navailbale techniques: %(choices)s')
if __name__ == '__main__':
args = parser.parse_args()
factory = SimpleFactory()
handleImbObj = factory.createHandleImbObj(args.method)
neuralNetworkModel = factory.createNeuralNetworkModel(args.model)
if isinstance(neuralNetworkModel, CNN):
(train_data, train_labels), (test_data, test_labels) = datasets.cifar10.load_data() # CIFAR-10
concatenatedData = np.concatenate((train_data, test_data))
elif isinstance(neuralNetworkModel, RNN):
(train_data, train_labels), (test_data, test_labels) = datasets.imdb.load_data(num_words=20000) # IMBD - REVIEWS
concatenatedData = np.concatenate((train_data, test_data))
concatenatedData = tf.keras.preprocessing.sequence.pad_sequences(concatenatedData, maxlen=80)
try:
data, labels = concatenatedData, np.concatenate((train_labels, test_labels))
# data, labels = DataProcessor.reduceDatasetSize(concatenatedData, np.concatenate((train_labels, test_labels)), 2)
data, labels = DataImbalancer.imbalanceImageDataset(data, labels, args.imbalance_ratio, [0])
except RuntimeError as re:
logger.error('Imbalance error {}'.format(re.__str__()))
history = handleImbObj.handleImbalanceProblem(data, labels, neuralNetworkModel, epochs=10)
ResultPlotter.plotResults(history)
def log(self):
logger.info(self.stdout)
if self.stderr:
logger.error(self.stderr)
