def __init__(self):
parser = Parser(CONFIG_PATH)
self._host = parser.server()['host']
self._port = parser.server()['port']
self._endpoints = {}
for ep in parser.endpoints():
self._endpoints[ep['endpoint_url']] = Endpoint(ep)
def __init__(self):
parser = Parser()
args = parser.get_section('FLAIR')
self.corpus_path = args['flair_data_path']
self.word_emb_path = args['flair_emb_path']
self.model_path = args['flair_model_path']
self.hidden_size = int(args['flair_num_hidden'])
self.epochs = int(args['flair_epochs'])
def __init__(self):
parser = Parser()
args = parser.get_section('SVM')
self.model_path = args['svm_model_path']
self.pipeline = Pipeline([('vect',
CountVectorizer(tokenizer=self.do_nothing,
preprocessor=None,
lowercase=False)),
('tfidf', TfidfTransformer()),
('clf', LinearSVC(class_weight='balanced'))])
def main():
parser = Parser()
config = parser.parse_config("client")
client = Client(config)
tasks = parser.parse_tasks()
client.register_task(tasks)
try:
client.execute()
except Exception as e:
raise e
finally:
client.close()
def __init__(self, dataset, is_train=True):
parser = Parser()
args = parser.get_sections(['GENERAL', 'RNN'])
self.dataset = dataset
self.num_hidden = int(args['num_hidden'])
self.hidden_size = int(args['hidden_size'])
self.num_classes = int(args['num_classes'])
self.num_epochs = int(args['epochs'])
self.max_sent_length = int(args['max_sent_length'])
self.saved_dir = args['saved_dir']
self.display_freq = int(args['display_frequency'])
self.rnn_keep_prob = float(args['rnn_keep_prob'])
self.dense_keep_prob = float(args['dense_keep_prob'])
self.x = tf.placeholder(tf.int32, [None, self.max_sent_length], name="x")
self.y = tf.placeholder(tf.int32, [None], name="y")
self.rnn_keep_prob_ph = tf.placeholder(tf.float32, name="rnn_keep_prob")
self.dense_keep_prob_ph = tf.placeholder(tf.float32, name="dense_keep_prob")
self.class_weight = tf.placeholder(tf.float32, name="class_weight")
self.global_step = tf.Variable(0, trainable=False)
if is_train:
with tf.name_scope("embeddings"):
init_embeddings = tf.constant(self.dataset.embeddings, dtype=tf.float32)
embeddings = tf.get_variable("embeddings", initializer=init_embeddings, trainable=False)
x_emb = tf.nn.embedding_lookup(embeddings, self.x)
with tf.name_scope("encoder"):
forward_cells = GRUCell(self.num_hidden)
backward_cells = GRUCell(self.num_hidden)
forward_cells = DropoutWrapper(forward_cells, output_keep_prob=self.rnn_keep_prob_ph)
backward_cells = DropoutWrapper(backward_cells, output_keep_prob=self.rnn_keep_prob_ph)
(encoder_outputs_fw, encoder_outputs_bw), _ = tf.nn.bidirectional_dynamic_rnn(forward_cells,
backward_cells,
x_emb, dtype=tf.float32)
output = tf.concat([encoder_outputs_fw, encoder_outputs_bw], axis=2)[:, -1, :]
with tf.name_scope("dense"):
dense = fully_connected(output, self.hidden_size, activation_fn=tf.nn.relu)
drop = dropout(dense, self.dense_keep_prob_ph)
logits = fully_connected(drop, self.num_classes, activation_fn=tf.nn.relu)
self.y_pred = tf.argmax(logits, axis=-1, output_type=tf.int32, name="y_pred")
with tf.name_scope("loss"):
weights = tf.gather(self.class_weight, self.y)
xent = tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=self.y, weights=weights)
self.loss = tf.reduce_mean(xent)
self.optimizer = tf.train.AdamOptimizer().minimize(self.loss, global_step=self.global_step)
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(self.y_pred, self.y)
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def __init__(self):
Parser.__init__(self)
os_dist = platform.dist()[0].lower()
self.os = os_dist
if os_dist == 'centos' or os_dist == 'redhat':
self.os_dist = 'rpm'
self.pkg_dist = 'el' + platform.dist()[1][0]
elif os_dist == 'ubuntu' or os_dist == 'debian':
self.os_dist = 'deb'
self.pkg_dist = platform.dist()[2] # to do
else:
self.os_dist = 'pkg'
self.arch = platform.machine()
def __init__(self, texts_file, tags_file, clean_data=True, remove_stopwords=False, is_train=True):
self.args = Parser().get_sections(['GENERAL', 'RNN', 'FLAIR'])
self.max_sent_length = int(self.args['max_sent_length'])
self.batch_size = int(self.args['batch_size'])
self.emb_size = int(self.args['emb_size'])
self.clean_data = clean_data
self.remove_stopwords = remove_stopwords
self.is_train = is_train
self.nlp = Polish()
self.df = self.build_dataframe(texts_file, tags_file)
self.unk_emb = self.get_random_emb(self.emb_size)
self.word2idx, self.idx2word = self.build_dict()
if self.is_train:
self.embeddings = self.get_embeddings(self.args['emb_path'])
def evaluate_baselines(true_y):
args = Parser().get_section('GENERAL')
most_frequent_label = stats.mode(true_y).mode[0]
most_frequent_label_baseline = [
most_frequent_label for _ in range(len(true_y))
]
random_baseline = [
randint(0,
int(args['num_classes']) - 1) for _ in range(len(true_y))
]
return (
"Baselines: \n"
f"random acc: {accuracy_score(true_y, random_baseline):.01%}, "
f"random F1: {f1_score(true_y, random_baseline, average='macro'):.01%}, \n"
f"most frequent label acc: {accuracy_score(true_y, most_frequent_label_baseline):.01%}, "
f"most frequent label F1: {f1_score(true_y, most_frequent_label_baseline, average='macro'):.01%}"
)
class Server(object):
__maxListen = 1
__runServer = None
@property
def serverShouldRun(self):
if self.__runServer == None:
self.__runServer = True
return self.__runServer
def stop_server(self):
self.__runServer = False
return self.serverShouldRun
def __init__(self):
self.config = Parser()
self.scale = USBScaleSingleton()
self.db = MySQLDataBase()
def get_weight(self):
return self.scale.get_weight(self.scale.DATA_MODE_GRAMS)
@property
def host(self):
return self.config.get('server', 'host')
@property
def port(self):
return int(self.config.get('server', 'port'))
def handle_request(self, request):
rVal = ''
if request == 'close server':
self.stop_server()
rVal = 'server about to stop..'
elif request == 'get weight':
rVal = str(self.get_weight())
elif request.startswith('store weight'):
args = re.split('\s+', request.replace('store result', ''))
if args == ['']:
args = ()
self.store_value(*args)
elif request == 'get all':
self.db.get_all_values()
else:
rVal = 'request unknown.'
return rVal
def do_server_loop(self):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setblocking(1)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.socket.bind( (self.host, self.port) )
self.socket.listen(self.__maxListen)
while self.serverShouldRun:
try:
request = ''
response = ''
clientsocket, address = self.socket.accept()
clientsocket.setblocking(1)
request = clientsocket.recv(4096)
try:
response = self.handle_request(request)
except Exception, e:
response = e
clientsocket.send(str(response))
clientsocket.close()
except socket.error as e:
print(
'server closed with error:\n\t'+\
'current request was: {0}\n\t'+\
'current requestor was:{1}\n\t'+\
'error was:{2}'.format(
request,
clientsocket,
e
)
)
break
def __init__(self): self.config = Parser() self.scale = USBScaleSingleton() self.db = MySQLDataBase()
subparsers = parser.add_subparsers(dest="mode")
# subparser for training
subparsers.add_parser("train", parents=[parent_parser])
# subparser for tagging
subparsers.add_parser("tagging", parents=[parent_parser])
return parser.parse_args()
def train(dataset):
model = GRU(dataset)
model.train()
if __name__ == "__main__":
args = parse_args()
config_args = Parser().get_section('RNN')
if args.mode == "train":
dataset = Dataset(args.text_corpus, args.tag_corpus, True, False, True)
train(dataset)
else:
dataset = Dataset(args.text_corpus, args.tag_corpus, False, False,
False)
model = GRU(None, False)
tagged = model.tagging(dataset)
tagged = [str(tag) for tag in tagged]
with open("results.txt", 'w') as file:
file.write("\n".join(tagged))
class Server(object):
__maxListen = 1
__runServer = None
@property
def serverShouldRun(self):
if self.__runServer == None:
self.__runServer = True
return self.__runServer
def stop_server(self):
self.__runServer = False
return self.serverShouldRun
def __init__(self):
self.config = Parser()
self.scale = USBScaleSingleton()
self.db = MySQLDataBase()
def get_weight(self):
return self.scale.get_weight(self.scale.DATA_MODE_GRAMS)
@property
def host(self):
return self.config.get('server', 'host')
@property
def port(self):
return int(self.config.get('server', 'port'))
def handle_request(self, request):
rVal = ''
if request == 'close server':
self.stop_server()
rVal = 'server about to stop..'
elif request == 'get weight':
rVal = str(self.get_weight())
elif request.startswith('store weight'):
args = re.split('\s+', request.replace('store result', ''))
if args == ['']:
args = ()
self.store_value(*args)
elif request == 'get all':
self.db.get_all_values()
else:
rVal = 'request unknown.'
return rVal
def do_server_loop(self):
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.setblocking(1)
self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.socket.bind((self.host, self.port))
self.socket.listen(self.__maxListen)
while self.serverShouldRun:
try:
request = ''
response = ''
clientsocket, address = self.socket.accept()
clientsocket.setblocking(1)
request = clientsocket.recv(4096)
try:
response = self.handle_request(request)
except Exception, e:
response = e
clientsocket.send(str(response))
clientsocket.close()
except socket.error as e:
print(
'server closed with error:\n\t'+\
'current request was: {0}\n\t'+\
'current requestor was:{1}\n\t'+\
'error was:{2}'.format(
request,
clientsocket,
e
)
)
break
def __init__(self):
self.config = Parser()
self.scale = USBScaleSingleton()
self.db = MySQLDataBase()
def __init__(self):
self.config = Parser()
self.connection = None
class MySQLDataBase(object):
def __init__(self):
self.config = Parser()
self.connection = None
def __connect(self):
host = self.config.get('database', 'host')
user = self.config.get('database', 'user')
password = self.config.get('database', 'password')
dbName = self.config.get('database', 'name')
self.connection = _mysql.connect(host=host,
user=user,
passwd=password,
db=dbName)
return self.connection()
def __disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
def save_changes(self):
if not self.connection:
return False
self.connection.commit()
self.__disconnect()
self.__connect()
return True
def __del__(self):
self.__disconnect()
def make_request(self, request, save=False, expectResult=True):
if not self.connection:
self.__connect()
cursor = self.connection.cursor()
state = cursor.execute(request)
if not state:
cursor.close()
return None
if expectResult:
result = cursor.fetchall()
if not result:
result = []
cursor.close()
if save:
self.save_changes()
return result
@property
def tableName(self):
return self.config.get('database', 'tablename')
@property
def primaryKey(self):
return self.config.get('database', 'pkey')
@property
def types(self):
return self.config.get('database', 'coltypes').split(',')
@property
def columns(self):
return self.config.get('database', 'columns').split(',')
def create_table(self):
values = []
for i, eachColumn in enumerate(self.columns):
value = '%s %s ' % (eachColumn, self.types[i])
if eachColumn == self.primaryKey:
value += 'AUTO_INCRECEMENT PRIMARY_KEY'
values.append(value)
sql = ''' CREATE TABLE %s (%s) ''' % (self.tableName,
', '.join(values))
# remove the following prints
print sql
return self.make_request(sql, save=True)
def get_all_values(self):
sql = '''SELECT * FROM %s ''' % self.tableName
# remove the following prints
print sql
return self.make_request(sql)
def add_entry(self, username, value):
timestamp = repr(time.strftime('%Y-%m-%d %H:%M:%S'))
cols = self.columns
vals = [username, value]
if cols.count(self.primaryKey):
cols.remove(self.primaryKey)
if 'timestamp' in cols:
vals.insert(cols.index('timestamp'), timestamp)
sql = '''INSERT INTO %s (%s) VALUES (%s) ''' % (
self.tableName, ', '.join(cols[:len(vals)]), ', '.join(vals))
# remove the following prints
print sql
self.make_request(sql, save=True)
def __init__(self): self.config = Parser() self.connection = None
class MySQLDataBase(object):
def __init__(self):
self.config = Parser()
self.connection = None
def __connect(self):
host = self.config.get('database', 'host')
user = self.config.get('database', 'user')
password = self.config.get('database', 'password')
dbName = self.config.get('database', 'name')
self.connection = _mysql.connect(
host=host,user=user,
passwd=password,db=dbName
)
return self.connection()
def __disconnect(self):
if self.connection:
self.connection.close()
self.connection = None
def save_changes(self):
if not self.connection:
return False
self.connection.commit()
self.__disconnect()
self.__connect()
return True
def __del__(self):
self.__disconnect()
def make_request(self, request, save=False, expectResult=True):
if not self.connection:
self.__connect()
cursor = self.connection.cursor()
state = cursor.execute(request)
if not state:
cursor.close()
return None
if expectResult:
result = cursor.fetchall()
if not result:
result = []
cursor.close()
if save:
self.save_changes()
return result
@property
def tableName(self):
return self.config.get('database', 'tablename')
@property
def primaryKey(self):
return self.config.get('database', 'pkey')
@property
def types(self):
return self.config.get('database', 'coltypes').split(',')
@property
def columns(self):
return self.config.get('database', 'columns').split(',')
def create_table(self):
values = []
for i, eachColumn in enumerate(self.columns):
value = '%s %s '%(eachColumn, self.types[i])
if eachColumn == self.primaryKey:
value += 'AUTO_INCRECEMENT PRIMARY_KEY'
values.append(value)
sql = ''' CREATE TABLE %s (%s) ''' % (self.tableName, ', '.join(values))
# remove the following prints
print sql
return self.make_request(sql, save=True)
def get_all_values(self):
sql = '''SELECT * FROM %s ''' % self.tableName
# remove the following prints
print sql
return self.make_request(sql)
def add_entry(self, username, value):
timestamp = repr(time.strftime('%Y-%m-%d %H:%M:%S'))
cols = self.columns
vals = [username, value]
if cols.count(self.primaryKey):
cols.remove(self.primaryKey)
if 'timestamp' in cols:
vals.insert(cols.index('timestamp'), timestamp)
sql = '''INSERT INTO %s (%s) VALUES (%s) '''%(self.tableName, ', '.join(cols[:len(vals)]), ', '.join(vals))
# remove the following prints
print sql
self.make_request(sql, save=True)