def intercept_unary_unary(self, continuation, client_call_details,
request):
with timer("query"):
ret = continuation(client_call_details, request)
return ret
from time import sleep
import grpc
import messages_pb2
import messages_pb2_grpc
from shared import timer
class LoggingInterceptor(grpc.UnaryUnaryClientInterceptor):
def intercept_unary_unary(self, continuation, client_call_details,
request):
with timer("query"):
ret = continuation(client_call_details, request)
return ret
channel = grpc.secure_channel("dgb-tests.coucher.org:8443",
credentials=grpc.ssl_channel_credentials())
intercepted_channel = grpc.intercept_channel(channel, LoggingInterceptor())
stub = messages_pb2_grpc.BenchStub(intercepted_channel)
while True:
with timer("outer"):
stub.GetGroupChat(messages_pb2.GetGroupChatReq(group_chat_id=15))
return token.lower() in GAZETTEER['B-LOC']
def is_ILOC_gaz(token):
return token.lower() in GAZETTEER['I-LOC']
def is_BORG_gaz(token):
return token.lower() in GAZETTEER['B-ORG']
def is_IORG_gaz(token):
return token.lower() in GAZETTEER['I-ORG']
def is_BMISC_gaz(token):
return token.lower() in GAZETTEER['B-MISC']
def is_IMISC_gaz(token):
return token.lower() in GAZETTEER['I-MISC']
def is_function_word(token):
return token.lower() in ['van', 'de', 'ter', 'der', 'te']
if __name__ == "__main__":
with timer("main"):
main()
def main(enable_cv, enable_gazetteer):
if enable_cv:
print('Cross-validation enabled!')
if enable_gazetteer:
print('Gazetter enabled!')
with timer('load data'):
train_sentences = load_sentences_from_csv('gro-ner-train.csv')
train_labels = extract_labels(train_sentences)
test_sentences = load_sentences_from_csv('gro-ner-test.csv')
test_labels = extract_labels(test_sentences)
with timer('featurize data'):
# 100% training data.
X = extract_features(train_sentences, enable_gazetteer)
y = np.array(train_labels)
# 100% test data.
X_test = extract_features(test_sentences, enable_gazetteer)
y_test = np.array(test_labels)
# 90% training data, 10% dev data.
X_train, X_dev, y_train, y_dev = train_test_split(
X,
y,
train_size=0.1,
random_state=1,
)
with timer('setup training model'):
model = Pipeline([
('dvect', DictVectorizer(sparse=True)),
('clf',
LinearSVC(
C=1.0,
max_iter=1000000,
class_weight='balanced',
random_state=1,
dual=False,
)),
])
print('training model parameters:')
for k, v in model.get_params().items():
print(f' {k}: {v}')
with timer('training model.fit'):
model.fit(X_train, y_train)
with timer('training model.predict'):
predictions = model.predict(X_dev)
with timer('training results'):
print('Training classification report:')
print(classification_report(y_dev, predictions, zero_division=0))
if enable_cv:
print('Cross-validation')
param_grid = {
'clf__C': [0.001, 0.10, 0.1, 10, 25, 50],
}
grid_search = GridSearchCV(
model,
param_grid=param_grid,
scoring='f1_macro',
n_jobs=2,
verbose=3,
cv=5,
)
with timer('grid_search.fit'):
grid_search.fit(X_train, y_train)
print(f'Best cross-validation score: {grid_search.best_score_}')
print('Tuned parameters:')
for k, v in grid_search.best_params_.items():
print(f' {k}: {v}')
with timer('setup final model'):
model = Pipeline([
('dvect', DictVectorizer(sparse=True)),
('clf',
LinearSVC(
class_weight='balanced',
max_iter=1000000,
random_state=1,
dual=False,
)),
])
model.set_params(**grid_search.best_params_)
print('final model parameters:')
for k, v in model.get_params().items():
print(f' {k}: {v}')
with timer('final model.fit'):
model.fit(X, y)
with timer('final model.predict'):
predictions = model.predict(X_test)
with timer('final results'):
print('Training classification report:')
print(classification_report(y_test, predictions, zero_division=0))
def timetaking_function(request, context):
with timer(method):
res = prev_func(request, context)
return res