class BrandTask(SequenceClassificationTask):
def label(self, document):
return document.brand_bio_labels
target = immutable(True)
@property
def scoring_function(self):
return 'f1'
def __init__(self, shared_layers: SharedLayers, all_data=False):
super().__init__(shared_layers, all_data=all_data)
self.best_score = float('-inf')
def is_best_score(self, f1):
return f1 > self.best_score
def filter_documents(self, documents: [HTMLDocument]) -> [HTMLDocument]:
return [
d for d in documents if d.brand_bio_labels is not None
and isinstance(d.brand_bio_labels, list)
]
@property
def name(self):
return 'brand'
def _score(self, windows, labels) -> float:
predictions = self._model.predict(windows)
predictions = ['O' if s < .5 else 'I_BRAND' for s in predictions]
labels = ['O' if l < .5 else 'I_BRAND' for l in labels]
return f1_score(labels, predictions, average='macro')
class InMemoryDatabase(SQLiteDatabase):
db_name = immutable(':memory:')
class C:
a = immutable('a')
class SQLiteDatabase(Database):
db_name = immutable('example.db')
connection = None
def connect(self):
if self.connection is None:
self.connection = sqlite3.connect(self.db_name)
def get_items(self) -> [ListItem]:
def save_item(self, item: ListItem) -> None:
pass
class InMemoryDatabase(SQLiteDatabase):
db_name = immutable(':memory:')
class SimpleLog(Log):
arguments = inject(Arguments)
def info(self, message: str):
if self.arguments.log_level > 1:
print('INFO:', message)
def error(self, message: str):
if self.arguments.log_level > 0:
print('ERROR:', message)
class ToDoApplication:
database = inject(Database)
log = inject(Log)
reader = inject(ItemReader)
writer = inject(ItemWriter)
def run(self):
self.report_items()
self.read_items()
def read_items(self):
self.log.info('Reading items...')
while self.reader.more_items():
try:
item = self.read_item()
self.database.save_item(item)
except:
self.log.error('Could not read item')
def read_item(self):
item = self.reader.next_item()
self.log.info('Got item: '.format(str(item)))
return item
def report_items(self):
self.log.info('Reporting Items...')
for item in self.database.get_items():
self.writer.write_item(item)
def production_environment() -> Environment:
return Environment(
SQLiteDatabase,
SimpleLog,
ConsoleArguments,
ConsoleItemReader,
ConsoleItemWriter
)
def dev_environment() -> Environment:
return Environment(
InMemoryDatabase,
SimpleLog,
ConsoleArguments,
ConsoleItemReader,
ConsoleItemWriter
)
def test_environment() -> Environment:
return Environment(
InMemoryDatabase,
DummyLog,
DummyArguments,
DummyItemReader,
DummyItemWriter,
)
if __name__ == '__main__':
with production_environment():
app = ToDoApplication()
app.run()