def convert_decimal(self, data):
context = DecimalContext(prec=10, rounding=ROUND_DOWN)
for prices in data['prices']:
for fld in self.__must_be_decimal:
v = data['prices'][prices][fld]
if v:
v = round(v, 2)
v = context.create_decimal_from_float(v)
else:
v = context.create_decimal_from_float(0)
data['prices'][prices][fld] = v
return data
def get_metrics(scores):
metrics = [get_precision(scores), get_recall(scores), get_f1(scores)]
rounder = Context(rounding=ROUND_HALF_UP, prec=4)
metrics_string = [
str(rounder.create_decimal_from_float(num * 100)) for num in metrics
]
return ("\t".join(metrics_string))
def test_run_external(cur, user, user2, tr_external):
try:
# fixed all data
cur.connection.commit()
tr_external.run()
assert tr_external.get_status() == SUCCESS
inv_from_new = Invoice.find_by_id(tr_external.invoice_id_from)
inv_to_new = Invoice.find_by_id(tr_external.invoice_id_to)
user_invoice_usd = user.invoices.get(USD, None)
context = Context(prec=1, rounding=ROUND_DOWN)
com = context.create_decimal_from_float(COMMISSION)
assert inv_from_new.balance == (
user_invoice_usd.balance - (tr_external.amount + tr_external.amount * com)).quantize(TWOPLACES)
user_invoice_eur = user2.invoices.get(EUR, None)
assert inv_to_new.balance == (user_invoice_eur.balance + tr_external.convert_amount(user_invoice_usd,
user_invoice_eur)).quantize(
TWOPLACES)
finally:
with get_db_cursor(commit=True) as cur:
cur.execute(f'''DELETE FROM public.transaction WHERE id={tr_external.id}::int;''')
cur.execute(f'''DELETE FROM public.invoice WHERE user_id={user.id}::int;''')
cur.execute(f'''DELETE FROM public.invoice WHERE user_id={user2.id}::int;''')
cur.execute(f'''DELETE FROM public.user WHERE id={user.id}::int;''')
cur.execute(f'''DELETE FROM public.user WHERE id={user2.id}::int;''')
def formatNumberPair(value, *errors, precision=2, separator=" +- "): c = Context(prec=precision) largest_error = None for error in errors: e = c.create_decimal_from_float(error) if e.is_finite(): if largest_error: largest_error = largest_error.max_mag(e) else: largest_error = e results = [] value = Decimal(value) if largest_error and value.is_finite(): value = value.quantize(largest_error) results.append(value) for error in errors: error = Decimal(error) if largest_error and error.is_finite(): error = error.quantize(largest_error) results.append(error) return tuple((str(r) for r in results))
async def writer(ws: ClientWriterStub, sequence_id: int) -> None:
while True:
sequence_id += 1
buy = random.choice([True, False])
context = Context(prec=8, rounding=ROUND_DOWN)
amount = context.create_decimal_from_float(random.random() * 0.001 +
0.00000001)
amount = amount.quantize(Decimal(10)**-8)
trade_pair = random.choice((
'BTC_USD',
'ETH_USD',
'BCH_USD',
'LTC_USD',
'BTC_EUR',
'ETH_EUR',
'BCH_EUR',
'LTC_EUR',
'ETH_BTC',
))
if buy:
logger.info(f'buying {amount} of {trade_pair}')
else:
logger.info(f'selling {amount} of {trade_pair}')
msg = {
'@type': 'PlaceBuyFoKOrder' if buy else 'PlaceSellFoKOrder',
'trade_pair': trade_pair,
'amount': str(amount),
'price': '1000000000' if buy else '0.00000001',
}
await ws.send_signed_message(sequence_id=sequence_id, payload=msg)
await asyncio.sleep(0.19)
def print_results(prf1):
# Always round .5 up, not towards even numbers as is the default
rounder = Context(rounding=ROUND_HALF_UP, prec=4)
print("Tag\tPrec\tRec\tF1")
for ent_type, score in sorted(prf1.items()):
if ent_type == "":
ent_type = "ALL"
metrics = [
str(float(rounder.create_decimal_from_float(num * 100)))
for num in score
]
#print("{:30s} {:30s}".format(ent_type, "\t".join(metrics)))
print(ent_type + "\t" + "\t".join(metrics))
def main() -> None:
train_dev_data = "annotated_data.jsonl"
test_data = "fics_annotated_all.jsonl"
nlp = spacy.load("en_core_web_sm", disable=["ner"])
train, dev = read_in_train_dev_data(train_dev_data, nlp)
test = read_in_test_data(test_data, nlp)
# Delete annotation on test
for doc in test:
doc.ents = []
crf = CRFsuiteEntityRecognizer(
WindowedTokenFeatureExtractor(
[
BiasFeature(),
TokenFeature(),
UppercaseFeature(),
TitlecaseFeature(),
InitialTitlecaseFeature(),
PunctuationFeature(),
DigitFeature(),
WordShapeFeature(),
WordVectorFeature('trek_w2v.model', scaling=1.0)
],
1,
),
BILOUEncoder(),
)
crf.train(train, "ap", {"max_iterations": 40}, "tmp.model")
test = [crf(doc) for doc in test]
# Load valid again to eval
test_gold = read_in_test_data(test_data, nlp)
print("Type\tPrec\tRec\tF1", file=sys.stderr)
# Always round .5 up, not towards even numbers as is the default
rounder = Context(rounding=ROUND_HALF_UP, prec=4)
# Set typed=False for untyped scores
scores = span_prf1(test_gold, test, typed=True)
for ent_type, score in sorted(scores.items()):
if ent_type == "":
ent_type = "ALL"
fields = [ent_type] + [
str(rounder.create_decimal_from_float(num * 100)) for num in score
]
print("\t".join(fields), file=sys.stderr)
counts = span_scoring_counts(test_gold, test, typed=True)
print("True positives ", sum(counts.true_positives.values()))
print("False positives ", sum(counts.false_positives.values()))
print("False negatives ", sum(counts.false_negatives.values()))
def evaluate_and_print(ref_doc: Sequence[Doc],
test_doc: Sequence[Doc],
type_map: Optional[Mapping[str, str]] = None):
print("Type\tPrec\tRec\tF1", file=sys.stderr)
# Always round .5 up, not towards even numbers as is the default
rounder = Context(rounding=ROUND_HALF_UP, prec=4)
scores = span_prf1_type_map(ref_doc, test_doc, type_map)
for ent_type, score in sorted(scores.items()):
if ent_type == "":
ent_type = "ALL"
fields = [ent_type] + [
str(rounder.create_decimal_from_float(num * 100)) for num in score
]
print("\t".join(fields), file=sys.stderr)
def __move_external(self, invoice_from, invoice_to, cur):
context = Context(prec=1, rounding=ROUND_DOWN)
com = context.create_decimal_from_float(COMMISSION)
amount_with_commission = self.amount + self.amount * com
if invoice_from.balance >= amount_with_commission:
invoice_from.update_balance_add(-amount_with_commission, cur)
else:
raise Exception('Insufficient funds')
if invoice_from.currency == invoice_to.currency:
invoice_to.update_balance_add(self.amount, cur)
else:
amount = self.convert_amount(invoice_from, invoice_to)
if amount > 0:
invoice_to.update_balance_add(amount, cur)
else:
raise Exception('Insufficient funds')
def formatNumber(number, *, precision=2):
c = Context(prec=precision)
n = c.create_decimal_from_float(number)
return n.to_eng_string(context=c)
def main():
global nlp, crf, train, gold_dev
nlp = spacy.load("en_core_web_sm", disable=["ner"])
annotated_data = ['Annotation_task/Annotated_data/Jenny_annotations.jsonl',
'Annotation_task/Annotated_data/micaela_annotation.jsonl',
'Annotation_task/Annotated_data/molly_annotations.jsonl',
'Annotation_task/Annotated_data/qingwen_annotations.jsonl']
crf = CRFsuiteEntityRecognizer(
WindowedTokenFeatureExtractor(
[
WordVectorFeature("wiki-news-300d-1M-subword.magnitude", 1.0),
TokenFeature(),
SuffixFeature(),
SentimentFeature(),
WordShapeFeature(),
# BiasFeature(),
POSFeature(),
CountFeature(),
PositionFeature()
],
1,
),
BILOUEncoder(),
)
crf.tagger = pycrfsuite.Tagger()
train_size = 500
algo = 'l2sgd'
train, gold_dev = compile_tagged_train_data(annotated_data, train_size)
crf.train(train, algo, {"max_iterations": 100}, "tmp.model")
dev_predicted = predict_dev_labels(gold_dev)
scores = span_prf1_type_map(gold_dev, dev_predicted,type_map={"ANTAG":"FLAT_ANTAG", "FLAT": "FLAT_ANTAG"})
rounder = Context(rounding=ROUND_HALF_UP, prec=4)
for ent_type, score in sorted(scores.items()):
if ent_type == "":
ent_type = "ALL"
fields = [ent_type] + [
str(rounder.create_decimal_from_float(num * 100)) for num in score
]
print("\t".join(fields), file=sys.stderr)
# Readable output for Span Scores ###
span_scores = span_scoring_counts(gold_dev, dev_predicted,type_map ={"ANTAG":"FLAT_ANTAG", "FLAT": "FLAT_ANTAG"})
falsepos = span_scores[1]
falseneg = span_scores[2]
occups = Counter([ent for ent in falseneg if ent[1]=='OCCUP'])
locs = Counter([ent for ent in falseneg if ent[1] == 'LOC'])
print("False positives:")
print(Counter([entity[1] for entity in falsepos]))
print()
print("False negatives:")
print(Counter([entity[1] for entity in falseneg]))
print()
print("15 most common false positive entities:")
print(falsepos.most_common(15))
print()
print("15 most common false negative entities:")
print(falseneg.most_common(15))
print()
print("15 most common false negative for occupations:")
print(occups.most_common(15))
print()
print("15 most common false negative for locations:")
print(locs.most_common(15))
print()
export_templates(dev_predicted)
export_entities(dev_predicted)
