class Price(object): gross = Decimal('NaN') net = Decimal('NaN') currency = None tax_name = None def __init__(self, net=None, gross=None, currency=None, tax_name=''): if net is not None: self.net = Decimal(net) if gross is None: self.gross = self.net else: self.gross = Decimal(gross) self.currency = currency self.tax_name = tax_name def __unicode__(self): if self.tax_name: return (u"net=%s,gross=%s (%s)" % (self.net, self.gross, self.tax_name)) else: return u"net=%s,gross=%s" % (self.net, self.gross) def __repr__(self): return "net=%s,gross=%s" % (self.net, self.gross) def __eq__(self, other): if isinstance(other, Price): return (self.gross == other.gross and self.net == other.net and self.currency == other.currency) return False def __ne__(self, other): return not self == other def __mul__(self, other): price_net = self.net * other price_gross = self.gross * other return Price(net=price_net, gross=price_gross, tax_name=self.tax_name) def __add__(self, other): if not isinstance(other, Price): raise TypeError("Cannot add %s object to Price" % type(other)) if other.currency != self.currency: raise ValueError("Cannot add Price in %s to %s" % (self.currency, other.currency)) price_net = self.net + other.net price_gross = self.gross + other.gross if self.tax_name == other.tax_name: return Price(net=price_net, gross=price_gross, currency=self.currency, tax_name=self.tax_name) return Price(net=price_net, gross=price_gross, currency=self.currency) def has_value(self): return not (self.net.is_nan() and self.gross.is_nan())
def write_output(self, key, value): if not str(key) in RCE_LIST_OUTPUTNAMES: raise ValueError("Output '" + str(key) + "' is not defined") elif (isinstance(value, complex)): raise ValueError("Value '" + str(value) + "' for Output '" + key + "' is complex, which is not supported by RCE") else: from decimal import Decimal if (type(value) is Decimal and value == Decimal('Infinity')): dictOut[key].append(float("Infinity")) elif (type(value) is Decimal and value == Decimal('-Infinity')): dictOut[key].append(float("-Infinity")) elif (type(value) is Decimal and Decimal.is_nan(value)): dictOut[key].append(float("nan")) elif isinstance(value, list): for index, elem in enumerate(value): if isinstance(elem, list): for index2, elem2 in enumerate(elem): if (type(elem2) is Decimal and elem2 == Decimal('Infinity')): elem[index2] = "+Infinity" elif (type(elem2) is Decimal and elem2 == Decimal('-Infinity')): elem[index2] = "-Infinity" elif (type(elem2) is Decimal and Decimal.is_nan(elem2)): elem[index2] = float("nan") else: if (type(elem) is Decimal and elem == Decimal('Infinity')): value[index] = "+Infinity" if (type(elem) is Decimal and elem == Decimal('-Infinity')): value[index] = "-Infinity" if (type(elem) is Decimal and Decimal.is_nan(elem)): value[index] = float("nan") if key in dictOut: dictOut[key].append(value) else: newlist = [value] dictOut.update({key: newlist}) elif key in dictOut: dictOut[key].append(value) else: newlist = [value] dictOut.update({key: newlist})
def to_bounded_decimal(value, what: str, lower_bounds: int, upper_bounds: int) -> Decimal: try: decimal_value = Decimal(value) if decimal_value.is_nan() or lower_bounds >= decimal_value or decimal_value >= upper_bounds: raise ValueError("{} must be between {} and {}.".format(what, lower_bounds, upper_bounds)) return decimal_value except DecimalException as _: raise ValueError("The {} value '{}' is not valid.".format(what, value))
def format_decimal(obj: Decimal) -> str: if obj.is_nan(): return "nan" if obj == Decimal("inf"): return "inf" if obj == Decimal("-inf"): return "-inf" return str(obj)
def to_string(self, value: Decimal) -> str: if value.is_nan(): raise ValueError("NaN values are not supported") if value.is_infinite(): raise ValueError("Infinite values are not supported") if Decimal("0.0") > value: raise ValueError("Negative decimal are not supported") return str(value).rstrip("0").rstrip(".")
def test_load_numeric_binary(conn, expr): cur = conn.cursor(binary=1) res = cur.execute(f"select '{expr}'::numeric").fetchone()[0] val = Decimal(expr) if val.is_nan(): assert res.is_nan() else: assert res == val if "e" not in expr: assert str(res) == str(val)
def test_roundtrip_numeric(conn, val, fmt_in, fmt_out): cur = conn.cursor(binary=fmt_out) val = Decimal(val) cur.execute(f"select %{fmt_in}", (val,)) result = cur.fetchone()[0] assert isinstance(result, Decimal) if val.is_nan(): assert result.is_nan() else: assert result == val
def test_roundtrip_numeric(conn, val): cur = conn.cursor() val = Decimal(val) cur.execute("select %s", (val,)) result = cur.fetchone()[0] assert isinstance(result, Decimal) if val.is_nan(): assert result.is_nan() else: assert result == val
def write_output(self,key,value): if not str(key) in RCE_LIST_OUTPUTNAMES: raise ValueError("Output '" + str(key) + "' is not defined") elif(isinstance(value, complex)): raise ValueError("Value '" + str(value) + "' for Output '" + key + "' is complex, which is not supported by RCE") else : from decimal import Decimal if (type(value) is Decimal and value == Decimal('Infinity')): dictOut[key].append(float("Infinity")) elif (type(value) is Decimal and value == Decimal('-Infinity')): dictOut[key].append(float("-Infinity")) elif (type(value) is Decimal and Decimal.is_nan(value)): dictOut[key].append(float("nan")) elif isinstance(value, list): for index, elem in enumerate(value): if isinstance(elem, list): for index2, elem2 in enumerate(elem): if (type(elem2) is Decimal and elem2 == Decimal('Infinity')): elem[index2] = "+Infinity"; elif(type(elem2) is Decimal and elem2 == Decimal('-Infinity')): elem[index2] = "-Infinity"; elif(type(elem2) is Decimal and Decimal.is_nan(elem2)): elem[index2] = float("nan"); else: if (type(elem) is Decimal and elem == Decimal('Infinity')): value[index] = "+Infinity"; if (type(elem) is Decimal and elem == Decimal('-Infinity')): value[index] = "-Infinity"; if (type(elem) is Decimal and Decimal.is_nan(elem)): value[index] = float ("nan"); if key in dictOut: dictOut[key].append(value) else: newlist = [value] dictOut.update({key:newlist}) elif key in dictOut: dictOut[key].append(value) else: newlist = [value] dictOut.update({key:newlist})
def test_numeric_as_float(conn, val): cur = conn.cursor() FloatLoader.register(conn.adapters.types["numeric"].oid, cur) val = Decimal(val) cur.execute("select %s as val", (val,)) result = cur.fetchone()[0] assert isinstance(result, float) if val.is_nan(): assert isnan(result) else: assert result == pytest.approx(float(val)) # the customization works with arrays too cur.execute("select %s as arr", ([val],)) result = cur.fetchone()[0] assert isinstance(result, list) assert isinstance(result[0], float) if val.is_nan(): assert isnan(result[0]) else: assert result[0] == pytest.approx(float(val))
def verify_p_and_l( self, p_and_l: ProfitAndLoss, size: Decimal, p_l: Decimal, expected_taxed_p_l: Decimal = Decimal("NaN"), ) -> None: self.assertEqual(p_and_l.size, size) self.assertEqual(p_and_l.profit_and_loss, p_l) if expected_taxed_p_l.is_nan(): self.assertEqual(p_and_l.taxed_profit_and_loss, p_l) else: self.assertEqual(p_and_l.taxed_profit_and_loss, expected_taxed_p_l)
def test_quote_numeric(conn, val, expr): val = Decimal(val) tx = Transformer() assert tx.get_dumper(val, Format.TEXT).quote(val) == expr cur = conn.cursor() cur.execute(sql.SQL("select {v}, -{v}").format(v=sql.Literal(val))) r = cur.fetchone() if val.is_nan(): assert isnan(r[0]) and isnan(r[1]) else: assert r == (val, -val)
def _represent_decimal(self: ruamel.yaml.BaseRepresenter, data: decimal.Decimal) -> ruamel.yaml.ScalarNode: if data.is_finite(): s = str( _POINT_ZERO_DECIMAL + data ) # The zero addition is to force float-like string representation elif data.is_nan(): s = ".nan" elif data.is_infinite(): s = ".inf" if data > 0 else "-.inf" else: assert False return self.represent_scalar("tag:yaml.org,2002:float", s)
def get_exact_ln_modelCount(problem, SOLUTION_COUNTS_DIR = "/atlas/u/jkuck/learn_BP/data/exact_SAT_counts_noIndSets/"): count_file = problem_name + '.txt' ###### get the exact log solution count ###### exact_ln_solution_count = None with open(SOLUTION_COUNTS_DIR + "/" + count_file, 'r') as f_solution_count: for line in f_solution_count: if line.strip().split(" ")[0] == 'dsharp': dsharp_solution_count = Decimal(line.strip().split(" ")[4]) dsharp_time = float(line.strip().split(" ")[6]) if not Decimal.is_nan(dsharp_solution_count): exact_ln_solution_count = float(dsharp_solution_count.ln()) assert(exact_ln_solution_count is not None) #we should have the exact count for all problems in the training/test set return exact_ln_solution_count, dsharp_time
def testInvalidConstruction(self): self.assertRaises(SimpleTypeValueError, xsd.decimal, 'bogus') nan = Decimal('NaN') self.assertTrue(nan.is_nan()) self.assertEqual('NaN', str(nan)) self.assertRaises(SimpleTypeValueError, xsd.decimal, nan) self.assertRaises(SimpleTypeValueError, xsd.decimal, 'NaN') inf = Decimal('Infinity') self.assertTrue(inf.is_infinite()) self.assertEqual('Infinity', str(inf)) self.assertRaises(SimpleTypeValueError, xsd.decimal, inf) self.assertRaises(SimpleTypeValueError, xsd.decimal, 'Infinity')
def testInvalidConstruction (self): self.assertRaises(SimpleTypeValueError, xsd.decimal, 'bogus') nan = Decimal('NaN') self.assertTrue(nan.is_nan()) self.assertEqual('NaN', str(nan)) self.assertRaises(SimpleTypeValueError, xsd.decimal, nan) self.assertRaises(SimpleTypeValueError, xsd.decimal, 'NaN') inf = Decimal('Infinity') self.assertTrue(inf.is_infinite()) self.assertEqual('Infinity', str(inf)) self.assertRaises(SimpleTypeValueError, xsd.decimal, inf) self.assertRaises(SimpleTypeValueError, xsd.decimal, 'Infinity')
def import_dict(self, Coindict): """ used for parsing data that comes from JSON type structure or DynamoDB """ for key in Coindict.keys(): val = Coindict[key] classname = val.__class__.__name__ if 'float' in classname: decvalue = Decimal(str(val)) if decvalue.is_nan(): decvalue = Decimal(0) clean_value = decvalue else: clean_value = val self.__dict__[key] = clean_value if 'Benchmarks' not in self.__dict__: self.Benchmarks = []
def smart_round(value: Decimal, precision: Optional[int] = None) -> Decimal: if value is None or value.is_nan(): return value if precision is not None: precision = 1 / (10**precision) return Decimal(str(value)).quantize(Decimal(str(precision))) step = Decimal("1") if Decimal("10000") > abs(value) > Decimal("100"): step = Decimal("0.1") elif Decimal("100") > abs(value) > Decimal("1"): step = Decimal("0.01") elif Decimal("1") > abs(value) > Decimal("0.01"): step = Decimal("0.0001") elif Decimal("0.01") > abs(value) > Decimal("0.0001"): step = Decimal("0.00001") elif Decimal("0.0001") > abs(value) > s_decimal_0: step = Decimal("0.00000001") return (value // step) * step
def test_nan_c(): nan = float("nan") dumped = rj.Encoder()(nan) loaded = rj.Decoder()(dumped) assert math.isnan(nan) assert math.isnan(loaded) with pytest.raises(ValueError): rj.Encoder(number_mode=None)(nan) d = Decimal(nan) assert d.is_nan() with pytest.raises(ValueError): rj.Encoder(number_mode=rj.NM_DECIMAL)(d) dumped = rj.Encoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(d) loaded = rj.Decoder(number_mode=rj.NM_DECIMAL|rj.NM_NAN)(dumped) assert loaded.is_nan()
def test_order_creation_with_default_values(self): order = LimitOrder(client_order_id="HBOT_1", trading_pair="HBOT-USDT", is_buy=False, base_currency="HBOT", quote_currency="USDT", price=Decimal("100"), quantity=Decimal("1.5") ) self.assertEqual("HBOT_1", order.client_order_id) self.assertEqual("HBOT-USDT", order.trading_pair) self.assertEqual(False, order.is_buy) self.assertEqual("HBOT", order.base_currency) self.assertEqual("USDT", order.quote_currency) self.assertEqual(Decimal("100"), order.price) self.assertEqual(Decimal("1.5"), order.quantity) self.assertTrue(Decimal.is_nan(order.filled_quantity)) self.assertEqual(0, order.creation_timestamp) self.assertEqual(LimitOrderStatus.UNKNOWN, order.status) self.assertEqual(-1, order.age())
def smart_round(value: Decimal, precision: Optional[int] = None) -> Decimal: # AttributeError: 'int' object has no attribute 'is_nan' # This happens when exporting history as json and one of the values is int instead of Decimal. if value is None or value.is_nan(): return value if precision is not None: precision = 1 / (10**precision) return Decimal(str(value)).quantize(Decimal(str(precision))) step = Decimal("1") if Decimal("10000") > abs(value) > Decimal("100"): step = Decimal("0.1") elif Decimal("100") > abs(value) > Decimal("1"): step = Decimal("0.01") elif Decimal("1") > abs(value) > Decimal("0.01"): step = Decimal("0.0001") elif Decimal("0.01") > abs(value) > Decimal("0.0001"): step = Decimal("0.00001") elif Decimal("0.0001") > abs(value) > s_decimal_0: step = Decimal("0.00000001") return (value // step) * step
def __new__(cls, value, dec_places=0): if dec_places < 0 or not isinstance(dec_places, int): raise ValueError('dec_places must be nonnegative integer') if isinstance(value, int): return _qpdf._new_real(value, 0) if isinstance(value, float) and isfinite(value): return _qpdf._new_real(value, dec_places) try: dec = Decimal(value) except InvalidOperation: raise TypeError( 'Could not convert object to int, float or Decimal') if dec.is_infinite() or dec.is_nan(): raise ValueError('NaN and infinity are not valid PDF objects') return _qpdf._new_real(str(dec))
def from_decimal(cls, dec): """Converts a finite Decimal instance to a rational number, exactly.""" from decimal import Decimal if isinstance(dec, numbers.Integral): dec = Decimal(int(dec)) elif not isinstance(dec, Decimal): raise TypeError( "%s.from_decimal() only takes Decimals, not %r (%s)" % (cls.__name__, dec, type(dec).__name__)) if dec.is_infinite(): raise OverflowError( "Cannot convert %s to %s." % (dec, cls.__name__)) if dec.is_nan(): raise ValueError("Cannot convert %s to %s." % (dec, cls.__name__)) sign, digits, exp = dec.as_tuple() digits = int(''.join(map(str, digits))) if sign: digits = -digits if exp >= 0: return cls(digits * 10 ** exp) else: return cls(digits, 10 ** -exp)
def test_nan_f(): nan = float("nan") dumped = rj.dumps(nan) loaded = rj.loads(dumped) assert math.isnan(nan) assert math.isnan(loaded) with pytest.raises(ValueError): rj.dumps(nan, number_mode=None) with pytest.raises(ValueError): rj.dumps(nan, allow_nan=False) d = Decimal(nan) assert d.is_nan() with pytest.raises(ValueError): rj.dumps(d, number_mode=rj.NM_DECIMAL) dumped = rj.dumps(d, number_mode=rj.NM_DECIMAL|rj.NM_NAN) loaded = rj.loads(dumped, number_mode=rj.NM_DECIMAL|rj.NM_NAN) assert loaded.is_nan()
class Price(object): gross = Decimal('NaN') net = Decimal('NaN') currency = None tax_name = None def __init__(self, net=None, gross=None, currency=None, tax_name=''): if net is not None: self.net = Decimal(net) if gross is None: self.gross = self.net else: self.gross = Decimal(gross) self.currency = currency self.tax_name = tax_name def __repr__(self): return ("Price(net=%s, gross=%s, currency=%s)" % (repr(self.net), repr(self.gross), repr(self.currency))) def __cmp__(self, other): if not isinstance(other, Price): raise TypeError('Cannot compare Price to %s' % other) if self.currency != other.currency: raise ValueError('Cannot compare Prices in %s and %s' % (self.currency, other.currency)) if self.net < other.net: return -1 elif self.net > other.net: return 1 return 0 def __eq__(self, other): if isinstance(other, Price): return (self.gross == other.gross and self.net == other.net and self.currency == other.currency) return False def __ne__(self, other): return not self == other def __mul__(self, other): price_net = self.net * other price_gross = self.gross * other return Price(net=price_net, gross=price_gross, currency=self.currency, tax_name=self.tax_name) def __add__(self, other): if isinstance(other, Tax): return other.apply(self) if not isinstance(other, Price): raise TypeError("Cannot add %s object to Price" % type(other)) if other.currency != self.currency: raise ValueError("Cannot add Price in %s to %s" % (self.currency, other.currency)) price_net = self.net + other.net price_gross = self.gross + other.gross if self.tax_name == other.tax_name: return Price(net=price_net, gross=price_gross, currency=self.currency, tax_name=self.tax_name) return Price(net=price_net, gross=price_gross, currency=self.currency) def has_value(self): return not (self.net.is_nan() and self.gross.is_nan()) @property def tax(self): return self.gross - self.net
def adopt_values(fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count): #print(wlb_row) #print(type(wlb_row)) # adopt the rec. sales price and the "Lieferbarkeit" directly and completely # See # http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy, # very bottom wlb_neu.loc[name, 'Empf. VK-Preis'] = str(fhz_preis) wlb_neu.loc[name, 'Sofort lieferbar'] = fhz_row['Sofort lieferbar'] #print(wlb_row['VK-Preis']) #print(type(wlb_row['VK-Preis'])) wlb_preis = Decimal(wlb_row['VK-Preis']) setgroesse = int(wlb_row['Setgröße']) # adopt the sale price only if significantly changed: if not fhz_preis.is_nan() and not wlb_preis.is_nan(): if (fhz_preis >= Decimal('2.')*wlb_preis) or (setgroesse > 1): # price is at least twice, this indicates that this is a set: print("Setgröße > 1 detektiert.") print("WLB-Preis:", wlb_preis, "FHZ-Preis:", fhz_preis, "(%s)" % fhz_row['Bezeichnung | Einheit']) fhz_preis = round(fhz_preis / setgroesse, 2) print("Alte (WLB) setgroesse:", setgroesse, " (Bitte prüfen, ob korrekt!)") print("FHZ-Preis wird zu FHZ-Preis / %s = %s" % (setgroesse, fhz_preis)) print("") sth_printed = True fhz_preis = returnRoundedPrice(fhz_preis) #if ( abs(fhz_preis - wlb_preis) > 0.021 ): if ( abs(fhz_preis - wlb_preis) > 0. ): # price seems to deviate more than usual count += 1 if wlb_row['Sortiment'] == 'Ja': print("Alter (WLB) Preis: %s Neuer (FHZ) Preis: %s\n" "FHZ: %s (%s) Sortiment: %s\n" "WLB: %s (%s) Sortiment: %s" % (wlb_preis, fhz_preis, fhz_row['Bezeichnung | Einheit'], name, fhz_row['Sortiment'], wlb_row['Bezeichnung | Einheit'], name, wlb_row['Sortiment'])) print("Ändere Preis von:", str(wlb_preis), " zu:", str(fhz_preis)) sth_printed = True wlb_neu.loc[name, 'VK-Preis'] = str(fhz_preis) geaenderte_preise = geaenderte_preise.append(wlb_neu.loc[name]) irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) price_changed = True sth_changed = True if options.ADOPT_NAMES: #### adopt the article name wlb_neu.loc[name, 'Bezeichnung | Einheit'] = fhz_row['Bezeichnung | Einheit'] # adopt VPE if fhz_row['VPE'] != wlb_row['VPE']: print("Ändere VPE für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], wlb_row['VPE'], fhz_row['VPE'])) wlb_neu.loc[name, 'VPE'] = fhz_row['VPE'] sth_printed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'VPE'] = fhz_row['VPE'] # adopt Menge fhz_menge = float(fhz_row['Menge (kg/l/St.)']) / setgroesse if fhz_menge != float(wlb_row['Menge (kg/l/St.)']): print("Ändere Menge für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], float(wlb_row['Menge (kg/l/St.)']), fhz_menge)) wlb_neu.loc[name, 'Menge (kg/l/St.)'] = '%.5f' % fhz_menge sth_printed = True if not price_changed: geaenderte_preise = geaenderte_preise.append(wlb_neu.loc[name]) price_changed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'Menge (kg/l/St.)'] = '%.5f' % fhz_menge # adopt Einheit if fhz_row['Einheit'] != wlb_row['Einheit']: print("Ändere Einheit für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], wlb_row['Einheit'], fhz_row['Einheit'])) wlb_neu.loc[name, 'Einheit'] = fhz_row['Einheit'] sth_printed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'Einheit'] = fhz_row['Einheit'] return (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count)
def main(): from optparse import OptionParser # install parser usage = "Usage: %prog [OPTIONS]" parser = OptionParser(usage) parser.add_option("--fhz", type="string", default='Artikelliste_Bestellvorlage_Lebensmittelpreisliste_1.2-2015.csv', dest="FHZ", help="The path to FHZ .csv file.") parser.add_option("--wlb", type="string", default='Artikelliste_DB_Dump_2015_KW40_LM.csv', dest="WLB", help="The path to WLB .csv file.") parser.add_option("-n", action="store_true", default=False, dest="ADOPT_NAMES", help="Artikelnamen ('Bezeichnung | Einheit') vom FHZ übernehmen?") # get parsed args (options, args) = parser.parse_args() ############# # Load data # ############# import numpy as np import pandas as pd fhz = pd.read_csv(options.FHZ, sep=';', dtype=str, index_col=(1, 2)) # input CSV list must contain only Lebensmittel and Getränke (LM) and not # Kunsthandwerk and other (KHW), if that's the case also for the FHZ CSV list wlb = pd.read_csv(options.WLB, sep=';', dtype=str, index_col=(1, 2)) # Check for duplicates in fhz: fhz_dup_indices = indexDuplicationCheck(fhz) print("Folgende Artikel kommen mehrfach in '%s' vor:" % options.FHZ) for i in fhz_dup_indices: print(fhz.loc[i]) print("---------------") print("---------------") # Check for duplicates in wlb: wlb_dup_indices = indexDuplicationCheck(wlb) print("Folgende Artikel kommen mehrfach in '%s' vor:" % options.WLB) for i in wlb_dup_indices: print(wlb.loc[i]) print("---------------") print("---------------") # Remove all newlines ('\n') from all fields fhz.replace(to_replace='\n', value=', ', inplace=True, regex=True) wlb.replace(to_replace='\n', value=', ', inplace=True, regex=True) fhz.replace(to_replace=';', value=',', inplace=True, regex=True) wlb.replace(to_replace=';', value=',', inplace=True, regex=True) ################### # Homogenize data # ################### # Remove all newlines ('\n') from all fields fhz.replace(to_replace='\n', value=', ', inplace=True, regex=True) wlb.replace(to_replace='\n', value=', ', inplace=True, regex=True) fhz.replace(to_replace=';', value=',', inplace=True, regex=True) wlb.replace(to_replace=';', value=',', inplace=True, regex=True) # homogenize Lieferanten: print('\n\n\n') print('Lieferanten-Vergleich:') print("WLB:", sorted(set(map(lambda i: i[0], wlb.index)))) print("FHZ:", set(map(lambda i: i[0], fhz.index))) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('El Puente', i[1]) if i[0] == 'EP' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('El Puente', i[1]) if i[0] == 'EP\n(fairfood)' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Fairtrade Center Breisgau', i[1]) if i[0] == 'ftc' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Café Libertad', i[1]) if i[0] == 'Café\nLibertad' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Ethiquable', i[1]) if i[0] == 'ethiquable' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Libera Terra', i[1]) if i[0] == 'Libera\nTerra' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('WeltPartner', i[1]) if i[0] == 'Welt Partner' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('WeltPartner', i[1]) if i[0] == 'Welt\nPartner' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('FAIR Handelshaus Bayern', i[1]) if i[0] == 'Fair Han-delshaus Bayern' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('unbekannt', i[1]) if type(i[0]) == float and np.isnan(i[0]) else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('FHZ Rheinland', i[1]) if i[0] == 'unbekannt' else i, fhz.index.tolist())), names=fhz.index.names) print("FHZ neu:", sorted(set(map(lambda i: i[0], fhz.index)))) # add '-' sign to article numbers in FHZ: # El Puente: fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('El Puente', convert_art_number_ep(i[1])) if i[0] == 'El Puente' else i, fhz.index.tolist())), names=fhz.index.names) # wp: fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('WeltPartner', convert_art_number_wp(i[1])) if i[0] == 'WeltPartner' else i, fhz.index.tolist())), names=fhz.index.names) # Make all article numbers lower case for better comparison: # First store the original article numbers: fhz = fhz.rename(columns={'Artikelnummer': 'Artikelnummer kleingeschrieben'}) wlb = wlb.rename(columns={'Artikelnummer': 'Artikelnummer kleingeschrieben'}) fhz.index.names = ['Lieferant', 'Artikelnummer kleingeschrieben'] wlb.index.names = ['Lieferant', 'Artikelnummer kleingeschrieben'] fhz['Artikelnummer'] = list(map(lambda i: i[1], fhz.index.tolist())) wlb['Artikelnummer'] = list(map(lambda i: i[1], wlb.index.tolist())) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: (i[0], i[1].lower()), fhz.index.tolist())), names=fhz.index.names) wlb.index = pd.MultiIndex.from_tuples(list(map(lambda i: (i[0], i[1].lower()), wlb.index.tolist())), names=wlb.index.names) ################################################# # Adopt values from FHZ (for existing articles) # ################################################# def adopt_values(fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count): #print(wlb_row) #print(type(wlb_row)) # adopt the rec. sales price and the "Lieferbarkeit" directly and completely # See # http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy, # very bottom wlb_neu.loc[name, 'Empf. VK-Preis'] = str(fhz_preis) wlb_neu.loc[name, 'Sofort lieferbar'] = fhz_row['Sofort lieferbar'] #print(wlb_row['VK-Preis']) #print(type(wlb_row['VK-Preis'])) wlb_preis = Decimal(wlb_row['VK-Preis']) setgroesse = int(wlb_row['Setgröße']) # adopt the sale price only if significantly changed: if not fhz_preis.is_nan() and not wlb_preis.is_nan(): if (fhz_preis >= Decimal('2.')*wlb_preis) or (setgroesse > 1): # price is at least twice, this indicates that this is a set: print("Setgröße > 1 detektiert.") print("WLB-Preis:", wlb_preis, "FHZ-Preis:", fhz_preis, "(%s)" % fhz_row['Bezeichnung | Einheit']) fhz_preis = round(fhz_preis / setgroesse, 2) print("Alte (WLB) setgroesse:", setgroesse, " (Bitte prüfen, ob korrekt!)") print("FHZ-Preis wird zu FHZ-Preis / %s = %s" % (setgroesse, fhz_preis)) print("") sth_printed = True fhz_preis = returnRoundedPrice(fhz_preis) #if ( abs(fhz_preis - wlb_preis) > 0.021 ): if ( abs(fhz_preis - wlb_preis) > 0. ): # price seems to deviate more than usual count += 1 if wlb_row['Sortiment'] == 'Ja': print("Alter (WLB) Preis: %s Neuer (FHZ) Preis: %s\n" "FHZ: %s (%s) Sortiment: %s\n" "WLB: %s (%s) Sortiment: %s" % (wlb_preis, fhz_preis, fhz_row['Bezeichnung | Einheit'], name, fhz_row['Sortiment'], wlb_row['Bezeichnung | Einheit'], name, wlb_row['Sortiment'])) print("Ändere Preis von:", str(wlb_preis), " zu:", str(fhz_preis)) sth_printed = True wlb_neu.loc[name, 'VK-Preis'] = str(fhz_preis) geaenderte_preise = geaenderte_preise.append(wlb_neu.loc[name]) irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) price_changed = True sth_changed = True if options.ADOPT_NAMES: #### adopt the article name wlb_neu.loc[name, 'Bezeichnung | Einheit'] = fhz_row['Bezeichnung | Einheit'] # adopt VPE if fhz_row['VPE'] != wlb_row['VPE']: print("Ändere VPE für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], wlb_row['VPE'], fhz_row['VPE'])) wlb_neu.loc[name, 'VPE'] = fhz_row['VPE'] sth_printed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'VPE'] = fhz_row['VPE'] # adopt Menge fhz_menge = float(fhz_row['Menge (kg/l/St.)']) / setgroesse if fhz_menge != float(wlb_row['Menge (kg/l/St.)']): print("Ändere Menge für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], float(wlb_row['Menge (kg/l/St.)']), fhz_menge)) wlb_neu.loc[name, 'Menge (kg/l/St.)'] = '%.5f' % fhz_menge sth_printed = True if not price_changed: geaenderte_preise = geaenderte_preise.append(wlb_neu.loc[name]) price_changed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'Menge (kg/l/St.)'] = '%.5f' % fhz_menge # adopt Einheit if fhz_row['Einheit'] != wlb_row['Einheit']: print("Ändere Einheit für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], wlb_row['Einheit'], fhz_row['Einheit'])) wlb_neu.loc[name, 'Einheit'] = fhz_row['Einheit'] sth_printed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'Einheit'] = fhz_row['Einheit'] return (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count) count = 0 print('\n\n\n') wlb_neu = wlb.copy() geaenderte_preise = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) irgendeine_aenderung = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) # Loop over fhz numerical index for i in range(len(fhz)): fhz_row = fhz.iloc[i] fhz_preis = Decimal(fhz_row['Empf. VK-Preis']) name = fhz_row.name sth_printed = False price_changed = False sth_changed = False try: wlb_match = wlb_neu.loc[name] if type(wlb_match) == pd.DataFrame: for j in range(len(wlb_match)): wlb_row = wlb_match.iloc[j] (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count) = adopt_values( fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count) else: (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_match, geaenderte_preise, irgendeine_aenderung, count) = adopt_values( fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_match, geaenderte_preise, irgendeine_aenderung, count) except KeyError: pass if sth_printed: # this ends processing of this article print("---------------") print(count, "Artikel haben geänderten Preis.") ################################# # Round up all articles' prices # ################################# # Round up all articles' prices: count = 0 print('\n\n\n') for i in range(len(wlb_neu)): wlb_row = wlb_neu.iloc[i] name = wlb_row.name alter_preis = Decimal(wlb_row['VK-Preis']) neuer_preis = returnRoundedPrice(alter_preis) if (not alter_preis.is_nan() and not neuer_preis.is_nan() and neuer_preis != alter_preis): count += 1 print("Runde Preis von:", str(alter_preis), " zu:", str(neuer_preis), '(%s, %s)' % (wlb_row['Bezeichnung | Einheit'], wlb_row['Sortiment'])) wlb_neu.loc[name, 'VK-Preis'] = str(neuer_preis) geaenderte_preise = geaenderte_preise.append(wlb_row) irgendeine_aenderung = irgendeine_aenderung.append(wlb_row) print(count, "VK-Preise wurden gerundet.") geaenderte_preise = removeEmptyRow(geaenderte_preise) irgendeine_aenderung = removeEmptyRow(irgendeine_aenderung) # Check for duplicates in geaenderte_preise: gp_dup_indices = indexDuplicationCheck(geaenderte_preise) print("Folgende Artikel kommen mehrfach in 'geaenderte_preise' vor:") for i in gp_dup_indices: print(geaenderte_preise.loc[i]) writeOutAsCSV(geaenderte_preise, 'preisänderung_geänderte_preise.csv', only_index=True) mask = geaenderte_preise['Sortiment'] == 'Ja' gp_sortiment = geaenderte_preise[mask] writeOutAsCSV(gp_sortiment, 'preisänderung_geänderte_preise_sortiment.csv', only_index=True) writeOutAsCSV(gp_sortiment, 'preisänderung_geänderte_preise_sortiment_alle_felder.csv') # Check for duplicates in irgendeine_aenderung: gp_dup_indices = indexDuplicationCheck(irgendeine_aenderung) print("Folgende Artikel kommen mehrfach in 'irgendeine_aenderung' vor:") for i in gp_dup_indices: print(irgendeine_aenderung.loc[i]) writeOutAsCSV(irgendeine_aenderung, 'preisänderung_irgendeine_änderung.csv') ##################### # Consistency check # ##################### count = 0 print('\n\n\n') for i in range(len(fhz)): fhz_row = fhz.iloc[i] name = fhz_row.name fhz_preis = Decimal(fhz_row['Empf. VK-Preis']) try: wlb_match = wlb_neu.loc[name] if type(wlb_match) == pd.DataFrame: for j in range(len(wlb_match)): wlb_row = wlb_match.iloc[j] wlb_preis = Decimal(wlb_row['Empf. VK-Preis']) else: wlb_preis = Decimal(wlb_match['Empf. VK-Preis']) except KeyError: wlb_preis = Decimal(np.nan) if ( not fhz_preis.is_nan() and not wlb_preis.is_nan() and #abs(fhz_preis - wlb_preis) > 0.021 ): abs(fhz_preis - wlb_preis) > 0. ): count += 1 print('FHZ: %s WLB: %s' % (fhz_preis, wlb_preis), '(%s) (%s)' % (fhz_row['Bezeichnung | Einheit'], wlb_row['Bezeichnung | Einheit'])) print(count, "Differenzen gefunden.") writeOutAsCSV(wlb_neu, 'preisänderung.csv') ####################### # Articles not in WLB # ####################### # Add new, so far unknown articles from FHZ print('\n\n\n') print("Neue Artikel (in FHZ vorhanden, nicht in WLB):") # Get empty df: wlb_neue_artikel = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) count = 0 for i in range(len(fhz)): fhz_row = fhz.iloc[i] fhz_preis = Decimal(fhz_row['Empf. VK-Preis']) name = fhz_row.name try: wlb_neu.loc[name] except KeyError: count += 1 # From: # http://stackoverflow.com/questions/10715965/add-one-row-in-a-pandas-dataframe wlb_neue_artikel = wlb_neue_artikel.append(fhz_row) fhz_preis = returnRoundedPrice(fhz_preis) wlb_neue_artikel.loc[name, 'VK-Preis'] = str(fhz_preis) print('"%s" nicht in WLB. (%s)' % (fhz_row['Bezeichnung | Einheit'], name)) print(count, "Artikel nicht in WLB.") wlb_neue_artikel = removeEmptyRow(wlb_neue_artikel) writeOutAsCSV(wlb_neue_artikel, 'preisänderung_neue_artikel.csv') ####################### # Articles not in FHZ # ####################### # Show list of articles missing in FHZ (so not orderable!) print('\n\n\n') print("Alte Artikel (in WLB vorhanden, nicht in FHZ, können nicht mehr bestellt werden!):") # Get empty df: wlb_alte_artikel = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) count = 0 # Loop over wlb numerical index for i in range(len(wlb_neu)): wlb_row = wlb_neu.iloc[i] name = wlb_row.name try: fhz.loc[name] except KeyError: count += 1 wlb_alte_artikel = wlb_alte_artikel.append(wlb_row) # Change 'popularity' to 'ausgelistet' so that it will not be ordered any more: wlb_alte_artikel.loc[name, 'Beliebtheit'] = 'ausgelistet' print('"%s" nicht in FHZ. (%s)' % (wlb_row['Bezeichnung | Einheit'], name)) print(count, "Artikel nicht in FHZ.") wlb_alte_artikel = removeEmptyRow(wlb_alte_artikel) writeOutAsCSV(wlb_alte_artikel, 'preisänderung_alte_artikel.csv')
fhz_row = fhz.iloc[i] fhz_preis = Decimal(fhz_row["Empf. VK-Preis"]) name = fhz_row.name try: wlb_row = wlb_neu.loc[name] # adopt the rec. sale price and the "Lieferbarkeit" directly and completely # See # http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy, # very bottom wlb_neu.loc[name, "Empf. VK-Preis"] = str(fhz_preis) wlb_neu.loc[name, "Sofort lieferbar"] = fhz_row["Sofort lieferbar"] wlb_preis = Decimal(wlb_row["VK-Preis"]) setgroesse = int(wlb_row["Setgröße"]) # adopt the sale price only if significantly changed: if not fhz_preis.is_nan() and not wlb_preis.is_nan(): if (fhz_preis >= Decimal("2.") * wlb_preis) or (setgroesse > 1): # price is at least twice, this indicates that this is a set: print("Setgröße > 1 detektiert.") print("WLB-Preis:", wlb_preis, "FHZ-Preis:", fhz_preis, "(%s)" % fhz_row["Bezeichnung | Einheit"]) fhz_preis = round(fhz_preis / setgroesse, 2) print("Alte (WLB) setgroesse:", setgroesse, " (Bitte prüfen, ob korrekt!)") print("Ändere FHZ-Preis zu FHZ-Preis / %s = %s" % (setgroesse, fhz_preis)) print("") if abs(fhz_preis - wlb_preis) > 0.021: # price seems to deviate more than usual count += 1 if wlb_row["Sortiment"] == "Ja": print( "Alter (WLB) Preis: %s Neuer (FHZ) Preis: %s\n" "FHZ: %s (%s) Sortiment: %s\n"
def main(): from optparse import OptionParser # install parser usage = "Usage: %prog [OPTIONS]" parser = OptionParser(usage) parser.add_option("--fhz", type="string", default='Artikelliste_Bestellvorlage_Lebensmittelpreisliste_1.2-2015.csv', dest="FHZ", help="The path to FHZ .csv file.") parser.add_option("--wlb", type="string", default='Artikelliste_DB_Dump_2015_KW40_LM.csv', dest="WLB", help="The path to WLB .csv file.") parser.add_option("-n", action="store_true", default=False, dest="ADOPT_NAMES", help="Artikelnamen ('Bezeichnung | Einheit') vom FHZ übernehmen?") # get parsed args (options, args) = parser.parse_args() ############# # Load data # ############# import numpy as np import pandas as pd fhz = pd.read_csv(options.FHZ, sep=';', dtype=str, index_col=(1, 2)) # input CSV list must contain only Lebensmittel and Getränke (LM) and not # Kunsthandwerk and other (KHW), if that's the case also for the FHZ CSV list wlb = pd.read_csv(options.WLB, sep=';', dtype=str, index_col=(1, 2)) # Check for duplicates in fhz: fhz_dup_indices = indexDuplicationCheck(fhz) print("Folgende Artikel kommen mehrfach in '%s' vor:" % options.FHZ) for i in fhz_dup_indices: print(fhz.loc[i]) print("---------------") print("---------------") # Check for duplicates in wlb: wlb_dup_indices = indexDuplicationCheck(wlb) print("Folgende Artikel kommen mehrfach in '%s' vor:" % options.WLB) for i in wlb_dup_indices: print(wlb.loc[i]) print("---------------") print("---------------") # Remove all newlines ('\n') from all fields fhz.replace(to_replace='\n', value=', ', inplace=True, regex=True) wlb.replace(to_replace='\n', value=', ', inplace=True, regex=True) fhz.replace(to_replace=';', value=',', inplace=True, regex=True) wlb.replace(to_replace=';', value=',', inplace=True, regex=True) ################### # Homogenize data # ################### # Remove all newlines ('\n') from all fields fhz.replace(to_replace='\n', value=', ', inplace=True, regex=True) wlb.replace(to_replace='\n', value=', ', inplace=True, regex=True) fhz.replace(to_replace=';', value=',', inplace=True, regex=True) wlb.replace(to_replace=';', value=',', inplace=True, regex=True) # homogenize Lieferanten: print('\n\n\n') print('Lieferanten-Vergleich:') print("WLB:", sorted(set(map(lambda i: i[0], wlb.index)))) print("FHZ:", set(map(lambda i: i[0], fhz.index))) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('El Puente', i[1]) if i[0] == 'EP' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Bannmühle', i[1]) if i[0] == 'Bannmühle/dwp' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Fairtrade Center Breisgau', i[1]) if i[0] == 'ftc' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Café Libertad', i[1]) if i[0] == 'Café\nLibertad' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Ethiquable', i[1]) if i[0] == 'ethiquable' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('Libera Terra', i[1]) if i[0] == 'Libera\nTerra' else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('unbekannt', i[1]) if type(i[0]) == float and np.isnan(i[0]) else i, fhz.index.tolist())), names=fhz.index.names) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('FHZ Rheinland', i[1]) if i[0] == 'unbekannt' else i, fhz.index.tolist())), names=fhz.index.names) print("FHZ neu:", sorted(set(map(lambda i: i[0], fhz.index)))) # add '-' sign to article numbers in FHZ: # El Puente: fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('El Puente', convert_art_number_ep(i[1])) if i[0] == 'El Puente' else i, fhz.index.tolist())), names=fhz.index.names) # dwp: fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: ('dwp', convert_art_number_dwp(i[1])) if i[0] == 'dwp' else i, fhz.index.tolist())), names=fhz.index.names) # Make all article numbers lower case for better comparison: # First store the original article numbers: fhz = fhz.rename(columns={'Artikelnummer': 'Artikelnummer kleingeschrieben'}) wlb = wlb.rename(columns={'Artikelnummer': 'Artikelnummer kleingeschrieben'}) fhz.index.names = ['Lieferant', 'Artikelnummer kleingeschrieben'] wlb.index.names = ['Lieferant', 'Artikelnummer kleingeschrieben'] fhz['Artikelnummer'] = list(map(lambda i: i[1], fhz.index.tolist())) wlb['Artikelnummer'] = list(map(lambda i: i[1], wlb.index.tolist())) fhz.index = pd.MultiIndex.from_tuples(list(map(lambda i: (i[0], i[1].lower()), fhz.index.tolist())), names=fhz.index.names) wlb.index = pd.MultiIndex.from_tuples(list(map(lambda i: (i[0], i[1].lower()), wlb.index.tolist())), names=wlb.index.names) ################################################# # Adopt values from FHZ (for existing articles) # ################################################# def adopt_values(fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count): #print(wlb_row) #print(type(wlb_row)) # adopt the rec. sales price and the "Lieferbarkeit" directly and completely # See # http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy, # very bottom wlb_neu.loc[name, 'Empf. VK-Preis'] = str(fhz_preis) wlb_neu.loc[name, 'Sofort lieferbar'] = fhz_row['Sofort lieferbar'] #print(wlb_row['VK-Preis']) #print(type(wlb_row['VK-Preis'])) wlb_preis = Decimal(wlb_row['VK-Preis']) setgroesse = int(wlb_row['Setgröße']) # adopt the sale price only if significantly changed: if not fhz_preis.is_nan() and not wlb_preis.is_nan(): if (fhz_preis >= Decimal('2.')*wlb_preis) or (setgroesse > 1): # price is at least twice, this indicates that this is a set: print("Setgröße > 1 detektiert.") print("WLB-Preis:", wlb_preis, "FHZ-Preis:", fhz_preis, "(%s)" % fhz_row['Bezeichnung | Einheit']) fhz_preis = round(fhz_preis / setgroesse, 2) print("Alte (WLB) setgroesse:", setgroesse, " (Bitte prüfen, ob korrekt!)") print("FHZ-Preis wird zu FHZ-Preis / %s = %s" % (setgroesse, fhz_preis)) print("") sth_printed = True fhz_preis = returnRoundedPrice(fhz_preis) #if ( abs(fhz_preis - wlb_preis) > 0.021 ): if ( abs(fhz_preis - wlb_preis) > 0. ): # price seems to deviate more than usual count += 1 if wlb_row['Sortiment'] == 'Ja': print("Alter (WLB) Preis: %s Neuer (FHZ) Preis: %s\n" "FHZ: %s (%s) Sortiment: %s\n" "WLB: %s (%s) Sortiment: %s" % (wlb_preis, fhz_preis, fhz_row['Bezeichnung | Einheit'], name, fhz_row['Sortiment'], wlb_row['Bezeichnung | Einheit'], name, wlb_row['Sortiment'])) print("Ändere Preis von:", str(wlb_preis), " zu:", str(fhz_preis)) sth_printed = True wlb_neu.loc[name, 'VK-Preis'] = str(fhz_preis) geaenderte_preise = geaenderte_preise.append(wlb_neu.loc[name]) irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) price_changed = True sth_changed = True if options.ADOPT_NAMES: #### adopt the article name wlb_neu.loc[name, 'Bezeichnung | Einheit'] = fhz_row['Bezeichnung | Einheit'] # adopt VPE if fhz_row['VPE'] != wlb_row['VPE']: print("Ändere VPE für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], wlb_row['VPE'], fhz_row['VPE'])) wlb_neu.loc[name, 'VPE'] = fhz_row['VPE'] sth_printed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'VPE'] = fhz_row['VPE'] # adopt Menge fhz_menge = float(fhz_row['Menge (kg/l/St.)']) / setgroesse if fhz_menge != float(wlb_row['Menge (kg/l/St.)']): print("Ändere Menge für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], float(wlb_row['Menge (kg/l/St.)']), fhz_menge)) wlb_neu.loc[name, 'Menge (kg/l/St.)'] = '%.5f' % fhz_menge sth_printed = True if not price_changed: geaenderte_preise = geaenderte_preise.append(wlb_neu.loc[name]) price_changed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'Menge (kg/l/St.)'] = '%.5f' % fhz_menge # adopt Einheit if fhz_row['Einheit'] != wlb_row['Einheit']: print("Ändere Einheit für %s (%s) von %s (WLB) zu %s (FHZ)" % (name, wlb_row['Bezeichnung | Einheit'], wlb_row['Einheit'], fhz_row['Einheit'])) wlb_neu.loc[name, 'Einheit'] = fhz_row['Einheit'] sth_printed = True if not sth_changed: irgendeine_aenderung = irgendeine_aenderung.append(wlb_neu.loc[name]) sth_changed = True else: irgendeine_aenderung.loc[name, 'Einheit'] = fhz_row['Einheit'] return (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count) count = 0 print('\n\n\n') wlb_neu = wlb.copy() geaenderte_preise = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) irgendeine_aenderung = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) # Loop over fhz numerical index for i in range(len(fhz)): fhz_row = fhz.iloc[i] fhz_preis = Decimal(fhz_row['Empf. VK-Preis']) name = fhz_row.name sth_printed = False price_changed = False sth_changed = False try: wlb_match = wlb_neu.loc[name] if type(wlb_match) == pd.DataFrame: for j in range(len(wlb_match)): wlb_row = wlb_match.iloc[j] (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count) = adopt_values( fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_row, geaenderte_preise, irgendeine_aenderung, count) else: (fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_match, geaenderte_preise, irgendeine_aenderung, count) = adopt_values( fhz_row, fhz_preis, name, sth_printed, price_changed, sth_changed, wlb_neu, wlb_match, geaenderte_preise, irgendeine_aenderung, count) except KeyError: pass if sth_printed: # this ends processing of this article print("---------------") print(count, "Artikel haben geänderten Preis.") ################################# # Round up all articles' prices # ################################# # Round up all articles' prices: count = 0 print('\n\n\n') for i in range(len(wlb_neu)): wlb_row = wlb_neu.iloc[i] name = wlb_row.name alter_preis = Decimal(wlb_row['VK-Preis']) neuer_preis = returnRoundedPrice(alter_preis) if (not alter_preis.is_nan() and not neuer_preis.is_nan() and neuer_preis != alter_preis): count += 1 print("Runde Preis von:", str(alter_preis), " zu:", str(neuer_preis), '(%s, %s)' % (wlb_row['Bezeichnung | Einheit'], wlb_row['Sortiment'])) wlb_neu.loc[name, 'VK-Preis'] = str(neuer_preis) geaenderte_preise = geaenderte_preise.append(wlb_row) irgendeine_aenderung = irgendeine_aenderung.append(wlb_row) print(count, "VK-Preise wurden gerundet.") geaenderte_preise = removeEmptyRow(geaenderte_preise) irgendeine_aenderung = removeEmptyRow(irgendeine_aenderung) # Check for duplicates in geaenderte_preise: gp_dup_indices = indexDuplicationCheck(geaenderte_preise) print("Folgende Artikel kommen mehrfach in 'geaenderte_preise' vor:") for i in gp_dup_indices: print(geaenderte_preise.loc[i]) writeOutAsCSV(geaenderte_preise, 'preisänderung_geänderte_preise.csv', only_index=True) mask = geaenderte_preise['Sortiment'] == 'Ja' gp_sortiment = geaenderte_preise[mask] writeOutAsCSV(gp_sortiment, 'preisänderung_geänderte_preise_sortiment.csv', only_index=True) writeOutAsCSV(gp_sortiment, 'preisänderung_geänderte_preise_sortiment_alle_felder.csv') # Check for duplicates in irgendeine_aenderung: gp_dup_indices = indexDuplicationCheck(irgendeine_aenderung) print("Folgende Artikel kommen mehrfach in 'irgendeine_aenderung' vor:") for i in gp_dup_indices: print(irgendeine_aenderung.loc[i]) writeOutAsCSV(irgendeine_aenderung, 'preisänderung_irgendeine_änderung.csv') #################### # Consistecy check # #################### count = 0 print('\n\n\n') for i in range(len(fhz)): fhz_row = fhz.iloc[i] name = fhz_row.name fhz_preis = Decimal(fhz_row['Empf. VK-Preis']) try: wlb_match = wlb_neu.loc[name] if type(wlb_match) == pd.DataFrame: for j in range(len(wlb_match)): wlb_row = wlb_match.iloc[j] wlb_preis = Decimal(wlb_row['Empf. VK-Preis']) else: wlb_preis = Decimal(wlb_match['Empf. VK-Preis']) except KeyError: wlb_preis = Decimal(np.nan) if ( not fhz_preis.is_nan() and not wlb_preis.is_nan() and #abs(fhz_preis - wlb_preis) > 0.021 ): abs(fhz_preis - wlb_preis) > 0. ): count += 1 print('FHZ: %s WLB: %s' % (fhz_preis, wlb_preis), '(%s) (%s)' % (fhz_row['Bezeichnung | Einheit'], wlb_row['Bezeichnung | Einheit'])) print(count, "Differenzen gefunden.") writeOutAsCSV(wlb_neu, 'preisänderung.csv') ####################### # Articles not in WLB # ####################### # Add new, so far unknown articles from FHZ print('\n\n\n') print("Neue Artikel (in FHZ vorhanden, nicht in WLB):") # Get empty df: wlb_neue_artikel = pd.DataFrame(columns=wlb_neu.columns, index=pd.MultiIndex.from_tuples([('','')], names=wlb_neu.index.names)) count = 0 for i in range(len(fhz)): fhz_row = fhz.iloc[i] fhz_preis = Decimal(fhz_row['Empf. VK-Preis']) name = fhz_row.name try: wlb_neu.loc[name] except KeyError: count += 1 # From: # http://stackoverflow.com/questions/10715965/add-one-row-in-a-pandas-dataframe wlb_neue_artikel = wlb_neue_artikel.append(fhz_row) fhz_preis = returnRoundedPrice(fhz_preis) wlb_neue_artikel.loc[name, 'VK-Preis'] = str(fhz_preis) print('"%s" nicht in WLB. (%s)' % (fhz_row['Bezeichnung | Einheit'], name)) print(count, "Artikel nicht in WLB.") wlb_neue_artikel = removeEmptyRow(wlb_neue_artikel) writeOutAsCSV(wlb_neue_artikel, 'preisänderung_neue_artikel.csv') ####################### # Articles not in FHZ # ####################### # Show list of articles missing in FHZ (so not orderable!) print('\n\n\n') print("Alte Artikel (in WLB vorhanden, nicht in FHZ, können nicht mehr bestellt werden!):") count = 0 # Loop over wlb numerical index for i in range(len(wlb_neu)): wlb_row = wlb_neu.iloc[i] name = wlb_row.name try: fhz.loc[name] except KeyError: count += 1 print('"%s" nicht in FHZ. (%s)' % (wlb_row['Bezeichnung | Einheit'], name)) print(count, "Artikel nicht in FHZ.")
def step_gradient(k1_current, k2_current, k3_current, points, learningRate): decimal.setcontext(decimal.Context(prec=10)) k1_gradient = decimal.Decimal(0.0) k2_gradient = decimal.Decimal(0.0) k3_gradient = decimal.Decimal(0.0) N = decimal.Decimal( float(len(points)) ) for i in range(0, len(points)): x = decimal.Decimal( points[i, 0] ) y = decimal.Decimal( points[i, 1] ) try: common_part_1 = -(2/N) * (y - (k1_current + k2_current * x) ** k3_current) except Exception as e: print('Exception #1') print('(y - (k1_current + k2_current * x)) : ', (y - (k1_current + k2_current * x))) print('k3_current : ', k3_current) try: common_part_2 = (k3_current * (k1_current + k2_current * x) ** (k3_current - 1)) except Exception as e: print('Exception #2') print('x : ', x) print('k3_current - 1 : ', (k3_current - 1)) print('k3_current * (k1_current + k2_current * x) : ', (k3_current * (k1_current + k2_current * x))) try: common_part_3 = decimal.Decimal.ln( k1_current + k2_current * x) except Exception as e: print('Exception #3') print('x : ', x) print('k1_current + k2_current * x : ', (k1_current + k2_current * x)) try: common_part_4 = (k1_current + k2_current * x) ** k3_current except Exception as e: print('Exception #4') print('x : ', x) print('k3_current : ', k3_current) print('k1_current + k2_current * x : ', (k1_current + k2_current * x)) # common_part_1 = -(2/N) * (y - (k1_current + k2_current * x) ** k3_current) # common_part_2 = (k3_current * (k1_current + k2_current * x) ** (k3_current - 1)) # common_part_3 = decimal.Decimal.ln( k1_current + k2_current * x) # common_part_4 = (k1_current + k2_current * x) ** k3_current k1_gradient += common_part_1 * common_part_2 k2_gradient += common_part_1 * common_part_2 * x k3_gradient += common_part_1 * common_part_3 * common_part_4 # k1_gradient += -(2/N) * (y - (k1_current + k2_current * x) ** k3_current) * (k3_current * (k1_current + k2_current * x) ** (k3_current - 1)) # k2_gradient += -(2/N) * (y - (k1_current + k2_current * x) ** k3_current) * (k3_current * (k1_current + k2_current * x) ** (k3_current - 1)) * x # k3_gradient += -(2/N) * (y - (k1_current + k2_current * x) ** k3_current) * np.log(k1_current + k2_current * x) * (k1_current + k2_current * x) ** k3_current if(dc.is_nan(k1_gradient) or dc.is_nan(k2_gradient) or dc.is_nan(k1_gradient) or dc.is_infinite(k1_gradient) or dc.is_infinite(k2_gradient) or dc.is_infinite(k1_gradient)): print(' i=', i) print(' x = ', x, ', y = ', y) print(' comPart_2_1 = ', (k3_current * (k1_current + k2_current * x))) print(' comPart_2_2 = ', (k3_current - 1)) print(' comPart_2_3 = ', k3_current * (k1_current + k2_current * x) ** (k3_current - 1)) print(' k1_cur=', k1_current, ' k2_cur=', k2_current, ' k3_cur=', k3_current) print(' common_part_1 = ', common_part_1) print(' common_part_2 = ', common_part_2) print(' common_part_3 = ', common_part_3) print(' k1_gradient = ', k1_gradient) print(' k2_gradient = ', k2_gradient) print(' k3_gradient = ', k3_gradient, '\n') k1_gradient = 1001.0 break new_k1 = k1_current - (learningRate * k1_gradient) new_k2 = k2_current - (learningRate * k2_gradient) new_k3 = k3_current - (learningRate * k3_gradient) return [new_k1, new_k2, new_k3, k1_gradient, k2_gradient, k3_gradient]
def format_fiat(self, value: Decimal) -> str: if value.is_nan(): return _("No data") return "%s" % (self.ccy_amount_str(value, True))
class ComplexDecimal(object): def __init__(self,real,i="0.0"): if type(real) != Decimal: self.real = Decimal(real) else: self.real = real if type(i) != Decimal: self.imaginary = Decimal(i) else: self.imaginary = i self.e = constants.e self.pi = constants.pi #Basic mathamtical functions ---------------------------------------- def conj(self): """Returns the conjugate for a number""" return ComplexDecimal(self.real,-self.imaginary) def re(self): """Returns real portion""" return self.real def im(self): """Returns imaginary portion""" return self.imaginary def floor(self): """Floors a number""" return ComplexDecimal(self.real.to_integral(), self.imaginary.to_integral()) def ceil(self): """Ceilings a number""" return ComplexDecimal(math.ceil(self.real),math.ceil(self.imaginary)) def round(self,prec=0): """Rounds a number to prec decimal places""" return ComplexDecimal(round(self.real,prec),round(self.imaginary,prec)) def exp(self): """Returns e^self""" if self.imaginary == 0: return ComplexDecimal(str(self.e ** self.real )) return ComplexDecimal(self.e**(self.real) * Decimal(self.cos_raw(self.imaginary)), self.e**(self.real) * Decimal(self.sin_raw(self.imaginary))) def isPrime_raw(self,x): if x<=1: return False for i in range(2,int(math.floor(math.sqrt(x))) + 1): if x%i==0: return False return True def isPrime(self): """Gets if self is prime""" if self.imaginary == 0: return self.isPrime_raw(self.real) if self.real == 0: return self.isPrime_raw(abs(self.imaginary)) return self.isPrime_raw(self.real * self.real + self.imaginary*self.imaginary) def phase(self): """Returns angle between x axis""" return Decimal(math.atan2(self.imaginary , self.real)) def polar(self): #Converts to polar return (abs(self), self.phase()) def toRect(self,r,phi): return ComplexDecimal(r) * ComplexDecimal(self.cos_raw(phi) , self.sin_raw(phi)) #Trig -------------------------------------------- #1/functions, log, def cos_raw(self,x,prec=Decimal("0.0000000000000000000001")): """Raw cosine function for real numbers only""" p = Decimal("0") s = t = Decimal("1.0") while abs(t/s) > prec: p+=Decimal("1") t = (-t * x * x) / ((Decimal("2") * p - Decimal("1")) * (Decimal("2") * p)) s += t if round(s,50) in [1,0,-1]: #If it's close enough round to get perfect answers (No one likes 1E-70 as cos(pi)) return Decimal(round(s,50)) return s def sin_raw(self,x,prec=Decimal("0.00000000000000000000001")): return self.cos_raw(x+self.pi/Decimal("2")) def tan_raw(self,x,prec=Decimal("0.00000000000000000000001")): return self.sin_raw(x,prec) / self.cos_raw(x,prec) def atan_raw(self,x): #Not relatively accurate returned = Decimal("0") for j in range(0,50): i = Decimal(j) a = Decimal("-1")**i * x ** (Decimal("2")*i + Decimal("1")) b = Decimal("2")*i + Decimal("1") returned += a/b return returned def cos(self): """Compute cos of self""" if self.imaginary == 0: return ComplexDecimal(self.cos_raw(self.real)) try: returned = ComplexDecimal( self.cos_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real, -self.sin_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real ) returned.imaginary = -returned.imaginary return returned except: return ComplexDecimal("inf") def sin(self): """Compute sin of self""" if self.imaginary == 0: return ComplexDecimal(self.sin_raw(self.real)) try: returned = ComplexDecimal( self.sin_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real, -self.cos_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real ) returned.imaginary = -returned.imaginary return returned except: return ComplexDecimal("inf") def tan(self): """Compute tangent of self""" try: return self.sin() / self.cos() except: return ComplexDecimal("inf") def acos(self): """Attempt to compute arcosine""" if self.imaginary == 0: return ComplexDecimal(math.acos(self)) A = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) B = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) return ComplexDecimal(A.acos(), -(B+(B*B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln() ) def asin(self): """Attempt to compute arcsine""" if self.imaginary == 0: return ComplexDecimal(math.asin(self)) A = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) B = ComplexDecimal(((Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ((Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) return ComplexDecimal(A.asin(), (B+(B*B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln() ) def atan(self): """Attempt to compute arctangent""" result = (ComplexDecimal(0,1)+self) / (ComplexDecimal(0,1) - self) return ComplexDecimal(0,0.5) * result.ln() def sinh(self): """Hyperbolic sine of self""" returned = (self.exp() - (-self).exp()) / ComplexDecimal("2") returned.imaginary = -returned.imaginary return returned def cosh(self): """Hyperbolic cosine of self""" returned = (self.exp() + (-self).exp()) / ComplexDecimal("2") returned.imaginary = -returned.imaginary return returned def tanh(self): """Hyperbolic tangent of self""" return self.sinh()/self.cosh() def acosh(self): """Arc hyperbolic cosine""" returned = self + (self*self - ComplexDecimal(1))**0.5 return returned.ln() def asinh(self): """Arc hyperbolic sine""" returned = self + (self*self + ComplexDecimal(1))**0.5 return returned.ln() def atanh(self): """Arc hyperbolic tan""" a = (ComplexDecimal(1)+self).ln() - (ComplexDecimal(1)-self).ln() return ComplexDecimal(0.5) * a def ln_raw(self,x): returned = Decimal("0") for i in range(0,25): a = Decimal("1") / (Decimal(i) * Decimal("2") + Decimal("1")) b = ( (x - Decimal("1"))/(x + Decimal("1")) ) ** (Decimal(i) * Decimal("2") + Decimal("1")) returned += a*b return returned*2 def ln(self): """Natural logarithim of self""" if self.imaginary == 0: try: return ComplexDecimal(self.ln_raw(self.real)) except: pass p = self.polar() return ComplexDecimal(self.ln_raw(p[0]), p[1]) def log(self,x=None): """Compute log of number in base x""" if not x: x = self.e if self.imaginary == 0: try: return ComplexDecimal(math.log(self.real,x)) except: pass p = self.polar() return ComplexDecimal(math.log(p[0],x), p[1]) def log10(self): """Compute log base 10""" if self.imaginary == 0: return ComplexDecimal(math.log10(self.real)) return self.ln() / ComplexDecimal("10").ln() #Replace builtins -------------------------------- # power, comparasion, #To float, int, complex def __str__(self): """Converts to str""" if self.imaginary == 0: return str(self.real) return (str(self.real)+"+"+str(self.imaginary)+"i").replace("+-","-") def __abs__(self): """Absolute value""" return (self.real**2 + self.imaginary**2).sqrt() def __add__(self,other): """Add 2 numbers together""" return ComplexDecimal(self.real+other.real, self.imaginary+other.imaginary) def __sub__(self,other): """Subtract 2 numbers""" return ComplexDecimal(self.real-other.real, self.imaginary-other.imaginary) def __mul__(self,other): """Multiply 2 numbers""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real*other.real,"0") return ComplexDecimal( self.real*other.real - self.imaginary*other.imaginary,self.real*other.imaginary + self.imaginary*other.real ) def __div__(self,other): """Divide 2 numbers""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real/other.real,"0") a = (self.real*other.real + self.imaginary*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary) b = (self.imaginary*other.real - self.real*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary) return ComplexDecimal(a,b) def __truediv__(self,other): """Divide 2 numbers""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real/other.real,"0") a = (self.real*other.real + self.imaginary*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary) b = (self.imaginary*other.real - self.real*other.imaginary)/(other.real*other.real + other.imaginary*other.imaginary) return ComplexDecimal(a,b) def __neg__(self): """Negates a number""" return ComplexDecimal(-self.real,-self.imaginary) def __pos__(self): """Positive number""" return ComplexDecimal(abs(self.real),abs(self.imaginary)) def __inverse__(self): """1/ number""" return ComplexDecimal(1)/self def __mod__(self,other):#a%b = a + b * ciel(-a/b) """Modolus""" return self+ other* ((-self/other).ceil()) #Powers def __pow__(self,other): """Powers :D""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real**other.real) if other.imaginary == 0: polar = self.polar() return self.toRect(polar[0]**other.real, polar[1]*other.real) elif other.real == 0: a = ComplexDecimal(self.real); b = ComplexDecimal(self.imaginary) c = ComplexDecimal(other.real); d = ComplexDecimal(other.imaginary) x = (-d * (b/a).atan()).exp() * ((a*a+b*b).ln() * d / ComplexDecimal(2)).cos() y = ComplexDecimal(0,1) * (-d * (b/a).atan()).exp() * ((a*a+b*b).ln() * d / ComplexDecimal(2)).sin() return x+y b = other.real; c = other.imaginary returned = self**(ComplexDecimal(b)) * self**(ComplexDecimal(0,1) * ComplexDecimal(c)) returned.imaginary = -returned.imaginary return returned #Additional conversions def __complex__(self): """Convert to complex""" return complex(float(self.real),float(self.imaginary)) def __int__(self): """Convert to int""" return int(self.real) def __float__(self): """Convert to float""" return float(self.real) #Comparasions def __lt__(self,other): #> if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError("Complex comparasion is not supported") return self.real < other.real def __le__(self,other): #>= if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError("Complex comparasion is not supported") return self.real <= other.real def __eq__(self,other): #== if self.real == other.real and self.imaginary == other.imaginary: return True return False def __ne__(self,other): #!= return not self.__eq__(other) def __gt__(self,other): #< if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError("Complex comparasion is not supported") return self.real > other.real def __ge__(self,other): #<= if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError("Complex comparasion is not supported") return self.real >= other.real #Some things reimplemented from decimal class for complex ComplexDecimals #================================================================ def copy_abs(self): return abs(self) def copy_negate(self): return -self def copy_sign(self,other): if other < 0: return -abs(self) return abs(self) def is_finite(self): return self.real.is_finite() and self.imaginary.is_finite() def is_infinite(self): return not self.is_finite() def is_nan(self): return self.real.is_nan() or self.imaginary.is_nan() def is_signed(self): return self < ComplexDecimal(0) def is_zero(self): return self.real.is_zero() and self.imaginary.is_zero() def radix(self): return Decimal(10) def sqrt(self): if self.imaginary.is_zero(): return ComplexDecimal(self.real.sqrt()) return self ** ComplexDecimal("0.5") def to_eng_string(self): returned = self.real.to_eng_string() + " + " + self.imaginary.to_eng_string + "i" return returned.replace(" + -"," - ")
def get_SATproblems_list(problems_to_load, counts_dir_name, problems_dir_name, dataset_size, begin_idx=0, verbose=True, epsilon=0, max_factor_dimensions=5, return_logZ_list=False, belief_repeats=None): ''' Inputs: - problems_to_load (list of strings): problems to load - problems_dir_name (string): directory containing problems in cnf form - counts_dir_name (string): directory containing .txt files with model counts for problems File name format: problem1.txt File content format: "sharpSAT time_out: False solution_count: 2097152 sharp_sat_time: 0.0" - begin_idx: (int) discard the first begin_idx problems, e.g. for validation - epsilon (float): set factor states with potential 0 to epsilon for numerical stability - max_factor_dimensions (int): do not construct a factor graph if the largest factor (clause) contains more than this many variables ''' sat_problems = [] ln_solution_counts = [] discarded_count = 0 load_failure_count = 0 # the number of SAT problems we failed to load properly no_solution_count = 0 # the number of SAT problems with no solutions unsolved_count = 0 # the number of SAT problems that we don't have an exact solution count for factors_to_large_count = 0 # the number of SAT problems with more than max_factor_dimensions variables in a clause dsharp_sharpsat_disagree = 0 # the number of SAT problems where dsharp and sharpsat disagree on the number of satisfying solutions problems_in_cnf_dir = os.listdir(problems_dir_name) # print("problems_in_cnf_dir:", problems_in_cnf_dir) problems_in_counts_dir = os.listdir(counts_dir_name) for problem_name in problems_to_load: if len(sat_problems) == dataset_size: break # problem_file = problem_name[:-19] + '.cnf' problem_file = problem_name + '.cnf.gz.no_w.cnf' if problem_file not in problems_in_cnf_dir: if verbose: print('no corresponding cnf file for', problem_file, "problem_name:", problem_name) continue count_file = problem_name + '.txt' if count_file not in problems_in_counts_dir: if verbose: print('no corresponding sat count file for', count_file, "problem_name:", problem_name) continue with open(counts_dir_name + "/" + count_file, 'r') as f_solution_count: sharpSAT_solution_count = None dsharp_solution_count = None for line in f_solution_count: #Only use dsharp counts because dsharp and sharpSAT seem to disagree on some benchmarks #dsharp is consistent with randomized hashing methods while sharpSAT is not #this is with sampling sets removed, not sure what is going on with sharpSAT # if line.strip().split(" ")[0] == 'sharpSAT': # sharpSAT_solution_count = Decimal(line.strip().split(" ")[4]) # if Decimal.is_nan(sharpSAT_solution_count): # sharpSAT_solution_count = None if line.strip().split(" ")[0] == 'dsharp': dsharp_solution_count = Decimal(line.strip().split(" ")[4]) if Decimal.is_nan(dsharp_solution_count): dsharp_solution_count = None if (dsharp_solution_count is not None) and (sharpSAT_solution_count is not None): # assert(dsharp_solution_count == sharpSAT_solution_count), (dsharp_solution_count, sharpSAT_solution_count) if dsharp_solution_count != sharpSAT_solution_count: dsharp_sharpsat_disagree += 1 continue if dsharp_solution_count is not None: solution_count = dsharp_solution_count elif sharpSAT_solution_count is not None: solution_count = sharpSAT_solution_count else: solution_count = None # assert(solution_count is not None) if solution_count is None: unsolved_count += 1 continue if solution_count == 0: no_solution_count += 1 continue ln_solution_count = float(solution_count.ln()) n_vars, clauses, load_successful = parse_dimacs(problems_dir_name + "/" + problem_file) if not load_successful: load_failure_count += 1 continue # print("factor_graph:", factor_graph) if discarded_count == begin_idx: # print('using problem:', problem_file) factor_graph = build_factorgraph_from_SATproblem( clauses, epsilon=epsilon, max_factor_dimensions=max_factor_dimensions, ln_Z=ln_solution_count, belief_repeats=belief_repeats) if factor_graph is None: #largest clause contains too many variables factors_to_large_count += 1 continue sat_problems.append(factor_graph) ln_solution_counts.append(ln_solution_count) print("successfully loaded:", problem_name) else: discarded_count += 1 assert (discarded_count <= begin_idx) assert (len(ln_solution_counts) == len(sat_problems)) print(len(ln_solution_counts), "SAT problems loaded successfully") print(unsolved_count, "unsolved SAT problems") print(no_solution_count, "SAT problems with no solution (not loaded)") print(load_failure_count, "SAT problems failed to load properly") print(factors_to_large_count, "SAT problems have more than 5 variables in a clause") if return_logZ_list: return sat_problems, ln_solution_counts else: return sat_problems
def format_picture(conv, value, picture): monetary = False decimal_point = conv['decimal_point'] thousands_sep = conv[monetary and 'mon_thousands_sep' or 'thousands_sep'] percent = '%' per_mille = '\u2030' minus_sign = '-' #grouping = conv[monetary and 'mon_grouping' or 'grouping'] if isinstance(value, float): value = Decimal.from_float(value) elif isinstance(value, _STR_NUM_TYPES): value = Decimal(value) elif not isinstance(value, Decimal): raise ValueError(_('Picture requires a number convertable to decimal or float').format(picture)) if value.is_nan(): return 'NaN' isNegative = value.is_signed() pic, sep, negPic = picture.partition(';') if negPic and ';' in negPic: raise ValueError(_('Picture contains multiple picture sepearators {0}').format(picture)) if isNegative and negPic: pic = negPic if len([c for c in pic if c in (percent, per_mille) ]) > 1: raise ValueError(_('Picture contains multiple percent or per_mille charcters {0}').format(picture)) if percent in pic: value *= 100 elif per_mille in pic: value *= 1000 intPart, sep, fractPart = pic.partition(decimal_point) prefix = '' numPlaces = 0 intPlaces = 0 grouping = 0 fractPlaces = 0 suffix = '' if fractPart: if decimal_point in fractPart: raise ValueError(_('Sub-picture contains decimal point sepearators {0}').format(pic)) for c in fractPart: if c.isdecimal(): numPlaces += 1 fractPlaces += 1 if suffix: raise ValueError(_('Sub-picture passive character {0} between active characters {1}').format(c, fractPart)) else: suffix += c intPosition = 0 for c in reversed(intPart): if c.isdecimal() or c == '#' or c == thousands_sep: if prefix: raise ValueError(_('Sub-picture passive character {0} between active characters {1}').format(c, intPart)) if c.isdecimal(): numPlaces += 1 intPlaces += 1 intPosition += 1 prefix = '' elif c == '#': numPlaces += 1 intPosition += 1 elif c == thousands_sep: if not grouping: grouping = intPosition else: prefix = c + prefix if not numPlaces and prefix != minus_sign: raise ValueError(_('Sub-picture must contain at least one digit position or sign character {0}').format(pic)) if intPlaces == 0 and fractPlaces == 0: intPlaces = 1 return format_decimal(value, intPlaces=intPlaces, fractPlaces=fractPlaces, sep=thousands_sep, dp=decimal_point, grouping=grouping, pos=prefix, neg=prefix if negPic else prefix + minus_sign, trailpos=suffix, trailneg=suffix)
class ComplexDecimal(object): def __init__(self, real, i="0.0"): if type(real) != Decimal: self.real = Decimal(real) else: self.real = real if type(i) != Decimal: self.imaginary = Decimal(i) else: self.imaginary = i self.e = constants.e self.pi = constants.pi #Basic mathamtical functions ---------------------------------------- def conj(self): """Returns the conjugate for a number""" return ComplexDecimal(self.real, -self.imaginary) def re(self): """Returns real portion""" return self.real def im(self): """Returns imaginary portion""" return self.imaginary def floor(self): """Floors a number""" return ComplexDecimal(self.real.to_integral(), self.imaginary.to_integral()) def ceil(self): """Ceilings a number""" return ComplexDecimal(math.ceil(self.real), math.ceil(self.imaginary)) def round(self, prec=0): """Rounds a number to prec decimal places""" return ComplexDecimal(round(self.real, prec), round(self.imaginary, prec)) def exp(self): """Returns e^self""" if self.imaginary == 0: return ComplexDecimal(str(self.e**self.real)) return ComplexDecimal( self.e**(self.real) * Decimal(self.cos_raw(self.imaginary)), self.e**(self.real) * Decimal(self.sin_raw(self.imaginary))) def isPrime_raw(self, x): if x <= 1: return False for i in range(2, int(math.floor(math.sqrt(x))) + 1): if x % i == 0: return False return True def isPrime(self): """Gets if self is prime""" if self.imaginary == 0: return self.isPrime_raw(self.real) if self.real == 0: return self.isPrime_raw(abs(self.imaginary)) return self.isPrime_raw(self.real * self.real + self.imaginary * self.imaginary) def phase(self): """Returns angle between x axis""" return Decimal(math.atan2(self.imaginary, self.real)) def polar(self): #Converts to polar return (Decimal(abs(self).real), self.phase()) def toRect(self, r, phi): return ComplexDecimal(r) * ComplexDecimal(self.cos_raw(phi), self.sin_raw(phi)) #Trig -------------------------------------------- #1/functions, log, def cos_raw(self, x, prec=Decimal("0.0000000000000000000001")): """Raw cosine function for real numbers only""" # p = Decimal("0") # s = t = Decimal("1.0") # while abs(t/s) > prec: # p+=Decimal("1") # t = (-t * x * x) / ((Decimal("2") * p - Decimal("1")) * (Decimal("2") * p)) # s += t # if round(s,50) in [1,0,-1]: #If it's close enough round to get perfect answers (No one likes 1E-70 as cos(pi)) # return Decimal(round(s,50)) # return s return Decimal(math.cos(x)) def sin_raw(self, x, prec=Decimal("0.00000000000000000000001")): #return self.cos_raw(x+self.pi/Decimal("2")) return Decimal(math.sin(x)) def tan_raw(self, x, prec=Decimal("0.00000000000000000000001")): return self.sin_raw(x, prec) / self.cos_raw(x, prec) def atan_raw(self, x): #Not relatively accurate returned = Decimal("0") for j in range(0, 50): i = Decimal(j) a = Decimal("-1")**i * x**(Decimal("2") * i + Decimal("1")) b = Decimal("2") * i + Decimal("1") returned += a / b return returned def cos(self): """Compute cos of self""" if self.imaginary == 0: return ComplexDecimal(self.cos_raw(self.real)) try: returned = ComplexDecimal( self.cos_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real, -self.sin_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real) returned.imaginary = -returned.imaginary return returned except: return ComplexDecimal("inf") def sin(self): """Compute sin of self""" if self.imaginary == 0: return ComplexDecimal(self.sin_raw(self.real)) try: returned = ComplexDecimal( self.sin_raw(self.real) * ComplexDecimal(self.imaginary).cosh().real, -self.cos_raw(self.real) * ComplexDecimal(self.imaginary).sinh().real) returned.imaginary = -returned.imaginary return returned except: return ComplexDecimal("inf") def tan(self): """Compute tangent of self""" try: return self.sin() / self.cos() except: return ComplexDecimal("inf") def acos(self): """Attempt to compute arcosine""" if self.imaginary == 0: return ComplexDecimal(math.acos(self)) A = ComplexDecimal(( (Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ( (Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) B = ComplexDecimal(( (Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ( (Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) return ComplexDecimal( A.acos(), -(B + (B * B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln()) def asin(self): """Attempt to compute arcsine""" if self.imaginary == 0: return ComplexDecimal(math.asin(self)) A = ComplexDecimal(( (Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) - ( (Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) B = ComplexDecimal(( (Decimal(1) + self.real)**Decimal(2) + self.imaginary**Decimal(2))**Decimal(0.5) + ( (Decimal(1) - self.real)**Decimal(0.5) + self.imaginary**Decimal(2))**Decimal(0.5)) / ComplexDecimal(2) return ComplexDecimal( A.asin(), (B + (B * B - ComplexDecimal(1))**ComplexDecimal(0.5)).ln()) def atan(self): """Attempt to compute arctangent""" result = (ComplexDecimal(0, 1) + self) / (ComplexDecimal(0, 1) - self) return ComplexDecimal(0, 0.5) * result.ln() def sinh(self): """Hyperbolic sine of self""" returned = (self.exp() - (-self).exp()) / ComplexDecimal("2") returned.imaginary = -returned.imaginary return returned def cosh(self): """Hyperbolic cosine of self""" returned = (self.exp() + (-self).exp()) / ComplexDecimal("2") returned.imaginary = -returned.imaginary return returned def tanh(self): """Hyperbolic tangent of self""" return self.sinh() / self.cosh() def acosh(self): """Arc hyperbolic cosine""" returned = self + (self * self - ComplexDecimal(1))**0.5 return returned.ln() def asinh(self): """Arc hyperbolic sine""" returned = self + (self * self + ComplexDecimal(1))**0.5 return returned.ln() def atanh(self): """Arc hyperbolic tan""" a = (ComplexDecimal(1) + self).ln() - (ComplexDecimal(1) - self).ln() return ComplexDecimal(0.5) * a def ln_raw(self, x): returned = Decimal("0") for i in range(0, 25): a = Decimal("1") / (Decimal(i) * Decimal("2") + Decimal("1")) b = ((x - Decimal("1")) / (x + Decimal("1")))**(Decimal(i) * Decimal("2") + Decimal("1")) returned += a * b return returned * 2 def ln(self): """Natural logarithim of self""" if self.imaginary == 0: try: return ComplexDecimal(self.ln_raw(self.real)) except: pass p = self.polar() return ComplexDecimal(self.ln_raw(p[0]), p[1]) def log(self, x=None): """Compute log of number in base x""" if not x: x = self.e if self.imaginary == 0: try: return ComplexDecimal(math.log(self.real, x)) except: pass p = self.polar() return ComplexDecimal(math.log(p[0], x), p[1]) def log10(self): """Compute log base 10""" if self.imaginary == 0: return ComplexDecimal(math.log10(self.real)) return self.ln() / ComplexDecimal("10").ln() #Replace builtins -------------------------------- # power, comparasion, #To float, int, complex def __str__(self): """Converts to str""" if self.imaginary == 0: return str(self.real) elif self.real == 0: if self.imaginary == 1: return "i" if self.imaginary == -1: return "-i" return str(self.imaginary) + "i" return (str(self.real) + "+" + str(self.imaginary) + "i").replace( "+-", "-") def __abs__(self): """Absolute value""" return ComplexDecimal((self.real**2 + self.imaginary**2).sqrt()) def __add__(self, other): """Add 2 numbers together""" return ComplexDecimal(self.real + other.real, self.imaginary + other.imaginary) def __sub__(self, other): """Subtract 2 numbers""" return ComplexDecimal(self.real - other.real, self.imaginary - other.imaginary) def __mul__(self, other): """Multiply 2 numbers""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real * other.real, "0") return ComplexDecimal( self.real * other.real - self.imaginary * other.imaginary, self.real * other.imaginary + self.imaginary * other.real) def __div__(self, other): """Divide 2 numbers""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real / other.real, "0") a = (self.real * other.real + self.imaginary * other.imaginary) / ( other.real * other.real + other.imaginary * other.imaginary) b = (self.imaginary * other.real - self.real * other.imaginary) / ( other.real * other.real + other.imaginary * other.imaginary) return ComplexDecimal(a, b) def __truediv__(self, other): """Divide 2 numbers""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real / other.real, "0") a = (self.real * other.real + self.imaginary * other.imaginary) / ( other.real * other.real + other.imaginary * other.imaginary) b = (self.imaginary * other.real - self.real * other.imaginary) / ( other.real * other.real + other.imaginary * other.imaginary) return ComplexDecimal(a, b) def __neg__(self): """Negates a number""" return ComplexDecimal(-self.real, -self.imaginary) def __pos__(self): """Positive number""" return ComplexDecimal(abs(self.real), abs(self.imaginary)) def __inverse__(self): """1/ number""" return ComplexDecimal(1) / self def __mod__(self, other): #a%b = a + b * ciel(-a/b) """Modolus""" return self + other * ((-self / other).ceil()) #Powers def __pow__(self, other): """Powers :D""" if self.imaginary == 0 and other.imaginary == 0: return ComplexDecimal(self.real**other.real) if other.imaginary == 0: polar = self.polar() return self.toRect(polar[0]**other.real, polar[1] * other.real) elif other.real == 0: a = ComplexDecimal(self.real) b = ComplexDecimal(self.imaginary) c = ComplexDecimal(other.real) d = ComplexDecimal(other.imaginary) x = (-d * (b / a).atan()).exp() * ( (a * a + b * b).ln() * d / ComplexDecimal(2)).cos() y = ComplexDecimal(0, 1) * (-d * (b / a).atan()).exp() * ( (a * a + b * b).ln() * d / ComplexDecimal(2)).sin() return x + y b = other.real c = other.imaginary returned = self**(ComplexDecimal(b)) * self**(ComplexDecimal(0, 1) * ComplexDecimal(c)) returned.imaginary = -returned.imaginary return returned #Additional conversions def __complex__(self): """Convert to complex""" return complex(float(self.real), float(self.imaginary)) def __int__(self): """Convert to int""" return int(self.real) def __float__(self): """Convert to float""" return float(self.real) #Comparasions def __lt__(self, other): #> if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError( "Complex comparasion is not supported") return self.real < other.real def __le__(self, other): #>= if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError( "Complex comparasion is not supported") return self.real <= other.real def __eq__(self, other): #== if self.real == other.real and self.imaginary == other.imaginary: return True return False def __ne__(self, other): #!= return not self.__eq__(other) def __gt__(self, other): #< if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError( "Complex comparasion is not supported") return self.real > other.real def __ge__(self, other): #<= if self.imaginary != 0 or other.imaginary != 0: raise errors.ComparasionError( "Complex comparasion is not supported") return self.real >= other.real #Some things reimplemented from decimal class for complex ComplexDecimals #================================================================ def copy_abs(self): return abs(self) def copy_negate(self): return -self def copy_sign(self, other): if other < 0: return -abs(self) return abs(self) def is_finite(self): return self.real.is_finite() and self.imaginary.is_finite() def is_infinite(self): return not self.is_finite() def is_nan(self): return self.real.is_nan() or self.imaginary.is_nan() def is_signed(self): return self < ComplexDecimal(0) def is_zero(self): return self.real.is_zero() and self.imaginary.is_zero() def radix(self): return Decimal(10) def sqrt(self): if self.imaginary.is_zero(): try: return ComplexDecimal(self.real.sqrt()) except: return ComplexDecimal(0, (-self.real).sqrt()) return self**ComplexDecimal("0.5") def to_eng_string(self): returned = self.real.to_eng_string( ) + " + " + self.imaginary.to_eng_string + "i" return returned.replace(" + -", " - ")
def format_picture(conv, value, picture): monetary = False decimal_point = conv['decimal_point'] thousands_sep = conv[monetary and 'mon_thousands_sep' or 'thousands_sep'] percent = '%' per_mille = '\u2030' minus_sign = '-' #grouping = conv[monetary and 'mon_grouping' or 'grouping'] if isinstance(value, float): value = Decimal.from_float(value) elif isinstance(value, _STR_NUM_TYPES): value = Decimal(value) elif not isinstance(value, Decimal): raise ValueError( _('Picture requires a number convertable to decimal or float'). format(picture)) if value.is_nan(): return 'NaN' isNegative = value.is_signed() pic, sep, negPic = picture.partition(';') if negPic and ';' in negPic: raise ValueError( _('Picture contains multiple picture sepearators {0}').format( picture)) if isNegative and negPic: pic = negPic if len([c for c in pic if c in (percent, per_mille)]) > 1: raise ValueError( _('Picture contains multiple percent or per_mille charcters {0}'). format(picture)) if percent in pic: value *= 100 elif per_mille in pic: value *= 1000 intPart, sep, fractPart = pic.partition(decimal_point) prefix = '' numPlaces = 0 intPlaces = 0 grouping = 0 fractPlaces = 0 suffix = '' if fractPart: if decimal_point in fractPart: raise ValueError( _('Sub-picture contains decimal point sepearators {0}').format( pic)) for c in fractPart: if c.isdecimal(): numPlaces += 1 fractPlaces += 1 if suffix: raise ValueError( _('Sub-picture passive character {0} between active characters {1}' ).format(c, fractPart)) else: suffix += c intPosition = 0 for c in reversed(intPart): if c.isdecimal() or c == '#' or c == thousands_sep: if prefix: raise ValueError( _('Sub-picture passive character {0} between active characters {1}' ).format(c, intPart)) if c.isdecimal(): numPlaces += 1 intPlaces += 1 intPosition += 1 prefix = '' elif c == '#': numPlaces += 1 intPosition += 1 elif c == thousands_sep: if not grouping: grouping = intPosition else: prefix = c + prefix if not numPlaces and prefix != minus_sign: raise ValueError( _('Sub-picture must contain at least one digit position or sign character {0}' ).format(pic)) if intPlaces == 0 and fractPlaces == 0: intPlaces = 1 return format_decimal(None, value, intPlaces=intPlaces, fractPlaces=fractPlaces, sep=thousands_sep, dp=decimal_point, grouping=grouping, pos=prefix, neg=prefix if negPic else prefix + minus_sign, trailpos=suffix, trailneg=suffix)
def __init__( self, amount: Optional[Union["Money", Decimal, int, float, str, object]] = None, currency: Optional[Union[Type[DefaultCurrency], Currency, str]] = DefaultCurrency, is_cents: Optional[bool] = None, units: Optional[int] = None, nanos: Optional[int] = None, **kwargs: Any, ) -> None: validate_amounts = [] if units is not None or nanos is not None: try: units = units or 0 nanos = nanos or 0 if (units > 0 and nanos < 0) or (units < 0 and nanos > 0): raise ValueError units_str = str(units).lstrip("-") nanos_str = str(nanos).lstrip("-").rjust(NANOS_LENGTH, "0") if len(units_str) > UNITS_MAX_LENGTH: raise ValueError if len(nanos_str) != NANOS_LENGTH: raise ValueError sign = "-" if nanos < 0 or units < 0 else "" new_amount = Decimal(f"{sign}{units_str}.{nanos_str}") if amount is None: amount = new_amount else: validate_amounts.append(new_amount) except Exception: raise ConversionError("Invalid values for units and nanos") if amount is None: raise ConversionError("Missing input values for monetary amount") if (isinstance(amount, Money) and currency is DefaultCurrency and is_cents is None and units is None and nanos is None): object.__setattr__(self, "_amount", amount._amount) object.__setattr__(self, "_currency", amount._currency) return if (currency is not DefaultCurrency and not isinstance(currency, str) and not isinstance(currency, Currency) and currency is not None): raise ConversionError("Invalid currency value") output_amount = None output_currency: Optional[Union[Currency, str]] = None if currency is not DefaultCurrency: if isinstance(currency, Currency): output_currency = currency else: output_currency = str(currency or "").strip().upper() or None if Money._is_unknown_amount_type(amount): try: match_amount = getattr(amount, "amount") match_amount = (match_amount( )) if match_amount and callable(match_amount) else match_amount if match_amount is None or Money._is_unknown_amount_type( match_amount): raise AttributeError match_currency = None try: match_currency = getattr(amount, "currency") match_currency = ((match_currency()) if match_currency and callable(match_currency) else match_currency) if not match_currency: raise AttributeError except AttributeError: matches = re.match(r"^(?:[-+]?[0-9.]+)[ ]+([a-zA-Z]+)$", str(amount)) if not matches: matches = re.match( r"^([a-zA-Z]+)[ ]+(?:[-+]?[0-9.]+)$", str(amount)) if matches: match_currency = matches.group(1) if match_currency is not None: match_currency = str(match_currency).strip().upper() if output_currency is not None and match_currency != output_currency: raise ConversionError( "Mismatching currency in input value and currency argument" ) output_currency = output_currency if isinstance( output_currency, Currency) else match_currency amount = match_amount except AttributeError: amount = str(amount) if amount is not None and isinstance( amount, int) and not isinstance(amount, bool): output_amount = Decimal(amount) elif amount is not None and isinstance(amount, float): output_amount = Decimal(str(amount)) elif amount is not None and isinstance(amount, str) and amount.strip(): amount = amount.strip() match_currency = None matches = re.match( r"^(?P<amount>[-+]?[0-9.]+)[ ]+(?P<currency>[a-zA-Z]+)$", amount) if not matches: matches = re.match( r"^(?P<currency>[a-zA-Z]+)[ ]+(?P<amount>[-+]?[0-9.]+)$", amount) if matches: amount = matches.group("amount").strip() match_currency = matches.group("currency").strip().upper() if match_currency is not None: if output_currency is not None and match_currency != output_currency: raise ConversionError( "Mismatching currency in input value and currency argument" ) output_currency = output_currency if isinstance( output_currency, Currency) else match_currency try: output_amount = Decimal(amount) except Exception: raise ConversionError( "Value cannot be used as monetary amount") elif amount is not None and isinstance(amount, Money): if amount.currency and not output_currency and currency is not DefaultCurrency: output_currency = amount.currency output_amount = amount._amount elif amount is not None and isinstance(amount, Decimal): output_amount = amount if output_amount is None: raise ConversionError("Missing input values for monetary amount") if output_amount.is_infinite(): raise ConversionError("Monetary amounts cannot be infinite") if output_amount.is_nan(): raise ConversionError("Input amount is not a number") if is_cents: output_amount = output_amount / 100 if output_amount > Decimal(HIGHEST_SUPPORTED_AMOUNT): raise ConversionError( f"Input amount is too high, max value is {HIGHEST_SUPPORTED_AMOUNT}" ) if output_amount < Decimal(LOWEST_SUPPORTED_AMOUNT): raise ConversionError( f"Input amount is too low, min value is {LOWEST_SUPPORTED_AMOUNT}" ) if output_currency and not re.match(r"^[A-Z]+$", str(output_currency)): raise ConversionError("Invalid currency") if output_amount == 0 and output_amount.is_signed(): output_amount = Decimal(0) if any([output_amount != a for a in validate_amounts]): raise ConversionError("Input arguments does not match") object.__setattr__(self, "_amount", output_amount) object.__setattr__(self, "_currency", output_currency)
class PerpetualMarketMakingStrategy(StrategyPyBase): OPTION_LOG_CREATE_ORDER = 1 << 3 OPTION_LOG_MAKER_ORDER_FILLED = 1 << 4 OPTION_LOG_STATUS_REPORT = 1 << 5 OPTION_LOG_ALL = 0x7fffffffffffffff _logger = None @classmethod def logger(cls): if cls._logger is None: cls._logger = logging.getLogger(__name__) return cls._logger def init_params( self, market_info: MarketTradingPairTuple, leverage: int, position_mode: str, bid_spread: Decimal, ask_spread: Decimal, order_amount: Decimal, long_profit_taking_spread: Decimal, short_profit_taking_spread: Decimal, stop_loss_spread: Decimal, time_between_stop_loss_orders: float, stop_loss_slippage_buffer: Decimal, order_levels: int = 1, order_level_spread: Decimal = s_decimal_zero, order_level_amount: Decimal = s_decimal_zero, order_refresh_time: float = 30.0, order_refresh_tolerance_pct: Decimal = s_decimal_neg_one, filled_order_delay: float = 60.0, order_optimization_enabled: bool = False, ask_order_optimization_depth: Decimal = s_decimal_zero, bid_order_optimization_depth: Decimal = s_decimal_zero, asset_price_delegate: AssetPriceDelegate = None, price_type: str = "mid_price", price_ceiling: Decimal = s_decimal_neg_one, price_floor: Decimal = s_decimal_neg_one, logging_options: int = OPTION_LOG_ALL, status_report_interval: float = 900, minimum_spread: Decimal = Decimal(0), hb_app_notification: bool = False, order_override: Dict[str, List[str]] = {}, ): if price_ceiling != s_decimal_neg_one and price_ceiling < price_floor: raise ValueError( "Parameter price_ceiling cannot be lower than price_floor.") self._sb_order_tracker = PerpetualMarketMakingOrderTracker() self._market_info = market_info self._leverage = leverage self._position_mode = PositionMode.HEDGE if position_mode == "Hedge" else PositionMode.ONEWAY self._bid_spread = bid_spread self._ask_spread = ask_spread self._minimum_spread = minimum_spread self._order_amount = order_amount self._long_profit_taking_spread = long_profit_taking_spread self._short_profit_taking_spread = short_profit_taking_spread self._stop_loss_spread = stop_loss_spread self._order_levels = order_levels self._buy_levels = order_levels self._sell_levels = order_levels self._order_level_spread = order_level_spread self._order_level_amount = order_level_amount self._order_refresh_time = order_refresh_time self._order_refresh_tolerance_pct = order_refresh_tolerance_pct self._filled_order_delay = filled_order_delay self._order_optimization_enabled = order_optimization_enabled self._ask_order_optimization_depth = ask_order_optimization_depth self._bid_order_optimization_depth = bid_order_optimization_depth self._asset_price_delegate = asset_price_delegate self._price_type = self.get_price_type(price_type) self._price_ceiling = price_ceiling self._price_floor = price_floor self._hb_app_notification = hb_app_notification self._order_override = order_override self._cancel_timestamp = 0 self._create_timestamp = 0 self._all_markets_ready = False self._logging_options = logging_options self._last_timestamp = 0 self._status_report_interval = status_report_interval self._last_own_trade_price = Decimal('nan') self._ts_peak_bid_price = Decimal('0') self._ts_peak_ask_price = Decimal('0') self._exit_orders = dict() self._next_buy_exit_order_timestamp = 0 self._next_sell_exit_order_timestamp = 0 self.add_markets([market_info.market]) self._close_order_type = OrderType.LIMIT self._time_between_stop_loss_orders = time_between_stop_loss_orders self._stop_loss_slippage_buffer = stop_loss_slippage_buffer def all_markets_ready(self): return all([market.ready for market in self.active_markets]) @property def order_refresh_tolerance_pct(self) -> Decimal: return self._order_refresh_tolerance_pct @order_refresh_tolerance_pct.setter def order_refresh_tolerance_pct(self, value: Decimal): self._order_refresh_tolerance_pct = value @property def order_amount(self) -> Decimal: return self._order_amount @order_amount.setter def order_amount(self, value: Decimal): self._order_amount = value @property def order_levels(self) -> int: return self._order_levels @order_levels.setter def order_levels(self, value: int): self._order_levels = value self._buy_levels = value self._sell_levels = value @property def buy_levels(self) -> int: return self._buy_levels @buy_levels.setter def buy_levels(self, value: int): self._buy_levels = value @property def sell_levels(self) -> int: return self._sell_levels @sell_levels.setter def sell_levels(self, value: int): self._sell_levels = value @property def order_level_amount(self) -> Decimal: return self._order_level_amount @order_level_amount.setter def order_level_amount(self, value: Decimal): self._order_level_amount = value @property def order_level_spread(self) -> Decimal: return self._order_level_spread @order_level_spread.setter def order_level_spread(self, value: Decimal): self._order_level_spread = value @property def bid_spread(self) -> Decimal: return self._bid_spread @bid_spread.setter def bid_spread(self, value: Decimal): self._bid_spread = value @property def ask_spread(self) -> Decimal: return self._ask_spread @ask_spread.setter def ask_spread(self, value: Decimal): self._ask_spread = value @property def order_optimization_enabled(self) -> bool: return self._order_optimization_enabled @order_optimization_enabled.setter def order_optimization_enabled(self, value: bool): self._order_optimization_enabled = value @property def order_refresh_time(self) -> float: return self._order_refresh_time @order_refresh_time.setter def order_refresh_time(self, value: float): self._order_refresh_time = value @property def filled_order_delay(self) -> float: return self._filled_order_delay @filled_order_delay.setter def filled_order_delay(self, value: float): self._filled_order_delay = value @property def price_ceiling(self) -> Decimal: return self._price_ceiling @price_ceiling.setter def price_ceiling(self, value: Decimal): self._price_ceiling = value @property def price_floor(self) -> Decimal: return self._price_floor @price_floor.setter def price_floor(self, value: Decimal): self._price_floor = value @property def base_asset(self): return self._market_info.base_asset @property def quote_asset(self): return self._market_info.quote_asset @property def trading_pair(self): return self._market_info.trading_pair def get_price(self) -> float: if self._asset_price_delegate is not None: price_provider = self._asset_price_delegate else: price_provider = self._market_info if self._price_type is PriceType.LastOwnTrade: price = self._last_own_trade_price else: price = price_provider.get_price_by_type(self._price_type) if price.is_nan(): price = price_provider.get_price_by_type(PriceType.MidPrice) return price def get_last_price(self) -> float: return self._market_info.get_last_price() def get_mid_price(self) -> Decimal: delegate: AssetPriceDelegate = self._asset_price_delegate if delegate is not None: mid_price = delegate.get_mid_price() else: mid_price = self._market_info.get_mid_price() return mid_price @property def active_orders(self) -> List[LimitOrder]: if self._market_info not in self._sb_order_tracker.market_pair_to_active_orders: return [] return self._sb_order_tracker.market_pair_to_active_orders[ self._market_info] @property def active_positions(self) -> Dict[str, Position]: return self._market_info.market.account_positions @property def active_buys(self) -> List[LimitOrder]: return [o for o in self.active_orders if o.is_buy] @property def active_sells(self) -> List[LimitOrder]: return [o for o in self.active_orders if not o.is_buy] @property def logging_options(self) -> int: return self._logging_options @logging_options.setter def logging_options(self, logging_options: int): self._logging_options = logging_options @property def asset_price_delegate(self) -> AssetPriceDelegate: return self._asset_price_delegate @asset_price_delegate.setter def asset_price_delegate(self, value): self._asset_price_delegate = value def perpetual_mm_assets_df(self) -> pd.DataFrame: market, trading_pair, base_asset, quote_asset = self._market_info quote_balance = float(market.get_balance(quote_asset)) available_quote_balance = float( market.get_available_balance(quote_asset)) data = [["", quote_asset], ["Total Balance", round(quote_balance, 4)], ["Available Balance", round(available_quote_balance, 4)]] df = pd.DataFrame(data=data) return df def active_orders_df(self) -> pd.DataFrame: price = self.get_price() active_orders = self.active_orders no_sells = len([o for o in active_orders if not o.is_buy]) active_orders.sort(key=lambda x: x.price, reverse=True) columns = [ "Level", "Type", "Price", "Spread", "Amount (Orig)", "Amount (Adj)", "Age" ] data = [] lvl_buy, lvl_sell = 0, 0 for idx in range(0, len(active_orders)): order = active_orders[idx] level = None if order.is_buy: level = lvl_buy + 1 lvl_buy += 1 else: level = no_sells - lvl_sell lvl_sell += 1 spread = 0 if price == 0 else abs(order.price - price) / price age = "n/a" # // indicates order is a paper order so 'n/a'. For real orders, calculate age. if "//" not in order.client_order_id: age = pd.Timestamp(int(time.time()) - int(order.client_order_id[-16:]) / 1e6, unit='s').strftime('%H:%M:%S') amount_orig = "" if level is None else self._order_amount + ( (level - 1) * self._order_level_amount) data.append([ level, "buy" if order.is_buy else "sell", float(order.price), f"{spread:.2%}", amount_orig, float(order.quantity), age ]) return pd.DataFrame(data=data, columns=columns) def active_positions_df(self) -> pd.DataFrame: columns = [ "Symbol", "Type", "Entry Price", "Amount", "Leverage", "Unrealized PnL" ] data = [] market, trading_pair = self._market_info.market, self._market_info.trading_pair for idx in self.active_positions.values(): is_buy = True if idx.amount > 0 else False unrealized_profit = ( (market.get_price(trading_pair, is_buy) - idx.entry_price) * idx.amount) data.append([ idx.trading_pair, idx.position_side.name, idx.entry_price, idx.amount, idx.leverage, unrealized_profit ]) return pd.DataFrame(data=data, columns=columns) def market_status_data_frame(self) -> pd.DataFrame: markets_data = [] markets_columns = [ "Exchange", "Market", "Best Bid", "Best Ask", f"Ref Price ({self._price_type.name})" ] if self._price_type is PriceType.LastOwnTrade and self._last_own_trade_price.is_nan( ): markets_columns[-1] = "Ref Price (MidPrice)" market_books = [(self._market_info.market, self._market_info.trading_pair)] if type(self._asset_price_delegate) is OrderBookAssetPriceDelegate: market_books.append((self._asset_price_delegate.market, self._asset_price_delegate.trading_pair)) for market, trading_pair in market_books: bid_price = market.get_price(trading_pair, False) ask_price = market.get_price(trading_pair, True) ref_price = float("nan") if market == self._market_info.market and self._asset_price_delegate is None: ref_price = self.get_price() elif market == self._asset_price_delegate.market and self._price_type is not PriceType.LastOwnTrade: ref_price = self._asset_price_delegate.get_price_by_type( self._price_type) markets_data.append([ market.display_name, trading_pair, float(bid_price), float(ask_price), float(ref_price) ]) return pd.DataFrame(data=markets_data, columns=markets_columns).replace(np.nan, '', regex=True) def format_status(self) -> str: if not self._all_markets_ready: return "Market connectors are not ready." lines = [] warning_lines = [] markets_df = self.market_status_data_frame() lines.extend(["", " Markets:"] + [ " " + line for line in markets_df.to_string(index=False).split("\n") ]) assets_df = map_df_to_str(self.perpetual_mm_assets_df()) first_col_length = max(*assets_df[0].apply(len)) df_lines = assets_df.to_string(index=False, header=False, formatters={ 0: ("{:<" + str(first_col_length) + "}").format }).split("\n") lines.extend(["", " Assets:"] + [" " + line for line in df_lines]) # See if there're any open orders. if len(self.active_orders) > 0: df = self.active_orders_df() lines.extend(["", " Orders:"] + [ " " + line for line in df.to_string(index=False).split("\n") ]) else: lines.extend(["", " No active maker orders."]) # See if there're any active positions. if len(self.active_positions) > 0: df = self.active_positions_df() lines.extend(["", " Positions:"] + [ " " + line for line in df.to_string(index=False).split("\n") ]) else: lines.extend(["", " No active positions."]) if len(warning_lines) > 0: lines.extend(["", "*** WARNINGS ***"] + warning_lines) return "\n".join(lines) def start(self, clock: Clock, timestamp: float): super().start(clock, timestamp) self._last_timestamp = timestamp self.apply_initial_settings(self.trading_pair, self._position_mode, self._leverage) def apply_initial_settings(self, trading_pair: str, position: Position, leverage: int): market: ExchangeBase = self._market_info.market market.set_leverage(trading_pair, leverage) market.set_position_mode(position) def tick(self, timestamp: float): market: ExchangeBase = self._market_info.market session_positions = [ s for s in self.active_positions.values() if s.trading_pair == self.trading_pair ] current_tick = timestamp // self._status_report_interval last_tick = self._last_timestamp // self._status_report_interval should_report_warnings = ((current_tick > last_tick) and (self._logging_options & self.OPTION_LOG_STATUS_REPORT)) try: if not self._all_markets_ready: self._all_markets_ready = all( [market.ready for market in self.active_markets]) if self._asset_price_delegate is not None and self._all_markets_ready: self._all_markets_ready = self._asset_price_delegate.ready if not self._all_markets_ready: # M({self.trading_pair}) Maker sell order {order_id}arkets not ready yet. Don't do anything. if should_report_warnings: self.logger().warning( "Markets are not ready. No market making trades are permitted." ) return if should_report_warnings: if not all([ market.network_status is NetworkStatus.CONNECTED for market in self.active_markets ]): self.logger().warning( "WARNING: Some markets are not connected or are down at the moment. Market " "making may be dangerous when markets or networks are unstable." ) if len(session_positions) == 0: self._exit_orders = dict( ) # Empty list of exit order at this point to reduce size proposal = None if self._create_timestamp <= self.current_timestamp: # 1. Create base order proposals proposal = self.create_base_proposal() # 2. Apply functions that limit numbers of buys and sells proposal self.apply_order_levels_modifiers(proposal) # 3. Apply functions that modify orders price self.apply_order_price_modifiers(proposal) # 4. Apply budget constraint, i.e. can't buy/sell more than what you have. self.apply_budget_constraint(proposal) self.filter_out_takers(proposal) self.cancel_active_orders(proposal) self.cancel_orders_below_min_spread() if self.to_create_orders(proposal): self.execute_orders_proposal(proposal, PositionAction.OPEN) # Reset peak ask and bid prices self._ts_peak_ask_price = market.get_price( self.trading_pair, False) self._ts_peak_bid_price = market.get_price( self.trading_pair, True) else: self.manage_positions(session_positions) finally: self._last_timestamp = timestamp def manage_positions(self, session_positions: List[Position]): mode = self._position_mode proposals = self.profit_taking_proposal(mode, session_positions) if proposals is not None: self.execute_orders_proposal(proposals, PositionAction.CLOSE) # check if stop loss needs to be placed proposals = self.stop_loss_proposal(mode, session_positions) if proposals is not None: self.execute_orders_proposal(proposals, PositionAction.CLOSE) def profit_taking_proposal(self, mode: PositionMode, active_positions: List) -> Proposal: market: ExchangeBase = self._market_info.market unwanted_exit_orders = [ o for o in self.active_orders if o.client_order_id not in self._exit_orders.keys() ] ask_price = market.get_price(self.trading_pair, True) bid_price = market.get_price(self.trading_pair, False) buys = [] sells = [] if mode == PositionMode.ONEWAY: # in one-way mode, only one active position is expected per time if len(active_positions) > 1: self.logger().error( f"More than one open position in {mode.name} position mode. " "Kindly ensure you do not interact with the exchange through " "other platforms and restart this strategy.") else: # Cancel open order that could potentially close position before reaching take_profit_limit for order in unwanted_exit_orders: if ((active_positions[0].amount < 0 and order.is_buy) or (active_positions[0].amount > 0 and not order.is_buy)): self.cancel_order(self._market_info, order.client_order_id) self.logger().info( f"Initiated cancellation of {'buy' if order.is_buy else 'sell'} order " f"{order.client_order_id} in favour of take profit order." ) for position in active_positions: if (ask_price > position.entry_price and position.amount > 0) or ( bid_price < position.entry_price and position.amount < 0): # check if there is an active order to take profit, and create if none exists profit_spread = self._long_profit_taking_spread if position.amount > 0 else self._short_profit_taking_spread take_profit_price = position.entry_price * (Decimal("1") + profit_spread) if position.amount > 0 \ else position.entry_price * (Decimal("1") - profit_spread) price = market.quantize_order_price(self.trading_pair, take_profit_price) size = market.quantize_order_amount(self.trading_pair, abs(position.amount)) old_exit_orders = [ o for o in self.active_orders if ((o.price != price or o.quantity != size) and o.client_order_id in self._exit_orders.keys() and ( (position.amount < 0 and o.is_buy) or (position.amount > 0 and not o.is_buy))) ] for old_order in old_exit_orders: self.cancel_order(self._market_info, old_order.client_order_id) self.logger().info( f"Initiated cancellation of previous take profit order {old_order.client_order_id} in favour of new take profit order." ) exit_order_exists = [ o for o in self.active_orders if o.price == price ] if len(exit_order_exists) == 0: if size > 0 and price > 0: if position.amount < 0: buys.append(PriceSize(price, size)) else: sells.append(PriceSize(price, size)) return Proposal(buys, sells) def _should_renew_stop_loss(self, stop_loss_order: LimitOrder) -> bool: stop_loss_creation_timestamp = self._exit_orders.get( stop_loss_order.client_order_id) time_since_stop_loss = self.current_timestamp - stop_loss_creation_timestamp return time_since_stop_loss >= self._time_between_stop_loss_orders def stop_loss_proposal(self, mode: PositionMode, active_positions: List[Position]) -> Proposal: market: ExchangeBase = self._market_info.market top_ask = market.get_price(self.trading_pair, False) top_bid = market.get_price(self.trading_pair, True) buys = [] sells = [] for position in active_positions: # check if stop loss order needs to be placed stop_loss_price = position.entry_price * (Decimal("1") + self._stop_loss_spread) if position.amount < 0 \ else position.entry_price * (Decimal("1") - self._stop_loss_spread) existent_stop_loss_orders = [ order for order in self.active_orders if order.client_order_id in self._exit_orders.keys() and ( (position.amount > 0 and not order.is_buy) or (position.amount < 0 and order.is_buy)) ] if (not existent_stop_loss_orders or (self._should_renew_stop_loss(existent_stop_loss_orders[0]))): previous_stop_loss_price = None for order in existent_stop_loss_orders: previous_stop_loss_price = order.price self.cancel_order(self._market_info, order.client_order_id) new_price = previous_stop_loss_price or stop_loss_price if (top_ask <= stop_loss_price and position.amount > 0): price = market.quantize_order_price( self.trading_pair, new_price * (Decimal(1) - self._stop_loss_slippage_buffer)) take_profit_orders = [ o for o in self.active_orders if (not o.is_buy and o.price > price and o.client_order_id in self._exit_orders.keys()) ] # cancel take profit orders if they exist for old_order in take_profit_orders: self.cancel_order(self._market_info, old_order.client_order_id) size = market.quantize_order_amount( self.trading_pair, abs(position.amount)) if size > 0 and price > 0: self.logger().info( "Creating stop loss sell order to close long position." ) sells.append(PriceSize(price, size)) elif (top_bid >= stop_loss_price and position.amount < 0): price = market.quantize_order_price( self.trading_pair, new_price * (Decimal(1) + self._stop_loss_slippage_buffer)) take_profit_orders = [ o for o in self.active_orders if (o.is_buy and o.price < price and o.client_order_id in self._exit_orders.keys()) ] # cancel take profit orders if they exist for old_order in take_profit_orders: self.cancel_order(self._market_info, old_order.client_order_id) size = market.quantize_order_amount( self.trading_pair, abs(position.amount)) if size > 0 and price > 0: self.logger().info( "Creating stop loss buy order to close short position." ) buys.append(PriceSize(price, size)) return Proposal(buys, sells) def create_base_proposal(self): market: ExchangeBase = self._market_info.market buys = [] sells = [] # First to check if a customized order override is configured, otherwise the proposal will be created according # to order spread, amount, and levels setting. order_override = self._order_override if order_override is not None and len(order_override) > 0: for key, value in order_override.items(): if str(value[0]) in ["buy", "sell"]: if str(value[0]) == "buy": price = self.get_price() * ( Decimal("1") - Decimal(str(value[1])) / Decimal("100")) price = market.quantize_order_price( self.trading_pair, price) size = Decimal(str(value[2])) size = market.quantize_order_amount( self.trading_pair, size) if size > 0 and price > 0: buys.append(PriceSize(price, size)) elif str(value[0]) == "sell": price = self.get_price() * ( Decimal("1") + Decimal(str(value[1])) / Decimal("100")) price = market.quantize_order_price( self.trading_pair, price) size = Decimal(str(value[2])) size = market.quantize_order_amount( self.trading_pair, size) if size > 0 and price > 0: sells.append(PriceSize(price, size)) else: for level in range(0, self._buy_levels): price = self.get_price() * (Decimal("1") - self._bid_spread - (level * self._order_level_spread)) price = market.quantize_order_price(self.trading_pair, price) size = self._order_amount + (self._order_level_amount * level) size = market.quantize_order_amount(self.trading_pair, size) if size > 0: buys.append(PriceSize(price, size)) for level in range(0, self._sell_levels): price = self.get_price() * (Decimal("1") + self._ask_spread + (level * self._order_level_spread)) price = market.quantize_order_price(self.trading_pair, price) size = self._order_amount + (self._order_level_amount * level) size = market.quantize_order_amount(self.trading_pair, size) if size > 0: sells.append(PriceSize(price, size)) return Proposal(buys, sells) def apply_order_levels_modifiers(self, proposal: Proposal): self.apply_price_band(proposal) def apply_price_band(self, proposal: Proposal): if self._price_ceiling > 0 and self.get_price() >= self._price_ceiling: proposal.buys = [] if self._price_floor > 0 and self.get_price() <= self._price_floor: proposal.sells = [] def apply_order_price_modifiers(self, proposal: Proposal): if self._order_optimization_enabled: self.apply_order_optimization(proposal) def apply_budget_constraint(self, proposal: Proposal): checker = self._market_info.market.budget_checker order_candidates = self.create_order_candidates_for_budget_check( proposal) adjusted_candidates = checker.adjust_candidates(order_candidates, all_or_none=True) self.apply_adjusted_order_candidates_to_proposal( adjusted_candidates, proposal) def create_order_candidates_for_budget_check(self, proposal: Proposal): order_candidates = [] is_maker = True order_candidates.extend([ PerpetualOrderCandidate( self.trading_pair, is_maker, OrderType.LIMIT, TradeType.BUY, buy.size, buy.price, leverage=Decimal(self._leverage), ) for buy in proposal.buys ]) order_candidates.extend([ PerpetualOrderCandidate( self.trading_pair, is_maker, OrderType.LIMIT, TradeType.SELL, sell.size, sell.price, leverage=Decimal(self._leverage), ) for sell in proposal.sells ]) return order_candidates def apply_adjusted_order_candidates_to_proposal( self, adjusted_candidates: List[PerpetualOrderCandidate], proposal: Proposal): for order in chain(proposal.buys, proposal.sells): adjusted_candidate = adjusted_candidates.pop(0) if adjusted_candidate.amount == s_decimal_zero: self.logger().info( f"Insufficient balance: {adjusted_candidate.order_side.name} order (price: {order.price}," f" size: {order.size}) is omitted.") self.logger().warning( "You are also at a possible risk of being liquidated if there happens to be an open loss." ) order.size = s_decimal_zero proposal.buys = [o for o in proposal.buys if o.size > 0] proposal.sells = [o for o in proposal.sells if o.size > 0] def filter_out_takers(self, proposal: Proposal): market: ExchangeBase = self._market_info.market top_ask = market.get_price(self.trading_pair, True) if not top_ask.is_nan(): proposal.buys = [ buy for buy in proposal.buys if buy.price < top_ask ] top_bid = market.get_price(self.trading_pair, False) if not top_bid.is_nan(): proposal.sells = [ sell for sell in proposal.sells if sell.price > top_bid ] # Compare the market price with the top bid and top ask price def apply_order_optimization(self, proposal: Proposal): market: ExchangeBase = self._market_info.market own_buy_size = s_decimal_zero own_sell_size = s_decimal_zero # If there are multiple orders, do not jump prices if self._order_levels > 1: return for order in self.active_orders: if order.is_buy: own_buy_size = order.quantity else: own_sell_size = order.quantity if len(proposal.buys) == 1: # Get the top bid price in the market using order_optimization_depth and your buy order volume top_bid_price = self._market_info.get_price_for_volume( False, self._bid_order_optimization_depth + own_buy_size).result_price price_quantum = market.get_order_price_quantum( self.trading_pair, top_bid_price) # Get the price above the top bid price_above_bid = (ceil(top_bid_price / price_quantum) + 1) * price_quantum # If the price_above_bid is lower than the price suggested by the pricing proposal, # lower your price to this lower_buy_price = min(proposal.buys[0].price, price_above_bid) proposal.buys[0].price = market.quantize_order_price( self.trading_pair, lower_buy_price) if len(proposal.sells) == 1: # Get the top ask price in the market using order_optimization_depth and your sell order volume top_ask_price = self._market_info.get_price_for_volume( True, self._ask_order_optimization_depth + own_sell_size).result_price price_quantum = market.get_order_price_quantum( self.trading_pair, top_ask_price) # Get the price below the top ask price_below_ask = (floor(top_ask_price / price_quantum) - 1) * price_quantum # If the price_below_ask is higher than the price suggested by the pricing proposal, # increase your price to this higher_sell_price = max(proposal.sells[0].price, price_below_ask) proposal.sells[0].price = market.quantize_order_price( self.trading_pair, higher_sell_price) def did_fill_order(self, order_filled_event: OrderFilledEvent): order_id = order_filled_event.order_id market_info = self._sb_order_tracker.get_shadow_market_pair_from_order_id( order_id) if market_info is not None: if self._logging_options & self.OPTION_LOG_MAKER_ORDER_FILLED: self.log_with_clock( logging.INFO, f"({market_info.trading_pair}) Maker " f"{'buy' if order_filled_event.trade_type is TradeType.BUY else 'sell'} order of " f"{order_filled_event.amount} {market_info.base_asset} filled." ) def did_complete_buy_order(self, order_completed_event: BuyOrderCompletedEvent): order_id = order_completed_event.order_id limit_order_record = self._sb_order_tracker.get_limit_order( self._market_info, order_id) if limit_order_record is None: return # delay order creation by filled_order_delay (in seconds) self._create_timestamp = self.current_timestamp + self._filled_order_delay self._cancel_timestamp = min(self._cancel_timestamp, self._create_timestamp) self._last_own_trade_price = limit_order_record.price self.log_with_clock( logging.INFO, f"({self.trading_pair}) Maker buy order {order_id} " f"({limit_order_record.quantity} {limit_order_record.base_currency} @ " f"{limit_order_record.price} {limit_order_record.quote_currency}) has been completely filled." ) self.notify_hb_app_with_timestamp( f"Maker BUY order {limit_order_record.quantity} {limit_order_record.base_currency} @ " f"{limit_order_record.price} {limit_order_record.quote_currency} is filled." ) def did_complete_sell_order( self, order_completed_event: SellOrderCompletedEvent): order_id = order_completed_event.order_id limit_order_record: LimitOrder = self._sb_order_tracker.get_limit_order( self._market_info, order_id) if limit_order_record is None: return # delay order creation by filled_order_delay (in seconds) self._create_timestamp = self.current_timestamp + self._filled_order_delay self._cancel_timestamp = min(self._cancel_timestamp, self._create_timestamp) self._last_own_trade_price = limit_order_record.price self.log_with_clock( logging.INFO, f"({self.trading_pair}) Maker sell order {order_id} " f"({limit_order_record.quantity} {limit_order_record.base_currency} @ " f"{limit_order_record.price} {limit_order_record.quote_currency}) has been completely filled." ) self.notify_hb_app_with_timestamp( f"Maker SELL order {limit_order_record.quantity} {limit_order_record.base_currency} @ " f"{limit_order_record.price} {limit_order_record.quote_currency} is filled." ) def is_within_tolerance(self, current_prices: List[Decimal], proposal_prices: List[Decimal]) -> bool: if len(current_prices) != len(proposal_prices): return False current_prices = sorted(current_prices) proposal_prices = sorted(proposal_prices) for current, proposal in zip(current_prices, proposal_prices): # if spread diff is more than the tolerance or order quantities are different, return false. if abs(proposal - current) / current > self._order_refresh_tolerance_pct: return False return True # Return value: whether order cancellation is deferred. def cancel_active_orders(self, proposal: Proposal): if self._cancel_timestamp > self.current_timestamp: return to_defer_canceling = False if len(self.active_orders) == 0: return if proposal is not None and self._order_refresh_tolerance_pct >= 0: active_buy_prices = [ Decimal(str(o.price)) for o in self.active_orders if o.is_buy ] active_sell_prices = [ Decimal(str(o.price)) for o in self.active_orders if not o.is_buy ] proposal_buys = [buy.price for buy in proposal.buys] proposal_sells = [sell.price for sell in proposal.sells] if self.is_within_tolerance(active_buy_prices, proposal_buys) and \ self.is_within_tolerance(active_sell_prices, proposal_sells): to_defer_canceling = True if not to_defer_canceling: for order in self.active_orders: self.cancel_order(self._market_info, order.client_order_id) else: self.logger().info( f"Not cancelling active orders since difference between new order prices " f"and current order prices is within " f"{self._order_refresh_tolerance_pct:.2%} order_refresh_tolerance_pct" ) self.set_timers() def cancel_orders_below_min_spread(self): price = self.get_price() for order in self.active_orders: negation = -1 if order.is_buy else 1 if (negation * (order.price - price) / price) < self._minimum_spread: self.logger().info( f"Order is below minimum spread ({self._minimum_spread})." f" Cancelling Order: ({'Buy' if order.is_buy else 'Sell'}) " f"ID - {order.client_order_id}") self.cancel_order(self._market_info, order.client_order_id) def to_create_orders(self, proposal: Proposal) -> bool: return (self._create_timestamp < self.current_timestamp and proposal is not None and len(self.active_orders) == 0) def execute_orders_proposal(self, proposal: Proposal, position_action: PositionAction): orders_created = False if len(proposal.buys) > 0: if position_action == PositionAction.CLOSE: if self.current_timestamp < self._next_buy_exit_order_timestamp: return else: self._next_buy_exit_order_timestamp = self.current_timestamp + self.filled_order_delay if self._logging_options & self.OPTION_LOG_CREATE_ORDER: price_quote_str = [ f"{buy.size.normalize()} {self.base_asset}, " f"{buy.price.normalize()} {self.quote_asset}" for buy in proposal.buys ] self.logger().info( f"({self.trading_pair}) Creating {len(proposal.buys)} {self._close_order_type.name} bid orders " f"at (Size, Price): {price_quote_str} to {position_action.name} position." ) for buy in proposal.buys: bid_order_id = self.buy_with_specific_market( self._market_info, buy.size, order_type=self._close_order_type, price=buy.price, position_action=position_action) if position_action == PositionAction.CLOSE: self._exit_orders[bid_order_id] = self.current_timestamp orders_created = True if len(proposal.sells) > 0: if position_action == PositionAction.CLOSE: if self.current_timestamp < self._next_sell_exit_order_timestamp: return else: self._next_sell_exit_order_timestamp = self.current_timestamp + self.filled_order_delay if self._logging_options & self.OPTION_LOG_CREATE_ORDER: price_quote_str = [ f"{sell.size.normalize()} {self.base_asset}, " f"{sell.price.normalize()} {self.quote_asset}" for sell in proposal.sells ] self.logger().info( f"({self.trading_pair}) Creating {len(proposal.sells)} {self._close_order_type.name} ask " f"orders at (Size, Price): {price_quote_str} to {position_action.name} position." ) for sell in proposal.sells: ask_order_id = self.sell_with_specific_market( self._market_info, sell.size, order_type=self._close_order_type, price=sell.price, position_action=position_action) if position_action == PositionAction.CLOSE: self._exit_orders[ask_order_id] = self.current_timestamp orders_created = True if orders_created: self.set_timers() def set_timers(self): next_cycle = self.current_timestamp + self._order_refresh_time if self._create_timestamp <= self.current_timestamp: self._create_timestamp = next_cycle if self._cancel_timestamp <= self.current_timestamp: self._cancel_timestamp = min(self._create_timestamp, next_cycle) def notify_hb_app(self, msg: str): if self._hb_app_notification: super().notify_hb_app(msg) def get_price_type(self, price_type_str: str) -> PriceType: if price_type_str == "mid_price": return PriceType.MidPrice elif price_type_str == "best_bid": return PriceType.BestBid elif price_type_str == "best_ask": return PriceType.BestAsk elif price_type_str == "last_price": return PriceType.LastTrade elif price_type_str == 'last_own_trade_price': return PriceType.LastOwnTrade elif price_type_str == "custom": return PriceType.Custom else: raise ValueError( f"Unrecognized price type string {price_type_str}.")