def test(self, d): """ Test, for purposes of type inference, if a value could possibly be valid for this column type. This will work with values that are native types and values that have been stringified. """ if d is None: return True if isinstance(d, Decimal): return True if type(d) is int or type(d) is float: return True if not isinstance(d, six.string_types): return False d = d.strip() d = d.strip('%') for symbol in CURRENCY_SYMBOLS: d = d.strip(symbol) if d.lower() in self.null_values: return True try: parse_decimal(d, self._locale) return True except: return False
def test_can_parse_decimals(self): self.assertEqual(decimal.Decimal('1099.98'), numbers.parse_decimal('1,099.98', locale='en_US')) self.assertEqual(decimal.Decimal('1099.98'), numbers.parse_decimal('1.099,98', locale='de')) self.assertRaises(numbers.NumberFormatError, lambda: numbers.parse_decimal('2,109,998', locale='de'))
def test_parse_decimal(): assert (numbers.parse_decimal('1,099.98', locale='en_US') == decimal.Decimal('1099.98')) assert numbers.parse_decimal('1.099,98', locale='de') == decimal.Decimal('1099.98') with pytest.raises(numbers.NumberFormatError) as excinfo: numbers.parse_decimal('2,109,998', locale='de') assert excinfo.value.args[0] == "'2,109,998' is not a valid decimal number"
def parse_value(value): """ Accepts a string value and attempts to parce it as a currency value. Returns the extracted numeric value converted to a string """ l_currency_language_code, l_currency_code = _getCodes() curSym = get_currency_symbol(l_currency_code, l_currency_language_code) grpSym = get_group_symbol(locale=l_currency_language_code.lower()) decSym = get_decimal_symbol(locale=l_currency_language_code.lower()) # Convert the Official characters into what comes from the keyboard. # This section may need to grow over time. # - Character 160 is a non-breaking space, which is different from a typed space if ord(grpSym) == 160: value = value.replace(u" ", grpSym) allSym = _getSymbols(value) invalidSym = allSym.replace(curSym, "").replace(grpSym, "").replace(decSym, "").replace(u"-", "") value = value.replace(curSym, "") if allSym.count(decSym) > 1: raise NumberFormatError(default_error_messages["decimal_symbol"] % decSym) elif (allSym.count(decSym) == 1 and allSym[-1] != decSym) or len(invalidSym) > 0: raise NumberFormatError(default_error_messages["invalid_format"] % (grpSym, decSym)) elif value.count(decSym) == 1: value = parse_decimal(value, locale=l_currency_language_code.lower()) else: value = parse_number(value, locale=l_currency_language_code.lower()) # The value is converted into a string because the parse functions return # floats return str(value)
def parse_decimal(string, locale=None): """Parses localized decimal string into a float. .. code-block:: python >>> parse_decimal('1,099.98', locale='en_US') 1099.98 >>> parse_decimal('1.099,98', locale='de') 1099.98 When the given string cannot be parsed, an exception is raised: .. code-block:: python >>> parse_decimal('2,109,998', locale='de') Traceback (most recent call last): ... NumberFormatError: '2,109,998' is not a valid decimal number :param string: The string to parse. :param locale: A locale code. If not set, uses the currently loaded locale. :returns: The parsed decimal number. :raises: ``NumberFormatError`` if the string can not be converted to a decimal number. """ locale = locale or get_locale() return numbers.parse_decimal(string, locale=locale)
def asValue(self, strValue, objType): ''' @see: Converter.asValue ''' assert isinstance(objType, Type), 'Invalid object type %s' % objType if strValue is None: return None if isinstance(objType, TypeModel): # If type model is provided we consider the model property type assert isinstance(objType, TypeModel) container = objType.container assert isinstance(container, Model) objType = container.properties[container.propertyId] if objType.isOf(str): return strValue if objType.isOf(Boolean): return strValue.strip().lower() == _('true').lower() if objType.isOf(Percentage): return float(strValue) / 100 if objType.isOf(int): return int(strValue) if objType.isOf(Number): return bn.parse_decimal(strValue, self.locale) if objType.isOf(Date): return datetime.strptime(strValue, '%Y-%m-%d').date() if objType.isOf(Time): return datetime.strptime(strValue, '%H:%M:%S').time() if objType.isOf(DateTime): return datetime.strptime(strValue, '%Y-%m-%d %H:%M:%S') raise TypeError('Invalid object type %s for Babel converter' % objType)
def hydrate_total_value(self, bundle): value = bundle.data.get('total_value', None) if value: bundle.data['total_value'] = parse_decimal(str(value), locale=bundle.request.locale) return bundle
def cast(self, d): """ Cast a single value to a :class:`decimal.Decimal`. :returns: :class:`decimal.Decimal` or :code:`None`. """ if isinstance(d, Decimal) or d is None: return d elif type(d) is int: return Decimal(d) elif type(d) is float: return Decimal(self.float_format % d) elif isinstance(d, six.string_types): d = d.strip() d = d.strip("%") for symbol in CURRENCY_SYMBOLS: d = d.strip(symbol) if d.lower() in self.null_values: return None else: raise CastError('Can not parse value "%s" as Decimal.' % d) try: return parse_decimal(d, self.locale) except: pass raise CastError('Can not parse value "%s" as Decimal.' % d)
def cast(self, d): """ Cast a single value to a :class:`decimal.Decimal`. :returns: :class:`decimal.Decimal` or :code:`None`. :raises: :exc:`.CastError` """ if isinstance(d, Decimal) or d is None: return d elif isinstance(d, int): return Decimal(d) elif isinstance(d, float): raise CastError('Can not convert float to Decimal. Convert data to string first!') elif isinstance(d, six.string_types): d = d.strip() d = d.strip('%') for symbol in CURRENCY_SYMBOLS: d = d.strip(symbol) if d.lower() in self.null_values: return None try: return parse_decimal(d, self._locale) except: raise CastError('Can not parse value "%s" as Decimal.' % d)
def execute(self, table_id, select_index, aggregation_index, conditions, lower=True): if not table_id.startswith('table'): table_id = 'table_{}'.format(table_id.replace('-', '_')) table_info = self.db.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql schema_str = schema_re.findall(table_info)[0] schema = {} for tup in schema_str.split(', '): c, t = tup.split() schema[c] = t select = 'col{}'.format(select_index) agg = Query.agg_ops[aggregation_index] if agg: select = '{}({})'.format(agg, select) where_clause = [] where_map = {} for col_index, op, val in conditions: if lower and isinstance(val, str): val = val.lower() if schema['col{}'.format(col_index)] == 'real' and not isinstance(val, (int, float)): try: val = float(parse_decimal(val)) except NumberFormatError as e: val = float(num_re.findall(val)[0]) where_clause.append('col{} {} :col{}'.format(col_index, Query.cond_ops[op], col_index)) where_map['col{}'.format(col_index)] = val where_str = '' if where_clause: where_str = 'WHERE ' + ' AND '.join(where_clause) query = 'SELECT {} AS result FROM {} {}'.format(select, table_id, where_str) out = self.db.query(query, **where_map) return [o.result for o in out]
def parselocal_float(txt, locale): """add??? :param txt: add??? :param locale: add??? :returns: add??? """ return numbers.parse_decimal(txt, locale)
def normalize_field(self, value): if isinstance(value, (int, float)): return value if value == '': value = None elif value is not None: value = float(parse_decimal(value, locale=settings.get_locale())) return super(FloatField, self).normalize_field(value)
def clean(self, value): locale=get_current_language() if value != "": try: value = force_unicode(parse_decimal(value, locale=locale)) except NumberFormatError: raise forms.ValidationError(self.error_messages['invalid']) return super(FloatField, self).clean(value)
def normalize_field(self, value): if value == '': value = None if isinstance(value, int): return value elif value is not None: locale = settings.get_locale() value = int(parse_decimal(value, locale=locale)) return super(IntegerField, self).normalize_field(value)
def test(self, d): """ Test, for purposes of type inference, if a string value could possibly be valid for this column type. """ d = d.strip() d = d.strip('%') for symbol in CURRENCY_SYMBOLS: d = d.strip(symbol) if d.lower() in self.null_values: return True try: parse_decimal(d, self._locale) return True except: return False
def normalize_field(self, value): if isinstance(value, Decimal): return value if value == '': value = None elif value is not None: value = parse_decimal(value, locale=settings.get_locale()) rounded = int(round(Decimal(value) * self.__multiplier)) value = Decimal(rounded) / self.__multiplier return super(DecimalField, self).normalize_field(value)
def _to_python(self, value, state): if not getattr(self, 'required', False) and not value: return getattr(self, 'if_missing', None) try: value = parse_decimal(value, locale = state._LOCALE_) if self.max and value > self.max: raise formencode.Invalid(self.message("amount_too_high", state, max_amount = format_decimal(self.max, locale=state._LOCALE_)), value, state) if self.min and value < self.min: raise formencode.Invalid(self.message("amount_too_low", state, min_amount = format_decimal(self.min, locale=state._LOCALE_)), value, state) except NumberFormatError, e: raise formencode.Invalid(self.message("invalid_amount", state, value = value), value, state)
def localize_money(money, currency=None, language=None): if language is None: language = preferred_language() if not isinstance(money, Decimal): money = parse_decimal(money) if currency: return format_currency(money, currency, locale=language) else: return format_decimal(money, locale=language, format="#,##0.00")
def parse_decimal(value): if isinstance(value, basestring): value = ustr(value) #deal with ' ' instead of u'\xa0' (SP instead of NBSP as grouping char) value = value.replace(' ', '') try: value = numbers.parse_decimal(value, locale=get_locale()) except ValueError, e: pass
def color_scale(name, text): # filter out everything that does not seem to be necessary to interpret the string as a number # this permits to transform "[ 95%]" to "95" before number conversion, # and thus allows to color a group larger than the matched number chars_in_numbers = "-+.,e/*" allowed = string.digits + chars_in_numbers nb = "".join([i for i in filter(allowed.__contains__, text)]) # interpret as decimal # First, try with the babel module, if available # if not, use python itself, # if thoses fails, try to `eval` the string # (this allow strings like "1/2+0.9*2") f = None try: # babel is a specialized module import babel.numbers as bn try: f = float(bn.parse_decimal(nb)) except bn.NumberFormatError: pass except ImportError: try: f = float(nb) except ValueError: pass if f is not None: # normalize with scale if it's a number f = (f - context["scale"][0]) / (context["scale"][1] - context["scale"][0]) else: # interpret as float between 0 and 1 otherwise f = eval(nb) # if out of scale, do not color if f < 0 or f > 1: return None # normalize and scale over the nb of colors in cmap colormap = context["colormaps"][name] i = int(math.ceil(f * (len(colormap) - 1))) color = colormap[i] # infer mode from the color in the colormap m = mode(color) if m == 8: color_code = str(30 + context["colors"][color]) else: color_code = str(color) return color_code
def render(self, name, value, attrs=None): from babel.numbers import parse_decimal from babel.numbers import format_decimal from babel.numbers import NumberFormatError locale=get_current_language() if value is None: value = "" if value and isinstance(value, basestring): try: value = parse_decimal(value, locale=locale) except NumberFormatError: pass if value is not "" and not isinstance(value, basestring): value = format_decimal(value, self.format, locale=locale) return super(DecimalWidget, self).render(name, value, attrs)
def add_helpers_to_context(tell_sentry, context, loc, request=None): context['locale'] = loc context['decimal_symbol'] = get_decimal_symbol(locale=loc) context['_'] = lambda s, *a, **kw: get_text(loc, s, *a, **kw) context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, request, loc, *a, **kw) context['format_number'] = lambda *a: format_number(*a, locale=loc) context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc) context['format_currency'] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw) context['format_percent'] = lambda *a: format_percent(*a, locale=loc) context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc) def _to_age(delta): try: return to_age(delta, loc) except: return to_age(delta, 'en') context['to_age'] = _to_age
def parse_decimal(self, string): """Parse localized decimal string into a float >>> Locale('en', 'US').parse_decimal('1,099.98') 1099.98 >>> Locale('de').parse_decimal('1.099,98') 1099.98 When the given string cannot be parsed, an exception is raised: >>> Locale('de').parse_decimal('2,109,998') Traceback (most recent call last): ... NumberFormatError: '2,109,998' is not a valid decimal number """ return numbers.parse_decimal(string, self)
def inbound(request): context = request.context loc = context.locale = get_locale_for_request(request) context.decimal_symbol = get_decimal_symbol(locale=loc) context._ = lambda s, *a, **kw: get_text(request, loc, s, *a, **kw) context.ngettext = lambda *a, **kw: n_get_text(request, loc, *a, **kw) context.format_number = lambda *a: format_number(*a, locale=loc) context.format_decimal = lambda *a: format_decimal(*a, locale=loc) context.format_currency = lambda *a: format_currency(*a, locale=loc) context.format_percent = lambda *a: format_percent(*a, locale=loc) context.parse_decimal = lambda *a: parse_decimal(*a, locale=loc) def _to_age(delta): try: return to_age(delta, loc) except: return to_age(delta, 'en') context.to_age = _to_age
def add_helpers_to_context(tell_sentry, context, loc): context['escape'] = lambda s: s # to be overriden by renderers context['locale'] = loc context['decimal_symbol'] = get_decimal_symbol(locale=loc) context['_'] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw) context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw) context['format_number'] = lambda *a: format_number(*a, locale=loc) context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc) context['format_currency'] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw) context['format_percent'] = lambda *a: format_percent(*a, locale=loc) context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc) def _to_age(delta, **kw): try: return to_age(delta, loc, **kw) except: return to_age(delta, 'en', **kw) context['to_age'] = _to_age
def add_helpers_to_context(tell_sentry, context, loc): context["escape"] = lambda s: s # to be overriden by renderers context["locale"] = loc context["decimal_symbol"] = get_decimal_symbol(locale=loc) context["_"] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw) context["ngettext"] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw) context["format_number"] = lambda *a: format_number(*a, locale=loc) context["format_decimal"] = lambda *a: format_decimal(*a, locale=loc) context["format_currency"] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw) context["format_percent"] = lambda *a: format_percent(*a, locale=loc) context["parse_decimal"] = lambda *a: parse_decimal(*a, locale=loc) def _to_age(delta, **kw): try: return to_age(delta, loc, **kw) except: return to_age(delta, "en", **kw) context["to_age"] = _to_age
def add_helpers_to_context(tell_sentry, context, loc): context['escape'] = lambda s: s # to be overriden by renderers context['locale'] = loc context['decimal_symbol'] = get_decimal_symbol(locale=loc) context['_'] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw) context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw) context['Money'] = Money context['format_number'] = lambda *a: format_number(*a, locale=loc) context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc) context['format_currency'] = lambda *a, **kw: format_money(*a, locale=loc, **kw) context['format_percent'] = lambda *a: format_percent(*a, locale=loc) context['format_datetime'] = lambda *a: format_datetime(*a, locale=loc) context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc) context['to_age'] = to_age def to_age_str(o, **kw): if not isinstance(o, datetime): kw.setdefault('granularity', 'day') return format_timedelta(to_age(o), locale=loc, **kw) context['to_age_str'] = to_age_str
def add_helpers_to_context(website, request): context = request.context loc = context["locale"] = get_locale_for_request(request, website) context["decimal_symbol"] = get_decimal_symbol(locale=loc) context["_"] = lambda s, *a, **kw: get_text(request, loc, s, *a, **kw) context["ngettext"] = lambda *a, **kw: n_get_text(website, request, loc, *a, **kw) context["format_number"] = lambda *a: format_number(*a, locale=loc) context["format_decimal"] = lambda *a: format_decimal(*a, locale=loc) context["format_currency"] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw) context["format_percent"] = lambda *a: format_percent(*a, locale=loc) context["parse_decimal"] = lambda *a: parse_decimal(*a, locale=loc) def _to_age(delta): try: return to_age(delta, loc) except: return to_age(delta, "en") context["to_age"] = _to_age
def detokenize_query(query, example_dict, table): detokenized_conds = [] for i, (col, op, val) in enumerate(query.conditions): val_tokens = val.split(' ') detokenized_cond_val = my_detokenize(val_tokens, example_dict['question']) if table.header[col].type == 'real' and not isinstance(detokenized_cond_val, (int, float)): if ',' not in detokenized_cond_val: try: detokenized_cond_val = float(parse_decimal(val)) except NumberFormatError as e: try: detokenized_cond_val = float(num_re.findall(val)[0]) except: pass detokenized_conds.append((col, op, detokenized_cond_val)) detokenized_query = Query(sel_index=query.sel_index, agg_index=query.agg_index, conditions=detokenized_conds) return detokenized_query
def parse_decimal(string): """Parse localized decimal string into a float. >>> parse_decimal('1,099.98', locale='en_US') 1099.98 >>> parse_decimal('1.099,98', locale='de') 1099.98 When the given string cannot be parsed, an exception is raised: >>> parse_decimal('2,109,998', locale='de') Traceback (most recent call last): ... NumberFormatError: '2,109,998' is not a valid decimal number :param string: The string to parse. :return: The parsed decimal number. :raise `NumberFormatError`: If the string can not be converted to a decimal number """ return numbers.parse_decimal(string, locale=local.locale)
def _to_python(self, value, state): if not getattr(self, 'required', False) and not value: return getattr(self, 'if_missing', None) try: value = parse_decimal(value, locale=state._LOCALE_) if self.max and value > self.max: raise formencode.Invalid( self.message("amount_too_high", state, max_amount=format_decimal( self.max, locale=state._LOCALE_)), value, state) if self.min and value < self.min: raise formencode.Invalid( self.message("amount_too_low", state, min_amount=format_decimal( self.min, locale=state._LOCALE_)), value, state) except NumberFormatError, e: raise formencode.Invalid( self.message("invalid_amount", state, value=value), value, state)
def add_helpers_to_context(tell_sentry, context, loc): context['escape'] = lambda s: s # to be overriden by renderers context['locale'] = loc context['decimal_symbol'] = get_decimal_symbol(locale=loc) context['_'] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw) context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw) context['Money'] = Money context['format_number'] = lambda *a: format_number(*a, locale=loc) context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc) context['format_currency'] = lambda *a, **kw: format_money( *a, locale=loc, **kw) context['format_percent'] = lambda *a: format_percent(*a, locale=loc) context['format_datetime'] = lambda *a: format_datetime(*a, locale=loc) context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc) context['to_age'] = to_age def to_age_str(o, **kw): if not isinstance(o, datetime): kw.setdefault('granularity', 'day') return format_timedelta(to_age(o), locale=loc, **kw) context['to_age_str'] = to_age_str
def parse_decimal(self, string): """Parses localized decimal string into a float. Example:: >>> parse_decimal('1,099.98', locale='en_US') 1099.98 >>> parse_decimal('1.099,98', locale='de') 1099.98 When the given string cannot be parsed, an exception is raised:: >>> parse_decimal('2,109,998', locale='de') Traceback (most recent call last): ... NumberFormatError: '2,109,998' is not a valid decimal number :param string: The string to parse. :returns: The parsed decimal number. :raises: ``NumberFormatError`` if the string can not be converted to a decimal number. """ return numbers.parse_decimal(string, locale=self.locale)
def cast(self, d): """ Cast a single value to a :class:`decimal.Decimal`. :returns: :class:`decimal.Decimal` or :code:`None`. """ if isinstance(d, Decimal) or d is None: return d t = type(d) if t is int: return Decimal(d) elif six.PY2 and t is long: return Decimal(d) elif t is float: return Decimal(repr(d)) elif isinstance(d, six.string_types): d = d.strip() d = d.strip('%') for symbol in CURRENCY_SYMBOLS: d = d.strip(symbol) if d.lower() in self.null_values: return None else: raise CastError('Can not parse value "%s" as Decimal.' % d) try: return parse_decimal(d, self.locale) except: pass raise CastError('Can not parse value "%s" as Decimal.' % d)
async def on_recognize( self, turn_context: TurnContext, state: Dict[str, object], options: PromptOptions, ) -> PromptRecognizerResult: if not turn_context: raise TypeError( "NumberPrompt.on_recognize(): turn_context cannot be None.") result = PromptRecognizerResult() if turn_context.activity.type == ActivityTypes.message: utterance = turn_context.activity.text if not utterance: return result culture = self._get_culture(turn_context) results: [ModelResult] = recognize_number(utterance, culture) if results: result.succeeded = True result.value = parse_decimal(results[0].resolution["value"], locale=culture.replace("-", "_")) return result
def serialize_prices_before_puts(mapper, connection, target): target.volume = int(parse_decimal(target.volume, locale=LOCALE))
def load_csv_record(csv_record: dict, solr_record: dict, search: Search, fields: dict, codes: dict, format: str): if format == 'NTR': solr_record['has_amendments'] = "0" if len(csv_record['fiscal_year']) > 4: try: year = int(csv_record['fiscal_year'][0:4]) if csv_record['quarter'] == 'Q1': start_date = date(year, 4, 1) elif csv_record['quarter'] == 'Q2': start_date = date(year, 7, 1) elif csv_record['quarter'] == 'Q3': start_date = date(year, 10, 1) elif csv_record['quarter'] == 'Q4': start_date = date(year + 1, 1, 1) solr_record['agreement_start_date'] = start_date.strftime( '%Y-%m-%d') solr_record['year'] = start_date.year solr_record['month'] = start_date.month except ValueError: pass else: solr_record['format_en'] = 'Grants and Contributions' solr_record['format_fr'] = 'Subventions et contributions' try: agreement_value = parse_decimal( csv_record['agreement_value'].replace('$', '').replace(',', ''), locale='en') # Set a value range if agreement_value < 0: solr_record['agreement_value_range_en'] = '(a) Negative' solr_record['agreement_value_range_fr'] = '(a) negatif' elif agreement_value < 10000: solr_record[ 'agreement_value_range_en'] = '(b) Less than $10,000' solr_record[ 'agreement_value_range_fr'] = '(b) moins de 10 000 $' elif 10000 <= agreement_value < 25000: solr_record[ 'agreement_value_range_en'] = '(c) $10,000 - $25,000' solr_record[ 'agreement_value_range_fr'] = '(c) de 10 000 $ à 25 000 $' elif 25000 <= agreement_value < 100000: solr_record[ 'agreement_value_range_en'] = '(d) $25,000 - $100,000' solr_record[ 'agreement_value_range_fr'] = '(d) de 25 000 $ à 100 000 $' elif 100000 <= agreement_value < 1000000: solr_record[ 'agreement_value_range_en'] = '(e) $100,000 - $1,000,000' solr_record[ 'agreement_value_range_fr'] = '(e) de 100 000 $ à 1 000 000 $' elif 1000000 <= agreement_value < 5000000: solr_record[ 'agreement_value_range_en'] = '(f) $1,000,000 - $5,000,000' solr_record[ 'agreement_value_range_fr'] = '(f) de 1 000 000 $ à 5 000 000 $' else: solr_record[ 'agreement_value_range_en'] = '(g) More than $5,000,000' solr_record[ 'agreement_value_range_fr'] = '(g) plus de cinq millions $' except NumberFormatError: pass # Flag the record for amendments if csv_record['amendment_number'] != "current": solr_record['has_amendments'] = "1" elif csv_record['amendment_number'] == "current" and csv_record[ 'amendment_date'] != "": solr_record['has_amendments'] = "1" else: solr_record['has_amendments'] = "0" # The coverage field is bilingual if 'coverage' in csv_record and csv_record['coverage']: solr_record = _set_bilingual_field('coverage', csv_record['coverage'], solr_record) if 'recipient_legal_name' in csv_record and csv_record[ 'recipient_legal_name']: solr_record = _set_bilingual_field( 'recipient_legal_name', csv_record['recipient_legal_name'], solr_record) if 'recipient_operating_name' in csv_record and csv_record[ 'recipient_operating_name']: solr_record = _set_bilingual_field( 'recipient_operating_name', csv_record['recipient_operating_name'], solr_record) if 'research_organization_name' in csv_record and csv_record[ 'research_organization_name']: solr_record = _set_bilingual_field( 'research_organization_name', csv_record['research_organization_name'], solr_record) if 'recipient_city' in csv_record and csv_record['recipient_city']: solr_record = _set_bilingual_field('recipient_city', csv_record['recipient_city'], solr_record) # Make the agreement number searchable solr_record['agreement_number_text'] = solr_record['agreement_number'] return solr_record
def colorin(text, color="red", style="normal"): """ Return the given text, surrounded by the given color ASCII markers. If the given color is a name that exists in available colors, a 8-colors mode is assumed, else, a 256-colors mode. The given style must exists in the available styles. >>> colorin("Fetchez la vache", "red", "bold") '\x1b[1;31mFetchez la vache\x1b[0m' >>> colout.colorin("Faites chier la vache", 41, "normal") '\x1b[0;38;5;41mFaites chier la vache\x1b[0m' """ assert (type(color) is str) global colormap_idx global debug # Special characters. start = "\033[" stop = "\033[0m" color_code = "" style_code = "" # Convert the style code if style == "random" or style == "Random": style = random.choice(list(styles.keys())) else: if style in styles: style_code = str(styles[style]) color = color.strip() if color == "none": # if no color, style cannot be applied if not debug: return text else: return "<none>" + text + "</none>" elif color == "random": mode = 8 color_code = random.choice(list(colors.values())) color_code = str(30 + color_code) elif color == "Random": mode = 256 color_nb = random.randint(0, 255) color_code = str(color_nb) elif color in colormaps.keys(): if color[0].islower(): # lower case first letter mode = 8 c = colormaps[color][colormap_idx] if c.isdigit(): color_code = str(30 + c) else: color_code = str(30 + colors[c]) else: # upper case mode = 256 color_nb = colormaps[color][colormap_idx] color_code = str(color_nb) if colormap_idx < len(colormaps[color]) - 1: colormap_idx += 1 else: colormap_idx = 0 elif color.lower() == "scale": # "scale" or "Scale" # filter out everything that does not seem to be necessary to interpret the string as a number # this permits to transform "[ 95%]" to "95" before number conversion, # and thus allows to color a group larger than the matched number chars_in_numbers = "-+.,e/*" allowed = string.digits + chars_in_numbers nb = "".join([i for i in filter(allowed.__contains__, text)]) # interpret as decimal # First, try with the babel module, if available # if not, use python itself, # if thoses fails, try to `eval` the string # (this allow strings like "1/2+0.9*2") try: # babel is a specialized module import babel.numbers as bn try: f = float(bn.parse_decimal(nb)) except NumberFormatError: f = eval( nb ) # Note: in python2, `eval(2/3)` would produce `0`, in python3 `0.666` except ImportError: try: f = float(nb) except ValueError: f = eval(nb) # if out of scale, do not color if f < scale[0] or f > scale[1]: return text if color[0].islower(): mode = 8 # Use the default colormap in lower case = 8-colors mode cmap = colormap # normalize and scale over the nb of colors in cmap i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1))) color = cmap[i] color_code = str(30 + colors[color]) else: mode = 256 cmap = colormap i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1))) color = cmap[i] color_code = str(color) # "hash" or "Hash"; useful to randomly but consistently color strings elif color.lower() == "hash": hasher = hashlib.md5() hasher.update(text.encode('utf-8')) hash = hasher.hexdigest() f = float(functools.reduce(lambda x, y: x + ord(y), hash, 0) % 101) if color[0].islower(): mode = 8 cmap = colormap # normalize and scale over the nb of colors in cmap i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1))) color = cmap[i] color_code = str(30 + colors[color]) else: mode = 256 cmap = colormap i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1))) color = cmap[i] color_code = str(color) # Really useful only when using colout as a library # thus you can change the "colormap" variable to your favorite one before calling colorin elif color == "colormap": color = colormap[colormap_idx] if color in colors: mode = 8 color_code = str(30 + colors[color]) else: mode = 256 color_nb = int(color) assert (0 <= color_nb <= 255) color_code = str(color_nb) if colormap_idx < len(colormap) - 1: colormap_idx += 1 else: colormap_idx = 0 # 8 colors modes elif color in colors: mode = 8 color_code = str(30 + colors[color]) # hexadecimal color elif color[0] == "#": mode = 256 color_nb = rgb_to_ansi(*hex_to_rgb(color)) assert (0 <= color_nb <= 255) color_code = str(color_nb) # 256 colors mode elif color.isdigit(): mode = 256 color_nb = int(color) assert (0 <= color_nb <= 255) color_code = str(color_nb) # programming language elif color.lower() in lexers: lexer = get_lexer_by_name(color.lower()) # Python => 256 colors, python => 8 colors ask_256 = color[0].isupper() if ask_256: try: formatter = Terminal256Formatter(style=style) except: # style not found formatter = Terminal256Formatter() else: if style not in ("light", "dark"): style = "dark" # dark color scheme by default formatter = TerminalFormatter(bg=style) # We should return all but the last character, # because Pygments adds a newline char. if not debug: return highlight(text, lexer, formatter)[:-1] else: return "<" + color + ">" + highlight( text, lexer, formatter)[:-1] + "</" + color + ">" # unrecognized else: raise UnknownColor(color) if not debug: return start + style_code + endmarks[ mode] + color_code + "m" + text + stop else: return start + style_code + endmarks[mode] + color_code + "m<" + str( color) + ">" + text + "</" + str(color) + ">" + stop
def execute(self, table_id, select_index, aggregation_index, conditions, condition_relation, lower=False): if not table_id.startswith('Table'): table_id = 'Table_{}'.format(table_id.replace('-', '_')) wr = "" if condition_relation == 1 or condition_relation == 0: wr = " AND " elif condition_relation == 2: wr = " OR " table_info = self.db.query( 'SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql schema_str = schema_re.findall(table_info)[0] schema = {} for tup in schema_str.split(','): c, t = tup.split(' ') schema[c] = t tmp = "" for sel, agg in zip(select_index, aggregation_index): select_str = 'col_{}'.format(sel + 1) agg_str = agg_dict[agg] if agg: tmp += '{}({}),'.format(agg_str, select_str) else: tmp += '({}),'.format(select_str) tmp = tmp[:-1] where_clause = [] where_map = {} for col_index, op, val in conditions: if lower and isinstance(val, str): val = val.lower() if schema['col_{}'.format(col_index + 1)] == 'real' and not isinstance( val, (int, float)): try: val = float(parse_decimal(val, locale='en_US')) except NumberFormatError as e: try: val = float(num_re.findall( val)[0]) # need to understand and debug this part. except: # Although column is of number, selected one is not number. Do nothing in this case. pass where_clause.append('col_{} {} :col_{}'.format( col_index + 1, cond_op_dict[op], col_index + 1)) where_map['col_{}'.format(col_index + 1)] = val where_str = '' if where_clause: where_str = 'WHERE ' + wr.join(where_clause) query = 'SELECT {} FROM {} {}'.format(tmp, table_id, where_str) #print(query) #print(where_map) out = self.db.query(query, **where_map) return out.as_dict()
def parse_decimal(self, string): """Parse localized decimal string into a float """ return numbers.parse_decimal(string, self)
def asGehalt(vStr): v = parse_decimal(vStr, locale="de") if v < 0: raise ValueError return dec(v)
def parse_decimal_or_400(s, *a): try: return parse_decimal(s, *a, locale=loc) except (InvalidOperation, NumberFormatError, ValueError): raise InvalidNumber(s)
def asPerc(vStr): v = parse_decimal(vStr, locale="de") if v > 100 or 0 > v: raise ValueError return v
def parse_dollars(cls, s: str) -> Decimal: assert s.startswith("$") return parse_decimal(s.strip("$"), locale='en_US')
def execute_return_query_new(self, data, table_header, lower=True): table_id = data['table_id'] select_index = data['sql']['sel'] aggregation_index = data['sql']['agg'] conditions = data['sql']['conds'] #table name conversion if not table_id.startswith('table'): table_id = 'table_{}'.format(table_id.replace('-', '_')) # this gets the creat table comment table_info = self.conn.execute( "SELECT sql from sqlite_master WHERE tbl_name = '{}'".format( table_id)).fetchall()[0] schema_str = schema_re.findall(table_info[0])[0] schema = {} for tup in schema_str.split(', '): c, t = tup.split() schema[c] = t #get execute select comment select = 'col{}'.format(select_index) #human readable query select_real_name = '[{}]'.format(table_header[select_index]) agg = agg_ops[aggregation_index] if agg: select = '{}({})'.format(agg, select) select_real_name = '{}({})'.format(agg, select_real_name) where_clause = [] where_clause_real_name = [] for col_index, op, val in conditions: if lower and (isinstance(val, str) or isinstance(val, str)): val = val.lower() if schema['col{}'.format(col_index)] == 'real' and not isinstance( val, (int, float)): try: # print('!!!!!!value of val is: ', val, 'type is: ', type(val)) # val = float(parse_decimal(val)) # somehow it generates error. val = float(parse_decimal(val, locale='en_US')) # print('!!!!!!After: val', val) except NumberFormatError as e: try: val = float(num_re.findall( val)[0]) # need to understand and debug this part. except: # Although column is of number, selected one is not number. Do nothing in this case. pass where_clause.append("col{} {} '{}'".format( col_index, cond_ops[op], str(val).replace("'", "''").replace("\\", "\\\\"))) where_clause_real_name.append("[{}] {} '{}'".format( table_header[col_index], cond_ops[op], val)) where_str = '' if where_clause: where_str = 'WHERE ' + ' AND '.join(where_clause) where_str_real_name = 'WHERE ' + ' AND '.join( where_clause_real_name) query = 'SELECT {} AS result FROM {} {}'.format( select, table_id, where_str) query_real_name = 'SELECT {} AS result FROM {} {}'.format( select_real_name, table_id, where_str_real_name) #print query #print(query) out = self.conn.execute(query) return data['question'], query_real_name, query, out.fetchall()
od_obj['contract_start_dt'] = contract_start_dt.strftime( '%Y-%m-%dT00:00:00Z') od_obj['contract_start_s'] = gc['contract_period_start'] if gc['delivery_date'] != "": delivery_dt: datetime = datetime.strptime( gc['delivery_date'], '%Y-%m-%d') od_obj['contract_delivery_dt'] = delivery_dt.strftime( '%Y-%m-%dT00:00:00Z') od_obj['contract_delivery_s'] = gc['delivery_date'] else: od_obj['contract_delivery_s'] = "-" if gc['contract_value'] != "": contract_value = parse_decimal(gc['contract_value'].replace( '$', '').replace(',', ''), locale='en') od_obj['contract_value_f'] = contract_value od_obj['contract_value_en_s'] = format_currency(contract_value, 'CAD', locale='en_CA') od_obj['contract_value_fr_s'] = format_currency(contract_value, 'CAD', locale='fr_CA') else: od_obj['contract_value_en_s'] = "-" od_obj['contract_value_fr_s'] = "-" if gc['original_value'] != "": original_value = parse_decimal(gc['original_value'].replace( '$', '').replace(',', ''),
print("Archivo: " + file) dataset['Win Time'] = dataset['Win Time'].apply(lambda x: x[0:10]) dataset['Win Time'] = dataset['Win Time'].apply( lambda x: dt.datetime.strptime(x, dateformat)) dataset['EGM'] = dataset['EGM'].apply(lambda x: re.sub( r"[_]|[ ]|server|SERVER", '', x)).apply(lambda x: str.upper(x)) dataset.loc[:, ['Bonus Win Amount' ]] = dataset.loc[:, ['Bonus Win Amount']].replace( to_replace="[%]", value="", regex=True).replace(to_replace='[nan]', value='0', regex=True) try: dataset.loc[:, ['Bonus Win Amount' ]] = dataset.loc[:, ['Bonus Win Amount']].applymap( lambda x: numbers.parse_decimal( x, 'en_US')).astype(float).round(2) except numbers.NumberFormatError as err: print('Error en parseo de número, intentando otro formato', err) dataset.loc[:, ['Bonus Win Amount' ]] = dataset.loc[:, ['Bonus Win Amount']].applymap( lambda x: numbers.parse_decimal(x, 'de')).astype( float).round(2) dataset.rename(index=str, columns={'Win Time': 'Date'}, inplace=True) dataframeOuptut = dataframeOuptut.append(dataset, sort=False) print("Archivo adjuntado.") dataset = pd.read_csv(join(path, file), sep=";", encoding='utf-8') os.remove(join(path, file)) dataset.to_csv(join(pathP, file), index=False, header=False,
def handle(self, *args, **options): total = 0 cycle = 0 try: # Retrieve the Search and Field models from the database solr = SolrClient(settings.SOLR_SERVER_URL) try: self.search_target = Search.objects.get( search_id=options['search']) self.solr_core = self.search_target.solr_core_name self.all_fields = Field.objects.filter( search_id=self.search_target) if options['nothing_to_report']: self.search_fields = Field.objects.filter( search_id=self.search_target, alt_format='ALL') | Field.objects.filter( search_id=self.search_target, alt_format='NTR') else: self.search_fields = Field.objects.filter( search_id=self.search_target, alt_format='ALL') | Field.objects.filter( search_id=self.search_target, alt_format='') for search_field in self.search_fields: self.csv_fields[search_field.field_id] = search_field codes = Code.objects.filter(field_id=search_field) # Most csv_fields will not have codes, so the queryset will be zero length if len(codes) > 0: code_dict = {} for code in codes: code_dict[code.code_id.lower()] = code self.field_codes[search_field.field_id] = code_dict except Search.DoesNotExist as x: self.logger.error('Search not found: "{0}"'.format(x)) exit(-1) except Field.DoesNotExist as x1: self.logger.error( 'Fields not found for search: "{0}"'.format(x1)) # Process the records in the CSV file one at a time with open(options['csv'], 'r', encoding='utf-8-sig', errors="ignore") as csv_file: csv_reader = csv.DictReader(csv_file, dialect='excel') solr_items = [] for csv_record in csv_reader: # Clear out the Solr core. on the first line if total == 0 and not options['nothing_to_report']: solr.delete_doc_by_query(self.solr_core, "*:*") print("Purging all records") elif total == 0 and options['nothing_to_report']: solr.delete_doc_by_query(self.solr_core, "format:NTR") solr.commit(self.solr_core, softCommit=True) print("Purging NTR records") total += 1 cycle += 1 # Call plugins if they exist for this search type. This is where a developer can introduce # code to customize the data that is loaded into Solr for a particular search. search_type_plugin = 'search.plugins.{0}'.format( options['search']) if search_type_plugin in self.discovered_plugins: include, filtered_record = self.discovered_plugins[ search_type_plugin].filter_csv_record( csv_record, self.search_target, self.csv_fields, self.field_codes, 'NTR' if options['nothing_to_report'] else '') if not include: continue else: csv_record = filtered_record # Create a dictionary for each record loaded into Solr solr_record = { 'format': 'NTR' if options['nothing_to_report'] else 'DEFAULT' } for csv_field in csv_reader.fieldnames: # Verify that it is a known field if csv_field not in self.csv_fields and csv_field not in ( 'owner_org_title', 'owner_org'): self.logger.error( "CSV files contains unknown field: {0}".format( csv_field)) exit(-1) if csv_field == 'owner_org_title': continue # Handle multi-valued fields here if self.csv_fields[csv_field].solr_field_multivalued: solr_record[csv_field] = csv_record[ csv_field].split(',') # Copy fields fo report cannot use multi-values - so directly populate with original string if self.csv_fields[csv_field].solr_field_export: for extra_field in self.csv_fields[ csv_field].solr_field_export.split( ','): solr_record[extra_field] = csv_record[ csv_field] else: solr_record[csv_field] = csv_record[csv_field] # Automatically expand out dates and numbers for use with Solr export handler if self.csv_fields[ csv_field].solr_field_type == 'pdate': try: if csv_record[csv_field]: csv_date = datetime.strptime( csv_record[csv_field], '%Y-%m-%d') solr_record[csv_field + '_en'] = format_date( csv_date, locale='en') solr_record[csv_field + '_fr'] = format_date( csv_date, locale='fr') if self.csv_fields[ csv_field].is_default_year: solr_record['year'] = csv_date.year if self.csv_fields[ csv_field].is_default_month: solr_record['month'] = csv_date.month else: solr_record[csv_field + '_en'] = '' solr_record[csv_field + '_fr'] = '' except ValueError as x2: self.logger.error( 'Invalid date: "{0}"'.format(x2)) solr_record[csv_field] = '' continue elif self.csv_fields[csv_field].solr_field_type in [ 'pint', 'pfloat' ]: if solr_record[csv_field]: if solr_record[csv_field] == '.': solr_record[csv_field] = "0" csv_decimal = parse_decimal( solr_record[csv_field], locale='en_US') if self.csv_fields[ csv_field].solr_field_is_currency: solr_record[csv_field + '_en'] = format_currency( csv_decimal, 'CAD', locale='en_CA') solr_record[csv_field + '_fr'] = format_currency( csv_decimal, 'CAD', locale='fr_CA') else: solr_record[csv_field + '_en'] = format_decimal( csv_decimal, locale='en_CA') solr_record[csv_field + '_fr'] = format_decimal( csv_decimal, locale='fr_CA') else: solr_record[csv_field + '_en'] = '' solr_record[csv_field + '_fr'] = '' # Lookup the expanded code value from the codes dict of dict if csv_field in self.field_codes: if csv_record[csv_field]: if self.csv_fields[ csv_field].solr_field_multivalued: codes_en = [] codes_fr = [] for code_value in csv_record[ csv_field].split(","): if code_value.lower( ) in self.field_codes[csv_field]: codes_en.append( self.field_codes[csv_field] [code_value.lower()].label_en) codes_fr.append( self.field_codes[csv_field] [code_value.lower()].label_fr) else: self.logger.info( "Unknown code value: {0} for field: {1}" .format(code_value, csv_field)) solr_record[csv_field + '_en'] = codes_en solr_record[csv_field + '_fr'] = codes_fr else: if csv_record[csv_field].lower( ) in self.field_codes[csv_field]: solr_record[csv_field + '_en'] = self.field_codes[ csv_field][csv_record[ csv_field].lower( )].label_en solr_record[csv_field + '_fr'] = self.field_codes[ csv_field][csv_record[ csv_field].lower( )].label_fr else: self.logger.info( "Unknown code value: {0} for field: {1}" .format(csv_record[csv_field], csv_field)) solr_record = self.set_empty_fields(solr_record) # Set the Solr ID field (Nothing To Report records are excluded) if not options['nothing_to_report']: if self.search_target.id_fields: id_values = [] for id_field in self.search_target.id_fields.split( ","): id_values.append(csv_record[id_field]) solr_record['id'] = ",".join(id_values) else: if 'month' in solr_record: solr_record['id'] = "{0}-{1}-{2}".format( solr_record['owner_org'], solr_record['year'], solr_record['month']) elif 'quarter' in solr_record: solr_record['id'] = "{0}-{1}-{2}".format( solr_record['owner_org'], solr_record['year'], solr_record['quarter']) # Call plugins if they exist for this search type. This is where a developer can introduce # code to customize the data that is loaded into Solr for a particular search. if search_type_plugin in self.discovered_plugins: solr_record = self.discovered_plugins[ search_type_plugin].load_csv_record( csv_record, solr_record, self.search_target, self.csv_fields, self.field_codes, 'NTR' if options['nothing_to_report'] else '') solr_items.append(solr_record) # Write to Solr whenever the cycle threshold is reached if cycle >= self.cycle_on: # try to connect to Solr up to 10 times for countdown in reversed(range(10)): try: solr.index(self.solr_core, solr_items) print("{0} rows processed".format(total)) cycle = 0 solr_items.clear() break except ConnectionError as cex: if not countdown: raise print( "Solr error: {0}. Waiting to try again ... {1}" .format(cex, countdown)) time.sleep((10 - countdown) * 5) # Write and remaining records to Solr and commit if cycle > 0: # try to connect to Solr up to 10 times for countdown in reversed(range(10)): try: solr.index(self.solr_core, solr_items) total += len(solr_items) print("{0} rows processed".format(cycle)) cycle = 0 solr_items.clear() break except ConnectionError as cex: if not countdown: raise print( "Solr error: {0}. Waiting to try again ... {1}" .format(cex, countdown)) time.sleep((10 - countdown) * 5) solr.commit(self.solr_core, softCommit=True, waitSearcher=True) print("Total rows processed: {0}".format(total)) except Exception as x: self.logger.error('Unexpected Error "{0}"'.format(x))
def _standardise_data(self, data): """ split accounting lines where needed Crésus writes one csv line per move when there are just two lines (take some money from one account and put all of it in another), and uses ellipses in more complex cases. What matters is the pce label, which is the same on all lines of a move. """ journal_id = self.journal_id.id previous_pce = None previous_date = None previous_tax_id = None lines = [] for self.index, line_cresus in enumerate(data, 1): if previous_pce is not None and previous_pce != line_cresus['pce']: yield self.prepare_move(lines, date=previous_date, ref=previous_pce, journal_id=journal_id) lines = [] previous_pce = line_cresus['pce'] previous_date = line_cresus['date'] from babel.numbers import parse_decimal recto_amount = float( parse_decimal(line_cresus['amount'], locale='de_CH')) verso_amount = 0.0 if recto_amount < 0: recto_amount, verso_amount = 0.0, -recto_amount tax_ids = [previous_tax_id] if previous_tax_id is not None else [] previous_tax_id = None if line_cresus['debit'] != '...': line = self.prepare_line( name=line_cresus['ref'], debit_amount=recto_amount, credit_amount=verso_amount, account_code=line_cresus['debit'], cresus_tax_code=line_cresus['typtvat'], analytic_account_code=line_cresus['analytic_account'], tax_ids=tax_ids) lines.append(line) if 'tax_line_id' in line: previous_tax_id = line['tax_line_id'] if line_cresus['credit'] != '...': line = self.prepare_line( name=line_cresus['ref'], debit_amount=verso_amount, credit_amount=recto_amount, account_code=line_cresus['credit'], cresus_tax_code=line_cresus['typtvat'], analytic_account_code=line_cresus['analytic_account'], tax_ids=tax_ids) lines.append(line) if 'tax_line_id' in line: previous_tax_id = line['tax_line_id'] yield self.prepare_move(lines, date=line_cresus['date'], ref=previous_pce, journal_id=journal_id)
def execute(self, table_id, select_index, aggregation_index, conditions, lower=True): if not table_id.startswith('Table'): table_id = 'Table_{}'.format(table_id.replace('-', '_')) # add # conn = records.Database('sqlite:///{}'.format(self.fdb)).get_connection() # add #table_info = self.db.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql.replace('\n','') #qu = self.db.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id) #qu = self.db.query("SELECT sql from sqlite_master WHERE tbl_name = 'Table_43ad6bdc1d7111e988a6f40f24344a08'") #qu = self.db.query("SELECT sql from sqlite_master WHERE tbl_name = 'table_f65ceb4f453d11e9b16ef40f24344a08'") qu = self.db.query( "SELECT sql from sqlite_master WHERE tbl_name = :name", name=table_id) qu_2 = qu.all() qu_3 = qu_2[0] table_info = qu_3.sql.replace('\n', '') #table_info = conn.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql.replace('\n', '') schema_str = schema_re.findall(table_info)[0] #获取表中每个字段的名字和其对应的类型 schema = {} for tup in schema_str.split(','): c, t = tup.split() #这里对c进行处理,col_1=>col_0 #c = degrad_c(c) schema[c] = t select = 'col_{}'.format(select_index + 1) agg = agg_ops[aggregation_index] if agg: select = '{}({})'.format(agg, select) where_clause = [] where_map = {} for col_index, op, val in conditions: # if lower and (isinstance(val, str) or isinstance(val, str)): # val = val.lower() #if schema['col{}'.format(col_index)] == 'real' and not isinstance(val, (int, float)): if schema['col_{}'.format( col_index + 1)] == 'real' and not isinstance( val, (int, float)): #如果数据列是real类型,且wv不是数字类型 try: # print('!!!!!!value of val is: ', val, 'type is: ', type(val)) # val = float(parse_decimal(val)) # somehow it generates error. val = float(parse_decimal(val, locale='en_US')) # print('!!!!!!After: val', val) except NumberFormatError as e: try: val = float(num_re.findall( val)[0]) # need to understand and debug this part. except: # Although column is of number, selected one is not number. Do nothing in this case. pass where_clause.append('col_{} {} :col_{}'.format( col_index + 1, cond_ops[op], col_index + 1)) where_map['col_{}'.format(col_index + 1)] = val where_str = '' if where_clause: where_str = 'WHERE ' + ' AND '.join(where_clause) query = 'SELECT {} AS result FROM {} {}'.format( select, table_id, where_str) #print query out = self.db.query(query, **where_map) #out = conn.query(query, **where_map) return [o.result for o in out]
def process_formdata(self, valuelist): if valuelist: try: self.data = parse_decimal(str(valuelist[0]), locale=LOCALE) except (decimal.InvalidOperation, ValueError): raise ValueError(self.gettext(u'Not a valid decimal value'))
def test_parse_decimal_strict_mode(self): # Numbers with a misplaced grouping symbol should be rejected with self.assertRaises(numbers.NumberFormatError) as info: numbers.parse_decimal('11.11', locale='de', strict=True) assert info.exception.suggestions == ['1.111', '11,11'] # Numbers with two misplaced grouping symbols should be rejected with self.assertRaises(numbers.NumberFormatError) as info: numbers.parse_decimal('80.00.00', locale='de', strict=True) assert info.exception.suggestions == ['800.000'] # Partially grouped numbers should be rejected with self.assertRaises(numbers.NumberFormatError) as info: numbers.parse_decimal('2000,000', locale='en_US', strict=True) assert info.exception.suggestions == ['2,000,000', '2,000'] # Numbers with duplicate grouping symbols should be rejected with self.assertRaises(numbers.NumberFormatError) as info: numbers.parse_decimal('0,,000', locale='en_US', strict=True) assert info.exception.suggestions == ['0'] # Return only suggestion for 0 on strict with self.assertRaises(numbers.NumberFormatError) as info: numbers.parse_decimal('0.00', locale='de', strict=True) assert info.exception.suggestions == ['0'] # Properly formatted numbers should be accepted assert str(numbers.parse_decimal('1.001', locale='de', strict=True)) == '1001' # Trailing zeroes should be accepted assert str(numbers.parse_decimal('3.00', locale='en_US', strict=True)) == '3.00' # Numbers without any grouping symbol should be accepted assert str(numbers.parse_decimal('2000.1', locale='en_US', strict=True)) == '2000.1' # High precision numbers should be accepted assert str(numbers.parse_decimal('5,000001', locale='fr', strict=True)) == '5.000001'
def execute(self, table_id, select_index, aggregation_index, conditions, lower=True): if not table_id.startswith('table'): table_id = 'table_{}'.format(table_id.replace('-', '_')) table_info = self.db.query( 'SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql.replace('\n', '') schema_str = schema_re.findall(table_info)[0] schema = {} for tup in schema_str.split(', '): c, t = tup.split() schema[c] = t if select_index == "*": select = "*" else: select = 'col{}'.format(select_index) agg = agg_ops[aggregation_index] if agg: select = '{}({})'.format(agg, select) where_clause = [] where_map = {} for col_index, op, val in conditions: if lower and isinstance(val, str): val = val.lower() if schema['col{}'.format(col_index)] == 'real' and not isinstance( val, (int, float)): try: # print('!!!!!!value of val is: ', val, 'type is: ', type(val)) # val = float(parse_decimal(val)) # somehow it generates error. val = float(parse_decimal(val, locale='en_US')) # print('!!!!!!After: val', val) except NumberFormatError as e: try: val = val # float(num_re.findall(val)[0]) # need to understand and debug this part. except: # Although column is of number, selected one is not number. Do nothing in this case. pass if isinstance(val, float): val = '%g' % val where_clause.append('col{} {} :col{}'.format( col_index, cond_ops[op], col_index)) where_map['col{}'.format(col_index)] = val where_str = '' if where_clause: where_str = 'WHERE ' + ' AND '.join(where_clause) if select_index == "*": query = 'SELECT {} FROM {} {}'.format(select, table_id, where_str) else: query = 'SELECT {} AS result FROM {} {}'.format( select, table_id, where_str) try: out = self.db.query(query, **where_map) except: return [] if out.dataset.headers == None: # no data result return [] result = [] if len(out.dataset.headers) > 1 and len( where_clause) != 0: # rows of result for row in out.dataset: for cell in row: result.append(cell) return result if len(out.dataset.headers) == 1: # one column of result for o in out: result.append(o.result) return result return result
def parselocal_float(txt, locale): """TODO :param txt: TODO :param locale: the current locale (e.g: en, en_us, it)""" return numbers.parse_decimal(txt, locale)
def parse_percent(cls, s: str) -> Decimal: assert s.endswith("%") return parse_decimal(s.strip("%"), locale='en_US') / 100
def __init__(self, textPage, datosSM=None): self.timestamp = gmtime() self.source = textPage['source'] self.PositionsCounter = defaultdict(int) self.PlayerData = {} self.PlayerByPos = defaultdict(list) self.Team2Player = defaultdict(set) self.noKiaLink = [] if (type(textPage['data']) is str): soup = BeautifulSoup(textPage['data'], "html.parser") elif (type(textPage['data']) is BeautifulSoup): soup = textPage['data'] else: raise NotImplementedError( "MercadoPageContent: type of content '%s' not supported" % type(textPage['data'])) positions = soup.find_all("table", {"class": "listajugadores"}) for pos in positions: position = pos['id'] for player in pos.find_all("tr"): player_data = player.find_all("td") if not player_data: continue fieldTrads = { 'foto': ['foto'], 'jugador': ['jugador'], 'equipo': ['equipo'], 'promedio': ['promVal', 'valJornada', 'seMantiene'], 'precio': ['precio', 'enEquipos%'], 'val': ['prom3Jornadas'], 'balance': ['sube15%'], 'baja': ['baja15%'], 'rival': ['rival'], 'iconos': ['iconos'] } result = { 'proxFuera': False, 'lesion': False, 'cupo': 'normal' } result['pos'] = position self.PositionsCounter[position] += 1 for data in player_data: classes = data.attrs['class'] # print("AQUI",data) if 'iconos' in classes: for icon in data.find_all("img"): if icon['title'] == "Extracomunitario": result['cupo'] = 'Extracomunitario' elif icon['title'] == "Español": result['cupo'] = "Español" elif icon['title'] == "Lesionado": result['lesion'] = True elif icon['alt'] == "Icono de más información": result['info'] = icon['title'] else: print("No debería llegar aquí: ", icon) continue elif "foto" in classes: img_link = data.img['src'] result['foto'] = img_link result['nombre'] = data.img['title'] # if result['nombre'] in datosSM.traducciones['jugadores']['j2c']: # tradsPosibles = datosSM.traducciones['jugadores']['j2c'] # elif traductor: # nombreRetoc = RetocaNombreJugador(result['nombre']) # tradsPosibles = traductor.BuscaTraduccion(nombreRetoc, 0) # # if isinstance(tradsPosibles, str): # Solo un codigo! # result['codJugador'] = tradsPosibles # else: # Set or None # print("Troublesome player", result['nombre'], result) # result['posibles'] = tradsPosibles continue elif 'jugador' in classes: if data.a: result['kiaLink'] = data.a['href'] # if traductor is None: # result['codJugador'] = result['kiaLink'] continue elif 'equipo' in classes: result['equipo'] = data.img['title'] if result['equipo'] in datosSM.traducciones['equipos'][ 'n2c']: result['CODequipo'] = sorted( datosSM.traducciones['equipos']['n2c'][ result['equipo']])[0] auxId = datosSM.traducciones['equipos']['c2i'].get( result['CODequipo'], None) result['IDequipo'] = onlySetElement(auxId) continue elif 'rival' in classes: for icon in data.find_all('img'): if icon['title'].lower() == "partido fuera": result['proxFuera'] = True else: result['rival'] = icon['title'] if result['rival'] in datosSM.traducciones[ 'equipos']['n2c']: result['CODrival'] = \ sorted(datosSM.traducciones['equipos']['n2c'][result['rival']])[0] auxId = datosSM.traducciones['equipos']['c2i'][ result['CODrival']] result['IDrival'] = onlySetElement(auxId) continue else: auxval = data.get_text().strip() classOrig = classes[0] for auxClass in fieldTrads[classOrig]: if auxClass not in result: classCel = auxClass break if '%' in auxval and classCel == 'enEquipos%': auxval = auxval.replace("%", "") result[classCel] = parse_decimal(auxval, locale="de") try: codJugador = result['kiaLink'] except KeyError: print( "Problemas con jugador '%s' (%s): no tiene kiaLink (?)" % (result.get('nombre', "Nombre desc"), result.get('equipo', "Equipo desc"))) self.noKiaLink.append(result) continue self.PlayerData[codJugador] = result self.PlayerByPos[position].append(codJugador) self.Team2Player[result['equipo']].add(codJugador)
def colorin(text, color="red", style="normal"): """ Return the given text, surrounded by the given color ASCII markers. If the given color is a name that exists in available colors, a 8-colors mode is assumed, else, a 256-colors mode. The given style must exists in the available styles. >>> colorin("Fetchez la vache", "red", "bold") '\x1b[1;31mFetchez la vache\x1b[0m' >>> colout.colorin("Faites chier la vache", 41, "normal") '\x1b[0;38;5;41mFaites chier la vache\x1b[0m' """ global colormap_idx # Special characters. start = "\033[" stop = "\033[0m" color_code = "" style_code = "" # Convert the style code if style == "random" or style == "Random": style = random.choice(list(styles.keys())) else: if style in styles: style_code = str(styles[style]) if color == "none": # if no color, style cannot be applied return text elif color == "random": mode = 8 color_code = random.choice(list(colors.values())) color_code = str(30 + color_code) elif color == "Random": mode = 256 color_nb = random.randint(0, 255) color_code = str(color_nb) elif color == "rainbow": mode = 8 color = colormap[colormap_idx] color_code = str(30 + colors[color]) if colormap_idx < len(colormap) - 1: colormap_idx += 1 else: colormap_idx = 0 elif color == "Rainbow": mode = 256 color_nb = rgb_to_ansi(*rgb_rainbow(colormap_idx)) color_code = str(color_nb) if colormap_idx < 255: colormap_idx += 1 else: colormap_idx = 0 elif color == "scale": try: import babel.numbers as bn f = float(bn.parse_decimal(text)) except ImportError: f = float(text) # if out of scale, do not color if f < scale[0] or f > scale[1]: return text # normalize and scale over the nb of colors in colormap i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(colormap) - 1))) mode = 8 color = colormap[i] color_code = str(30 + colors[color]) elif color == "colormap": color = colormap[colormap_idx] if color in colors: mode = 8 color_code = str(30 + colors[color]) else: mode = 256 color_nb = int(color) assert (0 <= color_nb <= 255) color_code = str(color_nb) if colormap_idx < len(colormap) - 1: colormap_idx += 1 else: colormap_idx = 0 # 8 colors modes elif color in colors: mode = 8 color_code = str(30 + colors[color]) # 256 colors mode elif color.isdigit(): mode = 256 color_nb = int(color) assert (0 <= color_nb <= 255) color_code = str(color_nb) # programming language elif color.lower() in lexers: lexer = get_lexer_by_name(color.lower()) # Python => 256 colors, python => 8 colors ask_256 = color[0].isupper() if ask_256: try: formatter = Terminal256Formatter(style=style) except: # style not found formatter = Terminal256Formatter() else: if style not in ("light", "dark"): style = "dark" # dark color scheme by default formatter = TerminalFormatter(bg=style) # We should return all but the last character, # because Pygments adds a newline char. return highlight(text, lexer, formatter)[:-1] # unrecognized else: raise Exception('Unrecognized color %s' % color) return start + style_code + endmarks[mode] + color_code + "m" + text + stop
def scan_search_alternate(self): log = [] browser = self.browser wait = self.wait presence_of = ec.presence_of_element_located visibility_of = ec.visibility_of_element_located wait = WebDriverWait(browser, timed_out) # Procedure: # 1. Find the product grid: "neemu-products-container" # 2. Get the grid products: "nm-product-item" # 3. Get product(s) prices and information # * Price: "nm-price-container" # * Info: "nm-product-name" # 4. Return info try: # Check it was found something or not and wait not_found = browser.find_elements_by_class_name( "nm-not-found-container") if len(not_found) > 0: raise "I cannot find the product requested" # Wait to "memu-products-container" to load wait.until(presence_of((By.CLASS_NAME, "productShowCaseContent"))) # Get grid grid = browser.find_elements_by_class_name( "productShowCaseContent") # Get grid products for i in range(10): products = grid[0].find_elements_by_class_name("product-li") if len(products) == 0: time.sleep(1) else: break # Scan products time.sleep(3) out = [] for p in products: # Get price # Occurs when it has discount price = p.find_elements_by_class_name("price-value") if len(price) == 0: price = p.find_elements_by_class_name("price") # Occurs when it does not have discount if len(price) == 0: continue price = price[0].text # Get info title = p.find_element_by_class_name("productTitle").text price = clean_price(price) info = title out += [{"info": info}] out[-1]["price"] = float( numbers.parse_decimal(price, locale="pt_BR")) success = True if len(out) > 0 else False except Exception as e: print("Fail to find the products container!") print(e) out = [] success = False return out, success
def colorin(text, color="red", style="normal"): """ Return the given text, surrounded by the given color ASCII markers. If the given color is a name that exists in available colors, a 8-colors mode is assumed, else, a 256-colors mode. The given style must exists in the available styles. >>> colorin("Fetchez la vache", "red", "bold") '\x1b[1;31mFetchez la vache\x1b[0m' >>> colout.colorin("Faites chier la vache", 41, "normal") '\x1b[0;38;5;41mFaites chier la vache\x1b[0m' """ assert (type(color) is str) global colormap_idx global debug # Special characters. start = "\033[" stop = "\033[0m" color_code = "" style_code = "" # Convert the style code if style == "random" or style == "Random": style = random.choice(list(styles.keys())) else: if style in styles: style_code = str(styles[style]) if color == "none": # if no color, style cannot be applied if not debug: return text else: return "<none>" + text + "</none>" elif color == "random": mode = 8 color_code = random.choice(list(colors.values())) color_code = str(30 + color_code) elif color == "Random": mode = 256 color_nb = random.randint(0, 255) color_code = str(color_nb) elif color in colormaps.keys(): if color[0].islower(): # lower case first letter mode = 8 c = colormaps[color][colormap_idx] if c.isdigit(): color_code = str(30 + c) else: color_code = str(30 + colors[c]) else: # upper case mode = 256 color_nb = colormaps[color][colormap_idx] color_code = str(color_nb) if colormap_idx < len(colormaps[color]) - 1: colormap_idx += 1 else: colormap_idx = 0 elif color.lower() == "scale": # "scale" or "Scale" try: import babel.numbers as bn f = float(bn.parse_decimal(text)) except ImportError: f = float(text) # if out of scale, do not color if f < scale[0] or f > scale[1]: return text if color[0].islower(): mode = 8 cmap = colormaps["spectrum"] # normalize and scale over the nb of colors in cmap i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1))) color = cmap[i] color_code = str(30 + colors[color]) else: mode = 256 cmap = colormaps["Spectrum"] i = int( math.ceil( (f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1))) color = cmap[i] color_code = str(color) # Really useful only when using colout as a library # thus you can change the "colormap" variable to your favorite one before calling colorin elif color == "colormap": color = colormap[colormap_idx] if color in colors: mode = 8 color_code = str(30 + colors[color]) else: mode = 256 color_nb = int(color) assert (0 <= color_nb <= 255) color_code = str(color_nb) if colormap_idx < len(colormap) - 1: colormap_idx += 1 else: colormap_idx = 0 # 8 colors modes elif color in colors: mode = 8 color_code = str(30 + colors[color]) # hexadecimal color elif color[0] == "#": mode = 256 color_nb = rgb_to_ansi(*hex_to_rgb(color)) assert (0 <= color_nb <= 255) color_code = str(color_nb) # 256 colors mode elif color.isdigit(): mode = 256 color_nb = int(color) assert (0 <= color_nb <= 255) color_code = str(color_nb) # programming language elif color.lower() in lexers: lexer = get_lexer_by_name(color.lower()) # Python => 256 colors, python => 8 colors ask_256 = color[0].isupper() if ask_256: try: formatter = Terminal256Formatter(style=style) except: # style not found formatter = Terminal256Formatter() else: if style not in ("light", "dark"): style = "dark" # dark color scheme by default formatter = TerminalFormatter(bg=style) # We should return all but the last character, # because Pygments adds a newline char. if not debug: return highlight(text, lexer, formatter)[:-1] else: return "<" + color + ">" + highlight( text, lexer, formatter)[:-1] + "</" + color + ">" # unrecognized else: raise UnknownColor(color) if not debug: return start + style_code + endmarks[ mode] + color_code + "m" + text + stop else: return start + style_code + endmarks[ mode] + color_code + "m<" + color + ">" + text + "</" + color + ">" + stop
def okButton_clicked(self): """ OK button has been clicked - save the values Returns: """ if self._selected_index is None: # Huh? return model = self._item_model stackedwidget_index = self.stackedWidget.currentIndex() the_item = self._get_item_for_stackedwidget(self._selected_index) new_name = nullify(self.nameLineEdit.text()) self.set_modified() if new_name is None: self.nameLineEdit.setText(the_item.name) else: # This will take care of saving the value model.setData(self._selected_index, new_name, QtCore.Qt.EditRole) # Trimmed name / rejected self.nameLineEdit.setText(the_item.name) # Items with descriptions - Author, Cuisine, Yield Units if stackedwidget_index == self.StackedItems.ITEM_WITH_DESCRIPTION: new_description = nullify(self.descriptionTextEdit.toPlainText()) the_item.description = new_description elif stackedwidget_index == self.StackedItems.INGREDIENT_UNIT: new_type = self.typeComboBox.currentIndex() if new_type >= 0: # new_type == -1 should never happen - it would mean no type has been selected which should be # impossible. However, better play it safe :-) the_item.type_ = new_type new_factor = nullify(self.factorLineEdit.text()) # Depending on the type of the factor (whatever the user entered) should either be None or have a # value. Unit Type GROUP isn't visible for the user so don't bother to check if new_type != data.IngredientUnit.UnitType.UNSPECIFIC: if new_factor is None: self.illegal_value(misc.IllegalValueEntered.ISEMPTY, None) new_factor = the_item.factor else: try: new_factor = parse_decimal(new_factor) if math.isclose(new_factor, 0.0): self.illegal_value( misc.IllegalValueEntered.ISZERO, "0") new_factor = the_item.factor if new_factor < 0.0: self.illegal_value( misc.IllegalValueEntered.ISNONUMBER, new_factor) new_factor = the_item.factor except NumberFormatError: # The user has entered something strange. self.illegal_value( misc.IllegalValueEntered.ISNONUMBER, new_factor) new_factor = the_item.factor else: # Unspecific -> no Factor new_factor = None if new_factor is not None: self.factorLineEdit.setText(str(new_factor)) else: self.factorLineEdit.clear() the_item.factor = new_factor elif stackedwidget_index == self.StackedItems.INGREDIENTS: the_item.icon = self._ingredient_icon self.okButton.setEnabled(False) self.cancelButton.setEnabled(False)