def test(self, d):
"""
Test, for purposes of type inference, if a value could possibly be valid
for this column type. This will work with values that are native types
and values that have been stringified.
"""
if d is None:
return True
if isinstance(d, Decimal):
return True
if type(d) is int or type(d) is float:
return True
if not isinstance(d, six.string_types):
return False
d = d.strip()
d = d.strip('%')
for symbol in CURRENCY_SYMBOLS:
d = d.strip(symbol)
if d.lower() in self.null_values:
return True
try:
parse_decimal(d, self._locale)
return True
except:
return False
def test_can_parse_decimals(self):
self.assertEqual(decimal.Decimal('1099.98'),
numbers.parse_decimal('1,099.98', locale='en_US'))
self.assertEqual(decimal.Decimal('1099.98'),
numbers.parse_decimal('1.099,98', locale='de'))
self.assertRaises(numbers.NumberFormatError,
lambda: numbers.parse_decimal('2,109,998', locale='de'))
def test_parse_decimal():
assert (numbers.parse_decimal('1,099.98', locale='en_US')
== decimal.Decimal('1099.98'))
assert numbers.parse_decimal('1.099,98', locale='de') == decimal.Decimal('1099.98')
with pytest.raises(numbers.NumberFormatError) as excinfo:
numbers.parse_decimal('2,109,998', locale='de')
assert excinfo.value.args[0] == "'2,109,998' is not a valid decimal number"
def parse_value(value):
"""
Accepts a string value and attempts to parce it as a currency value.
Returns the extracted numeric value converted to a string
"""
l_currency_language_code, l_currency_code = _getCodes()
curSym = get_currency_symbol(l_currency_code, l_currency_language_code)
grpSym = get_group_symbol(locale=l_currency_language_code.lower())
decSym = get_decimal_symbol(locale=l_currency_language_code.lower())
# Convert the Official characters into what comes from the keyboard.
# This section may need to grow over time.
# - Character 160 is a non-breaking space, which is different from a typed space
if ord(grpSym) == 160:
value = value.replace(u" ", grpSym)
allSym = _getSymbols(value)
invalidSym = allSym.replace(curSym, "").replace(grpSym, "").replace(decSym, "").replace(u"-", "")
value = value.replace(curSym, "")
if allSym.count(decSym) > 1:
raise NumberFormatError(default_error_messages["decimal_symbol"] % decSym)
elif (allSym.count(decSym) == 1 and allSym[-1] != decSym) or len(invalidSym) > 0:
raise NumberFormatError(default_error_messages["invalid_format"] % (grpSym, decSym))
elif value.count(decSym) == 1:
value = parse_decimal(value, locale=l_currency_language_code.lower())
else:
value = parse_number(value, locale=l_currency_language_code.lower())
# The value is converted into a string because the parse functions return
# floats
return str(value)
def parse_decimal(string, locale=None):
"""Parses localized decimal string into a float.
.. code-block:: python
>>> parse_decimal('1,099.98', locale='en_US')
1099.98
>>> parse_decimal('1.099,98', locale='de')
1099.98
When the given string cannot be parsed, an exception is raised:
.. code-block:: python
>>> parse_decimal('2,109,998', locale='de')
Traceback (most recent call last):
...
NumberFormatError: '2,109,998' is not a valid decimal number
:param string:
The string to parse.
:param locale:
A locale code. If not set, uses the currently loaded locale.
:returns:
The parsed decimal number.
:raises:
``NumberFormatError`` if the string can not be converted to a
decimal number.
"""
locale = locale or get_locale()
return numbers.parse_decimal(string, locale=locale)
def asValue(self, strValue, objType):
'''
@see: Converter.asValue
'''
assert isinstance(objType, Type), 'Invalid object type %s' % objType
if strValue is None: return None
if isinstance(objType, TypeModel): # If type model is provided we consider the model property type
assert isinstance(objType, TypeModel)
container = objType.container
assert isinstance(container, Model)
objType = container.properties[container.propertyId]
if objType.isOf(str):
return strValue
if objType.isOf(Boolean):
return strValue.strip().lower() == _('true').lower()
if objType.isOf(Percentage):
return float(strValue) / 100
if objType.isOf(int):
return int(strValue)
if objType.isOf(Number):
return bn.parse_decimal(strValue, self.locale)
if objType.isOf(Date):
return datetime.strptime(strValue, '%Y-%m-%d').date()
if objType.isOf(Time):
return datetime.strptime(strValue, '%H:%M:%S').time()
if objType.isOf(DateTime):
return datetime.strptime(strValue, '%Y-%m-%d %H:%M:%S')
raise TypeError('Invalid object type %s for Babel converter' % objType)
def hydrate_total_value(self, bundle):
value = bundle.data.get('total_value', None)
if value:
bundle.data['total_value'] = parse_decimal(str(value), locale=bundle.request.locale)
return bundle
def cast(self, d):
"""
Cast a single value to a :class:`decimal.Decimal`.
:returns:
:class:`decimal.Decimal` or :code:`None`.
"""
if isinstance(d, Decimal) or d is None:
return d
elif type(d) is int:
return Decimal(d)
elif type(d) is float:
return Decimal(self.float_format % d)
elif isinstance(d, six.string_types):
d = d.strip()
d = d.strip("%")
for symbol in CURRENCY_SYMBOLS:
d = d.strip(symbol)
if d.lower() in self.null_values:
return None
else:
raise CastError('Can not parse value "%s" as Decimal.' % d)
try:
return parse_decimal(d, self.locale)
except:
pass
raise CastError('Can not parse value "%s" as Decimal.' % d)
def cast(self, d):
"""
Cast a single value to a :class:`decimal.Decimal`.
:returns: :class:`decimal.Decimal` or :code:`None`.
:raises: :exc:`.CastError`
"""
if isinstance(d, Decimal) or d is None:
return d
elif isinstance(d, int):
return Decimal(d)
elif isinstance(d, float):
raise CastError('Can not convert float to Decimal. Convert data to string first!')
elif isinstance(d, six.string_types):
d = d.strip()
d = d.strip('%')
for symbol in CURRENCY_SYMBOLS:
d = d.strip(symbol)
if d.lower() in self.null_values:
return None
try:
return parse_decimal(d, self._locale)
except:
raise CastError('Can not parse value "%s" as Decimal.' % d)
def execute(self, table_id, select_index, aggregation_index, conditions, lower=True):
if not table_id.startswith('table'):
table_id = 'table_{}'.format(table_id.replace('-', '_'))
table_info = self.db.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql
schema_str = schema_re.findall(table_info)[0]
schema = {}
for tup in schema_str.split(', '):
c, t = tup.split()
schema[c] = t
select = 'col{}'.format(select_index)
agg = Query.agg_ops[aggregation_index]
if agg:
select = '{}({})'.format(agg, select)
where_clause = []
where_map = {}
for col_index, op, val in conditions:
if lower and isinstance(val, str):
val = val.lower()
if schema['col{}'.format(col_index)] == 'real' and not isinstance(val, (int, float)):
try:
val = float(parse_decimal(val))
except NumberFormatError as e:
val = float(num_re.findall(val)[0])
where_clause.append('col{} {} :col{}'.format(col_index, Query.cond_ops[op], col_index))
where_map['col{}'.format(col_index)] = val
where_str = ''
if where_clause:
where_str = 'WHERE ' + ' AND '.join(where_clause)
query = 'SELECT {} AS result FROM {} {}'.format(select, table_id, where_str)
out = self.db.query(query, **where_map)
return [o.result for o in out]
def parselocal_float(txt, locale):
"""add???
:param txt: add???
:param locale: add???
:returns: add???
"""
return numbers.parse_decimal(txt, locale)
def normalize_field(self, value):
if isinstance(value, (int, float)):
return value
if value == '':
value = None
elif value is not None:
value = float(parse_decimal(value, locale=settings.get_locale()))
return super(FloatField, self).normalize_field(value)
def clean(self, value):
locale=get_current_language()
if value != "":
try:
value = force_unicode(parse_decimal(value, locale=locale))
except NumberFormatError:
raise forms.ValidationError(self.error_messages['invalid'])
return super(FloatField, self).clean(value)
def normalize_field(self, value):
if value == '':
value = None
if isinstance(value, int):
return value
elif value is not None:
locale = settings.get_locale()
value = int(parse_decimal(value, locale=locale))
return super(IntegerField, self).normalize_field(value)
def test(self, d):
"""
Test, for purposes of type inference, if a string value could possibly
be valid for this column type.
"""
d = d.strip()
d = d.strip('%')
for symbol in CURRENCY_SYMBOLS:
d = d.strip(symbol)
if d.lower() in self.null_values:
return True
try:
parse_decimal(d, self._locale)
return True
except:
return False
def normalize_field(self, value):
if isinstance(value, Decimal):
return value
if value == '':
value = None
elif value is not None:
value = parse_decimal(value, locale=settings.get_locale())
rounded = int(round(Decimal(value) * self.__multiplier))
value = Decimal(rounded) / self.__multiplier
return super(DecimalField, self).normalize_field(value)
def _to_python(self, value, state):
if not getattr(self, 'required', False) and not value:
return getattr(self, 'if_missing', None)
try:
value = parse_decimal(value, locale = state._LOCALE_)
if self.max and value > self.max:
raise formencode.Invalid(self.message("amount_too_high", state, max_amount = format_decimal(self.max, locale=state._LOCALE_)), value, state)
if self.min and value < self.min:
raise formencode.Invalid(self.message("amount_too_low", state, min_amount = format_decimal(self.min, locale=state._LOCALE_)), value, state)
except NumberFormatError, e:
raise formencode.Invalid(self.message("invalid_amount", state, value = value), value, state)
def localize_money(money, currency=None, language=None):
if language is None:
language = preferred_language()
if not isinstance(money, Decimal):
money = parse_decimal(money)
if currency:
return format_currency(money, currency, locale=language)
else:
return format_decimal(money, locale=language, format="#,##0.00")
def parse_decimal(value):
if isinstance(value, basestring):
value = ustr(value)
#deal with ' ' instead of u'\xa0' (SP instead of NBSP as grouping char)
value = value.replace(' ', '')
try:
value = numbers.parse_decimal(value, locale=get_locale())
except ValueError, e:
pass
def color_scale(name, text):
# filter out everything that does not seem to be necessary to interpret the string as a number
# this permits to transform "[ 95%]" to "95" before number conversion,
# and thus allows to color a group larger than the matched number
chars_in_numbers = "-+.,e/*"
allowed = string.digits + chars_in_numbers
nb = "".join([i for i in filter(allowed.__contains__, text)])
# interpret as decimal
# First, try with the babel module, if available
# if not, use python itself,
# if thoses fails, try to `eval` the string
# (this allow strings like "1/2+0.9*2")
f = None
try:
# babel is a specialized module
import babel.numbers as bn
try:
f = float(bn.parse_decimal(nb))
except bn.NumberFormatError:
pass
except ImportError:
try:
f = float(nb)
except ValueError:
pass
if f is not None:
# normalize with scale if it's a number
f = (f - context["scale"][0]) / (context["scale"][1] - context["scale"][0])
else:
# interpret as float between 0 and 1 otherwise
f = eval(nb)
# if out of scale, do not color
if f < 0 or f > 1:
return None
# normalize and scale over the nb of colors in cmap
colormap = context["colormaps"][name]
i = int(math.ceil(f * (len(colormap) - 1)))
color = colormap[i]
# infer mode from the color in the colormap
m = mode(color)
if m == 8:
color_code = str(30 + context["colors"][color])
else:
color_code = str(color)
return color_code
def render(self, name, value, attrs=None):
from babel.numbers import parse_decimal
from babel.numbers import format_decimal
from babel.numbers import NumberFormatError
locale=get_current_language()
if value is None: value = ""
if value and isinstance(value, basestring):
try:
value = parse_decimal(value, locale=locale)
except NumberFormatError:
pass
if value is not "" and not isinstance(value, basestring):
value = format_decimal(value, self.format, locale=locale)
return super(DecimalWidget, self).render(name, value, attrs)
def add_helpers_to_context(tell_sentry, context, loc, request=None):
context['locale'] = loc
context['decimal_symbol'] = get_decimal_symbol(locale=loc)
context['_'] = lambda s, *a, **kw: get_text(loc, s, *a, **kw)
context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, request, loc, *a, **kw)
context['format_number'] = lambda *a: format_number(*a, locale=loc)
context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc)
context['format_currency'] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw)
context['format_percent'] = lambda *a: format_percent(*a, locale=loc)
context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc)
def _to_age(delta):
try:
return to_age(delta, loc)
except:
return to_age(delta, 'en')
context['to_age'] = _to_age
def parse_decimal(self, string):
"""Parse localized decimal string into a float
>>> Locale('en', 'US').parse_decimal('1,099.98')
1099.98
>>> Locale('de').parse_decimal('1.099,98')
1099.98
When the given string cannot be parsed, an exception is raised:
>>> Locale('de').parse_decimal('2,109,998')
Traceback (most recent call last):
...
NumberFormatError: '2,109,998' is not a valid decimal number
"""
return numbers.parse_decimal(string, self)
def inbound(request):
context = request.context
loc = context.locale = get_locale_for_request(request)
context.decimal_symbol = get_decimal_symbol(locale=loc)
context._ = lambda s, *a, **kw: get_text(request, loc, s, *a, **kw)
context.ngettext = lambda *a, **kw: n_get_text(request, loc, *a, **kw)
context.format_number = lambda *a: format_number(*a, locale=loc)
context.format_decimal = lambda *a: format_decimal(*a, locale=loc)
context.format_currency = lambda *a: format_currency(*a, locale=loc)
context.format_percent = lambda *a: format_percent(*a, locale=loc)
context.parse_decimal = lambda *a: parse_decimal(*a, locale=loc)
def _to_age(delta):
try:
return to_age(delta, loc)
except:
return to_age(delta, 'en')
context.to_age = _to_age
def add_helpers_to_context(tell_sentry, context, loc):
context['escape'] = lambda s: s # to be overriden by renderers
context['locale'] = loc
context['decimal_symbol'] = get_decimal_symbol(locale=loc)
context['_'] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw)
context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw)
context['format_number'] = lambda *a: format_number(*a, locale=loc)
context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc)
context['format_currency'] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw)
context['format_percent'] = lambda *a: format_percent(*a, locale=loc)
context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc)
def _to_age(delta, **kw):
try:
return to_age(delta, loc, **kw)
except:
return to_age(delta, 'en', **kw)
context['to_age'] = _to_age
def add_helpers_to_context(tell_sentry, context, loc):
context["escape"] = lambda s: s # to be overriden by renderers
context["locale"] = loc
context["decimal_symbol"] = get_decimal_symbol(locale=loc)
context["_"] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw)
context["ngettext"] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw)
context["format_number"] = lambda *a: format_number(*a, locale=loc)
context["format_decimal"] = lambda *a: format_decimal(*a, locale=loc)
context["format_currency"] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw)
context["format_percent"] = lambda *a: format_percent(*a, locale=loc)
context["parse_decimal"] = lambda *a: parse_decimal(*a, locale=loc)
def _to_age(delta, **kw):
try:
return to_age(delta, loc, **kw)
except:
return to_age(delta, "en", **kw)
context["to_age"] = _to_age
def add_helpers_to_context(tell_sentry, context, loc):
context['escape'] = lambda s: s # to be overriden by renderers
context['locale'] = loc
context['decimal_symbol'] = get_decimal_symbol(locale=loc)
context['_'] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw)
context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, context, loc, *a, **kw)
context['Money'] = Money
context['format_number'] = lambda *a: format_number(*a, locale=loc)
context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc)
context['format_currency'] = lambda *a, **kw: format_money(*a, locale=loc, **kw)
context['format_percent'] = lambda *a: format_percent(*a, locale=loc)
context['format_datetime'] = lambda *a: format_datetime(*a, locale=loc)
context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc)
context['to_age'] = to_age
def to_age_str(o, **kw):
if not isinstance(o, datetime):
kw.setdefault('granularity', 'day')
return format_timedelta(to_age(o), locale=loc, **kw)
context['to_age_str'] = to_age_str
def add_helpers_to_context(website, request):
context = request.context
loc = context["locale"] = get_locale_for_request(request, website)
context["decimal_symbol"] = get_decimal_symbol(locale=loc)
context["_"] = lambda s, *a, **kw: get_text(request, loc, s, *a, **kw)
context["ngettext"] = lambda *a, **kw: n_get_text(website, request, loc, *a, **kw)
context["format_number"] = lambda *a: format_number(*a, locale=loc)
context["format_decimal"] = lambda *a: format_decimal(*a, locale=loc)
context["format_currency"] = lambda *a, **kw: format_currency_with_options(*a, locale=loc, **kw)
context["format_percent"] = lambda *a: format_percent(*a, locale=loc)
context["parse_decimal"] = lambda *a: parse_decimal(*a, locale=loc)
def _to_age(delta):
try:
return to_age(delta, loc)
except:
return to_age(delta, "en")
context["to_age"] = _to_age
def detokenize_query(query, example_dict, table):
detokenized_conds = []
for i, (col, op, val) in enumerate(query.conditions):
val_tokens = val.split(' ')
detokenized_cond_val = my_detokenize(val_tokens, example_dict['question'])
if table.header[col].type == 'real' and not isinstance(detokenized_cond_val, (int, float)):
if ',' not in detokenized_cond_val:
try:
detokenized_cond_val = float(parse_decimal(val))
except NumberFormatError as e:
try: detokenized_cond_val = float(num_re.findall(val)[0])
except: pass
detokenized_conds.append((col, op, detokenized_cond_val))
detokenized_query = Query(sel_index=query.sel_index, agg_index=query.agg_index, conditions=detokenized_conds)
return detokenized_query
def parse_decimal(string):
"""Parse localized decimal string into a float.
>>> parse_decimal('1,099.98', locale='en_US')
1099.98
>>> parse_decimal('1.099,98', locale='de')
1099.98
When the given string cannot be parsed, an exception is raised:
>>> parse_decimal('2,109,998', locale='de')
Traceback (most recent call last):
...
NumberFormatError: '2,109,998' is not a valid decimal number
:param string:
The string to parse.
:return:
The parsed decimal number.
:raise `NumberFormatError`:
If the string can not be converted to a decimal number
"""
return numbers.parse_decimal(string, locale=local.locale)
def _to_python(self, value, state):
if not getattr(self, 'required', False) and not value:
return getattr(self, 'if_missing', None)
try:
value = parse_decimal(value, locale=state._LOCALE_)
if self.max and value > self.max:
raise formencode.Invalid(
self.message("amount_too_high",
state,
max_amount=format_decimal(
self.max, locale=state._LOCALE_)), value,
state)
if self.min and value < self.min:
raise formencode.Invalid(
self.message("amount_too_low",
state,
min_amount=format_decimal(
self.min, locale=state._LOCALE_)), value,
state)
except NumberFormatError, e:
raise formencode.Invalid(
self.message("invalid_amount", state, value=value), value,
state)
def add_helpers_to_context(tell_sentry, context, loc):
context['escape'] = lambda s: s # to be overriden by renderers
context['locale'] = loc
context['decimal_symbol'] = get_decimal_symbol(locale=loc)
context['_'] = lambda s, *a, **kw: get_text(context, loc, s, *a, **kw)
context['ngettext'] = lambda *a, **kw: n_get_text(tell_sentry, context,
loc, *a, **kw)
context['Money'] = Money
context['format_number'] = lambda *a: format_number(*a, locale=loc)
context['format_decimal'] = lambda *a: format_decimal(*a, locale=loc)
context['format_currency'] = lambda *a, **kw: format_money(
*a, locale=loc, **kw)
context['format_percent'] = lambda *a: format_percent(*a, locale=loc)
context['format_datetime'] = lambda *a: format_datetime(*a, locale=loc)
context['parse_decimal'] = lambda *a: parse_decimal(*a, locale=loc)
context['to_age'] = to_age
def to_age_str(o, **kw):
if not isinstance(o, datetime):
kw.setdefault('granularity', 'day')
return format_timedelta(to_age(o), locale=loc, **kw)
context['to_age_str'] = to_age_str
def parse_decimal(self, string):
"""Parses localized decimal string into a float. Example::
>>> parse_decimal('1,099.98', locale='en_US')
1099.98
>>> parse_decimal('1.099,98', locale='de')
1099.98
When the given string cannot be parsed, an exception is raised::
>>> parse_decimal('2,109,998', locale='de')
Traceback (most recent call last):
...
NumberFormatError: '2,109,998' is not a valid decimal number
:param string:
The string to parse.
:returns:
The parsed decimal number.
:raises:
``NumberFormatError`` if the string can not be converted to a
decimal number.
"""
return numbers.parse_decimal(string, locale=self.locale)
def cast(self, d):
"""
Cast a single value to a :class:`decimal.Decimal`.
:returns:
:class:`decimal.Decimal` or :code:`None`.
"""
if isinstance(d, Decimal) or d is None:
return d
t = type(d)
if t is int:
return Decimal(d)
elif six.PY2 and t is long:
return Decimal(d)
elif t is float:
return Decimal(repr(d))
elif isinstance(d, six.string_types):
d = d.strip()
d = d.strip('%')
for symbol in CURRENCY_SYMBOLS:
d = d.strip(symbol)
if d.lower() in self.null_values:
return None
else:
raise CastError('Can not parse value "%s" as Decimal.' % d)
try:
return parse_decimal(d, self.locale)
except:
pass
raise CastError('Can not parse value "%s" as Decimal.' % d)
async def on_recognize(
self,
turn_context: TurnContext,
state: Dict[str, object],
options: PromptOptions,
) -> PromptRecognizerResult:
if not turn_context:
raise TypeError(
"NumberPrompt.on_recognize(): turn_context cannot be None.")
result = PromptRecognizerResult()
if turn_context.activity.type == ActivityTypes.message:
utterance = turn_context.activity.text
if not utterance:
return result
culture = self._get_culture(turn_context)
results: [ModelResult] = recognize_number(utterance, culture)
if results:
result.succeeded = True
result.value = parse_decimal(results[0].resolution["value"],
locale=culture.replace("-", "_"))
return result
def serialize_prices_before_puts(mapper, connection, target):
target.volume = int(parse_decimal(target.volume, locale=LOCALE))
def load_csv_record(csv_record: dict, solr_record: dict, search: Search,
fields: dict, codes: dict, format: str):
if format == 'NTR':
solr_record['has_amendments'] = "0"
if len(csv_record['fiscal_year']) > 4:
try:
year = int(csv_record['fiscal_year'][0:4])
if csv_record['quarter'] == 'Q1':
start_date = date(year, 4, 1)
elif csv_record['quarter'] == 'Q2':
start_date = date(year, 7, 1)
elif csv_record['quarter'] == 'Q3':
start_date = date(year, 10, 1)
elif csv_record['quarter'] == 'Q4':
start_date = date(year + 1, 1, 1)
solr_record['agreement_start_date'] = start_date.strftime(
'%Y-%m-%d')
solr_record['year'] = start_date.year
solr_record['month'] = start_date.month
except ValueError:
pass
else:
solr_record['format_en'] = 'Grants and Contributions'
solr_record['format_fr'] = 'Subventions et contributions'
try:
agreement_value = parse_decimal(
csv_record['agreement_value'].replace('$',
'').replace(',', ''),
locale='en')
# Set a value range
if agreement_value < 0:
solr_record['agreement_value_range_en'] = '(a) Negative'
solr_record['agreement_value_range_fr'] = '(a) negatif'
elif agreement_value < 10000:
solr_record[
'agreement_value_range_en'] = '(b) Less than $10,000'
solr_record[
'agreement_value_range_fr'] = '(b) moins de 10 000 $'
elif 10000 <= agreement_value < 25000:
solr_record[
'agreement_value_range_en'] = '(c) $10,000 - $25,000'
solr_record[
'agreement_value_range_fr'] = '(c) de 10 000 $ à 25 000 $'
elif 25000 <= agreement_value < 100000:
solr_record[
'agreement_value_range_en'] = '(d) $25,000 - $100,000'
solr_record[
'agreement_value_range_fr'] = '(d) de 25 000 $ à 100 000 $'
elif 100000 <= agreement_value < 1000000:
solr_record[
'agreement_value_range_en'] = '(e) $100,000 - $1,000,000'
solr_record[
'agreement_value_range_fr'] = '(e) de 100 000 $ à 1 000 000 $'
elif 1000000 <= agreement_value < 5000000:
solr_record[
'agreement_value_range_en'] = '(f) $1,000,000 - $5,000,000'
solr_record[
'agreement_value_range_fr'] = '(f) de 1 000 000 $ à 5 000 000 $'
else:
solr_record[
'agreement_value_range_en'] = '(g) More than $5,000,000'
solr_record[
'agreement_value_range_fr'] = '(g) plus de cinq millions $'
except NumberFormatError:
pass
# Flag the record for amendments
if csv_record['amendment_number'] != "current":
solr_record['has_amendments'] = "1"
elif csv_record['amendment_number'] == "current" and csv_record[
'amendment_date'] != "":
solr_record['has_amendments'] = "1"
else:
solr_record['has_amendments'] = "0"
# The coverage field is bilingual
if 'coverage' in csv_record and csv_record['coverage']:
solr_record = _set_bilingual_field('coverage',
csv_record['coverage'],
solr_record)
if 'recipient_legal_name' in csv_record and csv_record[
'recipient_legal_name']:
solr_record = _set_bilingual_field(
'recipient_legal_name', csv_record['recipient_legal_name'],
solr_record)
if 'recipient_operating_name' in csv_record and csv_record[
'recipient_operating_name']:
solr_record = _set_bilingual_field(
'recipient_operating_name',
csv_record['recipient_operating_name'], solr_record)
if 'research_organization_name' in csv_record and csv_record[
'research_organization_name']:
solr_record = _set_bilingual_field(
'research_organization_name',
csv_record['research_organization_name'], solr_record)
if 'recipient_city' in csv_record and csv_record['recipient_city']:
solr_record = _set_bilingual_field('recipient_city',
csv_record['recipient_city'],
solr_record)
# Make the agreement number searchable
solr_record['agreement_number_text'] = solr_record['agreement_number']
return solr_record
def colorin(text, color="red", style="normal"):
"""
Return the given text, surrounded by the given color ASCII markers.
If the given color is a name that exists in available colors,
a 8-colors mode is assumed, else, a 256-colors mode.
The given style must exists in the available styles.
>>> colorin("Fetchez la vache", "red", "bold")
'\x1b[1;31mFetchez la vache\x1b[0m'
>>> colout.colorin("Faites chier la vache", 41, "normal")
'\x1b[0;38;5;41mFaites chier la vache\x1b[0m'
"""
assert (type(color) is str)
global colormap_idx
global debug
# Special characters.
start = "\033["
stop = "\033[0m"
color_code = ""
style_code = ""
# Convert the style code
if style == "random" or style == "Random":
style = random.choice(list(styles.keys()))
else:
if style in styles:
style_code = str(styles[style])
color = color.strip()
if color == "none":
# if no color, style cannot be applied
if not debug:
return text
else:
return "<none>" + text + "</none>"
elif color == "random":
mode = 8
color_code = random.choice(list(colors.values()))
color_code = str(30 + color_code)
elif color == "Random":
mode = 256
color_nb = random.randint(0, 255)
color_code = str(color_nb)
elif color in colormaps.keys():
if color[0].islower(): # lower case first letter
mode = 8
c = colormaps[color][colormap_idx]
if c.isdigit():
color_code = str(30 + c)
else:
color_code = str(30 + colors[c])
else: # upper case
mode = 256
color_nb = colormaps[color][colormap_idx]
color_code = str(color_nb)
if colormap_idx < len(colormaps[color]) - 1:
colormap_idx += 1
else:
colormap_idx = 0
elif color.lower() == "scale": # "scale" or "Scale"
# filter out everything that does not seem to be necessary to interpret the string as a number
# this permits to transform "[ 95%]" to "95" before number conversion,
# and thus allows to color a group larger than the matched number
chars_in_numbers = "-+.,e/*"
allowed = string.digits + chars_in_numbers
nb = "".join([i for i in filter(allowed.__contains__, text)])
# interpret as decimal
# First, try with the babel module, if available
# if not, use python itself,
# if thoses fails, try to `eval` the string
# (this allow strings like "1/2+0.9*2")
try:
# babel is a specialized module
import babel.numbers as bn
try:
f = float(bn.parse_decimal(nb))
except NumberFormatError:
f = eval(
nb
) # Note: in python2, `eval(2/3)` would produce `0`, in python3 `0.666`
except ImportError:
try:
f = float(nb)
except ValueError:
f = eval(nb)
# if out of scale, do not color
if f < scale[0] or f > scale[1]:
return text
if color[0].islower():
mode = 8
# Use the default colormap in lower case = 8-colors mode
cmap = colormap
# normalize and scale over the nb of colors in cmap
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1)))
color = cmap[i]
color_code = str(30 + colors[color])
else:
mode = 256
cmap = colormap
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1)))
color = cmap[i]
color_code = str(color)
# "hash" or "Hash"; useful to randomly but consistently color strings
elif color.lower() == "hash":
hasher = hashlib.md5()
hasher.update(text.encode('utf-8'))
hash = hasher.hexdigest()
f = float(functools.reduce(lambda x, y: x + ord(y), hash, 0) % 101)
if color[0].islower():
mode = 8
cmap = colormap
# normalize and scale over the nb of colors in cmap
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1)))
color = cmap[i]
color_code = str(30 + colors[color])
else:
mode = 256
cmap = colormap
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1)))
color = cmap[i]
color_code = str(color)
# Really useful only when using colout as a library
# thus you can change the "colormap" variable to your favorite one before calling colorin
elif color == "colormap":
color = colormap[colormap_idx]
if color in colors:
mode = 8
color_code = str(30 + colors[color])
else:
mode = 256
color_nb = int(color)
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
if colormap_idx < len(colormap) - 1:
colormap_idx += 1
else:
colormap_idx = 0
# 8 colors modes
elif color in colors:
mode = 8
color_code = str(30 + colors[color])
# hexadecimal color
elif color[0] == "#":
mode = 256
color_nb = rgb_to_ansi(*hex_to_rgb(color))
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
# 256 colors mode
elif color.isdigit():
mode = 256
color_nb = int(color)
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
# programming language
elif color.lower() in lexers:
lexer = get_lexer_by_name(color.lower())
# Python => 256 colors, python => 8 colors
ask_256 = color[0].isupper()
if ask_256:
try:
formatter = Terminal256Formatter(style=style)
except: # style not found
formatter = Terminal256Formatter()
else:
if style not in ("light", "dark"):
style = "dark" # dark color scheme by default
formatter = TerminalFormatter(bg=style)
# We should return all but the last character,
# because Pygments adds a newline char.
if not debug:
return highlight(text, lexer, formatter)[:-1]
else:
return "<" + color + ">" + highlight(
text, lexer, formatter)[:-1] + "</" + color + ">"
# unrecognized
else:
raise UnknownColor(color)
if not debug:
return start + style_code + endmarks[
mode] + color_code + "m" + text + stop
else:
return start + style_code + endmarks[mode] + color_code + "m<" + str(
color) + ">" + text + "</" + str(color) + ">" + stop
def execute(self,
table_id,
select_index,
aggregation_index,
conditions,
condition_relation,
lower=False):
if not table_id.startswith('Table'):
table_id = 'Table_{}'.format(table_id.replace('-', '_'))
wr = ""
if condition_relation == 1 or condition_relation == 0:
wr = " AND "
elif condition_relation == 2:
wr = " OR "
table_info = self.db.query(
'SELECT sql from sqlite_master WHERE tbl_name = :name',
name=table_id).all()[0].sql
schema_str = schema_re.findall(table_info)[0]
schema = {}
for tup in schema_str.split(','):
c, t = tup.split(' ')
schema[c] = t
tmp = ""
for sel, agg in zip(select_index, aggregation_index):
select_str = 'col_{}'.format(sel + 1)
agg_str = agg_dict[agg]
if agg:
tmp += '{}({}),'.format(agg_str, select_str)
else:
tmp += '({}),'.format(select_str)
tmp = tmp[:-1]
where_clause = []
where_map = {}
for col_index, op, val in conditions:
if lower and isinstance(val, str):
val = val.lower()
if schema['col_{}'.format(col_index +
1)] == 'real' and not isinstance(
val, (int, float)):
try:
val = float(parse_decimal(val, locale='en_US'))
except NumberFormatError as e:
try:
val = float(num_re.findall(
val)[0]) # need to understand and debug this part.
except:
# Although column is of number, selected one is not number. Do nothing in this case.
pass
where_clause.append('col_{} {} :col_{}'.format(
col_index + 1, cond_op_dict[op], col_index + 1))
where_map['col_{}'.format(col_index + 1)] = val
where_str = ''
if where_clause:
where_str = 'WHERE ' + wr.join(where_clause)
query = 'SELECT {} FROM {} {}'.format(tmp, table_id, where_str)
#print(query)
#print(where_map)
out = self.db.query(query, **where_map)
return out.as_dict()
def parse_decimal(self, string):
"""Parse localized decimal string into a float
"""
return numbers.parse_decimal(string, self)
def asGehalt(vStr):
v = parse_decimal(vStr, locale="de")
if v < 0: raise ValueError
return dec(v)
def parse_decimal_or_400(s, *a):
try:
return parse_decimal(s, *a, locale=loc)
except (InvalidOperation, NumberFormatError, ValueError):
raise InvalidNumber(s)
def asPerc(vStr):
v = parse_decimal(vStr, locale="de")
if v > 100 or 0 > v: raise ValueError
return v
def parse_dollars(cls, s: str) -> Decimal:
assert s.startswith("$")
return parse_decimal(s.strip("$"), locale='en_US')
def execute_return_query_new(self, data, table_header, lower=True):
table_id = data['table_id']
select_index = data['sql']['sel']
aggregation_index = data['sql']['agg']
conditions = data['sql']['conds']
#table name conversion
if not table_id.startswith('table'):
table_id = 'table_{}'.format(table_id.replace('-', '_'))
# this gets the creat table comment
table_info = self.conn.execute(
"SELECT sql from sqlite_master WHERE tbl_name = '{}'".format(
table_id)).fetchall()[0]
schema_str = schema_re.findall(table_info[0])[0]
schema = {}
for tup in schema_str.split(', '):
c, t = tup.split()
schema[c] = t
#get execute select comment
select = 'col{}'.format(select_index)
#human readable query
select_real_name = '[{}]'.format(table_header[select_index])
agg = agg_ops[aggregation_index]
if agg:
select = '{}({})'.format(agg, select)
select_real_name = '{}({})'.format(agg, select_real_name)
where_clause = []
where_clause_real_name = []
for col_index, op, val in conditions:
if lower and (isinstance(val, str) or isinstance(val, str)):
val = val.lower()
if schema['col{}'.format(col_index)] == 'real' and not isinstance(
val, (int, float)):
try:
# print('!!!!!!value of val is: ', val, 'type is: ', type(val))
# val = float(parse_decimal(val)) # somehow it generates error.
val = float(parse_decimal(val, locale='en_US'))
# print('!!!!!!After: val', val)
except NumberFormatError as e:
try:
val = float(num_re.findall(
val)[0]) # need to understand and debug this part.
except:
# Although column is of number, selected one is not number. Do nothing in this case.
pass
where_clause.append("col{} {} '{}'".format(
col_index, cond_ops[op],
str(val).replace("'", "''").replace("\\", "\\\\")))
where_clause_real_name.append("[{}] {} '{}'".format(
table_header[col_index], cond_ops[op], val))
where_str = ''
if where_clause:
where_str = 'WHERE ' + ' AND '.join(where_clause)
where_str_real_name = 'WHERE ' + ' AND '.join(
where_clause_real_name)
query = 'SELECT {} AS result FROM {} {}'.format(
select, table_id, where_str)
query_real_name = 'SELECT {} AS result FROM {} {}'.format(
select_real_name, table_id, where_str_real_name)
#print query
#print(query)
out = self.conn.execute(query)
return data['question'], query_real_name, query, out.fetchall()
od_obj['contract_start_dt'] = contract_start_dt.strftime(
'%Y-%m-%dT00:00:00Z')
od_obj['contract_start_s'] = gc['contract_period_start']
if gc['delivery_date'] != "":
delivery_dt: datetime = datetime.strptime(
gc['delivery_date'], '%Y-%m-%d')
od_obj['contract_delivery_dt'] = delivery_dt.strftime(
'%Y-%m-%dT00:00:00Z')
od_obj['contract_delivery_s'] = gc['delivery_date']
else:
od_obj['contract_delivery_s'] = "-"
if gc['contract_value'] != "":
contract_value = parse_decimal(gc['contract_value'].replace(
'$', '').replace(',', ''),
locale='en')
od_obj['contract_value_f'] = contract_value
od_obj['contract_value_en_s'] = format_currency(contract_value,
'CAD',
locale='en_CA')
od_obj['contract_value_fr_s'] = format_currency(contract_value,
'CAD',
locale='fr_CA')
else:
od_obj['contract_value_en_s'] = "-"
od_obj['contract_value_fr_s'] = "-"
if gc['original_value'] != "":
original_value = parse_decimal(gc['original_value'].replace(
'$', '').replace(',', ''),
print("Archivo: " + file)
dataset['Win Time'] = dataset['Win Time'].apply(lambda x: x[0:10])
dataset['Win Time'] = dataset['Win Time'].apply(
lambda x: dt.datetime.strptime(x, dateformat))
dataset['EGM'] = dataset['EGM'].apply(lambda x: re.sub(
r"[_]|[ ]|server|SERVER", '', x)).apply(lambda x: str.upper(x))
dataset.loc[:, ['Bonus Win Amount'
]] = dataset.loc[:, ['Bonus Win Amount']].replace(
to_replace="[%]", value="",
regex=True).replace(to_replace='[nan]',
value='0',
regex=True)
try:
dataset.loc[:, ['Bonus Win Amount'
]] = dataset.loc[:, ['Bonus Win Amount']].applymap(
lambda x: numbers.parse_decimal(
x, 'en_US')).astype(float).round(2)
except numbers.NumberFormatError as err:
print('Error en parseo de número, intentando otro formato', err)
dataset.loc[:,
['Bonus Win Amount'
]] = dataset.loc[:, ['Bonus Win Amount']].applymap(
lambda x: numbers.parse_decimal(x, 'de')).astype(
float).round(2)
dataset.rename(index=str, columns={'Win Time': 'Date'}, inplace=True)
dataframeOuptut = dataframeOuptut.append(dataset, sort=False)
print("Archivo adjuntado.")
dataset = pd.read_csv(join(path, file), sep=";", encoding='utf-8')
os.remove(join(path, file))
dataset.to_csv(join(pathP, file),
index=False,
header=False,
def handle(self, *args, **options):
total = 0
cycle = 0
try:
# Retrieve the Search and Field models from the database
solr = SolrClient(settings.SOLR_SERVER_URL)
try:
self.search_target = Search.objects.get(
search_id=options['search'])
self.solr_core = self.search_target.solr_core_name
self.all_fields = Field.objects.filter(
search_id=self.search_target)
if options['nothing_to_report']:
self.search_fields = Field.objects.filter(
search_id=self.search_target,
alt_format='ALL') | Field.objects.filter(
search_id=self.search_target, alt_format='NTR')
else:
self.search_fields = Field.objects.filter(
search_id=self.search_target,
alt_format='ALL') | Field.objects.filter(
search_id=self.search_target, alt_format='')
for search_field in self.search_fields:
self.csv_fields[search_field.field_id] = search_field
codes = Code.objects.filter(field_id=search_field)
# Most csv_fields will not have codes, so the queryset will be zero length
if len(codes) > 0:
code_dict = {}
for code in codes:
code_dict[code.code_id.lower()] = code
self.field_codes[search_field.field_id] = code_dict
except Search.DoesNotExist as x:
self.logger.error('Search not found: "{0}"'.format(x))
exit(-1)
except Field.DoesNotExist as x1:
self.logger.error(
'Fields not found for search: "{0}"'.format(x1))
# Process the records in the CSV file one at a time
with open(options['csv'],
'r',
encoding='utf-8-sig',
errors="ignore") as csv_file:
csv_reader = csv.DictReader(csv_file, dialect='excel')
solr_items = []
for csv_record in csv_reader:
# Clear out the Solr core. on the first line
if total == 0 and not options['nothing_to_report']:
solr.delete_doc_by_query(self.solr_core, "*:*")
print("Purging all records")
elif total == 0 and options['nothing_to_report']:
solr.delete_doc_by_query(self.solr_core, "format:NTR")
solr.commit(self.solr_core, softCommit=True)
print("Purging NTR records")
total += 1
cycle += 1
# Call plugins if they exist for this search type. This is where a developer can introduce
# code to customize the data that is loaded into Solr for a particular search.
search_type_plugin = 'search.plugins.{0}'.format(
options['search'])
if search_type_plugin in self.discovered_plugins:
include, filtered_record = self.discovered_plugins[
search_type_plugin].filter_csv_record(
csv_record, self.search_target,
self.csv_fields, self.field_codes,
'NTR' if options['nothing_to_report'] else '')
if not include:
continue
else:
csv_record = filtered_record
# Create a dictionary for each record loaded into Solr
solr_record = {
'format':
'NTR' if options['nothing_to_report'] else 'DEFAULT'
}
for csv_field in csv_reader.fieldnames:
# Verify that it is a known field
if csv_field not in self.csv_fields and csv_field not in (
'owner_org_title', 'owner_org'):
self.logger.error(
"CSV files contains unknown field: {0}".format(
csv_field))
exit(-1)
if csv_field == 'owner_org_title':
continue
# Handle multi-valued fields here
if self.csv_fields[csv_field].solr_field_multivalued:
solr_record[csv_field] = csv_record[
csv_field].split(',')
# Copy fields fo report cannot use multi-values - so directly populate with original string
if self.csv_fields[csv_field].solr_field_export:
for extra_field in self.csv_fields[
csv_field].solr_field_export.split(
','):
solr_record[extra_field] = csv_record[
csv_field]
else:
solr_record[csv_field] = csv_record[csv_field]
# Automatically expand out dates and numbers for use with Solr export handler
if self.csv_fields[
csv_field].solr_field_type == 'pdate':
try:
if csv_record[csv_field]:
csv_date = datetime.strptime(
csv_record[csv_field], '%Y-%m-%d')
solr_record[csv_field +
'_en'] = format_date(
csv_date, locale='en')
solr_record[csv_field +
'_fr'] = format_date(
csv_date, locale='fr')
if self.csv_fields[
csv_field].is_default_year:
solr_record['year'] = csv_date.year
if self.csv_fields[
csv_field].is_default_month:
solr_record['month'] = csv_date.month
else:
solr_record[csv_field + '_en'] = ''
solr_record[csv_field + '_fr'] = ''
except ValueError as x2:
self.logger.error(
'Invalid date: "{0}"'.format(x2))
solr_record[csv_field] = ''
continue
elif self.csv_fields[csv_field].solr_field_type in [
'pint', 'pfloat'
]:
if solr_record[csv_field]:
if solr_record[csv_field] == '.':
solr_record[csv_field] = "0"
csv_decimal = parse_decimal(
solr_record[csv_field], locale='en_US')
if self.csv_fields[
csv_field].solr_field_is_currency:
solr_record[csv_field +
'_en'] = format_currency(
csv_decimal,
'CAD',
locale='en_CA')
solr_record[csv_field +
'_fr'] = format_currency(
csv_decimal,
'CAD',
locale='fr_CA')
else:
solr_record[csv_field +
'_en'] = format_decimal(
csv_decimal,
locale='en_CA')
solr_record[csv_field +
'_fr'] = format_decimal(
csv_decimal,
locale='fr_CA')
else:
solr_record[csv_field + '_en'] = ''
solr_record[csv_field + '_fr'] = ''
# Lookup the expanded code value from the codes dict of dict
if csv_field in self.field_codes:
if csv_record[csv_field]:
if self.csv_fields[
csv_field].solr_field_multivalued:
codes_en = []
codes_fr = []
for code_value in csv_record[
csv_field].split(","):
if code_value.lower(
) in self.field_codes[csv_field]:
codes_en.append(
self.field_codes[csv_field]
[code_value.lower()].label_en)
codes_fr.append(
self.field_codes[csv_field]
[code_value.lower()].label_fr)
else:
self.logger.info(
"Unknown code value: {0} for field: {1}"
.format(code_value, csv_field))
solr_record[csv_field + '_en'] = codes_en
solr_record[csv_field + '_fr'] = codes_fr
else:
if csv_record[csv_field].lower(
) in self.field_codes[csv_field]:
solr_record[csv_field +
'_en'] = self.field_codes[
csv_field][csv_record[
csv_field].lower(
)].label_en
solr_record[csv_field +
'_fr'] = self.field_codes[
csv_field][csv_record[
csv_field].lower(
)].label_fr
else:
self.logger.info(
"Unknown code value: {0} for field: {1}"
.format(csv_record[csv_field],
csv_field))
solr_record = self.set_empty_fields(solr_record)
# Set the Solr ID field (Nothing To Report records are excluded)
if not options['nothing_to_report']:
if self.search_target.id_fields:
id_values = []
for id_field in self.search_target.id_fields.split(
","):
id_values.append(csv_record[id_field])
solr_record['id'] = ",".join(id_values)
else:
if 'month' in solr_record:
solr_record['id'] = "{0}-{1}-{2}".format(
solr_record['owner_org'], solr_record['year'],
solr_record['month'])
elif 'quarter' in solr_record:
solr_record['id'] = "{0}-{1}-{2}".format(
solr_record['owner_org'], solr_record['year'],
solr_record['quarter'])
# Call plugins if they exist for this search type. This is where a developer can introduce
# code to customize the data that is loaded into Solr for a particular search.
if search_type_plugin in self.discovered_plugins:
solr_record = self.discovered_plugins[
search_type_plugin].load_csv_record(
csv_record, solr_record, self.search_target,
self.csv_fields, self.field_codes,
'NTR' if options['nothing_to_report'] else '')
solr_items.append(solr_record)
# Write to Solr whenever the cycle threshold is reached
if cycle >= self.cycle_on:
# try to connect to Solr up to 10 times
for countdown in reversed(range(10)):
try:
solr.index(self.solr_core, solr_items)
print("{0} rows processed".format(total))
cycle = 0
solr_items.clear()
break
except ConnectionError as cex:
if not countdown:
raise
print(
"Solr error: {0}. Waiting to try again ... {1}"
.format(cex, countdown))
time.sleep((10 - countdown) * 5)
# Write and remaining records to Solr and commit
if cycle > 0:
# try to connect to Solr up to 10 times
for countdown in reversed(range(10)):
try:
solr.index(self.solr_core, solr_items)
total += len(solr_items)
print("{0} rows processed".format(cycle))
cycle = 0
solr_items.clear()
break
except ConnectionError as cex:
if not countdown:
raise
print(
"Solr error: {0}. Waiting to try again ... {1}"
.format(cex, countdown))
time.sleep((10 - countdown) * 5)
solr.commit(self.solr_core, softCommit=True, waitSearcher=True)
print("Total rows processed: {0}".format(total))
except Exception as x:
self.logger.error('Unexpected Error "{0}"'.format(x))
def _standardise_data(self, data):
""" split accounting lines where needed
Crésus writes one csv line per move when there are just two lines
(take some money from one account and put all of it in another),
and uses ellipses in more complex cases. What matters is the pce
label, which is the same on all lines of a move.
"""
journal_id = self.journal_id.id
previous_pce = None
previous_date = None
previous_tax_id = None
lines = []
for self.index, line_cresus in enumerate(data, 1):
if previous_pce is not None and previous_pce != line_cresus['pce']:
yield self.prepare_move(lines,
date=previous_date,
ref=previous_pce,
journal_id=journal_id)
lines = []
previous_pce = line_cresus['pce']
previous_date = line_cresus['date']
from babel.numbers import parse_decimal
recto_amount = float(
parse_decimal(line_cresus['amount'], locale='de_CH'))
verso_amount = 0.0
if recto_amount < 0:
recto_amount, verso_amount = 0.0, -recto_amount
tax_ids = [previous_tax_id] if previous_tax_id is not None else []
previous_tax_id = None
if line_cresus['debit'] != '...':
line = self.prepare_line(
name=line_cresus['ref'],
debit_amount=recto_amount,
credit_amount=verso_amount,
account_code=line_cresus['debit'],
cresus_tax_code=line_cresus['typtvat'],
analytic_account_code=line_cresus['analytic_account'],
tax_ids=tax_ids)
lines.append(line)
if 'tax_line_id' in line:
previous_tax_id = line['tax_line_id']
if line_cresus['credit'] != '...':
line = self.prepare_line(
name=line_cresus['ref'],
debit_amount=verso_amount,
credit_amount=recto_amount,
account_code=line_cresus['credit'],
cresus_tax_code=line_cresus['typtvat'],
analytic_account_code=line_cresus['analytic_account'],
tax_ids=tax_ids)
lines.append(line)
if 'tax_line_id' in line:
previous_tax_id = line['tax_line_id']
yield self.prepare_move(lines,
date=line_cresus['date'],
ref=previous_pce,
journal_id=journal_id)
def execute(self,
table_id,
select_index,
aggregation_index,
conditions,
lower=True):
if not table_id.startswith('Table'):
table_id = 'Table_{}'.format(table_id.replace('-', '_'))
# add
# conn = records.Database('sqlite:///{}'.format(self.fdb)).get_connection()
# add
#table_info = self.db.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql.replace('\n','')
#qu = self.db.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id)
#qu = self.db.query("SELECT sql from sqlite_master WHERE tbl_name = 'Table_43ad6bdc1d7111e988a6f40f24344a08'")
#qu = self.db.query("SELECT sql from sqlite_master WHERE tbl_name = 'table_f65ceb4f453d11e9b16ef40f24344a08'")
qu = self.db.query(
"SELECT sql from sqlite_master WHERE tbl_name = :name",
name=table_id)
qu_2 = qu.all()
qu_3 = qu_2[0]
table_info = qu_3.sql.replace('\n', '')
#table_info = conn.query('SELECT sql from sqlite_master WHERE tbl_name = :name', name=table_id).all()[0].sql.replace('\n', '')
schema_str = schema_re.findall(table_info)[0] #获取表中每个字段的名字和其对应的类型
schema = {}
for tup in schema_str.split(','):
c, t = tup.split()
#这里对c进行处理，col_1=>col_0
#c = degrad_c(c)
schema[c] = t
select = 'col_{}'.format(select_index + 1)
agg = agg_ops[aggregation_index]
if agg:
select = '{}({})'.format(agg, select)
where_clause = []
where_map = {}
for col_index, op, val in conditions:
# if lower and (isinstance(val, str) or isinstance(val, str)):
# val = val.lower()
#if schema['col{}'.format(col_index)] == 'real' and not isinstance(val, (int, float)):
if schema['col_{}'.format(
col_index + 1)] == 'real' and not isinstance(
val, (int, float)): #如果数据列是real类型，且wv不是数字类型
try:
# print('!!!!!!value of val is: ', val, 'type is: ', type(val))
# val = float(parse_decimal(val)) # somehow it generates error.
val = float(parse_decimal(val, locale='en_US'))
# print('!!!!!!After: val', val)
except NumberFormatError as e:
try:
val = float(num_re.findall(
val)[0]) # need to understand and debug this part.
except:
# Although column is of number, selected one is not number. Do nothing in this case.
pass
where_clause.append('col_{} {} :col_{}'.format(
col_index + 1, cond_ops[op], col_index + 1))
where_map['col_{}'.format(col_index + 1)] = val
where_str = ''
if where_clause:
where_str = 'WHERE ' + ' AND '.join(where_clause)
query = 'SELECT {} AS result FROM {} {}'.format(
select, table_id, where_str)
#print query
out = self.db.query(query, **where_map)
#out = conn.query(query, **where_map)
return [o.result for o in out]
def process_formdata(self, valuelist):
if valuelist:
try:
self.data = parse_decimal(str(valuelist[0]), locale=LOCALE)
except (decimal.InvalidOperation, ValueError):
raise ValueError(self.gettext(u'Not a valid decimal value'))
def test_parse_decimal_strict_mode(self):
# Numbers with a misplaced grouping symbol should be rejected
with self.assertRaises(numbers.NumberFormatError) as info:
numbers.parse_decimal('11.11', locale='de', strict=True)
assert info.exception.suggestions == ['1.111', '11,11']
# Numbers with two misplaced grouping symbols should be rejected
with self.assertRaises(numbers.NumberFormatError) as info:
numbers.parse_decimal('80.00.00', locale='de', strict=True)
assert info.exception.suggestions == ['800.000']
# Partially grouped numbers should be rejected
with self.assertRaises(numbers.NumberFormatError) as info:
numbers.parse_decimal('2000,000', locale='en_US', strict=True)
assert info.exception.suggestions == ['2,000,000', '2,000']
# Numbers with duplicate grouping symbols should be rejected
with self.assertRaises(numbers.NumberFormatError) as info:
numbers.parse_decimal('0,,000', locale='en_US', strict=True)
assert info.exception.suggestions == ['0']
# Return only suggestion for 0 on strict
with self.assertRaises(numbers.NumberFormatError) as info:
numbers.parse_decimal('0.00', locale='de', strict=True)
assert info.exception.suggestions == ['0']
# Properly formatted numbers should be accepted
assert str(numbers.parse_decimal('1.001', locale='de',
strict=True)) == '1001'
# Trailing zeroes should be accepted
assert str(numbers.parse_decimal('3.00', locale='en_US',
strict=True)) == '3.00'
# Numbers without any grouping symbol should be accepted
assert str(numbers.parse_decimal('2000.1', locale='en_US',
strict=True)) == '2000.1'
# High precision numbers should be accepted
assert str(numbers.parse_decimal('5,000001', locale='fr',
strict=True)) == '5.000001'
def execute(self,
table_id,
select_index,
aggregation_index,
conditions,
lower=True):
if not table_id.startswith('table'):
table_id = 'table_{}'.format(table_id.replace('-', '_'))
table_info = self.db.query(
'SELECT sql from sqlite_master WHERE tbl_name = :name',
name=table_id).all()[0].sql.replace('\n', '')
schema_str = schema_re.findall(table_info)[0]
schema = {}
for tup in schema_str.split(', '):
c, t = tup.split()
schema[c] = t
if select_index == "*":
select = "*"
else:
select = 'col{}'.format(select_index)
agg = agg_ops[aggregation_index]
if agg:
select = '{}({})'.format(agg, select)
where_clause = []
where_map = {}
for col_index, op, val in conditions:
if lower and isinstance(val, str):
val = val.lower()
if schema['col{}'.format(col_index)] == 'real' and not isinstance(
val, (int, float)):
try:
# print('!!!!!!value of val is: ', val, 'type is: ', type(val))
# val = float(parse_decimal(val)) # somehow it generates error.
val = float(parse_decimal(val, locale='en_US'))
# print('!!!!!!After: val', val)
except NumberFormatError as e:
try:
val = val # float(num_re.findall(val)[0]) # need to understand and debug this part.
except:
# Although column is of number, selected one is not number. Do nothing in this case.
pass
if isinstance(val, float):
val = '%g' % val
where_clause.append('col{} {} :col{}'.format(
col_index, cond_ops[op], col_index))
where_map['col{}'.format(col_index)] = val
where_str = ''
if where_clause:
where_str = 'WHERE ' + ' AND '.join(where_clause)
if select_index == "*":
query = 'SELECT {} FROM {} {}'.format(select, table_id, where_str)
else:
query = 'SELECT {} AS result FROM {} {}'.format(
select, table_id, where_str)
try:
out = self.db.query(query, **where_map)
except:
return []
if out.dataset.headers == None: # no data result
return []
result = []
if len(out.dataset.headers) > 1 and len(
where_clause) != 0: # rows of result
for row in out.dataset:
for cell in row:
result.append(cell)
return result
if len(out.dataset.headers) == 1: # one column of result
for o in out:
result.append(o.result)
return result
return result
def parselocal_float(txt, locale):
"""TODO
:param txt: TODO
:param locale: the current locale (e.g: en, en_us, it)"""
return numbers.parse_decimal(txt, locale)
def parse_percent(cls, s: str) -> Decimal:
assert s.endswith("%")
return parse_decimal(s.strip("%"), locale='en_US') / 100
def __init__(self, textPage, datosSM=None):
self.timestamp = gmtime()
self.source = textPage['source']
self.PositionsCounter = defaultdict(int)
self.PlayerData = {}
self.PlayerByPos = defaultdict(list)
self.Team2Player = defaultdict(set)
self.noKiaLink = []
if (type(textPage['data']) is str):
soup = BeautifulSoup(textPage['data'], "html.parser")
elif (type(textPage['data']) is BeautifulSoup):
soup = textPage['data']
else:
raise NotImplementedError(
"MercadoPageContent: type of content '%s' not supported" %
type(textPage['data']))
positions = soup.find_all("table", {"class": "listajugadores"})
for pos in positions:
position = pos['id']
for player in pos.find_all("tr"):
player_data = player.find_all("td")
if not player_data:
continue
fieldTrads = {
'foto': ['foto'],
'jugador': ['jugador'],
'equipo': ['equipo'],
'promedio': ['promVal', 'valJornada', 'seMantiene'],
'precio': ['precio', 'enEquipos%'],
'val': ['prom3Jornadas'],
'balance': ['sube15%'],
'baja': ['baja15%'],
'rival': ['rival'],
'iconos': ['iconos']
}
result = {
'proxFuera': False,
'lesion': False,
'cupo': 'normal'
}
result['pos'] = position
self.PositionsCounter[position] += 1
for data in player_data:
classes = data.attrs['class']
# print("AQUI",data)
if 'iconos' in classes:
for icon in data.find_all("img"):
if icon['title'] == "Extracomunitario":
result['cupo'] = 'Extracomunitario'
elif icon['title'] == "Español":
result['cupo'] = "Español"
elif icon['title'] == "Lesionado":
result['lesion'] = True
elif icon['alt'] == "Icono de más información":
result['info'] = icon['title']
else:
print("No debería llegar aquí: ", icon)
continue
elif "foto" in classes:
img_link = data.img['src']
result['foto'] = img_link
result['nombre'] = data.img['title']
# if result['nombre'] in datosSM.traducciones['jugadores']['j2c']:
# tradsPosibles = datosSM.traducciones['jugadores']['j2c']
# elif traductor:
# nombreRetoc = RetocaNombreJugador(result['nombre'])
# tradsPosibles = traductor.BuscaTraduccion(nombreRetoc, 0)
#
# if isinstance(tradsPosibles, str): # Solo un codigo!
# result['codJugador'] = tradsPosibles
# else: # Set or None
# print("Troublesome player", result['nombre'], result)
# result['posibles'] = tradsPosibles
continue
elif 'jugador' in classes:
if data.a:
result['kiaLink'] = data.a['href']
# if traductor is None:
# result['codJugador'] = result['kiaLink']
continue
elif 'equipo' in classes:
result['equipo'] = data.img['title']
if result['equipo'] in datosSM.traducciones['equipos'][
'n2c']:
result['CODequipo'] = sorted(
datosSM.traducciones['equipos']['n2c'][
result['equipo']])[0]
auxId = datosSM.traducciones['equipos']['c2i'].get(
result['CODequipo'], None)
result['IDequipo'] = onlySetElement(auxId)
continue
elif 'rival' in classes:
for icon in data.find_all('img'):
if icon['title'].lower() == "partido fuera":
result['proxFuera'] = True
else:
result['rival'] = icon['title']
if result['rival'] in datosSM.traducciones[
'equipos']['n2c']:
result['CODrival'] = \
sorted(datosSM.traducciones['equipos']['n2c'][result['rival']])[0]
auxId = datosSM.traducciones['equipos']['c2i'][
result['CODrival']]
result['IDrival'] = onlySetElement(auxId)
continue
else:
auxval = data.get_text().strip()
classOrig = classes[0]
for auxClass in fieldTrads[classOrig]:
if auxClass not in result:
classCel = auxClass
break
if '%' in auxval and classCel == 'enEquipos%':
auxval = auxval.replace("%", "")
result[classCel] = parse_decimal(auxval, locale="de")
try:
codJugador = result['kiaLink']
except KeyError:
print(
"Problemas con jugador '%s' (%s): no tiene kiaLink (?)"
% (result.get('nombre', "Nombre desc"),
result.get('equipo', "Equipo desc")))
self.noKiaLink.append(result)
continue
self.PlayerData[codJugador] = result
self.PlayerByPos[position].append(codJugador)
self.Team2Player[result['equipo']].add(codJugador)
def colorin(text, color="red", style="normal"):
"""
Return the given text, surrounded by the given color ASCII markers.
If the given color is a name that exists in available colors,
a 8-colors mode is assumed, else, a 256-colors mode.
The given style must exists in the available styles.
>>> colorin("Fetchez la vache", "red", "bold")
'\x1b[1;31mFetchez la vache\x1b[0m'
>>> colout.colorin("Faites chier la vache", 41, "normal")
'\x1b[0;38;5;41mFaites chier la vache\x1b[0m'
"""
global colormap_idx
# Special characters.
start = "\033["
stop = "\033[0m"
color_code = ""
style_code = ""
# Convert the style code
if style == "random" or style == "Random":
style = random.choice(list(styles.keys()))
else:
if style in styles:
style_code = str(styles[style])
if color == "none":
# if no color, style cannot be applied
return text
elif color == "random":
mode = 8
color_code = random.choice(list(colors.values()))
color_code = str(30 + color_code)
elif color == "Random":
mode = 256
color_nb = random.randint(0, 255)
color_code = str(color_nb)
elif color == "rainbow":
mode = 8
color = colormap[colormap_idx]
color_code = str(30 + colors[color])
if colormap_idx < len(colormap) - 1:
colormap_idx += 1
else:
colormap_idx = 0
elif color == "Rainbow":
mode = 256
color_nb = rgb_to_ansi(*rgb_rainbow(colormap_idx))
color_code = str(color_nb)
if colormap_idx < 255:
colormap_idx += 1
else:
colormap_idx = 0
elif color == "scale":
try:
import babel.numbers as bn
f = float(bn.parse_decimal(text))
except ImportError:
f = float(text)
# if out of scale, do not color
if f < scale[0] or f > scale[1]:
return text
# normalize and scale over the nb of colors in colormap
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(colormap) - 1)))
mode = 8
color = colormap[i]
color_code = str(30 + colors[color])
elif color == "colormap":
color = colormap[colormap_idx]
if color in colors:
mode = 8
color_code = str(30 + colors[color])
else:
mode = 256
color_nb = int(color)
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
if colormap_idx < len(colormap) - 1:
colormap_idx += 1
else:
colormap_idx = 0
# 8 colors modes
elif color in colors:
mode = 8
color_code = str(30 + colors[color])
# 256 colors mode
elif color.isdigit():
mode = 256
color_nb = int(color)
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
# programming language
elif color.lower() in lexers:
lexer = get_lexer_by_name(color.lower())
# Python => 256 colors, python => 8 colors
ask_256 = color[0].isupper()
if ask_256:
try:
formatter = Terminal256Formatter(style=style)
except: # style not found
formatter = Terminal256Formatter()
else:
if style not in ("light", "dark"):
style = "dark" # dark color scheme by default
formatter = TerminalFormatter(bg=style)
# We should return all but the last character,
# because Pygments adds a newline char.
return highlight(text, lexer, formatter)[:-1]
# unrecognized
else:
raise Exception('Unrecognized color %s' % color)
return start + style_code + endmarks[mode] + color_code + "m" + text + stop
def scan_search_alternate(self):
log = []
browser = self.browser
wait = self.wait
presence_of = ec.presence_of_element_located
visibility_of = ec.visibility_of_element_located
wait = WebDriverWait(browser, timed_out)
# Procedure:
# 1. Find the product grid: "neemu-products-container"
# 2. Get the grid products: "nm-product-item"
# 3. Get product(s) prices and information
# * Price: "nm-price-container"
# * Info: "nm-product-name"
# 4. Return info
try:
# Check it was found something or not and wait
not_found = browser.find_elements_by_class_name(
"nm-not-found-container")
if len(not_found) > 0:
raise "I cannot find the product requested"
# Wait to "memu-products-container" to load
wait.until(presence_of((By.CLASS_NAME, "productShowCaseContent")))
# Get grid
grid = browser.find_elements_by_class_name(
"productShowCaseContent")
# Get grid products
for i in range(10):
products = grid[0].find_elements_by_class_name("product-li")
if len(products) == 0:
time.sleep(1)
else:
break
# Scan products
time.sleep(3)
out = []
for p in products:
# Get price
# Occurs when it has discount
price = p.find_elements_by_class_name("price-value")
if len(price) == 0:
price = p.find_elements_by_class_name("price")
# Occurs when it does not have discount
if len(price) == 0:
continue
price = price[0].text
# Get info
title = p.find_element_by_class_name("productTitle").text
price = clean_price(price)
info = title
out += [{"info": info}]
out[-1]["price"] = float(
numbers.parse_decimal(price, locale="pt_BR"))
success = True if len(out) > 0 else False
except Exception as e:
print("Fail to find the products container!")
print(e)
out = []
success = False
return out, success
def colorin(text, color="red", style="normal"):
"""
Return the given text, surrounded by the given color ASCII markers.
If the given color is a name that exists in available colors,
a 8-colors mode is assumed, else, a 256-colors mode.
The given style must exists in the available styles.
>>> colorin("Fetchez la vache", "red", "bold")
'\x1b[1;31mFetchez la vache\x1b[0m'
>>> colout.colorin("Faites chier la vache", 41, "normal")
'\x1b[0;38;5;41mFaites chier la vache\x1b[0m'
"""
assert (type(color) is str)
global colormap_idx
global debug
# Special characters.
start = "\033["
stop = "\033[0m"
color_code = ""
style_code = ""
# Convert the style code
if style == "random" or style == "Random":
style = random.choice(list(styles.keys()))
else:
if style in styles:
style_code = str(styles[style])
if color == "none":
# if no color, style cannot be applied
if not debug:
return text
else:
return "<none>" + text + "</none>"
elif color == "random":
mode = 8
color_code = random.choice(list(colors.values()))
color_code = str(30 + color_code)
elif color == "Random":
mode = 256
color_nb = random.randint(0, 255)
color_code = str(color_nb)
elif color in colormaps.keys():
if color[0].islower(): # lower case first letter
mode = 8
c = colormaps[color][colormap_idx]
if c.isdigit():
color_code = str(30 + c)
else:
color_code = str(30 + colors[c])
else: # upper case
mode = 256
color_nb = colormaps[color][colormap_idx]
color_code = str(color_nb)
if colormap_idx < len(colormaps[color]) - 1:
colormap_idx += 1
else:
colormap_idx = 0
elif color.lower() == "scale": # "scale" or "Scale"
try:
import babel.numbers as bn
f = float(bn.parse_decimal(text))
except ImportError:
f = float(text)
# if out of scale, do not color
if f < scale[0] or f > scale[1]:
return text
if color[0].islower():
mode = 8
cmap = colormaps["spectrum"]
# normalize and scale over the nb of colors in cmap
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1)))
color = cmap[i]
color_code = str(30 + colors[color])
else:
mode = 256
cmap = colormaps["Spectrum"]
i = int(
math.ceil(
(f - scale[0]) / (scale[1] - scale[0]) * (len(cmap) - 1)))
color = cmap[i]
color_code = str(color)
# Really useful only when using colout as a library
# thus you can change the "colormap" variable to your favorite one before calling colorin
elif color == "colormap":
color = colormap[colormap_idx]
if color in colors:
mode = 8
color_code = str(30 + colors[color])
else:
mode = 256
color_nb = int(color)
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
if colormap_idx < len(colormap) - 1:
colormap_idx += 1
else:
colormap_idx = 0
# 8 colors modes
elif color in colors:
mode = 8
color_code = str(30 + colors[color])
# hexadecimal color
elif color[0] == "#":
mode = 256
color_nb = rgb_to_ansi(*hex_to_rgb(color))
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
# 256 colors mode
elif color.isdigit():
mode = 256
color_nb = int(color)
assert (0 <= color_nb <= 255)
color_code = str(color_nb)
# programming language
elif color.lower() in lexers:
lexer = get_lexer_by_name(color.lower())
# Python => 256 colors, python => 8 colors
ask_256 = color[0].isupper()
if ask_256:
try:
formatter = Terminal256Formatter(style=style)
except: # style not found
formatter = Terminal256Formatter()
else:
if style not in ("light", "dark"):
style = "dark" # dark color scheme by default
formatter = TerminalFormatter(bg=style)
# We should return all but the last character,
# because Pygments adds a newline char.
if not debug:
return highlight(text, lexer, formatter)[:-1]
else:
return "<" + color + ">" + highlight(
text, lexer, formatter)[:-1] + "</" + color + ">"
# unrecognized
else:
raise UnknownColor(color)
if not debug:
return start + style_code + endmarks[
mode] + color_code + "m" + text + stop
else:
return start + style_code + endmarks[
mode] + color_code + "m<" + color + ">" + text + "</" + color + ">" + stop
def okButton_clicked(self):
"""
OK button has been clicked - save the values
Returns:
"""
if self._selected_index is None:
# Huh?
return
model = self._item_model
stackedwidget_index = self.stackedWidget.currentIndex()
the_item = self._get_item_for_stackedwidget(self._selected_index)
new_name = nullify(self.nameLineEdit.text())
self.set_modified()
if new_name is None:
self.nameLineEdit.setText(the_item.name)
else:
# This will take care of saving the value
model.setData(self._selected_index, new_name, QtCore.Qt.EditRole)
# Trimmed name / rejected
self.nameLineEdit.setText(the_item.name)
# Items with descriptions - Author, Cuisine, Yield Units
if stackedwidget_index == self.StackedItems.ITEM_WITH_DESCRIPTION:
new_description = nullify(self.descriptionTextEdit.toPlainText())
the_item.description = new_description
elif stackedwidget_index == self.StackedItems.INGREDIENT_UNIT:
new_type = self.typeComboBox.currentIndex()
if new_type >= 0:
# new_type == -1 should never happen - it would mean no type has been selected which should be
# impossible. However, better play it safe :-)
the_item.type_ = new_type
new_factor = nullify(self.factorLineEdit.text())
# Depending on the type of the factor (whatever the user entered) should either be None or have a
# value. Unit Type GROUP isn't visible for the user so don't bother to check
if new_type != data.IngredientUnit.UnitType.UNSPECIFIC:
if new_factor is None:
self.illegal_value(misc.IllegalValueEntered.ISEMPTY,
None)
new_factor = the_item.factor
else:
try:
new_factor = parse_decimal(new_factor)
if math.isclose(new_factor, 0.0):
self.illegal_value(
misc.IllegalValueEntered.ISZERO, "0")
new_factor = the_item.factor
if new_factor < 0.0:
self.illegal_value(
misc.IllegalValueEntered.ISNONUMBER,
new_factor)
new_factor = the_item.factor
except NumberFormatError:
# The user has entered something strange.
self.illegal_value(
misc.IllegalValueEntered.ISNONUMBER,
new_factor)
new_factor = the_item.factor
else:
# Unspecific -> no Factor
new_factor = None
if new_factor is not None:
self.factorLineEdit.setText(str(new_factor))
else:
self.factorLineEdit.clear()
the_item.factor = new_factor
elif stackedwidget_index == self.StackedItems.INGREDIENTS:
the_item.icon = self._ingredient_icon
self.okButton.setEnabled(False)
self.cancelButton.setEnabled(False)
