def _retrieve_product(cls, url):
browser = mechanize.Browser()
product_data = browser.open(url).get_data()
product_soup = BeautifulSoup(product_data)
product_name = product_soup.find('td', 'tit-nar-bold')
product_name = product_name.contents[0].split('•')[0]
product_name = product_name.replace(' » ', '').strip()
prices = {}
cash_product_price = \
product_soup.find('td', {'background':
'images/ficha/bg_efectivo_d.gif'})
cash_product_price = Decimal(clean_price_string(
cash_product_price.find('a').string))
for p in ['cash', 'deposit', 'wire_transfer']:
prices[p] = cash_product_price
normal_product_price = \
product_soup.find('td', {'background':
'images/ficha/bg_precio_normal_d.gif'})
normal_product_price = Decimal(clean_price_string(
normal_product_price.find('a').string))
for p in ['debit_card', 'credit_card']:
prices[p] = normal_product_price
return [product_name, prices]
def format_satoshis(x,
is_diff=False,
num_zeros=0,
decimal_point=8,
whitespaces=False):
from decimal import Decimal
if x is None:
return 'unknown'
s = Decimal(x)
sign, digits, exp = s.as_tuple()
digits = map(str, digits)
while len(digits) < decimal_point + 1:
digits.insert(0, '0')
digits.insert(-decimal_point, '.')
s = ''.join(digits).rstrip('0')
if sign:
s = '-' + s
elif is_diff:
s = "+" + s
p = s.find('.')
s += "0" * (1 + num_zeros - (len(s) - p))
if whitespaces:
s += " " * (1 + decimal_point - (len(s) - p))
s = " " * (13 - decimal_point - (p)) + s
return s
def pretty_number_format(num, short=False):
if num == 0.:
return '0.0'
global prefixes
if len(prefixes) == 0:
logging.debug('initializing prefixes dictionary')
prefixes['-12'] = ('Trillionths', 'Trillionths')
prefixes['-9'] = ('Billionths', 'Billionths')
prefixes['-6'] = ('Millionths', 'Millionths')
prefixes['-3'] = ('Thousandths', 'Thousandths')
prefixes['0'] = ('', '')
prefixes['+3'] = ('Thousand', 'Th')
prefixes['+6'] = ('Million', 'Mi')
prefixes['+9'] = ('Billion', 'Bi')
prefixes['+12'] = ('Trillion', 'Tr')
eng_str = Decimal(num).normalize(Context(prec=4)).to_eng_string()
# return eng_str
e_pos = eng_str.find('E')
if e_pos == -1:
return eng_str
else:
postfix = eng_str[e_pos + 1:]
if short:
return eng_str[:e_pos] + ' ' + prefixes[postfix][1]
else:
return eng_str[:e_pos] + ' ' + prefixes[postfix][0]
def convert_bigdecimal_to_sqlbignum(cls,
numeric_bytes,
scale,
max_len,
precision,
is_unsigned=False):
try:
param_values = Decimal(numeric_bytes)
sign = 1 if param_values.is_signed() else 0
param_values = param_values.to_eng_string()
if sign:
param_values = param_values.lstrip('-')
except:
raise errors.DataError("decimal.ConversionSyntax")
if scale > 0:
pos_point = param_values.find('.')
if pos_point == -1:
param_values = ''.join((param_values, '0' * scale))
else:
remove_point = param_values.replace('.', '')
if pos_point + scale > len(remove_point):
param_values = ''.join(
(remove_point,
'0' * (pos_point + scale - len(remove_point))))
else:
param_values = remove_point[:pos_point + scale]
#keep precision in driver
param_values = param_values[:precision]
# iterate through 4 bytes at a time
val_len = len(param_values)
i = 0
ar = []
target_len = max_len // 2
target_list = [0] * target_len
tar_pos = 1
while i < val_len:
str_num = param_values[i:i + 4]
power = len(str_num)
num = int(str_num)
i += 4
temp = target_list[0] * 10**power + num
target_list[
0] = temp & 0xFFFF # we save only up to 16bits -- the rest gets carried over
# we do the same thing for the rest of the digits now that we have # an upper bound
for x in range(1, target_len, 1):
t = (temp & 0xFFFF0000) >> 16
temp = target_list[x] * 10**power + t
target_list[x] = temp & 0xFFFF
carry = (temp & 0xFFFF0000) >> 16
if carry > 0:
target_list[tar_pos] = carry
tar_pos += 1
return target_list, sign
def format_satoshis(x, is_diff=False, num_zeros = 0):
from decimal import Decimal
s = Decimal(x)
sign, digits, exp = s.as_tuple()
digits = map(str, digits)
while len(digits) < 9:
digits.insert(0,'0')
digits.insert(-8,'.')
s = ''.join(digits).rstrip('0')
if sign:
s = '-' + s
elif is_diff:
s = "+" + s
p = s.find('.')
s += "0"*( 1 + num_zeros - ( len(s) - p ))
s += " "*( 9 - ( len(s) - p ))
s = " "*( 5 - ( p )) + s
return s
def format_satoshis(x, is_diff=False, num_zeros=0):
from decimal import Decimal
s = Decimal(x)
sign, digits, exp = s.as_tuple()
digits = map(str, digits)
while len(digits) < 9:
digits.insert(0, '0')
digits.insert(-8, '.')
s = ''.join(digits).rstrip('0')
if sign:
s = '-' + s
elif is_diff:
s = "+" + s
p = s.find('.')
s += "0" * (1 + num_zeros - (len(s) - p))
s += " " * (9 - (len(s) - p))
s = " " * (5 - (p)) + s
return s
def format_satoshis(x, is_diff=False, num_zeros = 0, decimal_point = 8, whitespaces=False):
from decimal import Decimal
s = Decimal(x)
sign, digits, exp = s.as_tuple()
digits = map(str, digits)
while len(digits) < decimal_point + 1:
digits.insert(0,'0')
digits.insert(-decimal_point,'.')
s = ''.join(digits).rstrip('0')
if sign:
s = '-' + s
elif is_diff:
s = "+" + s
p = s.find('.')
s += "0"*( 1 + num_zeros - ( len(s) - p ))
if whitespaces:
s += " "*( 1 + decimal_point - ( len(s) - p ))
s = " "*( 13 - decimal_point - ( p )) + s
return s
def from_xml(cls, split_node, namespaces, account_objects):
"""
Creates an Split object from the GnuCash XML
:param split_node: XML node for the split
:type split_node: ElementTree.Element
:param namespaces: XML namespaces for GnuCash elements
:type namespaces: dict[str, str]
:param account_objects: Account objects already created from XML (used for assigning parent account)
:type account_objects: list[Account]
:return: Split object from XML
:rtype: Split
"""
account = split_node.find('split:account', namespaces).text
value = split_node.find('split:value', namespaces).text
value = Decimal(value[:value.find('/')]) / Decimal(100)
new_split = cls([x for x in account_objects if x.guid == account][0],
value,
split_node.find('split:reconciled-state',
namespaces).text)
new_split.guid = split_node.find('split:id', namespaces).text
split_memo = split_node.find('split:memo', namespaces)
if split_memo is not None:
new_split.memo = split_memo.text
split_action = split_node.find('split:action', namespaces)
if split_action is not None:
new_split.action = split_action.text
quantity_node = split_node.find('split:quantity', namespaces)
if quantity_node is not None:
quantity = quantity_node.text
if '/' in quantity:
new_split.quantity_denominator = quantity.split('/')[1]
return new_split
def getLoan(): #get user input for loan information and convert string as needed
print ("")
loan = input("Please enter loan amount: ")
print ("")
rate = input("Please enter interest rate: ")
print ("")
#convert user input as needed to clean-up strings before converting to Decimal
if loan.isdigit():
loan = Decimal(loan)
else:
loan = loan.replace("$", "")
loan = loan.replace(",", "")
k = loan.find("k")
if k != -1:
loan = loan.replace("k", "")
loan = (Decimal(loan))*1000
loan = Decimal(loan)
if rate.isdigit():
rate = (Decimal(rate))/100
else:
rate = rate.replace("%", "")
rate = (Decimal(rate))/100
return loan, rate
def products_for_url(cls, url, category=None, extra_args=None):
print(url)
session = session_with_proxy(extra_args)
soup = BeautifulSoup(session.get(url).text, 'html.parser')
name = soup.find('h1', 'product_name')
if not name:
return []
name = name.text.strip()
sku = soup.find('input', {'id': 'ArtId'})['value'].replace('prod', '')
availability = soup.find('span', {'itemprop': 'availability'})
if availability and availability['content'] == \
'http://schema.org/InStock':
stock = -1
else:
stock = 0
normal_price = soup.find('p', 'texto_gris_indiv')
if not normal_price:
return []
normal_price = Decimal(
normal_price.find('span').text.replace(',', '').replace(
'$', '').replace('.', ''))
price_container = soup.find('span', 'prodPlanCuo')
if not price_container:
price_container = soup.find('span', 'precio_big_indivGris')
offer_price = Decimal(
price_container.text.replace(',', '').replace('$',
'').replace('.', ''))
description = html_to_markdown(
str(soup.find('div', {'id': 'ContenedorDescripciones'})))
picture_urls = [
'https:' + tag.find('a')['data-zoom-image']
for tag in soup.findAll('div', {'id': 'imgAux'})
]
p = Product(name,
cls.__name__,
category,
url,
url,
sku,
stock,
normal_price,
offer_price,
'ARS',
sku=sku,
description=description,
picture_urls=picture_urls)
return [p]
def crawl_price(self):
self.bidPrice = self.html_obj.xpath(
"//input[@name='current_price']/@value")
self.originPrice = self.html_obj.xpath(
"//strong[@id='J_StrPrice']/em[@class='tb-rmb-num']/text()")
if not self.originPrice:
self.originPrice = self.html_obj.xpath(
"//strong[@class='J_originalPrice']/text()")
self.promoteUrl2 = get_val(self.data, "apiPromoData")
if self.promoteUrl2:
self.promoteUrl2 = self.promoteUrl2.replace(r'''\/''', "/")
price = ""
if self.is_tmall and self.tmallInitApi and self.tmallInitApijson:
try:
priceInfo = self.tmallInitApijson['defaultModel'][
'itemPriceResultDO']['priceInfo']
if priceInfo:
if priceInfo.has_key('def'):
defaultPriceInfo = priceInfo['def']
else:
defaultPriceInfo = priceInfo[priceInfo.keys()[0]]
# 2013-11-22 改为获取真实促销价格，而不是扣除佣金后的价格
if defaultPriceInfo.has_key(
'promotionList'
) and defaultPriceInfo['promotionList']:
price = defaultPriceInfo['promotionList'][0]['price']
if not price:
if defaultPriceInfo.has_key('price'):
price = defaultPriceInfo['price']
if not price:
if defaultPriceInfo.has_key('promPrice'):
price = defaultPriceInfo['promPrice']['price']
elif defaultPriceInfo.has_key(
'promotionList'
) and defaultPriceInfo['promotionList']:
price = str(
min([
float(x.get('price', '100000000.0'))
for x in defaultPriceInfo['promotionList']
]))
except:
logger.warn("Parse tmall json price failed, %s", self.item_id)
if not price:
if self.promoteUrl2:
self.promoteContent = self.crawl_page(
self.promoteUrl2).replace('"', '"')
tag = "low:"
if self.promoteContent.find(tag) > 0:
pos = self.promoteContent.find(tag) + len(tag)
pos2 = self.promoteContent.find(',', pos)
price = self.promoteContent[pos:pos2]
if not price:
price = get_num_val(self.promoteContent, 'price')
else:
self.promoteUrl = "http://marketing.taobao.com/home/promotion/item_promotion_list.do?itemId=%s" % self.num_id
self.promoteContent = self.crawl_page(self.promoteUrl)
if self.promoteContent:
self.promoteContent = self.promoteContent.replace(
'"', '"')
tag = "promPrice":""
if self.promoteContent.find(tag) > 0:
pos = self.promoteContent.find(tag) + len(tag)
pos2 = self.promoteContent.find('"', pos)
price = self.promoteContent[pos:pos2]
if not price:
tbPrice = self.html_obj.xpath("//strong[@class='tb-price']/text()")
tbPrice1 = self.html_obj.xpath("//span[@class='tb-price']/text()")
if tbPrice and not tbPrice[0].strip():
price = tbPrice[0].strip()
elif tbPrice1 and not tbPrice1[0].strip():
price = tbPrice1[0].strip()
if price.find("-") > 0:
price = price.split('-')[0].strip()
if not price:
rg_m = re.compile('price:\"[0-9]+[.][0-9]+\"', re.IGNORECASE
| re.DOTALL).search(self.dynamicStockData)
if rg_m:
price_str = rg_m.group(0).split(":")[1].replace("\"", "")
price = Decimal(price_str)
# 2013-09-03 get price url
if not price:
#这里稍微有点麻烦,主要针对string进行处理
pirce_url = "http://ajax.tbcdn.cn/json/umpStock.htm?itemId=%s&p=1" % self.num_id
response = download(pirce_url, self.headers)
rg = re.compile('price:\"[0-9]+[.][0-9]+\"',
re.IGNORECASE | re.DOTALL)
m = rg.search(response.decode('gb18030').encode('utf8'))
if m:
price_str = m.group(0).split(":")[1].replace("\"", "")
price = Decimal(price_str)
# not chuxiao price, set origin price
if not price:
if self.originPrice:
price = self.originPrice[0].strip()
elif self.bidPrice:
price = self.bidPrice[0].strip()
if price.find("-") > 0:
price = price.split('-')[0].strip()
self.price = float(price)
logger.debug("%s price is %s", self.item_id, self.price)
def crawl_price(self):
self.promoteUrl2 = get_val(self.data, "apiPromoData")
if self.promoteUrl2:
self.promoteUrl2 = self.promoteUrl2.replace(r'''\/''', "/")
price = ""
if self.is_tmall and self.tmallInitApi and self.tmallInitApijson:
try:
priceInfo = self.tmallInitApijson['defaultModel']['itemPriceResultDO']['priceInfo']
if priceInfo:
if priceInfo.has_key('def'):
defaultPriceInfo = priceInfo['def']
else:
defaultPriceInfo = priceInfo[priceInfo.keys()[0]]
if defaultPriceInfo.has_key('promPrice'):
price = defaultPriceInfo['promPrice']['price']
elif defaultPriceInfo.has_key('promotionList') and defaultPriceInfo['promotionList']:
price = str(min([float(x.get('price','100000000.0')) for x in defaultPriceInfo['promotionList']]))
else:
price = defaultPriceInfo['price']
except:
logger.warn("Parse tmall json price failed, %s", self.item_id)
if not price:
if self.promoteUrl2:
self.promoteContent = self.crawl_page(self.promoteUrl2).replace('"', '"')
tag = "low:"
if self.promoteContent.find(tag) > 0:
pos = self.promoteContent.find(tag) + len(tag)
pos2 = self.promoteContent.find(',', pos)
price = self.promoteContent[pos:pos2]
if not price:
price = get_num_val(self.promoteContent, 'price')
else:
self.promoteUrl = "http://marketing.taobao.com/home/promotion/item_promotion_list.do?itemId=%s" % self.num_id
self.promoteContent = self.crawl_page(self.promoteUrl).replace('"', '"')
tag = "promPrice":""
if self.promoteContent.find(tag) > 0:
pos = self.promoteContent.find(tag) + len(tag)
pos2 = self.promoteContent.find('"', pos)
price = self.promoteContent[pos:pos2]
if not price:
tbPrice = self.html_obj.xpath("//strong[@class='tb-price']/text()")
tbPrice1 = self.html_obj.xpath("//span[@class='tb-price']/text()")
if tbPrice and not tbPrice[0].strip():
price = tbPrice[0].strip()
elif tbPrice1 and not tbPrice1[0].strip():
price = tbPrice1[0].strip()
if price.find("-") > 0:
price = price.split('-')[0].strip()
# 2013-09-03 get price url
if not price:
#这里稍微有点麻烦,主要针对string进行处理
pirce_url = "http://ajax.tbcdn.cn/json/umpStock.htm?itemId=%s&p=1" % self.num_id
response = download(pirce_url, self.headers)
rg = re.compile('price:\"[0-9]+[.][0-9]+\"', re.IGNORECASE|re.DOTALL)
m = rg.search(response.decode('gb18030').encode('utf8'))
if m:
price_str = m.group(0).split(":")[1].replace("\"", "")
price = Decimal(price_str)
# not chuxiao price, set origin price
if not price:
if self.originPrice:
price = self.originPrice[0].strip()
elif self.bidPrice:
price = self.bidPrice[0].strip()
if price.find("-") > 0:
price = price.split('-')[0].strip()
self.price = float(price)
logger.debug("%s price is %s", self.item_id, self.price)
