def normalize_msg(line):
timestamp = dateutil.parser.parse(
line['Date'] + " " + line['Time'] + " " + line['Time Zone'],
).strftime("%s")
names = line['Name'].split(" ")
return {
'date': timestamp,
'email': line['From Email Address'],
'first_name': names[0],
'last_name': " ".join(names[1:]),
'street_address': line['Address Line 1'],
'supplemental_address_1': line['Address Line 2/District/Neighborhood'],
'city': line['Town/City'],
'state_province': line['State/Province/Region/County/Territory/Prefecture/Republic'],
'country': line['Country'],
'postal_code': line['Zip/Postal Code'],
'comment': line['Note'],
# somthing with: line['Subscription Number'],
'gross_currency': line['Currency'],
'gross': round(locale.atof(line['Gross']), 2),
'fee': round(locale.atof(line['Fee']), 2),
'net': round(locale.atof(line['Net']), 2),
'gateway': "paypal",
'gateway_txn_id': line['Transaction ID'],
}
def add_ebs_info(instances):
ebs_url = "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSOptimized.html"
tree = etree.parse(urllib2.urlopen(ebs_url), etree.HTMLParser())
table = tree.xpath('//div[@class="table-contents"]//table')[0]
rows = table.xpath('tr')
by_type = {i.instance_type: i for i in instances}
for row in rows:
if row.xpath('th'):
continue
cols = row.xpath('td')
# TODO: Support the asterisk, which means: "These instance types can support maximum
# performance for 30 minutes at least once every 24 hours. For example, c5.large
# instances can deliver 281 MB/s for 30 minutes at least once every 24 hours. If you
# have a workload that requires sustained maximum performance for longer than 30
# minutes, select an instance type based on the following baseline performance."
instance_type = sanitize_instance_type(totext(cols[0]).replace("*", ""))
ebs_optimized_by_default = totext(cols[1]) == 'Yes'
ebs_max_bandwidth = locale.atof(totext(cols[2]))
ebs_throughput = locale.atof(totext(cols[3]))
ebs_iops = locale.atof(totext(cols[4]))
if instance_type not in by_type:
print("Unknown instance type: {}".format(instance_type))
continue
by_type[instance_type].ebs_optimized_by_default = ebs_optimized_by_default
by_type[instance_type].ebs_throughput = ebs_throughput
by_type[instance_type].ebs_iops = ebs_iops
by_type[instance_type].ebs_max_bandwidth = ebs_max_bandwidth
def angle(coor,angleparam): angleenergy=0.0 ; ubenergy=0.0 deg2rad=numpy.pi/180.0 for i in xrange(len(angleparam)): vub=0.0 ttrio=angleparam[i][0] atm1=locale.atoi(ttrio[0]) atm2=locale.atoi(ttrio[1]) atm3=locale.atoi(ttrio[2]) coor1=coor[atm1]; coor2=coor[atm2] ; coor3=coor[atm3] theta=sasmath.calc_angle(coor1,coor2,coor3) ktheta=locale.atof(angleparam[i][1][0]) theta0=deg2rad*locale.atof(angleparam[i][1][1]) if(len(angleparam[i][1])>2): kub=locale.atof(angleparam[i][1][2]) so=locale.atof(angleparam[i][1][3]) v = coor3-coor1 s = math.sqrt(sum(v*v)) #s=sascalc.calc_dist(coor1,coor3) vub=kub*(s-so)**2.0 vangle=ktheta*(theta-theta0)**2.0 angleenergy=angleenergy+vangle ubenergy=ubenergy+vub return angleenergy,ubenergy
def _getExamplars(self):
traf = open(self.trainfile)
examplars = []
while(traf):
line = traf.readline()
if line!="":
tokens = line.strip().split(":")
print tokens
if 2!= len(tokens):
print "The format of the trainfile is wrong!--1"
sys.exit()
elif 8!= len(tokens[0].split(",")):
print "The format of the trainfile is wrong!--2"
sys.exit()
elif 1!= len(tokens[1].split(",")):
print "The format of the trainfile is wrong!--3"
sys.exit()
inputs = []
for i in range(8):
inputs.insert(i,atof(tokens[0].split(",")[i]))
outputs = []
outputs.insert(0,atof(tokens[1][0]))
examplar = (inputs,outputs)
examplars.append(examplar)
else:
break
print examplars
return examplars
def sig_insert_text(self, entry, new_text, new_text_length, position):
if not self.record:
entry.stop_emission('insert-text')
return
value = entry.get_text()
position = entry.get_position()
new_value = value[:position] + new_text + value[position:]
decimal_point = locale.localeconv()['decimal_point']
if new_value in ('-', decimal_point):
return
digits = self.field.digits(self.record)
try:
locale.atof(new_value)
except ValueError:
entry.stop_emission('insert-text')
return
new_int = new_value
new_decimal = ''
if decimal_point in new_value:
new_int, new_decimal = new_value.rsplit(decimal_point, 1)
if len(new_int) > digits[0] \
or len(new_decimal) > digits[1]:
entry.stop_emission('insert-text')
def time_to_dec(time):
"""Convert time interval from MM:SS to MM.S
Parameters
----------
string of time
time in MM:SS format
Returns
-------
time in minutes in decimal format
"""
from locale import setlocale, LC_ALL, atof
setlocale(LC_ALL, "en_US.UTF8")
try:
t = time
# no information provided
if t == "-":
return None
pl = t.split(":")
minutes = atof(pl[0])
secs = (atof(pl[1]) / 60)
ttime = minutes + secs
return ttime
except:
return None
def parse(self, response):
item = Item()
sel = response.xpath
sele = sel("//td[@class='formularioColumna']/text()").extract()
item['nombres'] = sele[2].strip().replace("\n", " -")
item['apellidos'] = sele[5].strip().replace("\n", " -")
item['dni'] = sele[8].strip().replace("\n", " -")
item['domicilio'] = sele[11].strip().replace("\n", " -")
item['delito'] = sele[14].strip().replace("\n", " -")
# check!
item['entidad_agraviada'] = sele[17].strip().replace("\n", " -")
item['fecha_sentencia'] = sele[20].strip().replace("\n", " -")
item['fecha_ejecutoria'] = sele[23].strip().replace("\n", " -")
item['juzgado'] = sele[26].strip().replace("\n", " -")
item['expediente'] = sele[29].strip().replace("\n", " -")
item['solidaria'] = sele[32].strip().replace("\n", " -")
sele = sel("//td[@class='formularioColumna']/div[@align='center']/text()").extract()
item['reparacion_civil'] = locale.atof(sele[0])
item['intereses'] = locale.atof(sele[1])
item['monto_total'] = locale.atof(sele[2])
item['pagos_realizados'] = locale.atof(sele[3])
item['pagos_pendientes'] = locale.atof(sele[4])
item['url'] = response.url.replace("\n", " -")
yield item
def parse_instance(tr, inst2family):
i = Instance()
cols = tr.xpath('td')
assert len(cols) == 12, "Expected 12 columns in the table, but got %d" % len(cols)
i.instance_type = totext(cols[0])
i.family = inst2family.get(i.instance_type, "Unknown")
# Some t2 instances support 32-bit arch
# http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-resize.html#resize-limitations
if i.instance_type in ('t2.micro', 't2.small'):
i.arch.append('i386')
i.vCPU = locale.atoi(totext(cols[1]))
i.memory = locale.atof(totext(cols[2]))
storage = totext(cols[3])
m = re.search(r'(\d+)\s*x\s*([0-9,]+)?', storage)
i.ssd = False
if m:
i.ebs_only = False
i.num_drives = locale.atoi(m.group(1))
i.drive_size = locale.atof(m.group(2))
i.ssd = 'SSD' in totext(cols[3])
else:
assert storage == 'EBS Only', "Unrecognized storage spec: %s" % (storage,)
i.ebs_only = True
i.ebs_optimized = totext(cols[10]).lower() == 'yes'
i.network_performance = totext(cols[4])
i.enhanced_networking = totext(cols[11]).lower() == 'yes'
i.generation = 'current'
# print "Parsed %s..." % (i.instance_type)
return i
def parse_instance(instance_type, product_attributes):
i = scrape.Instance()
i.instance_type = instance_type
pieces = instance_type.split('.')
if len(pieces) == 1:
# Dedicated host that is not u-*.metal, skipping
# May be a good idea to all dedicated hosts in the future
return
i.family = product_attributes.get('instanceFamily')
if '32-bit' in product_attributes.get('processorArchitecture'):
i.arch.append('i386')
i.vCPU = locale.atoi(product_attributes.get('vcpu'))
# Memory is given in form of "1,952 GiB", let's parse it
i.memory = locale.atof(product_attributes.get('memory').split(' ')[0])
i.network_performance = product_attributes.get('networkPerformance')
if product_attributes.get('currentGeneration') == 'Yes':
i.generation = 'current'
else:
i.generation = 'previous'
gpu = product_attributes.get('gpu')
if gpu is not None:
i.GPU = locale.atoi(gpu)
try:
ecu = product_attributes.get('ecu')
if ecu == 'Variable':
i.ECU = 'variable'
else:
i.ECU = locale.atof(ecu)
except:
pass
i.physical_processor = product_attributes.get('physicalProcessor')
# CPU features
processor_features = product_attributes.get('processorFeatures')
if processor_features is not None:
if "Intel AVX-512" in processor_features:
i.intel_avx512 = True
if "Intel AVX2" in processor_features:
i.intel_avx2 = True
if "Intel AVX" in processor_features:
i.intel_avx = True
if "Intel Turbo" in processor_features:
i.intel_turbo = True
i.clock_speed_ghz = product_attributes.get('clockSpeed')
enhanced_networking = product_attributes.get('enhancedNetworkingSupported')
if enhanced_networking is not None and enhanced_networking == 'Yes':
i.enhanced_networking = True
return i
def getConversion(body):
"""Calculate schmeckle to USD responses"""
values = []
msg_template_f = u"* {:,.2f} Schmeckles → **${:,.2f} USD**\n" # with decimals
msg_template_i = u"* {:,.0f} Schmeckles → **${:,.0f} USD**\n" # without decimals
msg_inf = u"* There's a lot of numbers there, I think you could probably go to Wall Street.\n\n*You ever hear about Wall Street, Morty? Y-Y-Y'know what those guys do i-in-in their fancy boardrooms? They take their balls and they dip 'em in cocaine and wipe 'em all over each other—y'know.*\n"
pairs = p.findall(body)
if len(pairs) > 0:
for match in pairs:
# '<number> schmeckle' -> match, float(<number>)
value_str = match.split()[0]
# Handle numbers with over 9000 characters. Yes, it's over 9000.
if (len(value_str)) > 9000:
values.append(locale.atof("inf"))
else:
values.append(locale.atof(value_str))
response = []
for schmeckle in values:
if isinf(schmeckle):
response.append(msg_inf)
else:
usd = schmeckle2usd(schmeckle)
if schmeckle.is_integer():
response.append(msg_template_i.format(schmeckle, usd))
else:
response.append(msg_template_f.format(schmeckle, usd))
return [response, values]
def currency_to_float(string):
try:
ret = locale.atof(string)
except ValueError:
ret = locale.atof(string.replace(
locale.localeconv()['mon_thousands_sep'], ''))
return float(ret)
def main():
datenum=[];
zenith=[];
azimuth=[];
tracker_angle=[];
file_pos=0;
day=0;
iDay=0;
morningIndex=[];
datum=0;
pi=22/7;
filename = 'e:\\BKK2010.txt';
outputfile = 'e:\\BKK2010.bin';
file = open(filename, 'r');
ofile = open(outputfile,'wb');
print ('--- Reading file %s ---\n' % filename);
while True:
line = file.readline()
if not line:
break
sline = line.split(' ');
# Wait for output buffer to drain.
datenum.append(locale.atof(sline[0]));
zenith.append(locale.atof(sline[1]));
azimuth.append(locale.atof(sline[2]));
print ('--- Finish ---\n');
nrow = len(datenum);
for i in range(0,nrow):
if ((zenith[i]<=90) & (day==0) ): # if the sun is above horizon
day=1; #toggle day
iDay=iDay+1;
datum=0;
morningIndex.append(i);
file_pos=ofile.tell()
data=struct.pack('I',900*i);
ofile.write(data);
ofile.flush();
if ((ofile.tell() - file_pos) != 4):
break;
elif ((zenith[i]>90) & (day==1) ): # if the sun is setting
day=0; #toggle night
for j in range(datum,63):
file_pos=ofile.tell()
data=struct.pack('I',0);
ofile.write(data);
ofile.flush();
if ((ofile.tell() - file_pos) != 4):
break;
print('Day %d, %d \r\n'%(iDay,datum));
if (day==1):
tracker_angle = math.atan((math.sin(zenith[i]*pi/180)* math.cos((azimuth[i]-90)*pi/180))/math.cos(zenith[i]*pi/180))*180/pi+90;
data=struct.pack('I',tracker_angle*(2**22)); # actual tracker angle data, max 63 data per page
file_pos=ofile.tell()
ofile.write(data);
ofile.flush();
if ((ofile.tell() - file_pos) != 4):
break;
datum = datum+1;
ofile.close()
def cur_parse(value):
"""
Parses input into a float value
Returns a tuple of (CURRENCY, value)
CURRENCY is the locale that was parsed, or 'BTC'
value is the ammount devoid of grouping and currency strings in a float
"""
locale_value, repl_count = LOCALE_CRNCY_RE.subn("", value)
if repl_count >= 1:
# we matched the current locale
locale_value = locale.atof(locale_value)
return (LOCALE[0], locale_value)
btc_value, repl_count = BTC_CRNCY_RE.subn("", value)
if repl_count >= 1:
# we matched BTC
btc_value = locale.atof(btc_value)
return ("BTC", btc_value)
btc_value, repl_count = BTC2_CRNCY_RE.subn("", value)
if repl_count >= 1:
# we matched BTC2
btc_value = locale.atof(btc_value)
return ("BTC", btc_value)
raise ValueError("Unknown locale for currency: %s" % value)
def salvar(self):
seta_locale(False)
if self.ui.checkSaida.isChecked():
print "checked"
self.val = -locale.atof(self.ui.textValor.text())
else:
print "nao checked"
self.val = locale.atof(self.ui.textValor.text())
cadastro = Caixa(
descricao=self.ui.textDescricao.text(),
data=datetime.datetime.strptime(self.ui.textData.text(), getDataString()).date(),
valor=self.val,
origem=self.ui.textOrigem.text(),
destino=self.ui.textDestino.text()
)
try:
self.session.add(cadastro)
self.session.commit()
mostrarMessageBox(
"Sucesso!",
"Cadastro de Caixa efetuado com sucesso!")
except Exception as e:
lineno, fname = get_filename_lnumber()
enviaEmail(e.message, fname, lineno)
mostrarMessageBox("Erro!", "Não foi possivel salvar os dados no banco. \n %s" % e.message)
def parse_prev_generation_instance(tr):
i = Instance()
cols = tr.xpath('td')
assert len(cols) == 8, "Expected 8 columns in the table, but got %d" % len(cols)
i.family = totext(cols[0])
i.instance_type = totext(cols[1])
archs = totext(cols[2])
i.arch = []
if '32-bit' in archs:
i.arch.append('i386')
if '64-bit' in archs:
i.arch.append('x86_64')
assert i.arch, "No archs detected: %s" % (archs,)
i.vCPU = locale.atoi(totext(cols[3]))
i.memory = locale.atof(totext(cols[4]))
storage = totext(cols[5])
m = re.search(r'(\d+)\s*x\s*([0-9,]+)?', storage)
i.ssd = False
if m:
i.ebs_only = False
i.num_drives = locale.atoi(m.group(1))
i.drive_size = locale.atof(m.group(2))
i.ssd = 'SSD' in totext(cols[5])
else:
assert storage == 'EBS Only', "Unrecognized storage spec: %s" % (storage,)
i.ebs_only = True
i.ebs_optimized = totext(cols[6]).lower() == 'yes'
i.network_performance = totext(cols[7])
i.enhanced_networking = False
i.generation = 'previous'
# print "Parsed %s..." % (i.instance_type)
return i
def _build_cpu_info(self):
try:
p = os.popen('lscpu')
cpuinfo = p.read()
p.close()
m = re.search('Model name:\s+(.*)', cpuinfo)
if m:
self._cpu['model'] = m.group(1)
m = re.search('Socket\(s\):\s+(\d+)', cpuinfo)
if m:
self._cpu['sockets'] = locale.atoi(m.group(1))
m = re.search('CPU MHz:\s+(.*)', cpuinfo)
if m:
self._cpu['clock'] = int(locale.atof(m.group(1)) * 1e6)
m = re.search('CPU max MHz:\s+(.*)', cpuinfo)
if m:
self._cpu['clockMax'] = int(locale.atof(m.group(1)) * 1e6)
m = re.search('CPU min MHz:\s+(.*)', cpuinfo)
if m:
self._cpu['clockMin'] = int(locale.atof(m.group(1)) * 1e6)
m = re.search('Core\(s\) per socket:\s+(.*)', cpuinfo)
if m:
self._cpu['cores_per_sockets'] = locale.atoi(m.group(1))
m = re.search('Thread\(s\) per core:\s+(.*)', cpuinfo)
if m:
self._cpu['threads_per_core'] = locale.atoi(m.group(1))
except:
pass
def indicadoreseconomicos():
if not request.ajax: return ''
session.forget()
#if request.ajax:
import urllib2
import locale
locale.setlocale(locale.LC_ALL, 'es_CL.UTF8')
uri = 'http://www.averigualo.cl/feed/indicadores.xml'
import re
locale.setlocale(locale.LC_ALL, 'es_CL.UTF8')
#uri = 'http://www.averigualo.cl/feed/indicadores.xml'
uri = 'http://indicador.eof.cl/xml'
try:
pag = urllib2.urlopen(uri).read()
pag = pag.decode('iso8859-1').encode('utf-8')
html = TAG(pag)
try:
eurocalculado = locale.atof(html.element('indicador',_nombre='Euro')[0])
except Exception,e:
eurocalculado = '-'
try:
#dolarcalculado = locale.atof(html.element('indicador',_nombre='Dólar Observado')[0])
dolarcalculado = locale.atof(html.element('indicador',_nombre=re.compile('Observado'))[0])
except Exception,e:
dolarcalculado = '-'
def _parse_contig_coverage(line):
"""
Parse contig coverage from a string, typically a line from the QualiMap result report.
If no contig coverage can be parsed, None is returned.
:param line: the input string
:return: the contig coverage as a dict, with "[CONTIG] coverage" as key and a dict with the contig coverage
information as value or None if no contig coverage could be parsed
"""
# identify contig coverage
try:
values = re.search(r'^\s+(\S+)\s+([0-9]+)\s+([0-9]+)\s+(\S+)\s+(\S+)\s*$', line).groups()
key = "{} coverage".format(values[0])
value = dict()
value["contig"] = values[0]
value["length"] = locale.atoi(values[1])
value["mapped bases"] = locale.atoi(values[2])
value["mean coverage"] = locale.atof(values[3])
value["standard deviation"] = locale.atof(values[4])
return {key: value}
except ValueError:
# problems with the conversion to numeric values
pass
except AttributeError:
# not a contig coverage row
pass
def __init__(self, fields, raw_csv, options):
"""Parameters:
fields: list of fields read from one line of the CSV file
raw_csv: unprocessed line from CSV file
options: from CLI args and config file
"""
if 'addons' in options:
self.addons = dict((k, fields[v - 1])
for k, v in options.addons.items())
else:
self.addons = dict()
# Get the date and convert it into a ledger formatted date.
self.date = fields[options.date - 1]
if options.ledger_date_format:
if options.ledger_date_format != options.csv_date_format:
self.date = (datetime
.strptime(self.date, options.csv_date_format)
.strftime(options.ledger_date_format))
# convert effective dates
if options.effective_date:
self.effective_date = fields[options.effective_date - 1]
if options.ledger_date_format:
if options.ledger_date_format != options.csv_date_format:
self.effective_date = (datetime
.strptime(self.effective_date, options.csv_date_format)
.strftime(options.ledger_date_format))
else:
self.effective_date = ""
desc = []
for index in re.compile(',\s*').split(options.desc):
desc.append(fields[int(index) - 1].strip())
self.desc = ' '.join(desc).strip()
self.credit = get_field_at_index(fields, options.credit, options.csv_decimal_comma, options.ledger_decimal_comma)
self.debit = get_field_at_index(fields, options.debit, options.csv_decimal_comma, options.ledger_decimal_comma)
if self.credit and self.debit and atof(self.credit) == 0:
self.credit = ''
elif self.credit and self.debit and atof(self.debit) == 0:
self.debit = ''
self.credit_account = options.account
self.currency = options.currency
self.cleared_character = options.cleared_character
if options.template_file:
with open(options.template_file, 'r') as f:
self.transaction_template = f.read()
else:
self.transaction_template = ""
self.raw_csv = raw_csv.strip()
# We also record this - in future we may use it to avoid duplication
self.md5sum = hashlib.md5(self.raw_csv).hexdigest()
def is_numeric(s):
if s.lower() in ['infinity', 'nan']:
return False
try:
setlocale(LC_NUMERIC, 'pl_PL.UTF-8')
atof(s)
return True
except ValueError:
return False
def group_by(records, key):
"""For a payout report, returns a structure like:
{'DE': {'charged': 15,
'received': 10,
'num_sales': 3},
'US': {...},
...
All values are in seller's currency (yours).
In case this is an "estimtaed sales" report, it will not contain the
"received" key, and the value in "charged" will be the buyer's currency.
"""
# itertools.groupby requires sorted input
records.sort(key=key)
result = collections.OrderedDict()
for country, sales in itertools.groupby(records, key=key):
sales = list(sales)
"""
TODO: There actually seems to be a bug in both payout and sales
reports, where I have buyers from the US paying in KRW, and no FX Rate
is given. In those cases, assuming 1 as FX rate is wrong. Example:
{'Merchant Currency': 'KRW', 'Country of Buyer': 'US', ..., 'Merchant Receives': '0.00', 'Item Price': '1,165.00', 'Charged Amount': '1,165.00', 'Order Charged Date': '2012-04-18', 'Currency of Sale': 'KRW', 'City of Buyer': 'Honolulu', 'Estimated FX Rate': '', 'State of Buyer': 'HI', ... 'Financial Status': 'Charged'}
"""
charged = lambda s: Decimal(str(locale.atof(s['Charged Amount'])))
received = lambda s: Decimal(str(locale.atof(s['Merchant Receives'])))
fx = lambda s: Decimal(str(locale.atof(s['Estimated FX Rate'] or '1')))
result[country] = {
'num_sales': len(list(sales)),
}
if is_payout_report(sales):
# This key indicates a payout report
result[country].update({
'charged': sum([charged(s) * fx(s) for s in sales]),
'received': sum([received(s) for s in sales])
})
else:
# A sales report, lacks info about actual monies payed out.
result[country].update({
'charged': sum([charged(s) for s in sales]),
# XXX It is actually wrong to assume a single currency for
# each country. Lots of people from KR pay in USD for example.
'currency': sales[0]['Currency of Sale']
})
if is_payout_report(records):
# Without a single currency, sum makes no sense
result['SUM'] = reduce(operator.add, map(collections.Counter, result.values()))
return result
def testAlternativeDecimalMark(self):
""" Test a locale which uses "," as its decimal separator."""
# NOTE: Windows locales aren't named the same as the `man locale 1`
# equivalents. For this code to be universal, it'd need to test os.name
# and convert to the appropriate locale name.
locale.setlocale(locale.LC_ALL, "german_germany")
# two tests to make sure that radix parsing is happening as expected
self.assertEqual(locale.atof("-11,00"), -11.0)
self.assertEqual(locale.atof("-11.000,50"), -11000.5)
def read_file(self, file_path):
f = open(file_path, 'r')
L = f.readlines()
f.close()
text = L[0][:-1] #Strip trailing "\n"
t = locale.atof(L[1][:-1])
h = locale.atof(L[2][:-1])
self.value.set_text(text)
self._t_h_bar.set_temp(t)
self._t_h_bar.set_humid(h)
def string2i(my_str):
my_str2 =my_str.replace(",","")
"Convert a string to an number"
if (my_str2 != '') and (my_str2 is not None):
locale.setlocale(locale.LC_ALL, '')
positive = my_str2.translate(None, '()')
result = locale.atof(positive) if positive == my_str2 else -locale.atof(positive)
return result
else:
return 0
def read_user_weights (properties, sWeights, dcWeight, dcRowSum, dDistAllSum, bExcludeII):
""" Reads FROM, TO, WEIGHT file into dictionary structure.
Header recorder must identify ID field. """
try:
bDefault = iFalse
f = open(sWeights, 'r')
# Make sure we have a valid ID field.
sFieldName = f.readline().strip()
if sFieldName.upper() == (properties.sFID).upper():
sIDField = None
sTokens = f.readline().split()
else:
sPathName = os.path.dirname(sWeights)
sIDField = pGP.ValidateFieldName(sFieldName, sPathName)
if not sIDField.upper() in properties.dcFields.keys():
# Maybe we didn't get a header record
try:
sTokens = sFieldName.split()
# If we have two integers and a weight, we'll assume the
# ID field is the FID field
id1 = int(sTokens[0])
id2 = int(sTokens[1])
dWt = locale.atof(sTokens[2])
sIDField = None
except:
raise ReportError(msgFileFormatErr)
else:
sTokens = f.readline().split()
pGP.AddMessage (sReadingWeightsFile)
# Read each record in weights file, creating a dictionary structure of distances.
bNegWts = iFalse
while len(sTokens) > 0:
sFrom = sTokens[0]
sTo = sTokens[1]
dWeight = locale.atof(sTokens[2])
if dWeight < 0.0: bNegWts = iTrue
sKey = sFrom + ":" + sTo
dcWeight[sKey] = dWeight
if not (sFrom == sTo and bExcludeII):
if (dcRowSum.has_key(sFrom)):
dcRowSum[sFrom] = dcRowSum[sFrom] + dWeight
else:
dcRowSum[sFrom] = dWeight
dDistAllSum = dDistAllSum + dWeight
sTokens = f.readline().split()
f.close()
if bNegWts: pGP.AddWarning(msgNegativeWts)
except: # If we encounter unexpected errors, we bail
bDefault = iTrue
sIDField = None
f.close()
raise ReportError(msgWtsMatrixErr)
return bDefault, sIDField, dcRowSum, dDistAllSum
def write_cloud_details_V2(cloud_sheet, row_dict, output_row):
output_col = 0
total_amount = 0.0
# Write Google data into Cloud sheet.
# Output 'Platform' field.
cloud_sheet.write(output_row, output_col, "Google Cloud Platform")
output_col += 1
# Output 'Account' field. (subacccount)
cloud_sheet.write(output_row, output_col, row_dict['Account ID'])
output_col += 1
# Output 'Project' field. (Project Name + Project ID)
cloud_sheet.write(output_row, output_col, row_dict['Source'])
output_col += 1
# Output 'Description' field. (SKU description of the charge)
sku_description = "%s %s" % (row_dict['Product'], row_dict['Resource Type'])
cloud_sheet.write(output_row, output_col, sku_description)
output_col += 1
# Output 'Dates' field.
date_range = "%s-%s" % (row_dict['Start Date'], row_dict['End Date'])
cloud_sheet.write(output_row, output_col, date_range)
output_col += 1
# Parse quantity.
quantity_str = row_dict['Quantity'].strip()
if len(quantity_str) > 0:
quantity = locale.atof(quantity_str)
else:
quantity = ''
# Output 'Quantity' field.
cloud_sheet.write(output_row, output_col, quantity, FLOAT_FORMAT)
output_col += 1
# Output 'Unit of Measure' field.
cloud_sheet.write(output_row, output_col, row_dict['Unit'])
output_col += 1
# Parse charge.
amount = locale.atof(row_dict['Amount'])
# Accumulate total charges.
total_amount += amount
# Output 'Charge' field.
cloud_sheet.write(output_row, output_col, amount, MONEY_FORMAT)
output_col += 1
return total_amount
def tablify(data):
lst = data.get_text().split("\n")
ret_list = lst[2:4]
ret_list.append(lst[6])
ret_list.append(lst[8])
ret_list.append(lst[13])
ret_list.append(str(locale.atof(lst[9])+locale.atof(lst[14])))
ret_list.append(str(locale.atof(lst[10])+locale.atof(lst[15])))
ret_list.extend(lst[16:18])
return ret_list
def el(coor,charge,exclusionlist): # print '\ncalculating electrostatic energy' natoms=len(charge) elenergy=0.0 # qi*qj/(e1*rij) e1=1.0 qconv=1.620177E-19 # eV to C qconv2=qconv*qconv # q*q --> C^2 eps=8.854187816E-12 # epsilon --> C^2/(J*m) ang2m=1E-10 # angstrom to m jtokjpmol=6.022137E+20 # J to KJ/mol kjtokcal=1.0/4.184 # KJ to kcal #qconv2=1.0 pi=numpy.pi conv=(qconv2/(4.0*pi*eps*ang2m))*jtokjpmol*kjtokcal #print 'conv = ',conv conv=332 conv=332.4683 np=0 stupid = True for i in xrange(natoms): qi=locale.atof(charge[i]) x1=coor[i][0] ; y1=coor[i][1] ; z1=coor[i][2] for j in xrange(i+1,natoms): #if str(j+1) not in exclusionlist[i][1]: if stupid: qj=locale.atof(charge[j]) x2=coor[j][0] ; y2=coor[j][1] ; z2=coor[j][2] dx2=(x1-x2)*(x1-x2) dy2=(y1-y2)*(y1-y2) dz2=(z1-z2)*(z1-z2) rij=numpy.sqrt(dx2+dy2+dz2) switch=getswitch(rij) vij=((qi*qj)/(e1*rij))*switch #vij=(qi*qj*qconv2)/(e1*eps*rij*ang2m) #vij=vij*jtokjpmol*kjtokcal elenergy=elenergy+vij np=np+1 else: pass #print 'atom ',i+1,' and atom ',j+1,' not counted in vdw calculation' elenergy=elenergy*conv return elenergy
def LayDuLieu(self):
result = {}
Bang = self.dv.find_element_by_css_selector('table.table-profit')
self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,'.col-1-1')))
for x in range(2,9):
row = Bang.find_element_by_css_selector('tr:nth-child('+ str(x) +')')
key = row.find_element_by_css_selector('td:first-child').text
result[key] = {}
result[key]['pre'] = locale.atof(row.find_element_by_css_selector('td:nth-child(2)').text)
result[key]['current'] = locale.atof(row.find_element_by_css_selector('td:nth-child(3)').text)
print result
return result
def on_exit_edit(edit_field, disk_win=None):
'''On exit, if the user has left the field blank, set the size to 0'''
text = edit_field.get_text()
if not text.strip():
text = "0"
enctext = text.encode(get_encoding())
# encode per locale for floating point conversion
edit_field.set_text("%.1f" % locale.atof(enctext))
part_order = disk_win.ui_obj.get_parts_in_use().index(edit_field.data_obj)
LOGGER.debug("Part being resized is at index: %s", part_order)
new_size_text = text.strip()
LOGGER.debug("Resizing text=%s", new_size_text)
# encode user input per locale for floating point conversion
enctext = new_size_text.encode(get_encoding())
new_size = Size(str(locale.atof(enctext)) + Size.gb_units)
old_size = edit_field.data_obj.size
new_size_byte = new_size.get(Size.byte_units)
old_size_byte = old_size.get(Size.byte_units)
precision = Size(UI_PRECISION).get(Size.byte_units)
if abs(new_size_byte - old_size_byte) > precision:
parent_doc_obj = edit_field.data_obj.doc_obj.parent
if isinstance(parent_doc_obj, Disk):
if isinstance(edit_field.data_obj.doc_obj, Partition):
resized_obj = parent_doc_obj.resize_partition(
edit_field.data_obj.doc_obj, new_size.get(Size.gb_units),
size_units=Size.gb_units)
else:
resized_obj = parent_doc_obj.resize_slice(
edit_field.data_obj.doc_obj, new_size.get(Size.gb_units),
size_units=Size.gb_units)
else:
resized_obj = parent_doc_obj.resize_slice(
edit_field.data_obj.doc_obj, new_size.get(Size.gb_units),
size_units=Size.gb_units)
if isinstance(resized_obj, Partition):
resized_obj.in_zpool = ROOT_POOL
else:
if resized_obj.in_zpool == ROOT_POOL:
resized_obj.tag = V_ROOT
if disk_win is not None:
disk_win.set_disk_info(ui_obj=disk_win.ui_obj)
disk_win.activate_index(part_order)
dump_doc("After resize")
def parse(self, response):
# Code to parse course pages ## Fill in dc_dict here
# parsing each entry
offer = dict()
# get main data from the html's head
head = response.xpath('/html/head')
offer['title'] = head.xpath(
'./meta[@property="og:title"]/@content').extract_first()
offer['image'] = head.xpath(
'./meta[@property="og:image"]/@content').extract_first()
offer['url'] = response.url
offer['description'] = head.xpath(
'./meta[@name="description"]/@content').extract_first()
quickfacts = response.css('div.quickfacts')
offer['summary_text'] = quickfacts.css('h1::text').extract_first()
offer['location'] = quickfacts.xpath('./div[@class="location"]/span[1]/text()').extract_first()
offer['merkmale'] = quickfacts.css('div.merkmale::text').extract_first()
hardfacts = response.css('div.hardfacts')
offer['total_price'] = float(hardfacts.xpath('./div[1]/strong/text()').extract_first().split(' ')[0].replace('.',''))
offer['area'] = atof(hardfacts.xpath('./div[2]/text()').extract_first().split(' ')[0].strip())
offer['num_rooms'] = atof(hardfacts.xpath('./div[3]/text()').extract_first().strip())
# PREISE UND KOSTEN
preise_container = response.xpath('//h2[contains(text(), "Preise")]/..')
## KALTMIETE
container = preise_container.xpath('.//strong[contains(text(), "Kaltmiete")]/../..')
if len(container) > 0:
offer['kaltmiete'] = atof(container.css('div.datacontent > strong::text').extract_first().split(' ')[0].strip())
## NEBENKOSTEN
container = preise_container.xpath('.//div[contains(text(), "Nebenkosten")]/../..')
if len(container) > 0:
offer['nebenkosten'] = atof(container.css('div.datacontent::text').extract_first().split(' ')[0].strip())
## HEIZKOSTEN
container = preise_container.xpath('.//div[contains(text(), "Heizkosten")]/../..')
if len(container) > 0:
offer['heizkosten'] = container.css('div.datacontent::text').extract_first()
## WARMMIETE
container = preise_container.xpath('.//div[contains(text(), "Warmmiete")]/../..')
if len(container) > 0:
offer['warmmiete'] = container.css('div.datacontent::text').extract_first()
## KAUTION
container = response.xpath('//div[contains(text(), "Kaution")]/..')
if len(container) > 0:
offer['kaution'] = atof(container.css('div.section_content > p::text').extract_first().strip().split(' ')[0])
# Get the parent div of the div whose content is 'Immobilie' to get the online ID
immobilie_container = response.xpath('//h2[contains(text(), "Immobilie")]/../..')
offer['online_id'] = immobilie_container.css('div.section_content > p::text').extract_first().strip().replace('Online-ID: ', '')
# Get the parent div of the div whose content is 'Die Wohnung' to get details
# TODO: Needs some text recognition to get main facts?
# whg_container = response.xpath('//div[contains(text(), "Die Wohnung")]/..')
# offer['type'] = whg_container.xpath('./div[2]/p/text()').extract_first().strip()
# offer['floor'] = whg_container.xpath('./div[2]/p/span/text()').extract_first().strip()
# offer['availability'] = whg_container.xpath('./div[2]/p/strong/text()').extract_first().strip()
# TODO: get more details from UL > LIs?
# Get the parent div of the div whose content is 'Wohnalage' to get details
whg_container = response.xpath('//div[contains(text(), "Wohnanlage")]/..')
offer['building_year'] = whg_container.xpath('./div[2]/p/text()').extract_first().strip()
# TODO: get more details from UL > LIs?
# Get the parent div of the div whose content is 'Objektbeschreibung' to get details
container = response.xpath('//div[contains(text(), "Objektbeschreibung")]/..')
text_content = container.xpath('./div[2]/p//text()').extract()
offer['object_description'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0])
# Get the parent div of the div whose content is 'Ausstattung' to get details
container = response.xpath('//div[contains(text(), "Ausstattung")]/..')
text_content = container.xpath('./div[2]/p//text()').extract()
offer['ausstattung'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0])
# Get the parent div of the div whose content is 'Sonstiges' to get details
container = response.xpath('//div[contains(text(), "Sonstiges")]/..')
if len(container) > 0:
text_content = container.xpath('./div[2]/p//text()').extract()
offer['sonstiges'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0])
# Get the parent div of the div whose content is 'KFZ Stellplatz' to get details
container = response.xpath('//div[contains(text(), "KFZ Stellplatz")]/..')
text_content = container.xpath('./div[2]/p//text()').extract()
offer['kfz_stellplatz'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0])
# Get the parent div of the div whose content is 'Stichworte' to get details
container = response.xpath('//div[contains(text(), "Stichworte")]/..')
text_content = container.xpath('./div[2]/p//text()').extract()
offer['keywords'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0])
# Get the parent div of the div whose content is 'Lagebeschreibung' to get details
container = response.xpath('//div[contains(text(), "Lagebeschreibung")]/..')
text_content = container.xpath('./div[2]/p//text()').extract()
offer['lagebeschreibung'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0])
# append to final list
offers_list.append(offer)
sheet = wbk["Sheet1"]
cells = sheet['A2':'H181']
i = -1
position = []
l1 = []
l2 = []
l3 = []
l4 = []
l5 = []
l6 = []
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
for pos1, pos2, c1, c2, c3, c4, c5, c6 in cells:
i = i + 1
l1.append(locale.atof(c1.value))
l2.append(locale.atof(c2.value))
l3.append(locale.atof(c3.value))
l4.append(locale.atof(c4.value))
l5.append(locale.atof(c5.value))
l6.append(locale.atof(c6.value))
if (i + 1) % 10 == 0:
position.append([])
position[int(i / 10)].append(locale.atof(pos1.value))
position[int(i / 10)].append(locale.atof(pos2.value))
print(round(variance(l1), 2), round(variance(l2), 2),
round(variance(l3), 2), round(variance(l4), 2),
round(variance(l5), 2), round(variance(l6), 2))
l1.clear()
l2.clear()
l3.clear()
polygonType = "SIMPLE_POLYS"
input_copies = []
#Convert constraint values from strings to number. If empty string use max Int
maxPointBarrierCount = locale.atoi(
strMaxPointBarrierCount) if strMaxPointBarrierCount else INFINITY
maxEdgeCountLineBarriers = locale.atoi(
strMaxEdgeCountLineBarriers
) if strMaxEdgeCountLineBarriers else INFINITY
maxEdgeCountPolygonBarriers = locale.atoi(
strMaxEdgeCountPolygonBarriers
) if strMaxEdgeCountPolygonBarriers else INFINITY
maxFacilities = locale.atoi(
strMaxFacilities) if strMaxFacilities else INFINITY
maxBreaks = locale.atoi(strMaxBreaks) if strMaxBreaks else INFINITY
maxBreakTime = locale.atof(
strMaxBreakTime) if strMaxBreakTime else INFINITY
maxBreakDistance = locale.atof(
strMaxBreakDistance) if strMaxBreakDistance else INFINITY
forceHierarchyTime = locale.atof(
strForceHierarchyTime) if strForceHierarchyTime else INFINITY
forceHierarchyDistance = locale.atof(
strForceHierarchyDistance) if strForceHierarchyDistance else INFINITY
descNDS = arcpy.Describe(networkDataset)
descNDSAttributes = descNDS.attributes
#Convert all input features to feature sets or recordsets if they are not
#This is required as if input is passed a feature layer or feature class
#We will end up directly modifying the inputs
facilities_obj = nau.InputFeatureClass(facilities)
#! /usr/bin/env/python
# -*- coding:utf-8 -*-
import locale
sample_locales = [
('USA', 'en_US.UTF-8', '1,234.56'),
('France', 'fr_FR.UTF-8', '1234,56'),
('Spain', 'es_ES.UTF-8', '1234,56'),
('Portugal', 'pt_PT.UTF-8', '1234.56'),
('Poland', 'pl_PL.UTF-8', '1234,56'),
]
for name, loc, a in sample_locales:
locale.setlocale(locale.LC_ALL, loc)
f = locale.atof(a)
print "{0:.20}s： {1:.9}s => {2}f".format(name, a, f)
def extract(self, file_):
entries = []
line_index = 0
closing_balance_index = -1
with change_locale(locale.LC_NUMERIC, self.numeric_locale):
with open(file_.name, encoding=self.file_encoding) as fd:
# Header
line = fd.readline().strip()
line_index += 1
if not self._expected_header_regex.match(line):
raise InvalidFormatError()
# Empty line
line = fd.readline().strip()
line_index += 1
if line:
raise InvalidFormatError()
# Meta
lines = [fd.readline().strip() for _ in range(3)]
reader = csv.reader(
lines,
delimiter=';',
quoting=csv.QUOTE_MINIMAL,
quotechar='"',
)
for line in reader:
key, value, _ = line
line_index += 1
if key.startswith('Von'):
self._date_from = datetime.strptime(value,
'%d.%m.%Y').date()
elif key.startswith('Bis'):
self._date_to = datetime.strptime(value,
'%d.%m.%Y').date()
elif key.startswith('Kontostand vom'):
# Beancount expects the balance amount to be from the
# beginning of the day, while the Tagessaldo entries in
# the DKB exports seem to be from the end of the day.
# So when setting the balance date, we add a timedelta
# of 1 day to the original value to make the balance
# assertions work.
self._balance_amount = Amount(
locale.atof(value.rstrip(' EUR'), Decimal),
self.currency,
)
self._balance_date = datetime.strptime(
key.lstrip('Kontostand vom ').rstrip(':'),
'%d.%m.%Y',
).date() + timedelta(days=1)
closing_balance_index = line_index
# Another empty line
line = fd.readline().strip()
line_index += 1
if line:
raise InvalidFormatError()
# Data entries
reader = csv.DictReader(fd,
delimiter=';',
quoting=csv.QUOTE_MINIMAL,
quotechar='"')
for line in reader:
meta = data.new_metadata(file_.name, line_index)
amount = None
if line['Betrag (EUR)']:
amount = Amount(
locale.atof(line['Betrag (EUR)'], Decimal),
self.currency,
)
date = datetime.strptime(line['Buchungstag'],
'%d.%m.%Y').date()
if line['Verwendungszweck'] == 'Tagessaldo':
if amount:
entries.append(
data.Balance(
meta,
date + timedelta(days=1),
self.account,
amount,
None,
None,
))
else:
description = '{} {}'.format(line['Buchungstext'],
line['Verwendungszweck'])
postings = [
data.Posting(self.account, amount, None, None,
None, None)
]
entries.append(
data.Transaction(
meta,
date,
self.FLAG,
line['Auftraggeber / Begünstigter'],
description,
data.EMPTY_SET,
data.EMPTY_SET,
postings,
))
line_index += 1
# Closing Balance
meta = data.new_metadata(file_.name, closing_balance_index)
entries.append(
data.Balance(
meta,
self._balance_date,
self.account,
self._balance_amount,
None,
None,
))
return entries
#!/usr/bin/env python
import csv
import locale
import datetime
locale.setlocale(locale.LC_ALL, 'en_US.UTF8')
infile = '201606_F3_1_8_2303.csv'
with open(infile, encoding='cp950') as csvfile:
reader = csv.reader(csvfile)
next(reader)
next(reader)
for row in reader:
datestr = row[0].split('/')
year = int(datestr[0]) + 1911
month = int(datestr[1])
day = int(datestr[2])
date = datetime.date(year, month, day)
print(date, [locale.atof(i) for i in row[1:]])
def send_value(self):
"""
Emit a :attr:`send_value_signal` to update channel value.
The text is cleaned of all units, user-formatting and scale values
before being sent back to the channel. This function is attached the
ReturnPressed signal of the PyDMLineEdit
"""
send_value = str(self.text())
# Clean text of unit string
if self._show_units and self._unit and self._unit in send_value:
send_value = send_value[:-len(self._unit)].strip()
try:
if self.channeltype not in [str, np.ndarray]:
scale = self._scale
if scale is None or scale == 0:
scale = 1.0
if self._display_format_type in [
DisplayFormat.Default, DisplayFormat.String
]:
if self.channeltype == float:
num_value = locale.atof(send_value)
else:
num_value = self.channeltype(send_value)
scale = self.channeltype(scale)
elif self._display_format_type == DisplayFormat.Hex:
num_value = int(send_value, 16)
elif self._display_format_type == DisplayFormat.Binary:
num_value = int(send_value, 2)
elif self._display_format_type in [
DisplayFormat.Exponential, DisplayFormat.Decimal
]:
num_value = locale.atof(send_value)
num_value = num_value / scale
self.send_value_signal[self.channeltype].emit(num_value)
elif self.channeltype == np.ndarray:
# Arrays will be in the [1.2 3.4 22.214] format
if self._display_format_type == DisplayFormat.String:
self.send_value_signal[str].emit(send_value)
else:
arr_value = list(
filter(
None,
send_value.replace("[",
"").replace("]",
"").split(" ")))
arr_value = np.array(arr_value, dtype=self.subtype)
self.send_value_signal[np.ndarray].emit(arr_value)
else:
# Channel Type is String
# Lets just send what we have after all
self.send_value_signal[str].emit(send_value)
except ValueError:
logger.exception(
"Error trying to set data '{0}' with type '{1}' and format '{2}' at widget '{3}'."
.format(self.text(), self.channeltype,
self._display_format_type, self.objectName()))
self.clearFocus()
self.set_display()
report_template['query']['start-date'] = dlabel
parseflag_start_date_parsed = True
report_template['query']['end-date'] = dlabel
elif report_template['name'] == 'visitor-location-city':
item['city'] = dlabel
elif report_template['name'] == 'visitor-location-region':
item['region'] = dlabel
elif report_template['name'] == 'visitor-location-country':
item['country'] = dlabel
else:
item['page_title'] = dlabel
# values
try:
vlabel = int(
locale.atof(bundled_dps[1 + index_offset].strip(',')))
except:
vlabel = bundled_dps[1 + index_offset].strip('%,')
if report_template['name'] == 'visits':
item['visits'] = vlabel
else:
item['active_visitors'] = vlabel
# hrefs
hyperlink_payload = rp.a
if hyperlink_payload is None:
pass
else:
item['page'] = str(rp.a.get('href'))
# push the data into its home.
name = pw.format_string(cells[1].findall("font/a")[0].text.strip(),None)
# get operational date
op_date = cells[2].findall("font")[0].text.strip()
if op_date:
try:
d = parse_date(op_date)
op_year = d.year
found_operational_year_count += 1
except:
op_year = pw.NO_DATA_NUMERIC
else:
op_year = pw.NO_DATA_NUMERIC
# get plant capacity
capacity = CAPACITY_CONVERSION_TO_MW * locale.atof(cells[4].findall("font")[0].text)
# get owner
owner = u""
owner_description = pw.format_string(cells[6].findall("font")[0].text.strip(), ENCODING)
if u"não identificado" in owner_description:
owner = pw.NO_DATA_UNICODE
break
# TODO: Read owner correctly
owner = owner_description
"""
for owner in owner_list:
if u"não identificado" in owner.text:
owner_full = pw.NO_DATA_UNICODE
def __init__(self, fields, raw_csv, options):
"""Parameters:
fields: list of fields read from one line of the CSV file
raw_csv: unprocessed line from CSV file
options: from CLI args and config file
"""
if 'addons' in options:
self.addons = dict(
(k, fields[v - 1]) for k, v in options.addons.items())
else:
self.addons = dict()
# Get the date and convert it into a ledger formatted date.
self.date = fields[options.date - 1]
entry_date = datetime.strptime(self.date, options.csv_date_format)
if options.ledger_date_format:
if options.ledger_date_format != options.csv_date_format:
self.date = (datetime.strptime(
self.date, options.csv_date_format).strftime(
options.ledger_date_format))
# determine how many days old this entry is
self.days_old = (datetime.now() - entry_date).days
# convert effective dates
if options.effective_date:
self.effective_date = fields[options.effective_date - 1]
if options.ledger_date_format:
if options.ledger_date_format != options.csv_date_format:
self.effective_date = (datetime.strptime(
self.effective_date, options.csv_date_format).strftime(
options.ledger_date_format))
else:
self.effective_date = ""
desc = []
for index in re.compile(',\s*').split(options.desc):
desc.append(fields[int(index) - 1].strip())
self.desc = ' '.join(desc).strip()
self.credit = get_field_at_index(fields, options.credit,
options.csv_decimal_comma,
options.ledger_decimal_comma)
self.debit = get_field_at_index(fields, options.debit,
options.csv_decimal_comma,
options.ledger_decimal_comma)
if self.credit and self.debit and atof(self.credit) == 0:
self.credit = ''
elif self.credit and self.debit and atof(self.debit) == 0:
self.debit = ''
self.credit_account = options.account
self.currency = options.currency
self.credit_currency = getattr(options, 'credit_currency',
self.currency)
self.cleared_character = options.cleared_character
if options.template_file:
with open(options.template_file, 'r', encoding='utf-8') as f:
self.transaction_template = f.read()
else:
self.transaction_template = ""
self.raw_csv = raw_csv.strip()
# We also record this - in future we may use it to avoid duplication
self.md5sum = hashlib.md5(self.raw_csv.encode('utf-8')).hexdigest()
loc_temp[0] = optimal_sum_temp_wwheat / 1
if optimal_sum_hum_wwheat == 0:
loc_hum[0] = 0
else:
loc_hum[0] = optimal_sum_hum_wwheat / 1
if optimal_sum_precip_wwheat == 0:
loc_precip[0] = 0
else:
loc_precip[0] = optimal_sum_precip_wwheat / 1
for i in range(0, 6):
arrayofcrops[i]["Location_Temperature"] = loc_temp[i]
arrayofcrops[i]["Location_Humidity"] = loc_hum[i]
arrayofcrops[i]["Location_Precipitation"] = loc_precip[i]
for i in range(0, 6):
weighted_ideal[i] = locale.atof(arrayofcrops[i].get(
"Ideal_Temperature")) * temp_weights[i] + locale.atof(arrayofcrops[
i].get("Ideal_Humidity")) * humidity_weights[i] + locale.atof(
arrayofcrops[i].get("Ideal_Precipitation")) * precip_weights[i]
weighted_location[i] = loc_temp[i] * temp_weights[i] + loc_hum[
i] * humidity_weights[i] + loc_precip[i] * precip_weights[i]
for i in range(0, 6):
per_change[i] = (abs(weighted_ideal[i] - weighted_location[i]) /
weighted_ideal[i]) * 100
if (per_change[i] > 0 and per_change[i] <= 5):
score[i] = 10
elif (per_change[i] > 5 and per_change[i] <= 10):
score[i] = 8
elif (per_change[i] > 10 and per_change[i] <= 15):
score[i] = 6
elif (per_change[i] > 15 and per_change[i] <= 20):
def _test_atof(self, value, out):
self.assertEqual(locale.atof(value), out)
def get_csil_prices(params=exparams, rval=None):
if rval is None:
rval = {}
delivery_codes = {
# 3: '3#333',
# 5: '5#334',
7: '7#529',
10: '10#1452',
12: '12#7271',
15: '15#1453',
18: '18#7272',
21: '21#7273'
}
delivery_times = sorted(delivery_codes.keys())
layers_codes = {
1: '2180',
2: '2181',
4: '2183',
6: '2184',
}
browser = splinter.Browser('firefox', profile=FIREFOX_PROFILE_PATH)
url = 'http://login.pcbpower.com/V2/login.aspx'
browser.visit(url)
values = {'txtUserName': dvobj.username, 'txtPassword': dvobj.password}
browser.fill_form(values)
button = browser.find_by_name('btnlogin')
button.click()
link = browser.find_by_id('ctl00_aPlaceOrder')
link.click()
values = OrderedDict()
values['ctl00$ContentPlaceHolder1$txtPCBName'] = params['pcbname']
values['ctl00$ContentPlaceHolder1$ddlLayers'] = layers_codes[
params['layers']] # noqa
values['ctl00$ContentPlaceHolder1$txtDimX'] = str(params['dX'])
values['ctl00$ContentPlaceHolder1$txtDimY'] = str(params['dY'])
if 'qty' in params.keys():
values['ctl00$ContentPlaceHolder1$txtQuantity'] = str(
params['qty'][1]) # noqa
else:
values['ctl00$ContentPlaceHolder1$txtQuantity'] = '1'
values['ctl00$ContentPlaceHolder1$DDLsurfacefinish'] = params['finish']
if 'time' in params.keys():
values['ctl00$ContentPlaceHolder1$ddlDelTerms'] = delivery_codes[
params['time']] # noqa
else:
values['ctl00$ContentPlaceHolder1$ddlDelTerms'] = delivery_codes[7]
if not browser.is_element_present_by_id('shortNotiText', wait_time=100):
raise Exception
ready = False
timeout = 10
while not ready and timeout:
el = browser.find_by_id('shortNotiText')
if el[0].text == u"We're online":
ready = True
timeout -= 1
time.sleep(1)
time.sleep(5)
browser.fill_form(values)
try:
oldt = browser.find_by_id('ctl00_ContentPlaceHolder1_lblUnitPrc').text
except AttributeError:
oldt = ''
# qty = str(params['qty'][1])
# oldv = qty
time.sleep(2)
button = browser.find_by_id('ctl00_ContentPlaceHolder1_btnCalculate')
button.click()
time.sleep(2)
button = browser.find_by_id('ctl00_ContentPlaceHolder1_btnCalculate')
button.click()
try:
newt = browser.find_by_id('ctl00_ContentPlaceHolder1_lblUnitPrc').text
except AttributeError:
newt = ''
try:
newtt = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblTotalPrice').text # noqa
except AttributeError:
newtt = ''
while oldt == newt:
try:
newt = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblUnitPrc').text # noqa
except AttributeError:
newt = ''
try:
newtt = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblTotalPrice').text # noqa
except AttributeError:
newtt = ''
time.sleep(0.5)
oldt = newt
oldtt = newtt
pb = TendrilProgressBar(max=len(params['qty']))
for qty in params['qty'][2:]:
lined = {}
while browser.find_by_name('ctl00$ContentPlaceHolder1$txtQuantity'
)[0].value != '': # noqa
browser.type('ctl00$ContentPlaceHolder1$txtQuantity', '\b')
time.sleep(0.1)
browser.type('ctl00$ContentPlaceHolder1$txtQuantity', str(qty))
time.sleep(0.1)
browser.type('ctl00$ContentPlaceHolder1$txtQuantity', '\t')
if qty > 4:
loi = [10]
else:
loi = [10]
pb.next(note="{0} {1}".format(qty, loi))
# "\n{0:>7.4f}% {1:<40} Qty: {2:<10} DTS: {3:<4}\nGenerating PCB Pricing".format( # noqa
# percentage, params['pcbname'], qty, loi)
for dt_s in loi:
dt_idx = delivery_times.index(dt_s)
dts = delivery_times[dt_idx:dt_idx + 3]
browser.select('ctl00$ContentPlaceHolder1$ddlDelTerms',
delivery_codes[dt_s]) # noqa
time.sleep(1)
while True:
try:
try:
newt = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblUnitPrc'
).text # noqa
except AttributeError:
newt = ''
try:
# newt1 = ''
# newt2 = ''
newt1 = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblnextunitprc1'
).text # noqa
newt2 = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblnextunitprc2'
).text # noqa
except AttributeError:
newt1 = ''
newt2 = ''
break
except selenium.common.exceptions.StaleElementReferenceException: # noqa
logger.warning("Selenium Exception Caught. Retrying")
continue
timeout = 25
while oldt == newt and oldtt == newtt and newt is not '' and timeout > 0: # noqa
timeout -= 1
time.sleep(1)
while True:
try:
try:
newt = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblUnitPrc'
).text # noqa
except AttributeError:
newt = ''
try:
# newt1 = ''
# newt2 = ''
newt1 = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblnextunitprc1'
).text # noqa
newt2 = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblnextunitprc2'
).text # noqa
except AttributeError:
newt1 = ''
newt2 = ''
break
except selenium.common.exceptions.StaleElementReferenceException: # noqa
logger.warning("Selenium Exception Caught. Retrying")
continue
try:
newtt = browser.find_by_id(
'ctl00_ContentPlaceHolder1_lblTotalPrice').text # noqa
except AttributeError:
newtt = ''
try:
lined[dts[0]] = locale.atof(newt)
if newt1 != '':
lined[dts[1]] = locale.atof(newt1)
if newt2 != '':
lined[dts[2]] = locale.atof(newt2)
except ValueError:
logger.warning("Caught Exception at CSIL Website. Retrying.")
browser.quit()
return get_csil_prices(params, rval)
oldt = newt
oldtt = newtt
params['qty'].remove(qty)
# print lined
rval[qty] = lined
browser.quit()
return rval
def convert_numeric():
try:
return locale.atof(value, Decimal)
except (decimal.InvalidOperation, AttributeError):
return
def itervals(self, record):
for name, fm in self.entries:
dt = fm['datatype']
val = record[fm['rec_index']]
if dt == 'composite':
sb = fm['display'].get('composite_sort', 'text')
if sb == 'date':
try:
val = parse_date(val)
except:
val = UNDEFINED_DATE
dt = 'datetime'
elif sb == 'number':
try:
p = 1
for i, candidate in enumerate(
('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')):
if val.endswith(candidate):
p = 1024**(i)
val = val[:-len(candidate)].strip()
break
val = locale.atof(val) * p
except:
val = 0.0
dt = 'float'
elif sb == 'bool':
val = force_to_bool(val)
dt = 'bool'
if dt == 'datetime':
if val is None:
val = UNDEFINED_DATE
if tweaks['sort_dates_using_visible_fields']:
format = None
if name == 'timestamp':
format = tweaks['gui_timestamp_display_format']
elif name == 'pubdate':
format = tweaks['gui_pubdate_display_format']
elif name == 'last_modified':
format = tweaks['gui_last_modified_display_format']
elif fm['is_custom']:
format = fm['display'].get('date_format', None)
val = clean_date_for_sort(val, format)
elif dt == 'series':
if val is None:
val = ('', 1)
else:
if self.library_order:
try:
lang = record[self.lang_idx].partition(',')[0]
except (AttributeError, ValueError, KeyError,
IndexError, TypeError):
lang = None
val = title_sort(val, order='library_order', lang=lang)
sidx_fm = self.field_metadata[name + '_index']
sidx = record[sidx_fm['rec_index']]
val = (self.string_sort_key(val), sidx)
elif dt in ('text', 'comments', 'composite', 'enumeration'):
if val:
if fm['is_multiple']:
jv = fm['is_multiple']['list_to_ui']
sv = fm['is_multiple']['cache_to_list']
if '&' in jv:
val = jv.join(
[author_to_author_sort(v) for v in val.split(sv)])
else:
val = jv.join(sorted(val.split(sv),
key=self.string_sort_key))
val = self.string_sort_key(val)
elif dt == 'bool':
if not self.db_prefs.get('bools_are_tristate'):
val = {True: 1, False: 2, None: 2}.get(val, 2)
else:
val = {True: 1, False: 2, None: 3}.get(val, 3)
yield val
def add_to_cart(request, product_id=None):
"""
Adds the passed product with passed product_id to the cart after
some validations have been taken place. The amount is taken from the query
string.
"""
if product_id is None:
product_id = request.REQUEST.get("product_id")
product = muecke_get_object_or_404(Product, pk=product_id)
# Only active and deliverable products can be added to the cart.
if (product.is_active() and product.is_deliverable()) == False:
raise Http404()
try:
value = request.POST.get("quantity", "1.0")
if isinstance(value, unicode):
# atof() on unicode string fails in some environments, like Czech
value = value.encode("utf-8")
quantity = locale.atof(value)
except (TypeError, ValueError):
quantity = 1.0
# Validate properties (They are added below)
properties_dict = {}
if product.is_configurable_product():
for key, value in request.POST.items():
if key.startswith("property-"):
try:
property_id = key.split("-")[1]
except IndexError:
continue
try:
property = Property.objects.get(pk=property_id)
except Property.DoesNotExist:
continue
if property.is_number_field:
try:
value = muecke.core.utils.atof(value)
except ValueError:
value = locale.atof("0.0")
properties_dict[property_id] = unicode(value)
# validate property's value
if property.is_number_field:
if (value < property.unit_min) or (value >
property.unit_max):
msg = _(
u"%(name)s must be between %(min)s and %(max)s %(unit)s."
) % {
"name": property.title,
"min": property.unit_min,
"max": property.unit_max,
"unit": property.unit
}
return muecke.core.utils.set_message_cookie(
product.get_absolute_url(), msg)
# calculate valid steps
steps = []
x = property.unit_min
while x < property.unit_max:
steps.append("%.2f" % x)
x = x + property.unit_step
steps.append("%.2f" % property.unit_max)
value = "%.2f" % value
if value not in steps:
msg = _(
u"Your entered value for %(name)s (%(value)s) is not in valid step width, which is %(step)s."
) % {
"name": property.title,
"value": value,
"step": property.unit_step
}
return muecke.core.utils.set_message_cookie(
product.get_absolute_url(), msg)
if product.get_active_packing_unit():
quantity = muecke.catalog.utils.calculate_real_amount(
product, quantity)
cart = cart_utils.get_or_create_cart(request)
cart_item = cart.add(product, properties_dict, quantity)
cart_items = [cart_item]
# Check stock amount
message = ""
if product.manage_stock_amount and cart_item.amount > product.stock_amount and not product.order_time:
if product.stock_amount == 0:
message = _(u"Sorry, but '%(product)s' is not available anymore." %
{"product": product.name})
elif product.stock_amount == 1:
message = _(
u"Sorry, but '%(product)s' is only one time available." %
{"product": product.name})
else:
message = _(
u"Sorry, but '%(product)s' is only %(amount)s times available."
) % {
"product": product.name,
"amount": product.stock_amount
}
cart_item.amount = product.stock_amount
cart_item.save()
# Add selected accessories to cart
for key, value in request.POST.items():
if key.startswith("accessory"):
accessory_id = key.split("-")[1]
try:
accessory = Product.objects.get(pk=accessory_id)
except ObjectDoesNotExist:
continue
# Get quantity
quantity = request.POST.get("quantity-%s" % accessory_id, 0)
try:
quantity = float(quantity)
except TypeError:
quantity = 1
cart_item = cart.add(product=accessory, amount=quantity)
cart_items.append(cart_item)
# Store cart items for retrieval within added_to_cart.
request.session["cart_items"] = cart_items
cart_changed.send(cart, request=request)
# Update the customer's shipping method (if appropriate)
customer = customer_utils.get_or_create_customer(request)
shipping_utils.update_to_valid_shipping_method(request,
customer,
save=True)
# Update the customer's payment method (if appropriate)
payment_utils.update_to_valid_payment_method(request, customer, save=True)
# Save the cart to update modification date
cart.save()
try:
url_name = settings.LFS_AFTER_ADD_TO_CART
except AttributeError:
url_name = "muecke.cart.views.added_to_cart"
if message:
return muecke.core.utils.set_message_cookie(reverse(url_name), message)
else:
return HttpResponseRedirect(reverse(url_name))
from bs4 import BeautifulSoup
import urllib.request
import locale
import numpy as np
import pandas as pd
import matplotlib
from ggplot import *
html=urllib.request.urlopen("http://www.twse.com.tw/exchangeReport/MI_5MINS_INDEX?response=html&date=20170609")
soup = BeautifulSoup(html, 'html.parser')
locale.setlocale( locale.LC_ALL, 'en_US.UTF-8')
#資料採集
#注意find("tbody")，此行若少會採集到表頭，表頭只有一格所以會產生error
a=[]
price=[locale.atof(td) for child in soup.find("tbody").find_all("tr") for td in child.find_all('td')[1] ]
time =[td for child in soup.find("tbody").find_all("tr") for td in child.find_all('td')[0] ]
# 採集的資料整理成可以處理的資料格式
pdPrice=pd.DataFrame(price)
pdTime=pd.DataFrame(time)
Price_dataFrame=pd.concat([pdTime,pdPrice],axis=1) # pd.concat裡面要放pandas dadframe
Price_dataFrame.columns=["time","price"]
# pandas DataFrame取用1到100row
Price_dataFrame.ix[1:100,:]# 可以用"名字"叫
Price_dataFrame.iloc[1:100,:]#只能用數字叫
Price_dataFrame['price']
Price_dataFrame['time']
Price_dataFrame.describe()
def process_row(row,
row_num,
column_map,
num_required_columns,
doaj_offline_analysis=False,
bypass_cert_verification=False):
"""
Enrich a single row of data and reformat it according to Open APC standards.
Take a csv row (a list) and a column mapping (a list of CSVColumn objects)
and return an enriched and re-arranged version which conforms to the Open
APC data schema.
Args:
row: A list of column values (as yielded by a UnicodeReader f.e.).
row_num: The line number in the csv file, for logging purposes.
column_map: An OrderedDict of CSVColumn Objects, mapping the row
cells to Open APC data schema fields.
num_required_columns: An int describing the required length of the row
list. If not matched, an error is logged and the
row is returned unchanged.
doaj_offline_analysis: If true, a local copy will be used for the DOAJ
lookup.
bypass_cert_verification: If true, certificate validation will be
skipped when connecting to metadata
providers via TLS.
Returns:
A list of values which represents the enriched and re-arranged variant
of the input row. If no errors were logged during the process, this
result will conform to the Open APC data schema.
"""
MESSAGES = {
"num_columns":
"Syntax: The number of values in this row (%s) " +
"differs from the number of columns (%s). Line left " +
"unchanged, the resulting CSV file will not be valid.",
"locale":
"Error: Could not process the monetary value '%s' in " +
"column %s. This will usually have one of two reasons:\n1) " +
"The value does not represent a number.\n2) The value " +
"represents a number, but its format differs from your " +
"current system locale - the most common source of error " +
"will be the decimal mark (1234.56 vs 1234,56). Try using " +
"another locale with the -l option.",
"unify":
"Normalisation: CrossRef-based {} changed from '{}' to '{}' " +
"to maintain consistency."
}
if len(row) != num_required_columns:
msg = "Line %s: " + MESSAGES["num_columns"]
logging.error(msg, row_num, len(row), num_required_columns)
return row
doi = row[column_map["doi"].index]
current_row = OrderedDict()
# Copy content of identified columns
for csv_column in column_map.values():
if csv_column.index is not None and len(row[csv_column.index]) > 0:
if csv_column.column_type == "euro":
# special case for monetary values: Cast to float to ensure
# the decimal point is a dot (instead of a comma)
euro_value = row[csv_column.index]
try:
euro = locale.atof(euro_value)
if euro.is_integer():
euro = int(euro)
current_row[csv_column.column_type] = str(euro)
except ValueError:
msg = "Line %s: " + MESSAGES["locale"]
logging.error(msg, row_num, euro_value, csv_column.index)
else:
current_row[csv_column.column_type] = row[csv_column.index]
else:
current_row[csv_column.column_type] = "NA"
if len(doi) == 0 or doi == 'NA':
msg = ("Line %s: No DOI found, entry could not enriched with " +
"Crossref or Pubmed metadata.")
logging.warning(msg, row_num)
current_row["indexed_in_crossref"] = "FALSE"
else:
# include crossref metadata
crossref_result = get_metadata_from_crossref(doi)
if crossref_result["success"]:
logging.info("Crossref: DOI resolved: " + doi)
current_row["indexed_in_crossref"] = "TRUE"
data = crossref_result["data"]
for key, value in data.iteritems():
if value is not None:
if key == "journal_full_title":
unified_value = get_unified_journal_title(value)
if unified_value != value:
msg = MESSAGES["unify"].format(
"journal title", value, unified_value)
logging.warning(msg)
new_value = unified_value
elif key == "publisher":
unified_value = get_unified_publisher_name(value)
if unified_value != value:
msg = MESSAGES["unify"].format(
"publisher name", value, unified_value)
logging.warning(msg)
new_value = unified_value
else:
new_value = value
else:
new_value = "NA"
msg = (
u"WARNING: Element '%s' not found in in response for "
+ "doi %s.")
logging.debug(msg, key, doi)
old_value = current_row[key]
current_row[key] = column_map[key].check_overwrite(
old_value, new_value)
else:
msg = "Line %s: Crossref: Error while trying to resolve DOI %s: %s"
logging.error(msg, row_num, doi, crossref_result["error_msg"])
current_row["indexed_in_crossref"] = "FALSE"
# include pubmed metadata
pubmed_result = get_metadata_from_pubmed(doi)
if pubmed_result["success"]:
logging.info("Pubmed: DOI resolved: " + doi)
data = pubmed_result["data"]
for key, value in data.iteritems():
if value is not None:
new_value = value
else:
new_value = "NA"
msg = (
u"WARNING: Element %s not found in in response for " +
"doi %s.")
logging.debug(msg, key, doi)
old_value = current_row[key]
current_row[key] = column_map[key].check_overwrite(
old_value, new_value)
else:
msg = "Line %s: Pubmed: Error while trying to resolve DOI %s: %s"
logging.error(msg, row_num, doi, pubmed_result["error_msg"])
# lookup in DOAJ. try the EISSN first, then ISSN and finally print ISSN
issns = []
if current_row["issn_electronic"] != "NA":
issns.append(current_row["issn_electronic"])
if current_row["issn"] != "NA":
issns.append(current_row["issn"])
if current_row["issn_print"] != "NA":
issns.append(current_row["issn_print"])
for issn in issns:
# look up in an offline copy of the DOAJ if requested...
if doaj_offline_analysis:
lookup_result = doaj_offline_analysis.lookup(issn)
if lookup_result:
msg = (u"DOAJ: Journal ISSN (%s) found in DOAJ " +
"offline copy ('%s').")
logging.info(msg, issn, lookup_result)
current_row["doaj"] = "TRUE"
break
else:
msg = (u"DOAJ: Journal ISSN (%s) not found in DOAJ " +
"offline copy.")
current_row["doaj"] = "FALSE"
logging.info(msg, issn)
# ...or query the online API
else:
doaj_res = lookup_journal_in_doaj(issn, bypass_cert_verification)
if doaj_res["data_received"]:
if doaj_res["data"]["in_doaj"]:
msg = u"DOAJ: Journal ISSN (%s) found in DOAJ ('%s')."
logging.info(msg, issn, doaj_res["data"]["title"])
current_row["doaj"] = "TRUE"
break
else:
msg = u"DOAJ: Journal ISSN (%s) not found in DOAJ."
logging.info(msg, issn)
current_row["doaj"] = "FALSE"
else:
msg = (u"Line %s: DOAJ: Error while trying to look up " +
"ISSN %s: %s")
logging.error(msg, row_num, issn, doaj_res["error_msg"])
return current_row.values()
def convert(self, value):
try:
return locale.atof(value, Decimal)
except decimal.InvalidOperation:
return self._default
def parseTransactions(driver, transactions):
if platform.system() == 'Darwin':
locale.setlocale(locale.LC_NUMERIC, "EN_US")
else:
locale.setlocale(locale.LC_NUMERIC, "")
transaction_arr = []
company_name_dict = {}
transactions_with_dates = []
for transaction in transactions:
header_text = transaction.find_element_by_xpath(
".//div[@class='_2VPzNpwfga_8Mcn-DCUwug']").text
canceled_text = transaction.find_element_by_xpath(
".//div[@class='_22YwnO0XVSevsIC6rD9HS3']").text
if transaction_types["CANCELED"] in canceled_text:
continue
if transaction_types["LIMIT_SELL"] in header_text or transaction_types[
"LIMIT_BUY"] in header_text or transaction_types[
"MARKET_SELL"] in header_text or transaction_types[
"MARKET_BUY"] in header_text:
info_children = transaction.find_elements_by_xpath(
".//div[@class='css-1qd1r5f']")
date_node = info_children[7]
date = date_node.get_attribute('textContent')
transaction_with_date = (transaction, date)
transactions_with_dates.append(transaction_with_date)
elif transaction_types["DIVIDEND"] in header_text:
date_node = transaction.find_elements_by_xpath(
".//span[@class='css-zy0xqa']")[1]
date = date_node.get_attribute('textContent')
transaction_with_date = (transaction, date)
transactions_with_dates.append(transaction_with_date)
transactions_with_dates.sort(
key=lambda date: datetime.strptime(date[1], '%b %d, %Y'))
for transaction_with_date in transactions_with_dates:
transaction = transaction_with_date[0]
header_text = transaction.find_element_by_xpath(
".//div[@class='_2VPzNpwfga_8Mcn-DCUwug']").text
canceled_text = transaction.find_element_by_xpath(
".//div[@class='_22YwnO0XVSevsIC6rD9HS3']").text
company_name_title = header_text.split("\n")
company_name_list = company_name_title[0].split(" ")
print(company_name_list)
if transaction_types["CANCELED"] in canceled_text:
continue
if transaction_types["LIMIT_SELL"] in header_text or transaction_types[
"LIMIT_BUY"] in header_text:
company_name = " ".join(company_name_list[:-2])
info_children = transaction.find_elements_by_xpath(
".//div[@class='css-1qd1r5f']")
ticker_symbol_node = info_children[1]
ticker_symbol = ticker_symbol_node.find_element_by_xpath(
".//a").get_attribute('textContent')
if company_name not in company_name_dict.keys():
company_name_dict[company_name] = ticker_symbol
transaction_type_node = info_children[3]
transaction_type = transaction_type_node.find_element_by_xpath(
".//span").get_attribute('textContent')
transaction_type_split = transaction_type.split(" ")
transaction_type_final = transaction_type_split[1]
transaction_date_node = info_children[7]
transaction_date = transaction_date_node.get_attribute(
'textContent')
filled_transaction_node = info_children[17]
filled_transaction = filled_transaction_node.get_attribute(
'textContent')
filled_transaction_split = filled_transaction.split(" ")
quantity = filled_transaction_split[0]
price = filled_transaction_split[3]
price = locale.atof(price.split("$")[1])
total_value_node = info_children[19]
total_value = locale.atof(
total_value_node.get_attribute('textContent').split("$")[1])
transaction_tuple = (ticker_symbol, transaction_type_final,
quantity, price, total_value,
transaction_date)
transaction_arr.append(transaction_tuple)
continue
if transaction_types["MARKET_SELL"] in header_text or transaction_types[
"MARKET_BUY"] in header_text:
company_name = " ".join(company_name_list[:-2])
info_children = transaction.find_elements_by_xpath(
".//div[@class='css-1qd1r5f']")
ticker_symbol_node = info_children[1]
ticker_symbol = ticker_symbol_node.find_element_by_xpath(
".//a").get_attribute('textContent')
if company_name not in company_name_dict.keys():
company_name_dict[company_name] = ticker_symbol
transaction_type_node = info_children[3]
transaction_type = transaction_type_node.find_element_by_xpath(
".//span").get_attribute('textContent')
transaction_type_split = transaction_type.split(" ")
transaction_type_final = transaction_type_split[1]
transaction_date_node = info_children[7]
transaction_date = transaction_date_node.get_attribute(
'textContent')
filled_transaction_node = info_children[15]
filled_transaction = filled_transaction_node.get_attribute(
'textContent')
filled_transaction_split = filled_transaction.split(" ")
quantity = filled_transaction_split[0]
price = filled_transaction_split[3]
price = locale.atof(price.split("$")[1])
total_value_node = info_children[17]
total_value = locale.atof(
total_value_node.get_attribute('textContent').split("$")[1])
transaction_tuple = (ticker_symbol, transaction_type_final,
quantity, price, total_value,
transaction_date)
transaction_arr.append(transaction_tuple)
continue
if transaction_types["DIVIDEND"] in header_text:
company_name = " ".join(company_name_list[2:])
ticker_symbol = company_name_dict[company_name]
info_children = transaction.find_elements_by_xpath(
".//div[@class='css-1qd1r5f']")
transaction_type = "Dividend"
transaction_date_node = transaction.find_elements_by_xpath(
".//span[@class='css-zy0xqa']")[1]
transaction_date = transaction_date_node.get_attribute(
'textContent')
quantity = info_children[1].get_attribute('textContent')
price = locale.atof(
info_children[3].get_attribute('textContent').split("$")[1])
total_value = locale.atof(
info_children[5].get_attribute('textContent').split("$")[1])
transaction_tuple = (ticker_symbol, transaction_type, quantity,
price, total_value, transaction_date)
transaction_arr.append(transaction_tuple)
return transaction_arr
def convert(self, value):
try:
return locale.atof(value)
except ValueError:
return self._default
def process_row(row,
row_num,
column_map,
num_required_columns,
no_crossref_lookup=False,
no_pubmed_lookup=False,
no_doaj_lookup=False,
doaj_offline_analysis=False,
bypass_cert_verification=False):
"""
Enrich a single row of data and reformat it according to Open APC standards.
Take a csv row (a list) and a column mapping (a list of CSVColumn objects)
and return an enriched and re-arranged version which conforms to the Open
APC data schema.
Args:
row: A list of column values (as yielded by a UnicodeReader f.e.).
row_num: The line number in the csv file, for logging purposes.
column_map: An OrderedDict of CSVColumn Objects, mapping the row
cells to Open APC data schema fields.
num_required_columns: An int describing the required length of the row
list. If not matched, an error is logged and the
row is returned unchanged.
no_crossref_lookup: If true, no metadata will be imported from crossref.
no_pubmed_lookup: If true, no_metadata will be imported from pubmed.
no_doaj_lookup: If true, journals will not be checked for being
listended in the DOAJ (default).
doaj_offline_analysis: If true, a local copy will be used for the DOAJ
lookup. Has no effect if no_doaj_lookup is set to
true.
bypass_cert_verification: If true, certificate validation will be
skipped when connecting to metadata
providers via TLS.
Returns:
A list of values which represents the enriched and re-arranged variant
of the input row. If no errors were logged during the process, this
result will conform to the Open APC data schema.
"""
MESSAGES = {
"num_columns":
u"Syntax: The number of values in this row (%s) " +
"differs from the number of columns (%s). Line left " +
"unchanged, the resulting CSV file will not be valid.",
"locale":
u"Error: Could not process the monetary value '%s' in " +
"column %s. This will usually have one of two reasons:\n1) " +
"The value does not represent a number.\n2) The value " +
"represents a number, but its format differs from your " +
"current system locale - the most common source of error " +
"will be the decimal mark (1234.56 vs 1234,56). Try using " +
"another locale with the -l option.",
"unify":
u"Normalisation: CrossRef-based {} changed from '{}' to '{}' " +
"to maintain consistency.",
"doi_norm":
u"Normalisation: DOI '{}' normalised to pure form ({}).",
"springer_distinction":
u"publisher 'Springer Nature' found " +
"for a pre-2015 article - publisher " +
"changed to '%s' based on prefix " + "discrimination ('%s')",
"unknown_prefix":
u"publisher 'Springer Nature' found for a " +
"pre-2015 article, but discrimination was " +
"not possible - unknown prefix ('%s')",
"issn_hyphen_fix":
u"Normalisation: Added hyphen to %s value (%s -> %s)"
}
if len(row) != num_required_columns:
msg = "Line %s: " + MESSAGES["num_columns"]
logging.error(msg, row_num, len(row), num_required_columns)
return row
doi = row[column_map["doi"].index]
current_row = OrderedDict()
# Copy content of identified columns
for csv_column in column_map.values():
if csv_column.column_type == "euro" and csv_column.index is not None:
# special case for monetary values: Cast to float to ensure
# the decimal point is a dot (instead of a comma)
euro_value = row[csv_column.index]
if len(euro_value) == 0:
msg = "Line %s: Empty monetary value in column %s."
logging.warning(msg, row_num, csv_column.index)
current_row[csv_column.column_type] = "NA"
else:
try:
euro = locale.atof(euro_value)
if euro.is_integer():
euro = int(euro)
current_row[csv_column.column_type] = str(euro)
except ValueError:
msg = "Line %s: " + MESSAGES["locale"]
logging.error(msg, row_num, euro_value, csv_column.index)
else:
if csv_column.index is not None and len(row[csv_column.index]) > 0:
current_row[csv_column.column_type] = row[csv_column.index]
else:
current_row[csv_column.column_type] = "NA"
if len(doi) == 0 or doi == 'NA':
msg = ("Line %s: No DOI found, entry could not enriched with " +
"Crossref or Pubmed metadata.")
logging.warning(msg, row_num)
current_row["indexed_in_crossref"] = "FALSE"
else:
# Normalise DOI
norm_doi = get_normalised_DOI(doi)
if norm_doi is not None and norm_doi != doi:
current_row["doi"] = norm_doi
msg = MESSAGES["doi_norm"].format(doi, norm_doi)
logging.warning(msg)
# include crossref metadata
if not no_crossref_lookup:
crossref_result = get_metadata_from_crossref(doi)
if crossref_result["success"]:
logging.info("Crossref: DOI resolved: " + doi)
current_row["indexed_in_crossref"] = "TRUE"
data = crossref_result["data"]
prefix = data.pop("prefix")
for key, value in data.iteritems():
if value is not None:
if key == "journal_full_title":
unified_value = get_unified_journal_title(value)
if unified_value != value:
msg = MESSAGES["unify"].format(
"journal title", value, unified_value)
logging.warning(msg)
new_value = unified_value
elif key == "publisher":
unified_value = get_unified_publisher_name(value)
if unified_value != value:
msg = MESSAGES["unify"].format(
"publisher name", value, unified_value)
logging.warning(msg)
new_value = unified_value
# Treat Springer Nature special case: crossref erroneously
# reports publisher "Springer Nature" even for articles
# published before 2015 (publishers fusioned only then)
if int(current_row["period"]
) < 2015 and new_value == "Springer Nature":
publisher = None
if prefix in [
"Springer (Biomed Central Ltd.)",
"Springer-Verlag",
"Springer - Psychonomic Society"
]:
publisher = "Springer Science + Business Media"
elif prefix in [
"Nature Publishing Group",
"Nature Publishing Group - Macmillan Publishers"
]:
publisher = "Nature Publishing Group"
if publisher:
msg = "Line %s: " + MESSAGES[
"springer_distinction"]
logging.warning(msg, row_num, publisher,
prefix)
new_value = publisher
else:
msg = "Line %s: " + MESSAGES[
"unknown_prefix"]
logging.error(msg, row_num, prefix)
# Fix ISSNs without hyphen
elif key in ["issn", "issn_print", "issn_electronic"]:
new_value = value
if re.match("^\d{7}[\dxX]$", value):
new_value = value[:4] + "-" + value[4:]
msg = "Line %s: " + MESSAGES["issn_hyphen_fix"]
logging.warning(msg, row_num, key, value,
new_value)
else:
new_value = value
else:
new_value = "NA"
msg = (
u"WARNING: Element '%s' not found in in response for "
+ "doi %s.")
logging.debug(msg, key, doi)
old_value = current_row[key]
current_row[key] = column_map[key].check_overwrite(
old_value, new_value)
else:
msg = "Line %s: Crossref: Error while trying to resolve DOI %s: %s"
logging.error(msg, row_num, doi, crossref_result["error_msg"])
current_row["indexed_in_crossref"] = "FALSE"
# include pubmed metadata
if not no_pubmed_lookup:
pubmed_result = get_metadata_from_pubmed(doi)
if pubmed_result["success"]:
logging.info("Pubmed: DOI resolved: " + doi)
data = pubmed_result["data"]
for key, value in data.iteritems():
if value is not None:
new_value = value
else:
new_value = "NA"
msg = (
u"WARNING: Element %s not found in in response for "
+ "doi %s.")
logging.debug(msg, key, doi)
old_value = current_row[key]
current_row[key] = column_map[key].check_overwrite(
old_value, new_value)
else:
msg = "Line %s: Pubmed: Error while trying to resolve DOI %s: %s"
logging.error(msg, row_num, doi, pubmed_result["error_msg"])
# lookup in DOAJ. try the EISSN first, then ISSN and finally print ISSN
if not no_doaj_lookup:
issns = []
new_value = "NA"
if current_row["issn_electronic"] != "NA":
issns.append(current_row["issn_electronic"])
if current_row["issn"] != "NA":
issns.append(current_row["issn"])
if current_row["issn_print"] != "NA":
issns.append(current_row["issn_print"])
for issn in issns:
# In some cases xref delievers ISSNs without a hyphen. Add it
# temporarily to prevent the DOAJ lookup from failing.
if re.match("^\d{7}[\dxX]$", issn):
issn = issn[:4] + "-" + issn[4:]
# look up in an offline copy of the DOAJ if requested...
if doaj_offline_analysis:
lookup_result = doaj_offline_analysis.lookup(issn)
if lookup_result:
msg = (u"DOAJ: Journal ISSN (%s) found in DOAJ " +
"offline copy ('%s').")
logging.info(msg, issn, lookup_result)
new_value = "TRUE"
break
else:
msg = (u"DOAJ: Journal ISSN (%s) not found in DOAJ " +
"offline copy.")
new_value = "FALSE"
logging.info(msg, issn)
# ...or query the online API
else:
doaj_res = lookup_journal_in_doaj(issn,
bypass_cert_verification)
if doaj_res["data_received"]:
if doaj_res["data"]["in_doaj"]:
msg = u"DOAJ: Journal ISSN (%s) found in DOAJ ('%s')."
logging.info(msg, issn, doaj_res["data"]["title"])
new_value = "TRUE"
break
else:
msg = u"DOAJ: Journal ISSN (%s) not found in DOAJ."
logging.info(msg, issn)
new_value = "FALSE"
else:
msg = (u"Line %s: DOAJ: Error while trying to look up " +
"ISSN %s: %s")
logging.error(msg, row_num, issn, doaj_res["error_msg"])
old_value = current_row["doaj"]
current_row["doaj"] = column_map["doaj"].check_overwrite(
old_value, new_value)
return current_row.values()
def refresh_cart(request):
"""
Refreshes the cart after some changes has been taken place, e.g.: the
amount of a product or shipping/payment method.
"""
cart = cart_utils.get_cart(request)
customer = customer_utils.get_or_create_customer(request)
# Update country
country_iso = request.POST.get("country")
if country_iso:
selected_country = Country.objects.get(code=country_iso.lower())
customer.selected_country_id = selected_country.id
if customer.selected_shipping_address:
customer.selected_shipping_address.country = selected_country
customer.selected_shipping_address.save()
customer.selected_shipping_address.save()
if customer.selected_invoice_address:
customer.selected_invoice_address.country = selected_country
customer.selected_invoice_address.save()
customer.selected_invoice_address.save()
# NOTE: The customer has to be saved already here in order to calculate
# a possible new valid shippig method below, which coulb be triggered by
# the changing of the shipping country.
customer.save()
# Update Amounts
message = ""
for item in cart.get_items():
try:
value = request.POST.get("amount-cart-item_%s" % item.id, "0.0")
if isinstance(value, unicode):
# atof() on unicode string fails in some environments, like Czech
value = value.encode("utf-8")
amount = locale.atof(value)
except (TypeError, ValueError):
amount = 1.0
if item.product.manage_stock_amount and amount > item.product.stock_amount and not item.product.order_time:
amount = item.product.stock_amount
if amount < 0:
amount = 0
if amount == 0:
message = _(
u"Sorry, but '%(product)s' is not available anymore." %
{"product": item.product.name})
elif amount == 1:
message = _(
u"Sorry, but '%(product)s' is only one time available." %
{"product": item.product.name})
else:
message = _(
u"Sorry, but '%(product)s' is only %(amount)s times available."
) % {
"product": item.product.name,
"amount": amount
}
if item.product.get_active_packing_unit():
item.amount = muecke.catalog.utils.calculate_real_amount(
item.product, float(amount))
else:
item.amount = amount
if amount == 0:
item.delete()
else:
item.save()
# IMPORTANT: We have to send the signal already here, because the valid
# shipping methods might be dependent on the price.
cart_changed.send(cart, request=request)
# Update shipping method
customer.selected_shipping_method_id = request.POST.get("shipping_method")
valid_shipping_methods = shipping_utils.get_valid_shipping_methods(request)
if customer.selected_shipping_method not in valid_shipping_methods:
customer.selected_shipping_method = shipping_utils.get_default_shipping_method(
request)
# Update payment method
customer.selected_payment_method_id = request.POST.get("payment_method")
# Last but not least we save the customer ...
customer.save()
result = simplejson.dumps(
{
"html": cart_inline(request),
"message": message,
}, cls=LazyEncoder)
return HttpResponse(result)
import math
locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')
df = pd.read_csv('/home/walker/hydrosphere/brakedata.csv', sep=',')
brakeData = df.ix[:, 0:3]
a = []
def parsePoint(w, k, h):
return LabeledPoint(worn, [km, heat])
for row in brakeData.itertuples():
worn = getattr(row, 'worn')
km = locale.atof(getattr(row, 'km'))
heat = getattr(row, 'heat')
lp = parsePoint(worn, km, heat)
a.append(lp)
lrm = LogisticRegressionWithLBFGS.train(sc.parallelize(a))
lrm.save(sc, "/tmp/brakeModel")
p = sc.parallelize(a)
valuesAndPreds = p.map(lambda p: (p.label, lrm.predict(p.features)))
accurate = 1 - valuesAndPreds.map(lambda (v, p): math.fabs(v - p)).reduce(
lambda x, y: x + y) / valuesAndPreds.count()
def processMenu(csvFile):
fileTuple3 = csvFile.rpartition('/')
fileName = fileTuple3[-1]
dateString_wext = fileName.lstrip('pmix_')
dateString = dateString_wext.rstrip('.csv')
print('Processing data for period: ' + dateString)
with open(csvFile) as f:
reader = csv.reader(f)
# Read and discard the first line, which is the column headings
next(reader)
# Iterate over the remaining lines in the CSV file
for line in reader:
# The elements of interest in the list 'line' are:
# line[0] Master ID
# line[1] Item ID ->[0] in dictionary
# line[2] Parent ID ->[1]
# line[3] Menu Name ->[2]
# line[4] Menu Group ->[3]
# line[5] Subgroup ->[4]
# line[6] Menu Item ->[5]
# line[7] Avg Price ->[6]
# line[12] Item Qty ->[7]
# line[13] Gross Amount ->[8]
# line[14] Void Qty ->[9]
# line[15] Void Amount ->[10]
# line[16] Discount Amount ->[11]
# line[17] Net Amount ->[12]
if line[0]:
# Convert Master ID text value to integer for use as dictionary key
# Strip off the leading 4 and all 0s that follow. This gets rid of
# the '4000000..' that all the IDs have, and leaves the numbers that
# form a unique ID for the item.
longID = line[0].lstrip('4')
shortID = longID.lstrip('0')
if shortID:
int_key = locale.atoi(shortID)
# This is a new key, so simply add it to the dictionary
if int_key not in masterDict and int_key > 0:
masterDict[int_key] = [
line[1], line[2], line[3], line[4], line[5], line[6],
locale.atof(line[7]),
locale.atof(line[12]),
locale.atof(line[13]),
locale.atof(line[14]),
locale.atof(line[15]),
locale.atof(line[16]),
locale.atof(line[17])
]
# There are some keys for open item buttons we don't care about
elif int_key > 0:
# This key is already in dictionary, so update the entry by adding
# this item's Qty/Amount to previous total.
# print ("Duplicate key: " + line[0] + ", for menu item " + masterDict[int_key][5])
# Note indexes in dictionary different than CVS file line
updatedLine = [
masterDict[int_key][0], masterDict[int_key][1],
masterDict[int_key][2], masterDict[int_key][3],
masterDict[int_key][4], masterDict[int_key][5],
masterDict[int_key][6],
masterDict[int_key][7] + locale.atof(line[12]),
masterDict[int_key][8] + locale.atof(line[13]),
masterDict[int_key][9] + locale.atof(line[14]),
masterDict[int_key][10] + locale.atof(line[15]),
masterDict[int_key][11] + locale.atof(line[16]),
masterDict[int_key][12] + locale.atof(line[17])
]
masterDict[int_key] = updatedLine
#print(masterDict)
return dateString
def convert_integer():
try:
return int(locale.atof(value))
except (ValueError, AttributeError):
return
def convert_float():
try:
return locale.atof(value)
except (ValueError, AttributeError):
return
import urllib2
import re
import locale
import request
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0',
'Cookie': 'd53db4de415c4e858dc761595623a898=+; Path=/\r\n'
}
request = request.RequestUrl()
site_status_1 = request.open_url_and_get_content(
'http://hughes.sieve.com.br:8000/level3/', headers=headers)
pattern = r'href="(.*?)"'
match = re.search(pattern, site_status_1)
site_status_2 = request.open_url_and_get_content(
'http://hughes.sieve.com.br:8000' + match.groups()[0], headers=headers)
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0',
'Cookie': 'd53db4de415c4e858dc761595623a898=+; Path=/\r\n',
'Cookie': '18=+;'
}
site_status_3 = request.open_url_and_get_content(
'http://hughes.sieve.com.br:8000/level3/', headers=headers)
locale.setlocale(locale.LC_ALL, 'pt_BR.UTF-8')
pattern = r'R\$ ([\d\.,]+)'
match = re.search(pattern, site_status_3)
price = locale.atof(match.groups()[0])
print 'R$ %f' % price
import sys
import locale
def find_text(element, match):
return element.find(match).text
locale.setlocale(locale.LC_ALL, 'ru_RU.utf8')
doc = urllib.request.urlopen('http://www.cbr.ru/scripts/XML_daily.asp')
val_curs = xml.etree.ElementTree.parse(doc).getroot()
min = sys.maxsize
max = 0
min_valute = None
max_valute = None
for valute in val_curs:
value = locale.atof(valute.find('Value').text) / locale.atof(valute.find('Nominal').text)
if value < min:
min = value
min_valute = valute
elif value > max:
max = value
max_valute = valute
assert min_valute != None
assert max_valute != None
print(f'Самая дешевая валюта: {find_text(min_valute, "Nominal")} {find_text(min_valute, "Name")}, cтоимость: {find_text(min_valute, "Value")} рублей')
print(f'Самая дорогая валюта: {find_text(max_valute, "Nominal")} {find_text(max_valute, "Name")}, cтоимость: {find_text(max_valute, "Value")} рублей')
def __brl_string_to_float(self, brl_string):
setlocale(LC_NUMERIC, 'pt_BR.UTF-8')
return atof(brl_string)
