def normalize_msg(line): timestamp = dateutil.parser.parse( line['Date'] + " " + line['Time'] + " " + line['Time Zone'], ).strftime("%s") names = line['Name'].split(" ") return { 'date': timestamp, 'email': line['From Email Address'], 'first_name': names[0], 'last_name': " ".join(names[1:]), 'street_address': line['Address Line 1'], 'supplemental_address_1': line['Address Line 2/District/Neighborhood'], 'city': line['Town/City'], 'state_province': line['State/Province/Region/County/Territory/Prefecture/Republic'], 'country': line['Country'], 'postal_code': line['Zip/Postal Code'], 'comment': line['Note'], # somthing with: line['Subscription Number'], 'gross_currency': line['Currency'], 'gross': round(locale.atof(line['Gross']), 2), 'fee': round(locale.atof(line['Fee']), 2), 'net': round(locale.atof(line['Net']), 2), 'gateway': "paypal", 'gateway_txn_id': line['Transaction ID'], }
def add_ebs_info(instances): ebs_url = "http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSOptimized.html" tree = etree.parse(urllib2.urlopen(ebs_url), etree.HTMLParser()) table = tree.xpath('//div[@class="table-contents"]//table')[0] rows = table.xpath('tr') by_type = {i.instance_type: i for i in instances} for row in rows: if row.xpath('th'): continue cols = row.xpath('td') # TODO: Support the asterisk, which means: "These instance types can support maximum # performance for 30 minutes at least once every 24 hours. For example, c5.large # instances can deliver 281 MB/s for 30 minutes at least once every 24 hours. If you # have a workload that requires sustained maximum performance for longer than 30 # minutes, select an instance type based on the following baseline performance." instance_type = sanitize_instance_type(totext(cols[0]).replace("*", "")) ebs_optimized_by_default = totext(cols[1]) == 'Yes' ebs_max_bandwidth = locale.atof(totext(cols[2])) ebs_throughput = locale.atof(totext(cols[3])) ebs_iops = locale.atof(totext(cols[4])) if instance_type not in by_type: print("Unknown instance type: {}".format(instance_type)) continue by_type[instance_type].ebs_optimized_by_default = ebs_optimized_by_default by_type[instance_type].ebs_throughput = ebs_throughput by_type[instance_type].ebs_iops = ebs_iops by_type[instance_type].ebs_max_bandwidth = ebs_max_bandwidth
def angle(coor,angleparam): angleenergy=0.0 ; ubenergy=0.0 deg2rad=numpy.pi/180.0 for i in xrange(len(angleparam)): vub=0.0 ttrio=angleparam[i][0] atm1=locale.atoi(ttrio[0]) atm2=locale.atoi(ttrio[1]) atm3=locale.atoi(ttrio[2]) coor1=coor[atm1]; coor2=coor[atm2] ; coor3=coor[atm3] theta=sasmath.calc_angle(coor1,coor2,coor3) ktheta=locale.atof(angleparam[i][1][0]) theta0=deg2rad*locale.atof(angleparam[i][1][1]) if(len(angleparam[i][1])>2): kub=locale.atof(angleparam[i][1][2]) so=locale.atof(angleparam[i][1][3]) v = coor3-coor1 s = math.sqrt(sum(v*v)) #s=sascalc.calc_dist(coor1,coor3) vub=kub*(s-so)**2.0 vangle=ktheta*(theta-theta0)**2.0 angleenergy=angleenergy+vangle ubenergy=ubenergy+vub return angleenergy,ubenergy
def _getExamplars(self): traf = open(self.trainfile) examplars = [] while(traf): line = traf.readline() if line!="": tokens = line.strip().split(":") print tokens if 2!= len(tokens): print "The format of the trainfile is wrong!--1" sys.exit() elif 8!= len(tokens[0].split(",")): print "The format of the trainfile is wrong!--2" sys.exit() elif 1!= len(tokens[1].split(",")): print "The format of the trainfile is wrong!--3" sys.exit() inputs = [] for i in range(8): inputs.insert(i,atof(tokens[0].split(",")[i])) outputs = [] outputs.insert(0,atof(tokens[1][0])) examplar = (inputs,outputs) examplars.append(examplar) else: break print examplars return examplars
def sig_insert_text(self, entry, new_text, new_text_length, position): if not self.record: entry.stop_emission('insert-text') return value = entry.get_text() position = entry.get_position() new_value = value[:position] + new_text + value[position:] decimal_point = locale.localeconv()['decimal_point'] if new_value in ('-', decimal_point): return digits = self.field.digits(self.record) try: locale.atof(new_value) except ValueError: entry.stop_emission('insert-text') return new_int = new_value new_decimal = '' if decimal_point in new_value: new_int, new_decimal = new_value.rsplit(decimal_point, 1) if len(new_int) > digits[0] \ or len(new_decimal) > digits[1]: entry.stop_emission('insert-text')
def time_to_dec(time): """Convert time interval from MM:SS to MM.S Parameters ---------- string of time time in MM:SS format Returns ------- time in minutes in decimal format """ from locale import setlocale, LC_ALL, atof setlocale(LC_ALL, "en_US.UTF8") try: t = time # no information provided if t == "-": return None pl = t.split(":") minutes = atof(pl[0]) secs = (atof(pl[1]) / 60) ttime = minutes + secs return ttime except: return None
def parse(self, response): item = Item() sel = response.xpath sele = sel("//td[@class='formularioColumna']/text()").extract() item['nombres'] = sele[2].strip().replace("\n", " -") item['apellidos'] = sele[5].strip().replace("\n", " -") item['dni'] = sele[8].strip().replace("\n", " -") item['domicilio'] = sele[11].strip().replace("\n", " -") item['delito'] = sele[14].strip().replace("\n", " -") # check! item['entidad_agraviada'] = sele[17].strip().replace("\n", " -") item['fecha_sentencia'] = sele[20].strip().replace("\n", " -") item['fecha_ejecutoria'] = sele[23].strip().replace("\n", " -") item['juzgado'] = sele[26].strip().replace("\n", " -") item['expediente'] = sele[29].strip().replace("\n", " -") item['solidaria'] = sele[32].strip().replace("\n", " -") sele = sel("//td[@class='formularioColumna']/div[@align='center']/text()").extract() item['reparacion_civil'] = locale.atof(sele[0]) item['intereses'] = locale.atof(sele[1]) item['monto_total'] = locale.atof(sele[2]) item['pagos_realizados'] = locale.atof(sele[3]) item['pagos_pendientes'] = locale.atof(sele[4]) item['url'] = response.url.replace("\n", " -") yield item
def parse_instance(tr, inst2family): i = Instance() cols = tr.xpath('td') assert len(cols) == 12, "Expected 12 columns in the table, but got %d" % len(cols) i.instance_type = totext(cols[0]) i.family = inst2family.get(i.instance_type, "Unknown") # Some t2 instances support 32-bit arch # http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-resize.html#resize-limitations if i.instance_type in ('t2.micro', 't2.small'): i.arch.append('i386') i.vCPU = locale.atoi(totext(cols[1])) i.memory = locale.atof(totext(cols[2])) storage = totext(cols[3]) m = re.search(r'(\d+)\s*x\s*([0-9,]+)?', storage) i.ssd = False if m: i.ebs_only = False i.num_drives = locale.atoi(m.group(1)) i.drive_size = locale.atof(m.group(2)) i.ssd = 'SSD' in totext(cols[3]) else: assert storage == 'EBS Only', "Unrecognized storage spec: %s" % (storage,) i.ebs_only = True i.ebs_optimized = totext(cols[10]).lower() == 'yes' i.network_performance = totext(cols[4]) i.enhanced_networking = totext(cols[11]).lower() == 'yes' i.generation = 'current' # print "Parsed %s..." % (i.instance_type) return i
def parse_instance(instance_type, product_attributes): i = scrape.Instance() i.instance_type = instance_type pieces = instance_type.split('.') if len(pieces) == 1: # Dedicated host that is not u-*.metal, skipping # May be a good idea to all dedicated hosts in the future return i.family = product_attributes.get('instanceFamily') if '32-bit' in product_attributes.get('processorArchitecture'): i.arch.append('i386') i.vCPU = locale.atoi(product_attributes.get('vcpu')) # Memory is given in form of "1,952 GiB", let's parse it i.memory = locale.atof(product_attributes.get('memory').split(' ')[0]) i.network_performance = product_attributes.get('networkPerformance') if product_attributes.get('currentGeneration') == 'Yes': i.generation = 'current' else: i.generation = 'previous' gpu = product_attributes.get('gpu') if gpu is not None: i.GPU = locale.atoi(gpu) try: ecu = product_attributes.get('ecu') if ecu == 'Variable': i.ECU = 'variable' else: i.ECU = locale.atof(ecu) except: pass i.physical_processor = product_attributes.get('physicalProcessor') # CPU features processor_features = product_attributes.get('processorFeatures') if processor_features is not None: if "Intel AVX-512" in processor_features: i.intel_avx512 = True if "Intel AVX2" in processor_features: i.intel_avx2 = True if "Intel AVX" in processor_features: i.intel_avx = True if "Intel Turbo" in processor_features: i.intel_turbo = True i.clock_speed_ghz = product_attributes.get('clockSpeed') enhanced_networking = product_attributes.get('enhancedNetworkingSupported') if enhanced_networking is not None and enhanced_networking == 'Yes': i.enhanced_networking = True return i
def getConversion(body): """Calculate schmeckle to USD responses""" values = [] msg_template_f = u"* {:,.2f} Schmeckles → **${:,.2f} USD**\n" # with decimals msg_template_i = u"* {:,.0f} Schmeckles → **${:,.0f} USD**\n" # without decimals msg_inf = u"* There's a lot of numbers there, I think you could probably go to Wall Street.\n\n*You ever hear about Wall Street, Morty? Y-Y-Y'know what those guys do i-in-in their fancy boardrooms? They take their balls and they dip 'em in cocaine and wipe 'em all over each other—y'know.*\n" pairs = p.findall(body) if len(pairs) > 0: for match in pairs: # '<number> schmeckle' -> match, float(<number>) value_str = match.split()[0] # Handle numbers with over 9000 characters. Yes, it's over 9000. if (len(value_str)) > 9000: values.append(locale.atof("inf")) else: values.append(locale.atof(value_str)) response = [] for schmeckle in values: if isinf(schmeckle): response.append(msg_inf) else: usd = schmeckle2usd(schmeckle) if schmeckle.is_integer(): response.append(msg_template_i.format(schmeckle, usd)) else: response.append(msg_template_f.format(schmeckle, usd)) return [response, values]
def currency_to_float(string): try: ret = locale.atof(string) except ValueError: ret = locale.atof(string.replace( locale.localeconv()['mon_thousands_sep'], '')) return float(ret)
def main(): datenum=[]; zenith=[]; azimuth=[]; tracker_angle=[]; file_pos=0; day=0; iDay=0; morningIndex=[]; datum=0; pi=22/7; filename = 'e:\\BKK2010.txt'; outputfile = 'e:\\BKK2010.bin'; file = open(filename, 'r'); ofile = open(outputfile,'wb'); print ('--- Reading file %s ---\n' % filename); while True: line = file.readline() if not line: break sline = line.split(' '); # Wait for output buffer to drain. datenum.append(locale.atof(sline[0])); zenith.append(locale.atof(sline[1])); azimuth.append(locale.atof(sline[2])); print ('--- Finish ---\n'); nrow = len(datenum); for i in range(0,nrow): if ((zenith[i]<=90) & (day==0) ): # if the sun is above horizon day=1; #toggle day iDay=iDay+1; datum=0; morningIndex.append(i); file_pos=ofile.tell() data=struct.pack('I',900*i); ofile.write(data); ofile.flush(); if ((ofile.tell() - file_pos) != 4): break; elif ((zenith[i]>90) & (day==1) ): # if the sun is setting day=0; #toggle night for j in range(datum,63): file_pos=ofile.tell() data=struct.pack('I',0); ofile.write(data); ofile.flush(); if ((ofile.tell() - file_pos) != 4): break; print('Day %d, %d \r\n'%(iDay,datum)); if (day==1): tracker_angle = math.atan((math.sin(zenith[i]*pi/180)* math.cos((azimuth[i]-90)*pi/180))/math.cos(zenith[i]*pi/180))*180/pi+90; data=struct.pack('I',tracker_angle*(2**22)); # actual tracker angle data, max 63 data per page file_pos=ofile.tell() ofile.write(data); ofile.flush(); if ((ofile.tell() - file_pos) != 4): break; datum = datum+1; ofile.close()
def cur_parse(value): """ Parses input into a float value Returns a tuple of (CURRENCY, value) CURRENCY is the locale that was parsed, or 'BTC' value is the ammount devoid of grouping and currency strings in a float """ locale_value, repl_count = LOCALE_CRNCY_RE.subn("", value) if repl_count >= 1: # we matched the current locale locale_value = locale.atof(locale_value) return (LOCALE[0], locale_value) btc_value, repl_count = BTC_CRNCY_RE.subn("", value) if repl_count >= 1: # we matched BTC btc_value = locale.atof(btc_value) return ("BTC", btc_value) btc_value, repl_count = BTC2_CRNCY_RE.subn("", value) if repl_count >= 1: # we matched BTC2 btc_value = locale.atof(btc_value) return ("BTC", btc_value) raise ValueError("Unknown locale for currency: %s" % value)
def salvar(self): seta_locale(False) if self.ui.checkSaida.isChecked(): print "checked" self.val = -locale.atof(self.ui.textValor.text()) else: print "nao checked" self.val = locale.atof(self.ui.textValor.text()) cadastro = Caixa( descricao=self.ui.textDescricao.text(), data=datetime.datetime.strptime(self.ui.textData.text(), getDataString()).date(), valor=self.val, origem=self.ui.textOrigem.text(), destino=self.ui.textDestino.text() ) try: self.session.add(cadastro) self.session.commit() mostrarMessageBox( "Sucesso!", "Cadastro de Caixa efetuado com sucesso!") except Exception as e: lineno, fname = get_filename_lnumber() enviaEmail(e.message, fname, lineno) mostrarMessageBox("Erro!", "Não foi possivel salvar os dados no banco. \n %s" % e.message)
def parse_prev_generation_instance(tr): i = Instance() cols = tr.xpath('td') assert len(cols) == 8, "Expected 8 columns in the table, but got %d" % len(cols) i.family = totext(cols[0]) i.instance_type = totext(cols[1]) archs = totext(cols[2]) i.arch = [] if '32-bit' in archs: i.arch.append('i386') if '64-bit' in archs: i.arch.append('x86_64') assert i.arch, "No archs detected: %s" % (archs,) i.vCPU = locale.atoi(totext(cols[3])) i.memory = locale.atof(totext(cols[4])) storage = totext(cols[5]) m = re.search(r'(\d+)\s*x\s*([0-9,]+)?', storage) i.ssd = False if m: i.ebs_only = False i.num_drives = locale.atoi(m.group(1)) i.drive_size = locale.atof(m.group(2)) i.ssd = 'SSD' in totext(cols[5]) else: assert storage == 'EBS Only', "Unrecognized storage spec: %s" % (storage,) i.ebs_only = True i.ebs_optimized = totext(cols[6]).lower() == 'yes' i.network_performance = totext(cols[7]) i.enhanced_networking = False i.generation = 'previous' # print "Parsed %s..." % (i.instance_type) return i
def _build_cpu_info(self): try: p = os.popen('lscpu') cpuinfo = p.read() p.close() m = re.search('Model name:\s+(.*)', cpuinfo) if m: self._cpu['model'] = m.group(1) m = re.search('Socket\(s\):\s+(\d+)', cpuinfo) if m: self._cpu['sockets'] = locale.atoi(m.group(1)) m = re.search('CPU MHz:\s+(.*)', cpuinfo) if m: self._cpu['clock'] = int(locale.atof(m.group(1)) * 1e6) m = re.search('CPU max MHz:\s+(.*)', cpuinfo) if m: self._cpu['clockMax'] = int(locale.atof(m.group(1)) * 1e6) m = re.search('CPU min MHz:\s+(.*)', cpuinfo) if m: self._cpu['clockMin'] = int(locale.atof(m.group(1)) * 1e6) m = re.search('Core\(s\) per socket:\s+(.*)', cpuinfo) if m: self._cpu['cores_per_sockets'] = locale.atoi(m.group(1)) m = re.search('Thread\(s\) per core:\s+(.*)', cpuinfo) if m: self._cpu['threads_per_core'] = locale.atoi(m.group(1)) except: pass
def indicadoreseconomicos(): if not request.ajax: return '' session.forget() #if request.ajax: import urllib2 import locale locale.setlocale(locale.LC_ALL, 'es_CL.UTF8') uri = 'http://www.averigualo.cl/feed/indicadores.xml' import re locale.setlocale(locale.LC_ALL, 'es_CL.UTF8') #uri = 'http://www.averigualo.cl/feed/indicadores.xml' uri = 'http://indicador.eof.cl/xml' try: pag = urllib2.urlopen(uri).read() pag = pag.decode('iso8859-1').encode('utf-8') html = TAG(pag) try: eurocalculado = locale.atof(html.element('indicador',_nombre='Euro')[0]) except Exception,e: eurocalculado = '-' try: #dolarcalculado = locale.atof(html.element('indicador',_nombre='Dólar Observado')[0]) dolarcalculado = locale.atof(html.element('indicador',_nombre=re.compile('Observado'))[0]) except Exception,e: dolarcalculado = '-'
def _parse_contig_coverage(line): """ Parse contig coverage from a string, typically a line from the QualiMap result report. If no contig coverage can be parsed, None is returned. :param line: the input string :return: the contig coverage as a dict, with "[CONTIG] coverage" as key and a dict with the contig coverage information as value or None if no contig coverage could be parsed """ # identify contig coverage try: values = re.search(r'^\s+(\S+)\s+([0-9]+)\s+([0-9]+)\s+(\S+)\s+(\S+)\s*$', line).groups() key = "{} coverage".format(values[0]) value = dict() value["contig"] = values[0] value["length"] = locale.atoi(values[1]) value["mapped bases"] = locale.atoi(values[2]) value["mean coverage"] = locale.atof(values[3]) value["standard deviation"] = locale.atof(values[4]) return {key: value} except ValueError: # problems with the conversion to numeric values pass except AttributeError: # not a contig coverage row pass
def __init__(self, fields, raw_csv, options): """Parameters: fields: list of fields read from one line of the CSV file raw_csv: unprocessed line from CSV file options: from CLI args and config file """ if 'addons' in options: self.addons = dict((k, fields[v - 1]) for k, v in options.addons.items()) else: self.addons = dict() # Get the date and convert it into a ledger formatted date. self.date = fields[options.date - 1] if options.ledger_date_format: if options.ledger_date_format != options.csv_date_format: self.date = (datetime .strptime(self.date, options.csv_date_format) .strftime(options.ledger_date_format)) # convert effective dates if options.effective_date: self.effective_date = fields[options.effective_date - 1] if options.ledger_date_format: if options.ledger_date_format != options.csv_date_format: self.effective_date = (datetime .strptime(self.effective_date, options.csv_date_format) .strftime(options.ledger_date_format)) else: self.effective_date = "" desc = [] for index in re.compile(',\s*').split(options.desc): desc.append(fields[int(index) - 1].strip()) self.desc = ' '.join(desc).strip() self.credit = get_field_at_index(fields, options.credit, options.csv_decimal_comma, options.ledger_decimal_comma) self.debit = get_field_at_index(fields, options.debit, options.csv_decimal_comma, options.ledger_decimal_comma) if self.credit and self.debit and atof(self.credit) == 0: self.credit = '' elif self.credit and self.debit and atof(self.debit) == 0: self.debit = '' self.credit_account = options.account self.currency = options.currency self.cleared_character = options.cleared_character if options.template_file: with open(options.template_file, 'r') as f: self.transaction_template = f.read() else: self.transaction_template = "" self.raw_csv = raw_csv.strip() # We also record this - in future we may use it to avoid duplication self.md5sum = hashlib.md5(self.raw_csv).hexdigest()
def is_numeric(s): if s.lower() in ['infinity', 'nan']: return False try: setlocale(LC_NUMERIC, 'pl_PL.UTF-8') atof(s) return True except ValueError: return False
def group_by(records, key): """For a payout report, returns a structure like: {'DE': {'charged': 15, 'received': 10, 'num_sales': 3}, 'US': {...}, ... All values are in seller's currency (yours). In case this is an "estimtaed sales" report, it will not contain the "received" key, and the value in "charged" will be the buyer's currency. """ # itertools.groupby requires sorted input records.sort(key=key) result = collections.OrderedDict() for country, sales in itertools.groupby(records, key=key): sales = list(sales) """ TODO: There actually seems to be a bug in both payout and sales reports, where I have buyers from the US paying in KRW, and no FX Rate is given. In those cases, assuming 1 as FX rate is wrong. Example: {'Merchant Currency': 'KRW', 'Country of Buyer': 'US', ..., 'Merchant Receives': '0.00', 'Item Price': '1,165.00', 'Charged Amount': '1,165.00', 'Order Charged Date': '2012-04-18', 'Currency of Sale': 'KRW', 'City of Buyer': 'Honolulu', 'Estimated FX Rate': '', 'State of Buyer': 'HI', ... 'Financial Status': 'Charged'} """ charged = lambda s: Decimal(str(locale.atof(s['Charged Amount']))) received = lambda s: Decimal(str(locale.atof(s['Merchant Receives']))) fx = lambda s: Decimal(str(locale.atof(s['Estimated FX Rate'] or '1'))) result[country] = { 'num_sales': len(list(sales)), } if is_payout_report(sales): # This key indicates a payout report result[country].update({ 'charged': sum([charged(s) * fx(s) for s in sales]), 'received': sum([received(s) for s in sales]) }) else: # A sales report, lacks info about actual monies payed out. result[country].update({ 'charged': sum([charged(s) for s in sales]), # XXX It is actually wrong to assume a single currency for # each country. Lots of people from KR pay in USD for example. 'currency': sales[0]['Currency of Sale'] }) if is_payout_report(records): # Without a single currency, sum makes no sense result['SUM'] = reduce(operator.add, map(collections.Counter, result.values())) return result
def testAlternativeDecimalMark(self): """ Test a locale which uses "," as its decimal separator.""" # NOTE: Windows locales aren't named the same as the `man locale 1` # equivalents. For this code to be universal, it'd need to test os.name # and convert to the appropriate locale name. locale.setlocale(locale.LC_ALL, "german_germany") # two tests to make sure that radix parsing is happening as expected self.assertEqual(locale.atof("-11,00"), -11.0) self.assertEqual(locale.atof("-11.000,50"), -11000.5)
def read_file(self, file_path): f = open(file_path, 'r') L = f.readlines() f.close() text = L[0][:-1] #Strip trailing "\n" t = locale.atof(L[1][:-1]) h = locale.atof(L[2][:-1]) self.value.set_text(text) self._t_h_bar.set_temp(t) self._t_h_bar.set_humid(h)
def string2i(my_str): my_str2 =my_str.replace(",","") "Convert a string to an number" if (my_str2 != '') and (my_str2 is not None): locale.setlocale(locale.LC_ALL, '') positive = my_str2.translate(None, '()') result = locale.atof(positive) if positive == my_str2 else -locale.atof(positive) return result else: return 0
def read_user_weights (properties, sWeights, dcWeight, dcRowSum, dDistAllSum, bExcludeII): """ Reads FROM, TO, WEIGHT file into dictionary structure. Header recorder must identify ID field. """ try: bDefault = iFalse f = open(sWeights, 'r') # Make sure we have a valid ID field. sFieldName = f.readline().strip() if sFieldName.upper() == (properties.sFID).upper(): sIDField = None sTokens = f.readline().split() else: sPathName = os.path.dirname(sWeights) sIDField = pGP.ValidateFieldName(sFieldName, sPathName) if not sIDField.upper() in properties.dcFields.keys(): # Maybe we didn't get a header record try: sTokens = sFieldName.split() # If we have two integers and a weight, we'll assume the # ID field is the FID field id1 = int(sTokens[0]) id2 = int(sTokens[1]) dWt = locale.atof(sTokens[2]) sIDField = None except: raise ReportError(msgFileFormatErr) else: sTokens = f.readline().split() pGP.AddMessage (sReadingWeightsFile) # Read each record in weights file, creating a dictionary structure of distances. bNegWts = iFalse while len(sTokens) > 0: sFrom = sTokens[0] sTo = sTokens[1] dWeight = locale.atof(sTokens[2]) if dWeight < 0.0: bNegWts = iTrue sKey = sFrom + ":" + sTo dcWeight[sKey] = dWeight if not (sFrom == sTo and bExcludeII): if (dcRowSum.has_key(sFrom)): dcRowSum[sFrom] = dcRowSum[sFrom] + dWeight else: dcRowSum[sFrom] = dWeight dDistAllSum = dDistAllSum + dWeight sTokens = f.readline().split() f.close() if bNegWts: pGP.AddWarning(msgNegativeWts) except: # If we encounter unexpected errors, we bail bDefault = iTrue sIDField = None f.close() raise ReportError(msgWtsMatrixErr) return bDefault, sIDField, dcRowSum, dDistAllSum
def write_cloud_details_V2(cloud_sheet, row_dict, output_row): output_col = 0 total_amount = 0.0 # Write Google data into Cloud sheet. # Output 'Platform' field. cloud_sheet.write(output_row, output_col, "Google Cloud Platform") output_col += 1 # Output 'Account' field. (subacccount) cloud_sheet.write(output_row, output_col, row_dict['Account ID']) output_col += 1 # Output 'Project' field. (Project Name + Project ID) cloud_sheet.write(output_row, output_col, row_dict['Source']) output_col += 1 # Output 'Description' field. (SKU description of the charge) sku_description = "%s %s" % (row_dict['Product'], row_dict['Resource Type']) cloud_sheet.write(output_row, output_col, sku_description) output_col += 1 # Output 'Dates' field. date_range = "%s-%s" % (row_dict['Start Date'], row_dict['End Date']) cloud_sheet.write(output_row, output_col, date_range) output_col += 1 # Parse quantity. quantity_str = row_dict['Quantity'].strip() if len(quantity_str) > 0: quantity = locale.atof(quantity_str) else: quantity = '' # Output 'Quantity' field. cloud_sheet.write(output_row, output_col, quantity, FLOAT_FORMAT) output_col += 1 # Output 'Unit of Measure' field. cloud_sheet.write(output_row, output_col, row_dict['Unit']) output_col += 1 # Parse charge. amount = locale.atof(row_dict['Amount']) # Accumulate total charges. total_amount += amount # Output 'Charge' field. cloud_sheet.write(output_row, output_col, amount, MONEY_FORMAT) output_col += 1 return total_amount
def tablify(data): lst = data.get_text().split("\n") ret_list = lst[2:4] ret_list.append(lst[6]) ret_list.append(lst[8]) ret_list.append(lst[13]) ret_list.append(str(locale.atof(lst[9])+locale.atof(lst[14]))) ret_list.append(str(locale.atof(lst[10])+locale.atof(lst[15]))) ret_list.extend(lst[16:18]) return ret_list
def el(coor,charge,exclusionlist): # print '\ncalculating electrostatic energy' natoms=len(charge) elenergy=0.0 # qi*qj/(e1*rij) e1=1.0 qconv=1.620177E-19 # eV to C qconv2=qconv*qconv # q*q --> C^2 eps=8.854187816E-12 # epsilon --> C^2/(J*m) ang2m=1E-10 # angstrom to m jtokjpmol=6.022137E+20 # J to KJ/mol kjtokcal=1.0/4.184 # KJ to kcal #qconv2=1.0 pi=numpy.pi conv=(qconv2/(4.0*pi*eps*ang2m))*jtokjpmol*kjtokcal #print 'conv = ',conv conv=332 conv=332.4683 np=0 stupid = True for i in xrange(natoms): qi=locale.atof(charge[i]) x1=coor[i][0] ; y1=coor[i][1] ; z1=coor[i][2] for j in xrange(i+1,natoms): #if str(j+1) not in exclusionlist[i][1]: if stupid: qj=locale.atof(charge[j]) x2=coor[j][0] ; y2=coor[j][1] ; z2=coor[j][2] dx2=(x1-x2)*(x1-x2) dy2=(y1-y2)*(y1-y2) dz2=(z1-z2)*(z1-z2) rij=numpy.sqrt(dx2+dy2+dz2) switch=getswitch(rij) vij=((qi*qj)/(e1*rij))*switch #vij=(qi*qj*qconv2)/(e1*eps*rij*ang2m) #vij=vij*jtokjpmol*kjtokcal elenergy=elenergy+vij np=np+1 else: pass #print 'atom ',i+1,' and atom ',j+1,' not counted in vdw calculation' elenergy=elenergy*conv return elenergy
def LayDuLieu(self): result = {} Bang = self.dv.find_element_by_css_selector('table.table-profit') self.wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR,'.col-1-1'))) for x in range(2,9): row = Bang.find_element_by_css_selector('tr:nth-child('+ str(x) +')') key = row.find_element_by_css_selector('td:first-child').text result[key] = {} result[key]['pre'] = locale.atof(row.find_element_by_css_selector('td:nth-child(2)').text) result[key]['current'] = locale.atof(row.find_element_by_css_selector('td:nth-child(3)').text) print result return result
def on_exit_edit(edit_field, disk_win=None): '''On exit, if the user has left the field blank, set the size to 0''' text = edit_field.get_text() if not text.strip(): text = "0" enctext = text.encode(get_encoding()) # encode per locale for floating point conversion edit_field.set_text("%.1f" % locale.atof(enctext)) part_order = disk_win.ui_obj.get_parts_in_use().index(edit_field.data_obj) LOGGER.debug("Part being resized is at index: %s", part_order) new_size_text = text.strip() LOGGER.debug("Resizing text=%s", new_size_text) # encode user input per locale for floating point conversion enctext = new_size_text.encode(get_encoding()) new_size = Size(str(locale.atof(enctext)) + Size.gb_units) old_size = edit_field.data_obj.size new_size_byte = new_size.get(Size.byte_units) old_size_byte = old_size.get(Size.byte_units) precision = Size(UI_PRECISION).get(Size.byte_units) if abs(new_size_byte - old_size_byte) > precision: parent_doc_obj = edit_field.data_obj.doc_obj.parent if isinstance(parent_doc_obj, Disk): if isinstance(edit_field.data_obj.doc_obj, Partition): resized_obj = parent_doc_obj.resize_partition( edit_field.data_obj.doc_obj, new_size.get(Size.gb_units), size_units=Size.gb_units) else: resized_obj = parent_doc_obj.resize_slice( edit_field.data_obj.doc_obj, new_size.get(Size.gb_units), size_units=Size.gb_units) else: resized_obj = parent_doc_obj.resize_slice( edit_field.data_obj.doc_obj, new_size.get(Size.gb_units), size_units=Size.gb_units) if isinstance(resized_obj, Partition): resized_obj.in_zpool = ROOT_POOL else: if resized_obj.in_zpool == ROOT_POOL: resized_obj.tag = V_ROOT if disk_win is not None: disk_win.set_disk_info(ui_obj=disk_win.ui_obj) disk_win.activate_index(part_order) dump_doc("After resize")
def parse(self, response): # Code to parse course pages ## Fill in dc_dict here # parsing each entry offer = dict() # get main data from the html's head head = response.xpath('/html/head') offer['title'] = head.xpath( './meta[@property="og:title"]/@content').extract_first() offer['image'] = head.xpath( './meta[@property="og:image"]/@content').extract_first() offer['url'] = response.url offer['description'] = head.xpath( './meta[@name="description"]/@content').extract_first() quickfacts = response.css('div.quickfacts') offer['summary_text'] = quickfacts.css('h1::text').extract_first() offer['location'] = quickfacts.xpath('./div[@class="location"]/span[1]/text()').extract_first() offer['merkmale'] = quickfacts.css('div.merkmale::text').extract_first() hardfacts = response.css('div.hardfacts') offer['total_price'] = float(hardfacts.xpath('./div[1]/strong/text()').extract_first().split(' ')[0].replace('.','')) offer['area'] = atof(hardfacts.xpath('./div[2]/text()').extract_first().split(' ')[0].strip()) offer['num_rooms'] = atof(hardfacts.xpath('./div[3]/text()').extract_first().strip()) # PREISE UND KOSTEN preise_container = response.xpath('//h2[contains(text(), "Preise")]/..') ## KALTMIETE container = preise_container.xpath('.//strong[contains(text(), "Kaltmiete")]/../..') if len(container) > 0: offer['kaltmiete'] = atof(container.css('div.datacontent > strong::text').extract_first().split(' ')[0].strip()) ## NEBENKOSTEN container = preise_container.xpath('.//div[contains(text(), "Nebenkosten")]/../..') if len(container) > 0: offer['nebenkosten'] = atof(container.css('div.datacontent::text').extract_first().split(' ')[0].strip()) ## HEIZKOSTEN container = preise_container.xpath('.//div[contains(text(), "Heizkosten")]/../..') if len(container) > 0: offer['heizkosten'] = container.css('div.datacontent::text').extract_first() ## WARMMIETE container = preise_container.xpath('.//div[contains(text(), "Warmmiete")]/../..') if len(container) > 0: offer['warmmiete'] = container.css('div.datacontent::text').extract_first() ## KAUTION container = response.xpath('//div[contains(text(), "Kaution")]/..') if len(container) > 0: offer['kaution'] = atof(container.css('div.section_content > p::text').extract_first().strip().split(' ')[0]) # Get the parent div of the div whose content is 'Immobilie' to get the online ID immobilie_container = response.xpath('//h2[contains(text(), "Immobilie")]/../..') offer['online_id'] = immobilie_container.css('div.section_content > p::text').extract_first().strip().replace('Online-ID: ', '') # Get the parent div of the div whose content is 'Die Wohnung' to get details # TODO: Needs some text recognition to get main facts? # whg_container = response.xpath('//div[contains(text(), "Die Wohnung")]/..') # offer['type'] = whg_container.xpath('./div[2]/p/text()').extract_first().strip() # offer['floor'] = whg_container.xpath('./div[2]/p/span/text()').extract_first().strip() # offer['availability'] = whg_container.xpath('./div[2]/p/strong/text()').extract_first().strip() # TODO: get more details from UL > LIs? # Get the parent div of the div whose content is 'Wohnalage' to get details whg_container = response.xpath('//div[contains(text(), "Wohnanlage")]/..') offer['building_year'] = whg_container.xpath('./div[2]/p/text()').extract_first().strip() # TODO: get more details from UL > LIs? # Get the parent div of the div whose content is 'Objektbeschreibung' to get details container = response.xpath('//div[contains(text(), "Objektbeschreibung")]/..') text_content = container.xpath('./div[2]/p//text()').extract() offer['object_description'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0]) # Get the parent div of the div whose content is 'Ausstattung' to get details container = response.xpath('//div[contains(text(), "Ausstattung")]/..') text_content = container.xpath('./div[2]/p//text()').extract() offer['ausstattung'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0]) # Get the parent div of the div whose content is 'Sonstiges' to get details container = response.xpath('//div[contains(text(), "Sonstiges")]/..') if len(container) > 0: text_content = container.xpath('./div[2]/p//text()').extract() offer['sonstiges'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0]) # Get the parent div of the div whose content is 'KFZ Stellplatz' to get details container = response.xpath('//div[contains(text(), "KFZ Stellplatz")]/..') text_content = container.xpath('./div[2]/p//text()').extract() offer['kfz_stellplatz'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0]) # Get the parent div of the div whose content is 'Stichworte' to get details container = response.xpath('//div[contains(text(), "Stichworte")]/..') text_content = container.xpath('./div[2]/p//text()').extract() offer['keywords'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0]) # Get the parent div of the div whose content is 'Lagebeschreibung' to get details container = response.xpath('//div[contains(text(), "Lagebeschreibung")]/..') text_content = container.xpath('./div[2]/p//text()').extract() offer['lagebeschreibung'] = '\n'.join([i.strip() for i in text_content if len(i.strip()) > 0]) # append to final list offers_list.append(offer)
sheet = wbk["Sheet1"] cells = sheet['A2':'H181'] i = -1 position = [] l1 = [] l2 = [] l3 = [] l4 = [] l5 = [] l6 = [] locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') for pos1, pos2, c1, c2, c3, c4, c5, c6 in cells: i = i + 1 l1.append(locale.atof(c1.value)) l2.append(locale.atof(c2.value)) l3.append(locale.atof(c3.value)) l4.append(locale.atof(c4.value)) l5.append(locale.atof(c5.value)) l6.append(locale.atof(c6.value)) if (i + 1) % 10 == 0: position.append([]) position[int(i / 10)].append(locale.atof(pos1.value)) position[int(i / 10)].append(locale.atof(pos2.value)) print(round(variance(l1), 2), round(variance(l2), 2), round(variance(l3), 2), round(variance(l4), 2), round(variance(l5), 2), round(variance(l6), 2)) l1.clear() l2.clear() l3.clear()
polygonType = "SIMPLE_POLYS" input_copies = [] #Convert constraint values from strings to number. If empty string use max Int maxPointBarrierCount = locale.atoi( strMaxPointBarrierCount) if strMaxPointBarrierCount else INFINITY maxEdgeCountLineBarriers = locale.atoi( strMaxEdgeCountLineBarriers ) if strMaxEdgeCountLineBarriers else INFINITY maxEdgeCountPolygonBarriers = locale.atoi( strMaxEdgeCountPolygonBarriers ) if strMaxEdgeCountPolygonBarriers else INFINITY maxFacilities = locale.atoi( strMaxFacilities) if strMaxFacilities else INFINITY maxBreaks = locale.atoi(strMaxBreaks) if strMaxBreaks else INFINITY maxBreakTime = locale.atof( strMaxBreakTime) if strMaxBreakTime else INFINITY maxBreakDistance = locale.atof( strMaxBreakDistance) if strMaxBreakDistance else INFINITY forceHierarchyTime = locale.atof( strForceHierarchyTime) if strForceHierarchyTime else INFINITY forceHierarchyDistance = locale.atof( strForceHierarchyDistance) if strForceHierarchyDistance else INFINITY descNDS = arcpy.Describe(networkDataset) descNDSAttributes = descNDS.attributes #Convert all input features to feature sets or recordsets if they are not #This is required as if input is passed a feature layer or feature class #We will end up directly modifying the inputs facilities_obj = nau.InputFeatureClass(facilities)
#! /usr/bin/env/python # -*- coding:utf-8 -*- import locale sample_locales = [ ('USA', 'en_US.UTF-8', '1,234.56'), ('France', 'fr_FR.UTF-8', '1234,56'), ('Spain', 'es_ES.UTF-8', '1234,56'), ('Portugal', 'pt_PT.UTF-8', '1234.56'), ('Poland', 'pl_PL.UTF-8', '1234,56'), ] for name, loc, a in sample_locales: locale.setlocale(locale.LC_ALL, loc) f = locale.atof(a) print "{0:.20}s: {1:.9}s => {2}f".format(name, a, f)
def extract(self, file_): entries = [] line_index = 0 closing_balance_index = -1 with change_locale(locale.LC_NUMERIC, self.numeric_locale): with open(file_.name, encoding=self.file_encoding) as fd: # Header line = fd.readline().strip() line_index += 1 if not self._expected_header_regex.match(line): raise InvalidFormatError() # Empty line line = fd.readline().strip() line_index += 1 if line: raise InvalidFormatError() # Meta lines = [fd.readline().strip() for _ in range(3)] reader = csv.reader( lines, delimiter=';', quoting=csv.QUOTE_MINIMAL, quotechar='"', ) for line in reader: key, value, _ = line line_index += 1 if key.startswith('Von'): self._date_from = datetime.strptime(value, '%d.%m.%Y').date() elif key.startswith('Bis'): self._date_to = datetime.strptime(value, '%d.%m.%Y').date() elif key.startswith('Kontostand vom'): # Beancount expects the balance amount to be from the # beginning of the day, while the Tagessaldo entries in # the DKB exports seem to be from the end of the day. # So when setting the balance date, we add a timedelta # of 1 day to the original value to make the balance # assertions work. self._balance_amount = Amount( locale.atof(value.rstrip(' EUR'), Decimal), self.currency, ) self._balance_date = datetime.strptime( key.lstrip('Kontostand vom ').rstrip(':'), '%d.%m.%Y', ).date() + timedelta(days=1) closing_balance_index = line_index # Another empty line line = fd.readline().strip() line_index += 1 if line: raise InvalidFormatError() # Data entries reader = csv.DictReader(fd, delimiter=';', quoting=csv.QUOTE_MINIMAL, quotechar='"') for line in reader: meta = data.new_metadata(file_.name, line_index) amount = None if line['Betrag (EUR)']: amount = Amount( locale.atof(line['Betrag (EUR)'], Decimal), self.currency, ) date = datetime.strptime(line['Buchungstag'], '%d.%m.%Y').date() if line['Verwendungszweck'] == 'Tagessaldo': if amount: entries.append( data.Balance( meta, date + timedelta(days=1), self.account, amount, None, None, )) else: description = '{} {}'.format(line['Buchungstext'], line['Verwendungszweck']) postings = [ data.Posting(self.account, amount, None, None, None, None) ] entries.append( data.Transaction( meta, date, self.FLAG, line['Auftraggeber / Begünstigter'], description, data.EMPTY_SET, data.EMPTY_SET, postings, )) line_index += 1 # Closing Balance meta = data.new_metadata(file_.name, closing_balance_index) entries.append( data.Balance( meta, self._balance_date, self.account, self._balance_amount, None, None, )) return entries
#!/usr/bin/env python import csv import locale import datetime locale.setlocale(locale.LC_ALL, 'en_US.UTF8') infile = '201606_F3_1_8_2303.csv' with open(infile, encoding='cp950') as csvfile: reader = csv.reader(csvfile) next(reader) next(reader) for row in reader: datestr = row[0].split('/') year = int(datestr[0]) + 1911 month = int(datestr[1]) day = int(datestr[2]) date = datetime.date(year, month, day) print(date, [locale.atof(i) for i in row[1:]])
def send_value(self): """ Emit a :attr:`send_value_signal` to update channel value. The text is cleaned of all units, user-formatting and scale values before being sent back to the channel. This function is attached the ReturnPressed signal of the PyDMLineEdit """ send_value = str(self.text()) # Clean text of unit string if self._show_units and self._unit and self._unit in send_value: send_value = send_value[:-len(self._unit)].strip() try: if self.channeltype not in [str, np.ndarray]: scale = self._scale if scale is None or scale == 0: scale = 1.0 if self._display_format_type in [ DisplayFormat.Default, DisplayFormat.String ]: if self.channeltype == float: num_value = locale.atof(send_value) else: num_value = self.channeltype(send_value) scale = self.channeltype(scale) elif self._display_format_type == DisplayFormat.Hex: num_value = int(send_value, 16) elif self._display_format_type == DisplayFormat.Binary: num_value = int(send_value, 2) elif self._display_format_type in [ DisplayFormat.Exponential, DisplayFormat.Decimal ]: num_value = locale.atof(send_value) num_value = num_value / scale self.send_value_signal[self.channeltype].emit(num_value) elif self.channeltype == np.ndarray: # Arrays will be in the [1.2 3.4 22.214] format if self._display_format_type == DisplayFormat.String: self.send_value_signal[str].emit(send_value) else: arr_value = list( filter( None, send_value.replace("[", "").replace("]", "").split(" "))) arr_value = np.array(arr_value, dtype=self.subtype) self.send_value_signal[np.ndarray].emit(arr_value) else: # Channel Type is String # Lets just send what we have after all self.send_value_signal[str].emit(send_value) except ValueError: logger.exception( "Error trying to set data '{0}' with type '{1}' and format '{2}' at widget '{3}'." .format(self.text(), self.channeltype, self._display_format_type, self.objectName())) self.clearFocus() self.set_display()
report_template['query']['start-date'] = dlabel parseflag_start_date_parsed = True report_template['query']['end-date'] = dlabel elif report_template['name'] == 'visitor-location-city': item['city'] = dlabel elif report_template['name'] == 'visitor-location-region': item['region'] = dlabel elif report_template['name'] == 'visitor-location-country': item['country'] = dlabel else: item['page_title'] = dlabel # values try: vlabel = int( locale.atof(bundled_dps[1 + index_offset].strip(','))) except: vlabel = bundled_dps[1 + index_offset].strip('%,') if report_template['name'] == 'visits': item['visits'] = vlabel else: item['active_visitors'] = vlabel # hrefs hyperlink_payload = rp.a if hyperlink_payload is None: pass else: item['page'] = str(rp.a.get('href')) # push the data into its home.
name = pw.format_string(cells[1].findall("font/a")[0].text.strip(),None) # get operational date op_date = cells[2].findall("font")[0].text.strip() if op_date: try: d = parse_date(op_date) op_year = d.year found_operational_year_count += 1 except: op_year = pw.NO_DATA_NUMERIC else: op_year = pw.NO_DATA_NUMERIC # get plant capacity capacity = CAPACITY_CONVERSION_TO_MW * locale.atof(cells[4].findall("font")[0].text) # get owner owner = u"" owner_description = pw.format_string(cells[6].findall("font")[0].text.strip(), ENCODING) if u"não identificado" in owner_description: owner = pw.NO_DATA_UNICODE break # TODO: Read owner correctly owner = owner_description """ for owner in owner_list: if u"não identificado" in owner.text: owner_full = pw.NO_DATA_UNICODE
def __init__(self, fields, raw_csv, options): """Parameters: fields: list of fields read from one line of the CSV file raw_csv: unprocessed line from CSV file options: from CLI args and config file """ if 'addons' in options: self.addons = dict( (k, fields[v - 1]) for k, v in options.addons.items()) else: self.addons = dict() # Get the date and convert it into a ledger formatted date. self.date = fields[options.date - 1] entry_date = datetime.strptime(self.date, options.csv_date_format) if options.ledger_date_format: if options.ledger_date_format != options.csv_date_format: self.date = (datetime.strptime( self.date, options.csv_date_format).strftime( options.ledger_date_format)) # determine how many days old this entry is self.days_old = (datetime.now() - entry_date).days # convert effective dates if options.effective_date: self.effective_date = fields[options.effective_date - 1] if options.ledger_date_format: if options.ledger_date_format != options.csv_date_format: self.effective_date = (datetime.strptime( self.effective_date, options.csv_date_format).strftime( options.ledger_date_format)) else: self.effective_date = "" desc = [] for index in re.compile(',\s*').split(options.desc): desc.append(fields[int(index) - 1].strip()) self.desc = ' '.join(desc).strip() self.credit = get_field_at_index(fields, options.credit, options.csv_decimal_comma, options.ledger_decimal_comma) self.debit = get_field_at_index(fields, options.debit, options.csv_decimal_comma, options.ledger_decimal_comma) if self.credit and self.debit and atof(self.credit) == 0: self.credit = '' elif self.credit and self.debit and atof(self.debit) == 0: self.debit = '' self.credit_account = options.account self.currency = options.currency self.credit_currency = getattr(options, 'credit_currency', self.currency) self.cleared_character = options.cleared_character if options.template_file: with open(options.template_file, 'r', encoding='utf-8') as f: self.transaction_template = f.read() else: self.transaction_template = "" self.raw_csv = raw_csv.strip() # We also record this - in future we may use it to avoid duplication self.md5sum = hashlib.md5(self.raw_csv.encode('utf-8')).hexdigest()
loc_temp[0] = optimal_sum_temp_wwheat / 1 if optimal_sum_hum_wwheat == 0: loc_hum[0] = 0 else: loc_hum[0] = optimal_sum_hum_wwheat / 1 if optimal_sum_precip_wwheat == 0: loc_precip[0] = 0 else: loc_precip[0] = optimal_sum_precip_wwheat / 1 for i in range(0, 6): arrayofcrops[i]["Location_Temperature"] = loc_temp[i] arrayofcrops[i]["Location_Humidity"] = loc_hum[i] arrayofcrops[i]["Location_Precipitation"] = loc_precip[i] for i in range(0, 6): weighted_ideal[i] = locale.atof(arrayofcrops[i].get( "Ideal_Temperature")) * temp_weights[i] + locale.atof(arrayofcrops[ i].get("Ideal_Humidity")) * humidity_weights[i] + locale.atof( arrayofcrops[i].get("Ideal_Precipitation")) * precip_weights[i] weighted_location[i] = loc_temp[i] * temp_weights[i] + loc_hum[ i] * humidity_weights[i] + loc_precip[i] * precip_weights[i] for i in range(0, 6): per_change[i] = (abs(weighted_ideal[i] - weighted_location[i]) / weighted_ideal[i]) * 100 if (per_change[i] > 0 and per_change[i] <= 5): score[i] = 10 elif (per_change[i] > 5 and per_change[i] <= 10): score[i] = 8 elif (per_change[i] > 10 and per_change[i] <= 15): score[i] = 6 elif (per_change[i] > 15 and per_change[i] <= 20):
def _test_atof(self, value, out): self.assertEqual(locale.atof(value), out)
def get_csil_prices(params=exparams, rval=None): if rval is None: rval = {} delivery_codes = { # 3: '3#333', # 5: '5#334', 7: '7#529', 10: '10#1452', 12: '12#7271', 15: '15#1453', 18: '18#7272', 21: '21#7273' } delivery_times = sorted(delivery_codes.keys()) layers_codes = { 1: '2180', 2: '2181', 4: '2183', 6: '2184', } browser = splinter.Browser('firefox', profile=FIREFOX_PROFILE_PATH) url = 'http://login.pcbpower.com/V2/login.aspx' browser.visit(url) values = {'txtUserName': dvobj.username, 'txtPassword': dvobj.password} browser.fill_form(values) button = browser.find_by_name('btnlogin') button.click() link = browser.find_by_id('ctl00_aPlaceOrder') link.click() values = OrderedDict() values['ctl00$ContentPlaceHolder1$txtPCBName'] = params['pcbname'] values['ctl00$ContentPlaceHolder1$ddlLayers'] = layers_codes[ params['layers']] # noqa values['ctl00$ContentPlaceHolder1$txtDimX'] = str(params['dX']) values['ctl00$ContentPlaceHolder1$txtDimY'] = str(params['dY']) if 'qty' in params.keys(): values['ctl00$ContentPlaceHolder1$txtQuantity'] = str( params['qty'][1]) # noqa else: values['ctl00$ContentPlaceHolder1$txtQuantity'] = '1' values['ctl00$ContentPlaceHolder1$DDLsurfacefinish'] = params['finish'] if 'time' in params.keys(): values['ctl00$ContentPlaceHolder1$ddlDelTerms'] = delivery_codes[ params['time']] # noqa else: values['ctl00$ContentPlaceHolder1$ddlDelTerms'] = delivery_codes[7] if not browser.is_element_present_by_id('shortNotiText', wait_time=100): raise Exception ready = False timeout = 10 while not ready and timeout: el = browser.find_by_id('shortNotiText') if el[0].text == u"We're online": ready = True timeout -= 1 time.sleep(1) time.sleep(5) browser.fill_form(values) try: oldt = browser.find_by_id('ctl00_ContentPlaceHolder1_lblUnitPrc').text except AttributeError: oldt = '' # qty = str(params['qty'][1]) # oldv = qty time.sleep(2) button = browser.find_by_id('ctl00_ContentPlaceHolder1_btnCalculate') button.click() time.sleep(2) button = browser.find_by_id('ctl00_ContentPlaceHolder1_btnCalculate') button.click() try: newt = browser.find_by_id('ctl00_ContentPlaceHolder1_lblUnitPrc').text except AttributeError: newt = '' try: newtt = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblTotalPrice').text # noqa except AttributeError: newtt = '' while oldt == newt: try: newt = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblUnitPrc').text # noqa except AttributeError: newt = '' try: newtt = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblTotalPrice').text # noqa except AttributeError: newtt = '' time.sleep(0.5) oldt = newt oldtt = newtt pb = TendrilProgressBar(max=len(params['qty'])) for qty in params['qty'][2:]: lined = {} while browser.find_by_name('ctl00$ContentPlaceHolder1$txtQuantity' )[0].value != '': # noqa browser.type('ctl00$ContentPlaceHolder1$txtQuantity', '\b') time.sleep(0.1) browser.type('ctl00$ContentPlaceHolder1$txtQuantity', str(qty)) time.sleep(0.1) browser.type('ctl00$ContentPlaceHolder1$txtQuantity', '\t') if qty > 4: loi = [10] else: loi = [10] pb.next(note="{0} {1}".format(qty, loi)) # "\n{0:>7.4f}% {1:<40} Qty: {2:<10} DTS: {3:<4}\nGenerating PCB Pricing".format( # noqa # percentage, params['pcbname'], qty, loi) for dt_s in loi: dt_idx = delivery_times.index(dt_s) dts = delivery_times[dt_idx:dt_idx + 3] browser.select('ctl00$ContentPlaceHolder1$ddlDelTerms', delivery_codes[dt_s]) # noqa time.sleep(1) while True: try: try: newt = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblUnitPrc' ).text # noqa except AttributeError: newt = '' try: # newt1 = '' # newt2 = '' newt1 = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblnextunitprc1' ).text # noqa newt2 = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblnextunitprc2' ).text # noqa except AttributeError: newt1 = '' newt2 = '' break except selenium.common.exceptions.StaleElementReferenceException: # noqa logger.warning("Selenium Exception Caught. Retrying") continue timeout = 25 while oldt == newt and oldtt == newtt and newt is not '' and timeout > 0: # noqa timeout -= 1 time.sleep(1) while True: try: try: newt = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblUnitPrc' ).text # noqa except AttributeError: newt = '' try: # newt1 = '' # newt2 = '' newt1 = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblnextunitprc1' ).text # noqa newt2 = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblnextunitprc2' ).text # noqa except AttributeError: newt1 = '' newt2 = '' break except selenium.common.exceptions.StaleElementReferenceException: # noqa logger.warning("Selenium Exception Caught. Retrying") continue try: newtt = browser.find_by_id( 'ctl00_ContentPlaceHolder1_lblTotalPrice').text # noqa except AttributeError: newtt = '' try: lined[dts[0]] = locale.atof(newt) if newt1 != '': lined[dts[1]] = locale.atof(newt1) if newt2 != '': lined[dts[2]] = locale.atof(newt2) except ValueError: logger.warning("Caught Exception at CSIL Website. Retrying.") browser.quit() return get_csil_prices(params, rval) oldt = newt oldtt = newtt params['qty'].remove(qty) # print lined rval[qty] = lined browser.quit() return rval
def convert_numeric(): try: return locale.atof(value, Decimal) except (decimal.InvalidOperation, AttributeError): return
def itervals(self, record): for name, fm in self.entries: dt = fm['datatype'] val = record[fm['rec_index']] if dt == 'composite': sb = fm['display'].get('composite_sort', 'text') if sb == 'date': try: val = parse_date(val) except: val = UNDEFINED_DATE dt = 'datetime' elif sb == 'number': try: p = 1 for i, candidate in enumerate( ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB')): if val.endswith(candidate): p = 1024**(i) val = val[:-len(candidate)].strip() break val = locale.atof(val) * p except: val = 0.0 dt = 'float' elif sb == 'bool': val = force_to_bool(val) dt = 'bool' if dt == 'datetime': if val is None: val = UNDEFINED_DATE if tweaks['sort_dates_using_visible_fields']: format = None if name == 'timestamp': format = tweaks['gui_timestamp_display_format'] elif name == 'pubdate': format = tweaks['gui_pubdate_display_format'] elif name == 'last_modified': format = tweaks['gui_last_modified_display_format'] elif fm['is_custom']: format = fm['display'].get('date_format', None) val = clean_date_for_sort(val, format) elif dt == 'series': if val is None: val = ('', 1) else: if self.library_order: try: lang = record[self.lang_idx].partition(',')[0] except (AttributeError, ValueError, KeyError, IndexError, TypeError): lang = None val = title_sort(val, order='library_order', lang=lang) sidx_fm = self.field_metadata[name + '_index'] sidx = record[sidx_fm['rec_index']] val = (self.string_sort_key(val), sidx) elif dt in ('text', 'comments', 'composite', 'enumeration'): if val: if fm['is_multiple']: jv = fm['is_multiple']['list_to_ui'] sv = fm['is_multiple']['cache_to_list'] if '&' in jv: val = jv.join( [author_to_author_sort(v) for v in val.split(sv)]) else: val = jv.join(sorted(val.split(sv), key=self.string_sort_key)) val = self.string_sort_key(val) elif dt == 'bool': if not self.db_prefs.get('bools_are_tristate'): val = {True: 1, False: 2, None: 2}.get(val, 2) else: val = {True: 1, False: 2, None: 3}.get(val, 3) yield val
def add_to_cart(request, product_id=None): """ Adds the passed product with passed product_id to the cart after some validations have been taken place. The amount is taken from the query string. """ if product_id is None: product_id = request.REQUEST.get("product_id") product = muecke_get_object_or_404(Product, pk=product_id) # Only active and deliverable products can be added to the cart. if (product.is_active() and product.is_deliverable()) == False: raise Http404() try: value = request.POST.get("quantity", "1.0") if isinstance(value, unicode): # atof() on unicode string fails in some environments, like Czech value = value.encode("utf-8") quantity = locale.atof(value) except (TypeError, ValueError): quantity = 1.0 # Validate properties (They are added below) properties_dict = {} if product.is_configurable_product(): for key, value in request.POST.items(): if key.startswith("property-"): try: property_id = key.split("-")[1] except IndexError: continue try: property = Property.objects.get(pk=property_id) except Property.DoesNotExist: continue if property.is_number_field: try: value = muecke.core.utils.atof(value) except ValueError: value = locale.atof("0.0") properties_dict[property_id] = unicode(value) # validate property's value if property.is_number_field: if (value < property.unit_min) or (value > property.unit_max): msg = _( u"%(name)s must be between %(min)s and %(max)s %(unit)s." ) % { "name": property.title, "min": property.unit_min, "max": property.unit_max, "unit": property.unit } return muecke.core.utils.set_message_cookie( product.get_absolute_url(), msg) # calculate valid steps steps = [] x = property.unit_min while x < property.unit_max: steps.append("%.2f" % x) x = x + property.unit_step steps.append("%.2f" % property.unit_max) value = "%.2f" % value if value not in steps: msg = _( u"Your entered value for %(name)s (%(value)s) is not in valid step width, which is %(step)s." ) % { "name": property.title, "value": value, "step": property.unit_step } return muecke.core.utils.set_message_cookie( product.get_absolute_url(), msg) if product.get_active_packing_unit(): quantity = muecke.catalog.utils.calculate_real_amount( product, quantity) cart = cart_utils.get_or_create_cart(request) cart_item = cart.add(product, properties_dict, quantity) cart_items = [cart_item] # Check stock amount message = "" if product.manage_stock_amount and cart_item.amount > product.stock_amount and not product.order_time: if product.stock_amount == 0: message = _(u"Sorry, but '%(product)s' is not available anymore." % {"product": product.name}) elif product.stock_amount == 1: message = _( u"Sorry, but '%(product)s' is only one time available." % {"product": product.name}) else: message = _( u"Sorry, but '%(product)s' is only %(amount)s times available." ) % { "product": product.name, "amount": product.stock_amount } cart_item.amount = product.stock_amount cart_item.save() # Add selected accessories to cart for key, value in request.POST.items(): if key.startswith("accessory"): accessory_id = key.split("-")[1] try: accessory = Product.objects.get(pk=accessory_id) except ObjectDoesNotExist: continue # Get quantity quantity = request.POST.get("quantity-%s" % accessory_id, 0) try: quantity = float(quantity) except TypeError: quantity = 1 cart_item = cart.add(product=accessory, amount=quantity) cart_items.append(cart_item) # Store cart items for retrieval within added_to_cart. request.session["cart_items"] = cart_items cart_changed.send(cart, request=request) # Update the customer's shipping method (if appropriate) customer = customer_utils.get_or_create_customer(request) shipping_utils.update_to_valid_shipping_method(request, customer, save=True) # Update the customer's payment method (if appropriate) payment_utils.update_to_valid_payment_method(request, customer, save=True) # Save the cart to update modification date cart.save() try: url_name = settings.LFS_AFTER_ADD_TO_CART except AttributeError: url_name = "muecke.cart.views.added_to_cart" if message: return muecke.core.utils.set_message_cookie(reverse(url_name), message) else: return HttpResponseRedirect(reverse(url_name))
from bs4 import BeautifulSoup import urllib.request import locale import numpy as np import pandas as pd import matplotlib from ggplot import * html=urllib.request.urlopen("http://www.twse.com.tw/exchangeReport/MI_5MINS_INDEX?response=html&date=20170609") soup = BeautifulSoup(html, 'html.parser') locale.setlocale( locale.LC_ALL, 'en_US.UTF-8') #資料採集 #注意find("tbody"),此行若少會採集到表頭,表頭只有一格所以會產生error a=[] price=[locale.atof(td) for child in soup.find("tbody").find_all("tr") for td in child.find_all('td')[1] ] time =[td for child in soup.find("tbody").find_all("tr") for td in child.find_all('td')[0] ] # 採集的資料整理成可以處理的資料格式 pdPrice=pd.DataFrame(price) pdTime=pd.DataFrame(time) Price_dataFrame=pd.concat([pdTime,pdPrice],axis=1) # pd.concat裡面要放pandas dadframe Price_dataFrame.columns=["time","price"] # pandas DataFrame取用1到100row Price_dataFrame.ix[1:100,:]# 可以用"名字"叫 Price_dataFrame.iloc[1:100,:]#只能用數字叫 Price_dataFrame['price'] Price_dataFrame['time'] Price_dataFrame.describe()
def process_row(row, row_num, column_map, num_required_columns, doaj_offline_analysis=False, bypass_cert_verification=False): """ Enrich a single row of data and reformat it according to Open APC standards. Take a csv row (a list) and a column mapping (a list of CSVColumn objects) and return an enriched and re-arranged version which conforms to the Open APC data schema. Args: row: A list of column values (as yielded by a UnicodeReader f.e.). row_num: The line number in the csv file, for logging purposes. column_map: An OrderedDict of CSVColumn Objects, mapping the row cells to Open APC data schema fields. num_required_columns: An int describing the required length of the row list. If not matched, an error is logged and the row is returned unchanged. doaj_offline_analysis: If true, a local copy will be used for the DOAJ lookup. bypass_cert_verification: If true, certificate validation will be skipped when connecting to metadata providers via TLS. Returns: A list of values which represents the enriched and re-arranged variant of the input row. If no errors were logged during the process, this result will conform to the Open APC data schema. """ MESSAGES = { "num_columns": "Syntax: The number of values in this row (%s) " + "differs from the number of columns (%s). Line left " + "unchanged, the resulting CSV file will not be valid.", "locale": "Error: Could not process the monetary value '%s' in " + "column %s. This will usually have one of two reasons:\n1) " + "The value does not represent a number.\n2) The value " + "represents a number, but its format differs from your " + "current system locale - the most common source of error " + "will be the decimal mark (1234.56 vs 1234,56). Try using " + "another locale with the -l option.", "unify": "Normalisation: CrossRef-based {} changed from '{}' to '{}' " + "to maintain consistency." } if len(row) != num_required_columns: msg = "Line %s: " + MESSAGES["num_columns"] logging.error(msg, row_num, len(row), num_required_columns) return row doi = row[column_map["doi"].index] current_row = OrderedDict() # Copy content of identified columns for csv_column in column_map.values(): if csv_column.index is not None and len(row[csv_column.index]) > 0: if csv_column.column_type == "euro": # special case for monetary values: Cast to float to ensure # the decimal point is a dot (instead of a comma) euro_value = row[csv_column.index] try: euro = locale.atof(euro_value) if euro.is_integer(): euro = int(euro) current_row[csv_column.column_type] = str(euro) except ValueError: msg = "Line %s: " + MESSAGES["locale"] logging.error(msg, row_num, euro_value, csv_column.index) else: current_row[csv_column.column_type] = row[csv_column.index] else: current_row[csv_column.column_type] = "NA" if len(doi) == 0 or doi == 'NA': msg = ("Line %s: No DOI found, entry could not enriched with " + "Crossref or Pubmed metadata.") logging.warning(msg, row_num) current_row["indexed_in_crossref"] = "FALSE" else: # include crossref metadata crossref_result = get_metadata_from_crossref(doi) if crossref_result["success"]: logging.info("Crossref: DOI resolved: " + doi) current_row["indexed_in_crossref"] = "TRUE" data = crossref_result["data"] for key, value in data.iteritems(): if value is not None: if key == "journal_full_title": unified_value = get_unified_journal_title(value) if unified_value != value: msg = MESSAGES["unify"].format( "journal title", value, unified_value) logging.warning(msg) new_value = unified_value elif key == "publisher": unified_value = get_unified_publisher_name(value) if unified_value != value: msg = MESSAGES["unify"].format( "publisher name", value, unified_value) logging.warning(msg) new_value = unified_value else: new_value = value else: new_value = "NA" msg = ( u"WARNING: Element '%s' not found in in response for " + "doi %s.") logging.debug(msg, key, doi) old_value = current_row[key] current_row[key] = column_map[key].check_overwrite( old_value, new_value) else: msg = "Line %s: Crossref: Error while trying to resolve DOI %s: %s" logging.error(msg, row_num, doi, crossref_result["error_msg"]) current_row["indexed_in_crossref"] = "FALSE" # include pubmed metadata pubmed_result = get_metadata_from_pubmed(doi) if pubmed_result["success"]: logging.info("Pubmed: DOI resolved: " + doi) data = pubmed_result["data"] for key, value in data.iteritems(): if value is not None: new_value = value else: new_value = "NA" msg = ( u"WARNING: Element %s not found in in response for " + "doi %s.") logging.debug(msg, key, doi) old_value = current_row[key] current_row[key] = column_map[key].check_overwrite( old_value, new_value) else: msg = "Line %s: Pubmed: Error while trying to resolve DOI %s: %s" logging.error(msg, row_num, doi, pubmed_result["error_msg"]) # lookup in DOAJ. try the EISSN first, then ISSN and finally print ISSN issns = [] if current_row["issn_electronic"] != "NA": issns.append(current_row["issn_electronic"]) if current_row["issn"] != "NA": issns.append(current_row["issn"]) if current_row["issn_print"] != "NA": issns.append(current_row["issn_print"]) for issn in issns: # look up in an offline copy of the DOAJ if requested... if doaj_offline_analysis: lookup_result = doaj_offline_analysis.lookup(issn) if lookup_result: msg = (u"DOAJ: Journal ISSN (%s) found in DOAJ " + "offline copy ('%s').") logging.info(msg, issn, lookup_result) current_row["doaj"] = "TRUE" break else: msg = (u"DOAJ: Journal ISSN (%s) not found in DOAJ " + "offline copy.") current_row["doaj"] = "FALSE" logging.info(msg, issn) # ...or query the online API else: doaj_res = lookup_journal_in_doaj(issn, bypass_cert_verification) if doaj_res["data_received"]: if doaj_res["data"]["in_doaj"]: msg = u"DOAJ: Journal ISSN (%s) found in DOAJ ('%s')." logging.info(msg, issn, doaj_res["data"]["title"]) current_row["doaj"] = "TRUE" break else: msg = u"DOAJ: Journal ISSN (%s) not found in DOAJ." logging.info(msg, issn) current_row["doaj"] = "FALSE" else: msg = (u"Line %s: DOAJ: Error while trying to look up " + "ISSN %s: %s") logging.error(msg, row_num, issn, doaj_res["error_msg"]) return current_row.values()
def convert(self, value): try: return locale.atof(value, Decimal) except decimal.InvalidOperation: return self._default
def parseTransactions(driver, transactions): if platform.system() == 'Darwin': locale.setlocale(locale.LC_NUMERIC, "EN_US") else: locale.setlocale(locale.LC_NUMERIC, "") transaction_arr = [] company_name_dict = {} transactions_with_dates = [] for transaction in transactions: header_text = transaction.find_element_by_xpath( ".//div[@class='_2VPzNpwfga_8Mcn-DCUwug']").text canceled_text = transaction.find_element_by_xpath( ".//div[@class='_22YwnO0XVSevsIC6rD9HS3']").text if transaction_types["CANCELED"] in canceled_text: continue if transaction_types["LIMIT_SELL"] in header_text or transaction_types[ "LIMIT_BUY"] in header_text or transaction_types[ "MARKET_SELL"] in header_text or transaction_types[ "MARKET_BUY"] in header_text: info_children = transaction.find_elements_by_xpath( ".//div[@class='css-1qd1r5f']") date_node = info_children[7] date = date_node.get_attribute('textContent') transaction_with_date = (transaction, date) transactions_with_dates.append(transaction_with_date) elif transaction_types["DIVIDEND"] in header_text: date_node = transaction.find_elements_by_xpath( ".//span[@class='css-zy0xqa']")[1] date = date_node.get_attribute('textContent') transaction_with_date = (transaction, date) transactions_with_dates.append(transaction_with_date) transactions_with_dates.sort( key=lambda date: datetime.strptime(date[1], '%b %d, %Y')) for transaction_with_date in transactions_with_dates: transaction = transaction_with_date[0] header_text = transaction.find_element_by_xpath( ".//div[@class='_2VPzNpwfga_8Mcn-DCUwug']").text canceled_text = transaction.find_element_by_xpath( ".//div[@class='_22YwnO0XVSevsIC6rD9HS3']").text company_name_title = header_text.split("\n") company_name_list = company_name_title[0].split(" ") print(company_name_list) if transaction_types["CANCELED"] in canceled_text: continue if transaction_types["LIMIT_SELL"] in header_text or transaction_types[ "LIMIT_BUY"] in header_text: company_name = " ".join(company_name_list[:-2]) info_children = transaction.find_elements_by_xpath( ".//div[@class='css-1qd1r5f']") ticker_symbol_node = info_children[1] ticker_symbol = ticker_symbol_node.find_element_by_xpath( ".//a").get_attribute('textContent') if company_name not in company_name_dict.keys(): company_name_dict[company_name] = ticker_symbol transaction_type_node = info_children[3] transaction_type = transaction_type_node.find_element_by_xpath( ".//span").get_attribute('textContent') transaction_type_split = transaction_type.split(" ") transaction_type_final = transaction_type_split[1] transaction_date_node = info_children[7] transaction_date = transaction_date_node.get_attribute( 'textContent') filled_transaction_node = info_children[17] filled_transaction = filled_transaction_node.get_attribute( 'textContent') filled_transaction_split = filled_transaction.split(" ") quantity = filled_transaction_split[0] price = filled_transaction_split[3] price = locale.atof(price.split("$")[1]) total_value_node = info_children[19] total_value = locale.atof( total_value_node.get_attribute('textContent').split("$")[1]) transaction_tuple = (ticker_symbol, transaction_type_final, quantity, price, total_value, transaction_date) transaction_arr.append(transaction_tuple) continue if transaction_types["MARKET_SELL"] in header_text or transaction_types[ "MARKET_BUY"] in header_text: company_name = " ".join(company_name_list[:-2]) info_children = transaction.find_elements_by_xpath( ".//div[@class='css-1qd1r5f']") ticker_symbol_node = info_children[1] ticker_symbol = ticker_symbol_node.find_element_by_xpath( ".//a").get_attribute('textContent') if company_name not in company_name_dict.keys(): company_name_dict[company_name] = ticker_symbol transaction_type_node = info_children[3] transaction_type = transaction_type_node.find_element_by_xpath( ".//span").get_attribute('textContent') transaction_type_split = transaction_type.split(" ") transaction_type_final = transaction_type_split[1] transaction_date_node = info_children[7] transaction_date = transaction_date_node.get_attribute( 'textContent') filled_transaction_node = info_children[15] filled_transaction = filled_transaction_node.get_attribute( 'textContent') filled_transaction_split = filled_transaction.split(" ") quantity = filled_transaction_split[0] price = filled_transaction_split[3] price = locale.atof(price.split("$")[1]) total_value_node = info_children[17] total_value = locale.atof( total_value_node.get_attribute('textContent').split("$")[1]) transaction_tuple = (ticker_symbol, transaction_type_final, quantity, price, total_value, transaction_date) transaction_arr.append(transaction_tuple) continue if transaction_types["DIVIDEND"] in header_text: company_name = " ".join(company_name_list[2:]) ticker_symbol = company_name_dict[company_name] info_children = transaction.find_elements_by_xpath( ".//div[@class='css-1qd1r5f']") transaction_type = "Dividend" transaction_date_node = transaction.find_elements_by_xpath( ".//span[@class='css-zy0xqa']")[1] transaction_date = transaction_date_node.get_attribute( 'textContent') quantity = info_children[1].get_attribute('textContent') price = locale.atof( info_children[3].get_attribute('textContent').split("$")[1]) total_value = locale.atof( info_children[5].get_attribute('textContent').split("$")[1]) transaction_tuple = (ticker_symbol, transaction_type, quantity, price, total_value, transaction_date) transaction_arr.append(transaction_tuple) return transaction_arr
def convert(self, value): try: return locale.atof(value) except ValueError: return self._default
def process_row(row, row_num, column_map, num_required_columns, no_crossref_lookup=False, no_pubmed_lookup=False, no_doaj_lookup=False, doaj_offline_analysis=False, bypass_cert_verification=False): """ Enrich a single row of data and reformat it according to Open APC standards. Take a csv row (a list) and a column mapping (a list of CSVColumn objects) and return an enriched and re-arranged version which conforms to the Open APC data schema. Args: row: A list of column values (as yielded by a UnicodeReader f.e.). row_num: The line number in the csv file, for logging purposes. column_map: An OrderedDict of CSVColumn Objects, mapping the row cells to Open APC data schema fields. num_required_columns: An int describing the required length of the row list. If not matched, an error is logged and the row is returned unchanged. no_crossref_lookup: If true, no metadata will be imported from crossref. no_pubmed_lookup: If true, no_metadata will be imported from pubmed. no_doaj_lookup: If true, journals will not be checked for being listended in the DOAJ (default). doaj_offline_analysis: If true, a local copy will be used for the DOAJ lookup. Has no effect if no_doaj_lookup is set to true. bypass_cert_verification: If true, certificate validation will be skipped when connecting to metadata providers via TLS. Returns: A list of values which represents the enriched and re-arranged variant of the input row. If no errors were logged during the process, this result will conform to the Open APC data schema. """ MESSAGES = { "num_columns": u"Syntax: The number of values in this row (%s) " + "differs from the number of columns (%s). Line left " + "unchanged, the resulting CSV file will not be valid.", "locale": u"Error: Could not process the monetary value '%s' in " + "column %s. This will usually have one of two reasons:\n1) " + "The value does not represent a number.\n2) The value " + "represents a number, but its format differs from your " + "current system locale - the most common source of error " + "will be the decimal mark (1234.56 vs 1234,56). Try using " + "another locale with the -l option.", "unify": u"Normalisation: CrossRef-based {} changed from '{}' to '{}' " + "to maintain consistency.", "doi_norm": u"Normalisation: DOI '{}' normalised to pure form ({}).", "springer_distinction": u"publisher 'Springer Nature' found " + "for a pre-2015 article - publisher " + "changed to '%s' based on prefix " + "discrimination ('%s')", "unknown_prefix": u"publisher 'Springer Nature' found for a " + "pre-2015 article, but discrimination was " + "not possible - unknown prefix ('%s')", "issn_hyphen_fix": u"Normalisation: Added hyphen to %s value (%s -> %s)" } if len(row) != num_required_columns: msg = "Line %s: " + MESSAGES["num_columns"] logging.error(msg, row_num, len(row), num_required_columns) return row doi = row[column_map["doi"].index] current_row = OrderedDict() # Copy content of identified columns for csv_column in column_map.values(): if csv_column.column_type == "euro" and csv_column.index is not None: # special case for monetary values: Cast to float to ensure # the decimal point is a dot (instead of a comma) euro_value = row[csv_column.index] if len(euro_value) == 0: msg = "Line %s: Empty monetary value in column %s." logging.warning(msg, row_num, csv_column.index) current_row[csv_column.column_type] = "NA" else: try: euro = locale.atof(euro_value) if euro.is_integer(): euro = int(euro) current_row[csv_column.column_type] = str(euro) except ValueError: msg = "Line %s: " + MESSAGES["locale"] logging.error(msg, row_num, euro_value, csv_column.index) else: if csv_column.index is not None and len(row[csv_column.index]) > 0: current_row[csv_column.column_type] = row[csv_column.index] else: current_row[csv_column.column_type] = "NA" if len(doi) == 0 or doi == 'NA': msg = ("Line %s: No DOI found, entry could not enriched with " + "Crossref or Pubmed metadata.") logging.warning(msg, row_num) current_row["indexed_in_crossref"] = "FALSE" else: # Normalise DOI norm_doi = get_normalised_DOI(doi) if norm_doi is not None and norm_doi != doi: current_row["doi"] = norm_doi msg = MESSAGES["doi_norm"].format(doi, norm_doi) logging.warning(msg) # include crossref metadata if not no_crossref_lookup: crossref_result = get_metadata_from_crossref(doi) if crossref_result["success"]: logging.info("Crossref: DOI resolved: " + doi) current_row["indexed_in_crossref"] = "TRUE" data = crossref_result["data"] prefix = data.pop("prefix") for key, value in data.iteritems(): if value is not None: if key == "journal_full_title": unified_value = get_unified_journal_title(value) if unified_value != value: msg = MESSAGES["unify"].format( "journal title", value, unified_value) logging.warning(msg) new_value = unified_value elif key == "publisher": unified_value = get_unified_publisher_name(value) if unified_value != value: msg = MESSAGES["unify"].format( "publisher name", value, unified_value) logging.warning(msg) new_value = unified_value # Treat Springer Nature special case: crossref erroneously # reports publisher "Springer Nature" even for articles # published before 2015 (publishers fusioned only then) if int(current_row["period"] ) < 2015 and new_value == "Springer Nature": publisher = None if prefix in [ "Springer (Biomed Central Ltd.)", "Springer-Verlag", "Springer - Psychonomic Society" ]: publisher = "Springer Science + Business Media" elif prefix in [ "Nature Publishing Group", "Nature Publishing Group - Macmillan Publishers" ]: publisher = "Nature Publishing Group" if publisher: msg = "Line %s: " + MESSAGES[ "springer_distinction"] logging.warning(msg, row_num, publisher, prefix) new_value = publisher else: msg = "Line %s: " + MESSAGES[ "unknown_prefix"] logging.error(msg, row_num, prefix) # Fix ISSNs without hyphen elif key in ["issn", "issn_print", "issn_electronic"]: new_value = value if re.match("^\d{7}[\dxX]$", value): new_value = value[:4] + "-" + value[4:] msg = "Line %s: " + MESSAGES["issn_hyphen_fix"] logging.warning(msg, row_num, key, value, new_value) else: new_value = value else: new_value = "NA" msg = ( u"WARNING: Element '%s' not found in in response for " + "doi %s.") logging.debug(msg, key, doi) old_value = current_row[key] current_row[key] = column_map[key].check_overwrite( old_value, new_value) else: msg = "Line %s: Crossref: Error while trying to resolve DOI %s: %s" logging.error(msg, row_num, doi, crossref_result["error_msg"]) current_row["indexed_in_crossref"] = "FALSE" # include pubmed metadata if not no_pubmed_lookup: pubmed_result = get_metadata_from_pubmed(doi) if pubmed_result["success"]: logging.info("Pubmed: DOI resolved: " + doi) data = pubmed_result["data"] for key, value in data.iteritems(): if value is not None: new_value = value else: new_value = "NA" msg = ( u"WARNING: Element %s not found in in response for " + "doi %s.") logging.debug(msg, key, doi) old_value = current_row[key] current_row[key] = column_map[key].check_overwrite( old_value, new_value) else: msg = "Line %s: Pubmed: Error while trying to resolve DOI %s: %s" logging.error(msg, row_num, doi, pubmed_result["error_msg"]) # lookup in DOAJ. try the EISSN first, then ISSN and finally print ISSN if not no_doaj_lookup: issns = [] new_value = "NA" if current_row["issn_electronic"] != "NA": issns.append(current_row["issn_electronic"]) if current_row["issn"] != "NA": issns.append(current_row["issn"]) if current_row["issn_print"] != "NA": issns.append(current_row["issn_print"]) for issn in issns: # In some cases xref delievers ISSNs without a hyphen. Add it # temporarily to prevent the DOAJ lookup from failing. if re.match("^\d{7}[\dxX]$", issn): issn = issn[:4] + "-" + issn[4:] # look up in an offline copy of the DOAJ if requested... if doaj_offline_analysis: lookup_result = doaj_offline_analysis.lookup(issn) if lookup_result: msg = (u"DOAJ: Journal ISSN (%s) found in DOAJ " + "offline copy ('%s').") logging.info(msg, issn, lookup_result) new_value = "TRUE" break else: msg = (u"DOAJ: Journal ISSN (%s) not found in DOAJ " + "offline copy.") new_value = "FALSE" logging.info(msg, issn) # ...or query the online API else: doaj_res = lookup_journal_in_doaj(issn, bypass_cert_verification) if doaj_res["data_received"]: if doaj_res["data"]["in_doaj"]: msg = u"DOAJ: Journal ISSN (%s) found in DOAJ ('%s')." logging.info(msg, issn, doaj_res["data"]["title"]) new_value = "TRUE" break else: msg = u"DOAJ: Journal ISSN (%s) not found in DOAJ." logging.info(msg, issn) new_value = "FALSE" else: msg = (u"Line %s: DOAJ: Error while trying to look up " + "ISSN %s: %s") logging.error(msg, row_num, issn, doaj_res["error_msg"]) old_value = current_row["doaj"] current_row["doaj"] = column_map["doaj"].check_overwrite( old_value, new_value) return current_row.values()
def refresh_cart(request): """ Refreshes the cart after some changes has been taken place, e.g.: the amount of a product or shipping/payment method. """ cart = cart_utils.get_cart(request) customer = customer_utils.get_or_create_customer(request) # Update country country_iso = request.POST.get("country") if country_iso: selected_country = Country.objects.get(code=country_iso.lower()) customer.selected_country_id = selected_country.id if customer.selected_shipping_address: customer.selected_shipping_address.country = selected_country customer.selected_shipping_address.save() customer.selected_shipping_address.save() if customer.selected_invoice_address: customer.selected_invoice_address.country = selected_country customer.selected_invoice_address.save() customer.selected_invoice_address.save() # NOTE: The customer has to be saved already here in order to calculate # a possible new valid shippig method below, which coulb be triggered by # the changing of the shipping country. customer.save() # Update Amounts message = "" for item in cart.get_items(): try: value = request.POST.get("amount-cart-item_%s" % item.id, "0.0") if isinstance(value, unicode): # atof() on unicode string fails in some environments, like Czech value = value.encode("utf-8") amount = locale.atof(value) except (TypeError, ValueError): amount = 1.0 if item.product.manage_stock_amount and amount > item.product.stock_amount and not item.product.order_time: amount = item.product.stock_amount if amount < 0: amount = 0 if amount == 0: message = _( u"Sorry, but '%(product)s' is not available anymore." % {"product": item.product.name}) elif amount == 1: message = _( u"Sorry, but '%(product)s' is only one time available." % {"product": item.product.name}) else: message = _( u"Sorry, but '%(product)s' is only %(amount)s times available." ) % { "product": item.product.name, "amount": amount } if item.product.get_active_packing_unit(): item.amount = muecke.catalog.utils.calculate_real_amount( item.product, float(amount)) else: item.amount = amount if amount == 0: item.delete() else: item.save() # IMPORTANT: We have to send the signal already here, because the valid # shipping methods might be dependent on the price. cart_changed.send(cart, request=request) # Update shipping method customer.selected_shipping_method_id = request.POST.get("shipping_method") valid_shipping_methods = shipping_utils.get_valid_shipping_methods(request) if customer.selected_shipping_method not in valid_shipping_methods: customer.selected_shipping_method = shipping_utils.get_default_shipping_method( request) # Update payment method customer.selected_payment_method_id = request.POST.get("payment_method") # Last but not least we save the customer ... customer.save() result = simplejson.dumps( { "html": cart_inline(request), "message": message, }, cls=LazyEncoder) return HttpResponse(result)
import math locale.setlocale(locale.LC_ALL, 'en_US.UTF-8') df = pd.read_csv('/home/walker/hydrosphere/brakedata.csv', sep=',') brakeData = df.ix[:, 0:3] a = [] def parsePoint(w, k, h): return LabeledPoint(worn, [km, heat]) for row in brakeData.itertuples(): worn = getattr(row, 'worn') km = locale.atof(getattr(row, 'km')) heat = getattr(row, 'heat') lp = parsePoint(worn, km, heat) a.append(lp) lrm = LogisticRegressionWithLBFGS.train(sc.parallelize(a)) lrm.save(sc, "/tmp/brakeModel") p = sc.parallelize(a) valuesAndPreds = p.map(lambda p: (p.label, lrm.predict(p.features))) accurate = 1 - valuesAndPreds.map(lambda (v, p): math.fabs(v - p)).reduce( lambda x, y: x + y) / valuesAndPreds.count()
def processMenu(csvFile): fileTuple3 = csvFile.rpartition('/') fileName = fileTuple3[-1] dateString_wext = fileName.lstrip('pmix_') dateString = dateString_wext.rstrip('.csv') print('Processing data for period: ' + dateString) with open(csvFile) as f: reader = csv.reader(f) # Read and discard the first line, which is the column headings next(reader) # Iterate over the remaining lines in the CSV file for line in reader: # The elements of interest in the list 'line' are: # line[0] Master ID # line[1] Item ID ->[0] in dictionary # line[2] Parent ID ->[1] # line[3] Menu Name ->[2] # line[4] Menu Group ->[3] # line[5] Subgroup ->[4] # line[6] Menu Item ->[5] # line[7] Avg Price ->[6] # line[12] Item Qty ->[7] # line[13] Gross Amount ->[8] # line[14] Void Qty ->[9] # line[15] Void Amount ->[10] # line[16] Discount Amount ->[11] # line[17] Net Amount ->[12] if line[0]: # Convert Master ID text value to integer for use as dictionary key # Strip off the leading 4 and all 0s that follow. This gets rid of # the '4000000..' that all the IDs have, and leaves the numbers that # form a unique ID for the item. longID = line[0].lstrip('4') shortID = longID.lstrip('0') if shortID: int_key = locale.atoi(shortID) # This is a new key, so simply add it to the dictionary if int_key not in masterDict and int_key > 0: masterDict[int_key] = [ line[1], line[2], line[3], line[4], line[5], line[6], locale.atof(line[7]), locale.atof(line[12]), locale.atof(line[13]), locale.atof(line[14]), locale.atof(line[15]), locale.atof(line[16]), locale.atof(line[17]) ] # There are some keys for open item buttons we don't care about elif int_key > 0: # This key is already in dictionary, so update the entry by adding # this item's Qty/Amount to previous total. # print ("Duplicate key: " + line[0] + ", for menu item " + masterDict[int_key][5]) # Note indexes in dictionary different than CVS file line updatedLine = [ masterDict[int_key][0], masterDict[int_key][1], masterDict[int_key][2], masterDict[int_key][3], masterDict[int_key][4], masterDict[int_key][5], masterDict[int_key][6], masterDict[int_key][7] + locale.atof(line[12]), masterDict[int_key][8] + locale.atof(line[13]), masterDict[int_key][9] + locale.atof(line[14]), masterDict[int_key][10] + locale.atof(line[15]), masterDict[int_key][11] + locale.atof(line[16]), masterDict[int_key][12] + locale.atof(line[17]) ] masterDict[int_key] = updatedLine #print(masterDict) return dateString
def convert_integer(): try: return int(locale.atof(value)) except (ValueError, AttributeError): return
def convert_float(): try: return locale.atof(value) except (ValueError, AttributeError): return
import urllib2 import re import locale import request headers = { 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0', 'Cookie': 'd53db4de415c4e858dc761595623a898=+; Path=/\r\n' } request = request.RequestUrl() site_status_1 = request.open_url_and_get_content( 'http://hughes.sieve.com.br:8000/level3/', headers=headers) pattern = r'href="(.*?)"' match = re.search(pattern, site_status_1) site_status_2 = request.open_url_and_get_content( 'http://hughes.sieve.com.br:8000' + match.groups()[0], headers=headers) headers = { 'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:25.0) Gecko/20100101 Firefox/25.0', 'Cookie': 'd53db4de415c4e858dc761595623a898=+; Path=/\r\n', 'Cookie': '18=+;' } site_status_3 = request.open_url_and_get_content( 'http://hughes.sieve.com.br:8000/level3/', headers=headers) locale.setlocale(locale.LC_ALL, 'pt_BR.UTF-8') pattern = r'R\$ ([\d\.,]+)' match = re.search(pattern, site_status_3) price = locale.atof(match.groups()[0]) print 'R$ %f' % price
import sys import locale def find_text(element, match): return element.find(match).text locale.setlocale(locale.LC_ALL, 'ru_RU.utf8') doc = urllib.request.urlopen('http://www.cbr.ru/scripts/XML_daily.asp') val_curs = xml.etree.ElementTree.parse(doc).getroot() min = sys.maxsize max = 0 min_valute = None max_valute = None for valute in val_curs: value = locale.atof(valute.find('Value').text) / locale.atof(valute.find('Nominal').text) if value < min: min = value min_valute = valute elif value > max: max = value max_valute = valute assert min_valute != None assert max_valute != None print(f'Самая дешевая валюта: {find_text(min_valute, "Nominal")} {find_text(min_valute, "Name")}, cтоимость: {find_text(min_valute, "Value")} рублей') print(f'Самая дорогая валюта: {find_text(max_valute, "Nominal")} {find_text(max_valute, "Name")}, cтоимость: {find_text(max_valute, "Value")} рублей')
def __brl_string_to_float(self, brl_string): setlocale(LC_NUMERIC, 'pt_BR.UTF-8') return atof(brl_string)