def parse_etf(code: str, tag: str, etf_type: str): url = NAVER + code print(url) tree = tree_from_url(url, 'euc-kr') try: title = tree.xpath('//*[@id="middle"]/div[1]/div[1]/h2/a')[0].text except: return month1 = parse_float(tree.xpath('//*[@id="tab_con1"]/div[5]/table/tbody/tr[1]/td/em')[0].text.strip()) month3 = parse_float(tree.xpath('//*[@id="tab_con1"]/div[5]/table/tbody/tr[2]/td/em')[0].text.strip()) month6 = parse_float(tree.xpath('//*[@id="tab_con1"]/div[5]/table/tbody/tr[3]/td/em')[0].text.strip()) month12 = parse_float(tree.xpath('//*[@id="tab_con1"]/div[5]/table/tbody/tr[4]/td/em')[0].text.strip()) company = tree.xpath('//table[contains(@class, "tbl_type1")]//td/span/text()')[2] cost = parse_float(tree.xpath('//table[contains(@class, "tbl_type1")]//td/em/text()')[0]) tags = tag.split(',') db.save_etf({ 'code': code, 'title': title, 'company': company, 'month1': month1, 'month3': month3, 'month6': month6, 'month12': month12, 'cost': cost, 'tags': tags, 'type': etf_type, })
async def run(self): print(f'{self.__class__.__name__}: running') msg = await self.receive(timeout=30 ) # wait for a message for 180 seconds if not msg: print( f'{self.__class__.__name__}: received no message, terminating!' ) return reply = msg.make_reply() parsed_msg = msg.body.lower().split(" ") if len(parsed_msg) == 2 and parse_float(parsed_msg[0]) and parse_float( parsed_msg[1]): print(f'{self.__class__.__name__}: got param!') length = float(parsed_msg[0]) max_vel = float(parsed_msg[1]) self.agent.track.append((length, max_vel)) reply.body = "Got Your params: length='{}', max_vel='{}'".format( self.agent.track[-1][0], self.agent.track[-1][1]) self.agent.track_length += length next_state = COLLECTING_PARAMS elif len(parsed_msg) == 1 and parsed_msg[0] == 'finish': print(f'{self.__class__.__name__}: finishing!') reply.body = "Finished passing parameters! Broadcasting race..." next_state = SUBSCRIBING_TO_DRIVER elif len(parsed_msg) == 2 and parsed_msg[0] == 'laps' and parse_int( parsed_msg[1]): print(f'{self.__class__.__name__}: got laps number!') self.agent.laps = int(parsed_msg[1]) reply.body = "Got Your laps number: laps='{}'".format( self.agent.laps) next_state = COLLECTING_PARAMS elif len(parsed_msg) == 1 and parsed_msg[0] == "default": self.agent.track.append((50, 10)) self.agent.track.append((40, 15)) self.agent.track.append((30, 5)) self.agent.track.append((70, 20)) self.agent.track.append((10, 3)) self.agent.track_length = 200 self.agent.laps = 5 reply.body = "Finished passing parameters! Broadcasting race..." next_state = SUBSCRIBING_TO_DRIVER else: reply.body = paramsHelpMessage next_state = COLLECTING_PARAMS await self.send(reply) self.set_next_state(next_state)
def fetch_details(self): jf = get_json_data(URL_MATCHES_DETAIL % self.detailID) details = jf[0] for key in IYMS_ORDER: self.ratios.append(parse_float(details[key])) for key in SK_ORDER: self.ratios.append(parse_float(details[key]))
def assign_marks(self, q): utils.printmd(f"**Assigning marks for {q.file}.**") print("Enter 'q', 'quit', or 'exit' at any time. ...\n\n") for part in q.contents: utils.printmd(f"**{part}**") _in, valid = utils.parse_float(input(">>> ")) while not valid: print("Please input a number.") _in, valid = utils.parse_float(input(">>> ")) self.marks[part] = float(_in)
def csv2html(fcsv): import csv f = open(fcsv) reader = csv.reader(f, delimiter=";", lineterminator="\n") html = "<title>Report Universal Pilates</title>\n" html = "<table border=1 bordercolor=gray>" for row in reader: html += "<tr>" numcol = 0 for column in row: if numcol == 1: try: column = '<p align="right">%s</p>' % (utils.float2str(utils.parse_float(column), 2)) except: pass else: if not ">" in column: column = "<p><b>%s</b></p>" % column else: column = column.replace(">", " ") html += "<td>" + column + "</td>\n" numcol += 1 html += "</tr>" html += "</table>" f.close() return html
def parse_naver_company(code): url = NAVER_COMPANY + code print('네이버 {}'.format(url)) tree = tree_from_url(url) element = tree.xpath( '//*[@id="pArea"]/div[1]/div/table/tr[3]/td/dl/dt[2]/b') if not element: print('수집 실패') return False bps = parse_int(element[0].text) print('BPS: {}'.format(bps)) element = tree.xpath( '//*[@id="pArea"]/div[1]/div/table/tr[3]/td/dl/dt[6]/b') if element: dividend_rate = parse_float(element[0].text) print('배당률: {}'.format(dividend_rate)) else: dividend_rate = 0 print('배당 수집 실패') return False stock = { 'code': code, 'bps': bps, 'dividend_rate': dividend_rate, 'use_fnguide': False, } stock = db.save_stock(stock) return stock
def on_message(self, msg): try: dl_msg_type = msg[1] if dl_msg_type == DataLoggerMessages.EXTRACT_GENERAL: self.uid = msg[2] self.n_columns = parse_uint32(msg[3:7]) self.n_entries = parse_uint32(msg[7:11]) self.schema_name = stringify(msg[11:]) elif dl_msg_type == DataLoggerMessages.EXTRACT_COLUMN: column_idx = msg[2] self.column_done[column_idx] = True self.column_type[column_idx] = msg[3] self.column_name[column_idx] = stringify(msg[4:]) elif dl_msg_type == DataLoggerMessages.EXTRACT_DATA: entry_idx = parse_uint32(msg[2:6]) self.entry_done[entry_idx] = True entry = [] offset = 6 for column_type in self.column_type: if column_type == ColumnType.FLOAT: entry.append(parse_float(msg[offset:(offset+4)])) offset += 4 if column_type == ColumnType.UINT32: entry.append(parse_uint32(msg[offset:(offset+4)])) offset += 4 self.entry[entry_idx] = tuple(entry) if (self.schema_name is not None and all(self.column_done) and all(self.entry_done)): self.done = True except Exception as e: print(e) self.failure = True
def cambiar_peso(self, cell, path, texto): """ Cambia el peso de la bala editada. """ try: peso = utils._float(texto) except ValueError: utils.dialogo_info(titulo = "ERROR", texto = "El valor tecleado %s no es correcto." % (peso), padre = self.wids['ventana']) else: model = self.wids['tv_balas'].get_model() try: bala = pclases.BalaCable.get(model[path][-1]) except: utils.dialogo_info(titulo = "ERROR", texto = "No se pudo acceder a la bala.", padre = self.wids['ventana']) self.actualizar_tabla() else: difpeso = bala.peso - peso bala.peso = peso bala.syncUpdate() model[path][2] = utils.float2str(bala.peso, 1) try: totpantalla = utils.parse_float( self.wids['e_pantalla'].get_text()) except: totpantalla = 0.0 totpantalla -= difpeso self.rellenar_totales(totpantalla)
def row_values_table_by_index(table, index: int) -> List[str]: try: return [parse_float(v) for v in table.xpath('tbody/tr')[index].xpath('td//text()')] except ValueError: return [] except IndexError: return []
def generate(self, ffrom, tto): period_type = self.get_period_type() ret, new_ffrom, new_tto = self._reused_data(ffrom, tto) if new_ffrom is None and new_tto is None: # full reuse - ret == old_data return ret gc = gspread.authorize(self.credentials) wks = getattr(gc.open(self.spreadsheet), self.sheet) date_column = self.config.get("date_column", 1) data_column = self.config.get("data_column", 2) start_row = self.config.get("start_row", 0) data_values = wks.col_values(data_column) data = {} for i, v in enumerate(wks.col_values(date_column)): if i < start_row: continue if not v: continue data[ensure_date(v)] = parse_float(data_values[i] or "0") return [{ "label": d.strftime("%Y-%m-%d"), "data": data.get(d, 0) } for d in generate_date_series(ffrom, tto, period_type)]
def __init__(self, md, weekid): # md is match_data self.weekID = weekid self.matchID = int(md[10]) self.detailID = int(md[0]) self.datetime = datetime.strptime(md[7] + " " + md[6], '%d.%m.%Y %H:%M') self.league = md[26] self.team_1 = md[1] self.team_2 = md[3] self.mbs = parse_int(md[13]) self.iy_goals_1 = parse_int(md[11]) self.iy_goals_2 = parse_int(md[12]) if self.iy_goals_1 is None or self.iy_goals_2 is None: self.iy_goals_1 = None self.iy_goals_2 = None self.ms_goals_1 = None self.ms_goals_2 = None else: self.ms_goals_1 = parse_int(md[8]) self.ms_goals_2 = parse_int(md[9]) self.was_played = self.ms_goals_1 is not None self.h1 = 0 if md[14] == '' else int(md[14]) self.h2 = 0 if md[15] == '' else int(md[15]) self.ratios = [] res = {} res['mac'] = [parse_float(x) for x in md[16:19]] res['ilk'] = [parse_float(x) for x in md[33:36]] res['han'] = [parse_float(x) for x in md[36:39]] res['kar'] = [parse_float(x) for x in md[39:41]] res['cif'] = [parse_float(x) for x in md[19:22]] res['iy'] = [parse_float(x) for x in md[42:44]] res['au1'] = [parse_float(x) for x in md[44:46]] res['au2'] = [parse_float(x) for x in md[22:24]] res['au3'] = [parse_float(x) for x in md[46:48]] res['top'] = [parse_float(x) for x in md[29:33]] if self.was_played: self.results = get_results(self.iy_goals_1, self.iy_goals_2, self.ms_goals_1, self.ms_goals_2, self.h1, self.h2) self.ratios = concat([res[bet] for bet in BET_ORDER]) if self.league != 'DUEL': self.fetch_details()
def row_values_table(table, row_headers: List[str], key: str) -> List[str]: try: i = row_headers.index(key) return [ parse_float(v) for v in table.xpath('tbody/tr')[i].xpath('td//text()') ] except ValueError: return []
def parse_basic(code): print('종목 {} 기본...'.format(code)) url = DAUM_BASIC + code print('다음 {}'.format(url)) tree = tree_from_url(url) if not tree.xpath('//*[@id="topWrap"]/div[1]/h2'): return False title = tree.xpath('//*[@id="topWrap"]/div[1]/h2')[0].text price = parse_float( tree.xpath('//*[@id="topWrap"]/div[1]/ul[2]/li[1]/em')[0].text) diff = tree.xpath('//*[@id="topWrap"]/div[1]/ul[2]/li[2]/span')[0] rate_diff = tree.xpath( '//*[@id="topWrap"]/div[1]/ul[2]/li[3]/span')[0].text exchange = tree.xpath('//*[@id="topWrap"]/div[1]/ul[1]/li[2]/a')[0].text price_diff = parse_float(diff.text) rate_diff = float( rate_diff.replace(',', '').replace('+', '').replace('-', '').replace( '%', '').replace('%', '')) is_price_down = diff.get('class').endswith('down') if is_price_down: price_diff = -abs(price_diff) rate_diff = -abs(rate_diff) per = parse_float( tree.xpath('//*[@id="stockContent"]/ul[2]/li[3]/dl[2]/dd') [0].text.split('/')[1]) pbr = parse_float( tree.xpath('//*[@id="stockContent"]/ul[2]/li[4]/dl[2]/dd') [0].text.split('/')[1]) trade_volume = parse_float( tree.xpath('//*[@id="topWrap"]/div[1]/ul[2]/li[5]/span[1]')[0].text) trade_value = parse_float( tree.xpath('//*[@id="topWrap"]/div[1]/ul[2]/li[6]/span')[0].text) agg_value = parse_float( tree.xpath('//*[@id="stockContent"]/ul[2]/li[2]/dl[2]/dd')[0].text) print('종목명: {title} 현재가: {price}'.format(title=title, price=price)) stock = { 'code': code, 'title': title, 'current_price': price, 'price_diff': price_diff, 'rate_diff': rate_diff, 'per': per, 'pbr': pbr, 'trade_volume': trade_volume, 'trade_value': trade_value, 'exchange': exchange, 'agg_value': agg_value, } db.save_stock(stock) return True
def rellenar_tabla_por_forma_de_pago(self, cobros_por_forma_de_pago): model_cobros = self.wids["tv_datos_cobros_formapago"].get_model() model_vtos = self.wids["tv_datos_vtos_formapago"].get_model() model_cobros.clear() model_vtos.clear() for formadepago in cobros_por_forma_de_pago: if cobros_por_forma_de_pago[formadepago]["cobros"]: padre = model_cobros.append(None, (formadepago, "", "", "0.0", None)) for cobro in cobros_por_forma_de_pago[formadepago]["cobros"]: fra = cobro.get_factura_o_prefactura() model_cobros.append( padre, ("", fra.numfactura, fra.cliente.nombre, cobro.importe, cobro.get_puid()) ) model_cobros[padre][3] = utils.float2str(utils.parse_float(model_cobros[padre][3]) + cobro.importe) if cobros_por_forma_de_pago[formadepago]["vencimientos"]: padre = model_vtos.append(None, (formadepago, "", "", "0.0", None)) for vto in cobros_por_forma_de_pago[formadepago]["vencimientos"]: fra = vto.get_factura_o_prefactura() model_vtos.append(padre, ("", fra.numfactura, fra.cliente.nombre, vto.importe, vto.get_puid())) model_vtos[padre][3] = utils.float2str(utils.parse_float(model_vtos[padre][3]) + vto.importe)
def crear_bala(self, boton = None, peso = None): """ Crea una bala del producto mostrado en pantalla e introduce su información en el TreeView. El peso lo solicita en una ventana de diálogo. Si se recibe peso, debe ser un float. """ # TODO: Hacer que el peso lo tome del puerto serie y se le pase a # esta función. producto = utils.combo_get_value(self.wids['cbe_producto']) if producto == None: utils.dialogo_info(titulo = "SELECCIONE UN PRODUCTO", texto = "Debe seleccionar un producto en el desplegable.", padre = self.wids['ventana']) else: if peso == None: peso = utils.dialogo_entrada(titulo = "PESO", texto = "Introduzca peso:", padre = self.wids['ventana'], valor_por_defecto = "0") try: peso = utils._float(peso) except ValueError: utils.dialogo_info(titulo = "ERROR", texto = "El valor tecleado %s no es correcto." % (peso), padre = self.wids['ventana']) peso = 0 nueva_bala = self.crear_objeto_bala(producto, peso) if nueva_bala == None: utils.dialogo_info(titulo = "ERROR", texto = "La bala no se pudo crear. Inténtelo de nuevo.", padre = self.wids['ventana']) else: self.add_nueva_bala_tv(nueva_bala) try: totpantalla = utils.parse_float( self.wids['e_pantalla'].get_text()) except: totpantalla = 0.0 totpantalla += peso self.rellenar_totales(totpantalla)
def edit_float(cell, path, newtext, tv, numcol, clase, atributo): """ Cambia el texto del model[path][numcol] y del objeto relacionado que saca a partir del ID de la última columna del model. """ try: numero = utils.parse_float(newtext) except (ValueError, TypeError): parent = tv.parent while parent != None: parent = parent.parent utils.dialogo_info(titulo = "ERROR EN NÚMERO", texto = "El texto «%s» no es un número válido." % (newtext), padre = parent) else: model = tv.get_model() id = model[path][-1] objeto = clase.get(id) setattr(objeto, atributo, numero) objeto.syncUpdate() model[path][numcol] = utils.float2str(getattr(objeto, atributo), autodec = True)
def parse_message(msg): parsed_message = [] i = 0 last_open_str = 0 def store_string_so_far(): if last_open_str != i: parsed_message.append(stringify(msg[last_open_str:i])) while i < len(msg): if stringify(msg[i:(i+2)]) == '%d': store_string_so_far() int_bytes = msg[(i+2):(i+4)] parsed_int = parse_int(int_bytes) parsed_message.append(parsed_int) last_open_str = i + 4 i = i + 4 elif stringify(msg[i:(i+2)]) == '%f': store_string_so_far() float_bytes = msg[(i+2):(i+6)] parsed_float = parse_float(float_bytes) parsed_message.append(parsed_float) last_open_str = i + 6 i = i + 6 elif stringify(msg[i:(i+2)]) == '%l': store_string_so_far() uint32_bytes = msg[(i+2):(i+6)] parsed_uint32 = parse_uint32(uint32_bytes) parsed_message.append(parsed_uint32) last_open_str = i + 6 i = i + 6 else: if i+1 == len(msg): i += 1 store_string_so_far() else: i += 1 return parsed_message
def get_attractions_from_list(url): print "Open %s" % url attractions = [] r = requests.get(url) soup = BeautifulSoup(r.text, 'html.parser') list_items = soup.find("div", { "id": "FILTERED_LIST"})\ .find_all("div", {"class": "entry"}) try: next_page = BASE_URL + soup.find("a", class_="next")["href"] except: next_page = None for li in list_items: attraction = {} title_tag = li.find("div", { "class": "property_title" }).find("a") attraction["name"] = title_tag.text attraction["url"] = BASE_URL + title_tag["href"] attraction["rank"] = parse_int(li.find("div", class_="popRanking").text.strip().split(" ")[1]) try: attraction["score"] = parse_float(li.find("img", class_="sprite-ratings")["alt"].split(" ")[0]) except TypeError: attraction["score"] = None try: attraction["n_reviews"] = parse_int(li.find("span", class_="more").text.strip().split(" ")[0]) except AttributeError: attraction["n_reviews"] = None print attraction attractions.append(attraction) print "Found %s attractions" % len(attractions) return { "data": attractions, "next_page_url": next_page, }
def handle(self, msg): if len(msg) == 0: print ('WARNING: Empty message') return msg_type = msg[0] if msg_type == ToComputer.DEBUG: # debug message subsystems = [ 'INFO', 'ERROR', 'CRON', ] if (0 > msg[1] or msg[1] >= len(subsystems)): print ("WARNING: Unknown debug category: %d.." % (msg[1],)) subsystem = 'UNKNOWN' print (stringify(msg[2:])) else: subsystem = subsystems[msg[1]] content = parse_message(msg[2:]) content = ''.join([str(m) for m in content]) content = '[%s] %s' % (subsystem, content) self.log(content) elif msg_type == ToComputer.GET_SETTINGS_REPLY: time_since_epoch_s = parse_uint32(msg[1:5]) date = datetime.fromtimestamp(time_since_epoch_s) box_uid = msg[5] box_node_type = chr(msg[6]) box_balance = parse_uint32(msg[7:11]) state_of_charge = parse_float(msg[11:15]) uncertainty_of_charge = parse_float(msg[15:19]) battery_capacity = parse_float(msg[19:23]) off_threshold = parse_float(msg[23:27]) red_threshold = parse_float(msg[27:31]) yellow_threshold = parse_float(msg[31:35]) balance_update_hours = parse_int(msg[35:37]) balance_update_minutes = parse_int(msg[37:39]) balance_update_ammount = parse_uint32(msg[39:43]) #self.log('Time on device is ' + str(date)) self.update_if_not_focused(self.ui_root.settings.box_time, str(date)) self.update_if_not_focused(self.ui_root.settings.box_uid, str(box_uid)) self.update_if_not_focused(self.ui_root.settings.box_node_type, str(box_node_type)) self.update_if_not_focused(self.ui_root.settings.box_balance, str(box_balance)) self.update_if_not_focused(self.ui_root.settings.state_of_charge, str(state_of_charge)) self.update_if_not_focused(self.ui_root.settings.uncertainty_of_charge, str(uncertainty_of_charge)) self.update_if_not_focused(self.ui_root.settings.battery_capacity, str(battery_capacity)) self.update_if_not_focused(self.ui_root.settings.off_threshold, str(off_threshold)[:6]) self.update_if_not_focused(self.ui_root.settings.red_threshold, str(red_threshold)[:6]) self.update_if_not_focused(self.ui_root.settings.yellow_threshold, str(yellow_threshold)[:6]) self.update_if_not_focused(self.ui_root.settings.balance_update_hours, str(balance_update_hours)) self.update_if_not_focused(self.ui_root.settings.balance_update_minutes, str(balance_update_minutes)) self.update_if_not_focused(self.ui_root.settings.balance_update_ammount, str(balance_update_ammount)) elif msg_type == ToComputer.DATA_LOGGER_REPLY: controller.get.data_logger.on_message(msg) else: print( 'WARNING: Uknown message type :', msg[0])
def treeview2pdf(tv, titulo = None, fecha = None, apaisado = None, pijama = False, graficos = [], numcols_a_totalizar = [], extra_data = []): """ A partir de un TreeView crea un PDF con su contenido. 1.- Asigna un nombre de archivo en función del nombre del TreeView. 2.- Si titulo es None, asigna como título el nombre del TreeView. 3.- El ancho de los campos será el ancho relativo en porcentaje que ocupa el ancho de la columna (get_width) a la que correspondería. El título del campo será el título (get_title) de la columna. 4.- Si fecha no es None debe ser una cadena de texto. Si es None, se usará la fecha actual del sistema. 5.- Si la suma del ancho de las columnas del TreeView es superior a 800 píxeles el PDF generado será apaisado, a no ser que se fuerce mediante el parámetro "apaisado" que recibe la función. numcols_a_totalizar es una lista de índices (empezando por 0) de las columnas a las que se va a intentar convertir a número y sumar para mostrar una última línea con el texto "TOTAL" o "TOTALES" si hay más de una. extra_data son líneas que se añadirán a las que tiene el TreeView *al final* del informe (incluso detrás de los totales, si los hubiera). """ archivo = get_nombre_archivo_from_tv(tv) if titulo == None: titulo = get_titulo_from_tv(tv) campos, pdf_apaisado, cols_a_derecha, cols_centradas=get_campos_from_tv(tv) datos = get_datos_from_tv(tv) totales = dict(zip(numcols_a_totalizar, len(numcols_a_totalizar) * [0])) for fila in datos: for numcol in totales: try: totales[numcol] += utils.parse_float(fila[numcol]) except ValueError: # ¿No hay dato en esa fila? Entonces cuento # instancias. totales[numcol] += 1 #print fila, numcol, fila[numcol] if totales and datos: last_i = len(datos) - 1 # Apuntará a la última línea no nula while (last_i > 0 and reduce(lambda x, y: str(x) + str(y), datos[last_i]) == ""): last_i -= 1 if (datos[last_i] and not reduce(lambda x, y: x == y == "---" and "---", datos[last_i])): datos.append(("---", ) * len(campos)) fila = ["TOTAL"] + [""] * (len(campos) - 1) if len(totales) > 1: fila[0] = "TOTALES" for total in totales: fila[total] = utils.float2str(totales[total] , precision = 2, autodec = True) datos.append(fila) if extra_data and not isinstance(extra_data[0], (tuple, list)): extra_data = [extra_data] for extra in extra_data: dif_len = len(campos) - len(extra) if dif_len <> 0 and not isinstance(extra, list): extra = list(extra) if dif_len > 0: extra += [""] * dif_len elif dif_len < 0: extra = extra[:len(campos)] datos.append(extra) if fecha == None: fecha = utils.str_fecha(datetime.date.today()) if apaisado != None: pdf_apaisado = apaisado return geninformes.imprimir2(archivo, titulo, campos, datos, fecha, apaisado = pdf_apaisado, cols_a_derecha = cols_a_derecha, cols_centradas = cols_centradas, pijama = pijama, graficos = graficos)
def parse_fnguide(code: str): print('종목 {} FnGuide...'.format(code)) url = FNGUIDE + code print('FnGuide {}'.format(url)) tree = tree_from_url(url) title = first_or_none(tree.xpath('//*[@id="giName"]/text()')) if not title: return False groups = first_or_none( tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[1]/text()')) groups = groups.split(' ') group = groups[1] if len(groups) > 1 else None subgroup = first_or_none( tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[4]/text()')) subgroup = subgroup.replace('\xa0', '') closing_month = first_or_none( tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[6]/text()')) closing_month = parse_int(closing_month.split(' ')[0][:-1]) forward_per = parse_float( first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[2]/dd/text()'))) group_per = parse_float( first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[3]/dd/text()'))) dividend_rate = parse_float( first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[5]/dd/text()'))) relative_earning_rate = parse_float( first_or_none(tree.xpath('//*[@id="svdMainChartTxt13"]/text()'))) momentums = tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[1]/span/text()') momentums = [parse_float(m) for m in momentums] month1 = momentums[0] if len(momentums) >= 1 else 0 month3 = momentums[1] if len(momentums) >= 2 else 0 month6 = momentums[2] if len(momentums) >= 3 else 0 month12 = momentums[3] if len(momentums) >= 4 else 0 foreigner_weight = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[2]/text()'))) beta = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[4]/td[2]/text()'))) stocks = first_or_none( tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[5]/td[1]/text()')) stocks = stocks.split('/ ') has_preferred_stock = False if stocks[1] == '0' else True floating_rate = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid1"]/table/tbody/tr[6]/td[2]/text()'))) YoY = parse_float( first_or_none( tree.xpath( '//*[@id="svdMainGrid2"]/table/tbody/tr/td[4]/span/text()'))) consensus_point = parse_float( first_or_none( tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[1]/text()'))) consensus_price = parse_int( first_or_none( tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[2]/text()'))) consensus_count = parse_int( first_or_none( tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[5]/text()'))) bps = parse_int( first_or_none( tree.xpath( '//*[@id="highlight_D_A"]/table/tbody/tr[19]/td[3]/text()'))) stock = { 'code': code, 'group': group, 'subgroup': subgroup, 'closing_month': closing_month, 'forward_per': forward_per, 'group_per': group_per, 'dividend_rate': dividend_rate, 'relative_earning_rate': relative_earning_rate, 'month1': month1, 'month3': month3, 'month6': month6, 'month12': month12, 'foreigner_weight': foreigner_weight, 'beta': beta, 'has_preferred_stock': has_preferred_stock, 'floating_rate': floating_rate, 'YoY': YoY, 'consensus_point': consensus_point, 'consensus_price': consensus_price, 'consensus_count': consensus_count, 'bps': bps, 'use_fnguide': True, } db.save_stock(stock) return True
def parse_snowball(code): if not parse_basic(code): print('수집 실패') return if not parse_fnguide(code): print('FnGuide 수집실패') if not parse_naver_company(code): return print('종목 {} 스노우볼...'.format(code)) url = NAVER_YEARLY % (code) tree = tree_from_url(url) try: years = list( filter( lambda x: x != '', map(lambda x: x.strip().split('/')[0], tree.xpath('/html/body/table/thead/tr[2]/th/text()')))) last_year_index = years.index(LAST_YEAR) except ValueError: return tds = tree.xpath('/html/body/table/tbody/tr[22]/td') ROEs = [first_or_none(td.xpath('span/text()')) for td in tds] while ROEs and ROEs[-1] is None: ROEs.pop() if len(ROEs) == 0: print('*** ROE 정보가 없음 >>>') return CAPEXs = tree.xpath('/html/body/table/tbody/tr[17]/td/span/text()') CAPEXs = [parse_float(x) for x in CAPEXs] ROEs = [float_or_none(x) for x in ROEs] DEPTs = tree.xpath('/html/body/table/tbody/tr[24]/td/span/text()') DEPTs = [parse_float(x) for x in DEPTs] EPSs = tree.xpath('/html/body/table/tbody/tr[26]/td/span/text()') EPSs = [parse_float(x) for x in EPSs] PERs = tree.xpath('/html/body/table/tbody/tr[27]/td/span/text()') PERs = [parse_float(x) for x in PERs] BPSs = tree.xpath('/html/body/table/tbody/tr[28]/td/span/text()') BPSs = [parse_int(x) for x in BPSs] PBRs = tree.xpath('/html/body/table/tbody/tr[29]/td/span/text()') PBRs = [parse_float(x) for x in PBRs] #자산총계 TAs = tree.xpath('/html/body/table/tbody/tr[8]/td/span/text()') TAs = [parse_int(x) for x in TAs] #당기순이익 NPs = tree.xpath('/html/body/table/tbody/tr[5]/td/span/text()') NPs = [parse_int(x) for x in NPs] #영업활동현금흐름 CFOs = tree.xpath('/html/body/table/tbody/tr[14]/td/span/text()') CFOs = [parse_int(x) for x in CFOs] #발행주식수 TIs = tree.xpath('/html/body/table/tbody/tr[33]/td/span/text()') TIs = [parse_int(x) for x in TIs] stock = { 'code': code, 'ROEs': ROEs, 'last_year_index': last_year_index, 'PBRs': PBRs, 'EPSs': EPSs, 'TAs': TAs, 'NPs': NPs, 'CFOs': CFOs, 'PERs': PERs, 'TIs': TIs, 'DEPTs': DEPTs, 'BPSs': BPSs, 'CAPEXs': CAPEXs, } stock = db.save_stock(stock) stock.save_record() parse_quarterly(code) parse_json(code)
def load_map(self, map): """Load a map.""" #Unload the current map first self.unload_map() #Locate the XML file for the map Logger.info("Loading map '{}'...".format(map)) map_file = os.path.join(map, os.path.basename(map) + ".xml") if not os.path.exists(map_file): Logger.error("Failed to load map file '{}'.".format(map_file)) return False #Load the map XML file xml = etree.parse(map_file) root = xml.getroot() for child in root: #Terrain? if child.tag == "terrain": #Validate terrain if not ("size" in child.attrib and "spawnpos" in child.attrib and "heightmap" in child.attrib): Logger.error( "Terrain section must define 'size', 'spawnpos', and 'heightmap'." ) return False #Load terrain self.size = parse_vec(child.attrib["size"], 3) self.spawnpos = parse_vec(child.attrib["spawnpos"], 2) heightmap = os.path.join(map, child.attrib["heightmap"]) self.terrain = GeoMipTerrain("Terrain") self.terrain.set_block_size(64) self.terrain.set_bruteforce(True) if not self.terrain.set_heightfield(heightmap): Logger.error("Failed to load heightmap for terrain.") self.terrain = None return False self.terrain_np = self.terrain.get_root() self.terrain_np.set_scale(self.size[0] / 512, self.size[1] / 512, self.size[2]) tex = loader.load_texture( "./data/textures/terrain/grass_tex2.png") self.terrain_np.set_texture(tex) self.terrain_np.set_tex_scale(TextureStage.get_default(), self.size[0] / 512, self.size[1] / 512) tex.set_wrap_u(Texture.WM_repeat) tex.set_wrap_v(Texture.WM_repeat) self.terrain_np.reparent_to(render) self.terrain.generate() base.camera.set_pos(self.size[0] / 2, self.size[1] / 2, self.size[2]) #Portal? elif child.tag == "portal": #Validate portal if not ("pos" in child.attrib and "destmap" in child.attrib): Logger.warning("Portal must define 'pos' and 'destmap'.") continue #Load portal pos = parse_vec(child.attrib["pos"], 3) radius = parse_float( child.attrib["radius"]) if "radius" in child.attrib else 1 destmap = child.attrib["destmap"] self.add_portal(pos, radius, destmap) #Gate? elif child.tag == "gate": #Validate gate if not ("pos" in child.attrib and "destmap" in child.attrib and "destvec" in child.attrib): Logger.warning( "Gate must define 'pos', 'destmap', and 'destvec'.") continue #Load gate pos = parse_vec(child.attrib["pos"], 3) destmap = child.attrib["destmap"] destvec = parse_vec(child.attrib["destvec"], 3) material = child.attrib[ "material"] if "material" in child.attrib else "" self.add_gate(pos, destmap, destvec, material) #Object? elif child.tag == "object": #Validate object if not ("mesh" in child.attrib and "pos" in child.attrib): Logger.warning("Object must define 'mesh' and 'pos'.") continue #Load object mesh = child.attrib["mesh"] pos = parse_vec(child.attrib["pos"], 3) rot = parse_vec(child.attrib["rot"], 3) if "rot" in child.attrib else [0, 0, 0] scale = parse_vec(child.attrib["scale"], 3) if "scale" in child.attrib else [1, 1, 1] material = child.attrib[ "material"] if "material" in child.attrib else "" sound = child.attrib["sound"] if "sound" in child.attrib else "" self.add_object(mesh, pos, rot, scale, material, sound) #Object Group? elif child.tag == "objectgroup": #Validate object group if not ("mesh" in child.attrib): Logger.warning("Object group must define 'mesh'.") continue #Load object group mesh = child.attrib["mesh"] material = child.attrib[ "material"] if "material" in child.attrib else "" self.load_object_group(child, mesh, material) #Unknown? else: Logger.warning( "Unknown tag '{}' encountered in map '{}'.".format( child.tag, map)) #Map loaded Logger.info("Map '{}' loaded.".format(map)) return True
def parse_fnguide(code: str): print('종목 {} FnGuide...'.format(code)) url = FNGUIDE + code print('FnGuide {}'.format(url)) tree = tree_from_url(url) title = first_or_none(tree.xpath('//*[@id="giName"]/text()')) if not title: return False groups = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[1]/text()')) groups = groups.split(' ') group = groups[1] if len(groups) > 1 else None subgroup = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[4]/text()')) subgroup = subgroup.replace('\xa0', '') closing_month = first_or_none(tree.xpath('//*[@id="compBody"]/div[1]/div[1]/p/span[6]/text()')) closing_month = parse_int(closing_month.split(' ')[0][:-1]) forward_per = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[2]/dd/text()'))) group_per = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[3]/dd/text()'))) dividend_rate = parse_float(first_or_none(tree.xpath('//*[@id="corp_group2"]/dl[5]/dd/text()'))) relative_earning_rate = parse_float(first_or_none(tree.xpath('//*[@id="svdMainChartTxt13"]/text()'))) momentums = tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[1]/span/text()') momentums = [parse_float(m) for m in momentums] month1 = momentums[0] if len(momentums) >= 1 else 0 month3 = momentums[1] if len(momentums) >= 2 else 0 month6 = momentums[2] if len(momentums) >= 3 else 0 month12 = momentums[3] if len(momentums) >= 4 else 0 foreigner_weight = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[3]/td[2]/text()'))) beta = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[4]/td[2]/text()'))) stocks = first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[7]/td[1]/text()')) stocks = stocks.split('/ ') has_preferred_stock = False if stocks[1] == '0' else True floating_rate = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid1"]/table/tbody/tr[6]/td[2]/text()'))) YoY = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid2"]/table/tbody/tr/td[4]/span/text()'))) consensus_point = parse_float(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[1]/text()'))) consensus_price = parse_int(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[2]/text()'))) consensus_count = parse_int(first_or_none(tree.xpath('//*[@id="svdMainGrid9"]/table/tbody/tr/td[5]/text()'))) bps = parse_int(first_or_none(tree.xpath('//*[@id="highlight_D_A"]/table/tbody/tr[19]/td[3]/text()'))) try: years = tree.xpath('//*[@id="highlight_D_Y"]/table/thead/tr[2]/th/div/text()') years = [x.split('/')[0] for x in years] last_year_index = years.index(LAST_YEAR) except ValueError: print("** 작년 데이터 없음 **") return NPs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[3]/td/text()') NPs = [parse_float(x) for x in NPs] TAs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[6]/td/text()') TAs = [parse_float(x) for x in TAs] ROEs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[17]/td/text()') ROEs = [parse_float(x) for x in ROEs] EPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[18]/td/text()') EPSs = [parse_float(x) for x in EPSs] BPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[19]/td/text()') BPSs = [parse_float(x) for x in BPSs] DPSs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[20]/td/text()') DPSs = [parse_float(x) for x in DPSs] PERs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[21]/td/text()') PERs = [parse_float(x) for x in PERs] PBRs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[22]/td/text()') PBRs = [parse_float(x) for x in PBRs] DEPTs = tree.xpath('//*[@id="highlight_D_Y"]/table/tbody/tr[7]/td/text()') DEPTs = [parse_float(x) for x in DEPTs] stock = { 'code': code, 'group': group, 'subgroup': subgroup, 'closing_month': closing_month, 'forward_per': forward_per, 'group_per': group_per, 'dividend_rate': dividend_rate, 'relative_earning_rate': relative_earning_rate, 'month1': month1, 'month3': month3, 'month6': month6, 'month12': month12, 'foreigner_weight': foreigner_weight, 'beta': beta, 'has_preferred_stock': has_preferred_stock, 'floating_rate': floating_rate, 'YoY': YoY, 'consensus_point': consensus_point, 'consensus_price': consensus_price, 'consensus_count': consensus_count, 'bps': bps, 'use_fnguide': True, 'last_year_index': last_year_index, 'NPs': NPs, 'TAs': TAs, 'ROEs': ROEs, 'EPSs': EPSs, 'BPSs': BPSs, 'DPSs': DPSs, 'PERs': PERs, 'PBRs': PBRs, 'DEPTs': DEPTs, } db.save_stock(stock) return True