def _parseDataFromFile(self, dataFile: str) -> DataFrame:
with open(dataFile, encoding="utf-8") as f:
json_data = json.load(f)
json_dict = json_data[0]
data_field = json_dict.get("data", None)
if data_field is None:
_LOGGER.warning("no 'data' field found")
return None
# print("xxx:", data_field)
## example data
# c h l o p t v
# 0 418.4 419.8 418.4 419.8 418.4 1599202800 11141
# 1 419.0 419.0 418.1 418.4 419.0 1599202801 334
# 2 418.0 419.5 418.0 419.0 418.0 1599202802 130
dataFrame = DataFrame(data_field)
# print( "xxx:\n", dataFrame )
apply_on_column(dataFrame, 't', convert_timestamp_datetime)
if self.rangeCode != "1D":
## add recent value to range other than "1D" (current)
currData = GpwCurrentStockIntradayData.DAO(self.isin)
currData.dataTime = self.dataTime
currWorksheet = currData.getWorksheetData()
if currWorksheet is not None:
lastRow = currWorksheet.iloc[-1]
dataFrame = dataFrame.append(lastRow)
dataFrame.reset_index(drop=True, inplace=True)
return dataFrame
return None
def parse_mb_transactions_data(sourceFile):
with fix_separator(sourceFile) as tmpfile:
# print( "file:\n" + tmpfile.read() )
# tmpfile.seek(0)
columns = count_separator(tmpfile, ";")
tmpfile.seek(0)
header = []
if columns == 8:
header = [
"trans_time", "name", "stock_id", "k_s", "amount",
"unit_price", "unit_currency", "price", "currency"
]
elif columns == 10:
header = [
"trans_time", "name", "stock_id", "k_s", "amount",
"unit_price", "unit_currency", "commision_value",
"commision_currency", "price", "currency"
]
# print( "header:", header, columns )
dataFrame = pandas.read_csv(tmpfile,
names=header,
sep=r'[;\t]',
decimal='.',
thousands=' ',
engine='python',
encoding='utf_8')
# print( "raw data:\n" + str( dataFrame ) )
apply_on_column(dataFrame, 'name', str)
#### fix names to match GPW names
## XTRADEBDM -> XTB
## CELONPHARMA -> CLNPHARMA
dataFrame["name"].replace({
"XTRADEBDM": "XTB",
"CELONPHARMA": "CLNPHARMA"
},
inplace=True)
apply_on_column(dataFrame, 'amount', convert_int)
apply_on_column(dataFrame, 'unit_price', convert_float)
apply_on_column(dataFrame, 'price', convert_float)
return dataFrame
def _parseDataFromFile(self, dataFile: str) -> DataFrame:
with open(dataFile, encoding="utf-8") as f:
json_data = json.load(f)
json_dict = json_data[0]
data_field = json_dict.get("data", None)
if data_field is None:
return None
# print("xxx:", data_field)
## example data
# c h l o p t
# 0 1765.37 1765.37 1765.37 1765.37 1765.37 1599462009
# 1 1768.42 1768.42 1768.42 1768.42 1768.42 1599462015
# 2 1768.49 1768.49 1768.49 1768.49 1768.49 1599462030
dataFrame = DataFrame(data_field)
# print( "xxx:\n", dataFrame )
apply_on_column(dataFrame, 't', convert_timestamp_datetime)
return dataFrame
return None
def _parseDataFromFile(self, dataFile) -> DataFrame:
# _LOGGER.debug( "opening workbook: %s", dataFile )
# fix HTML: handle multiple tbody inside single table
with open(dataFile, encoding="utf-8") as file:
soup = BeautifulSoup(file, "html.parser")
for body in soup("tbody"):
body.unwrap()
dataFrame = pandas.read_html(str(soup), flavor="bs4")
dataFrame = dataFrame[0]
dataFrame.dropna(how='all', inplace=True)
apply_on_column(dataFrame, 'Kurs AD', convert_float)
apply_on_column(dataFrame, 'Zmiana AD', convert_float)
apply_on_column(dataFrame, 'Zmianaprocentowa AD',
convert_percentage)
apply_on_column(dataFrame, 'Otwarcie AD', convert_float)
apply_on_column(dataFrame, 'Max AD', convert_float)
apply_on_column(dataFrame, 'Min AD', convert_float)
return dataFrame
def _parseDataFromFile(self, dataFile: str) -> DataFrame:
# _LOGGER.debug( "opening workbook: %s", dataFile )
allDataFrames = pandas.read_html(dataFile,
thousands='',
decimal=',',
encoding='utf-8')
dataFrame = DataFrame()
dataFrame = dataFrame.append(allDataFrames[1]) ## country
dataFrame = dataFrame.append(allDataFrames[2]) ## foreign
cleanup_column(dataFrame, 'Sektor')
apply_on_column(dataFrame, 'Liczba wyemitowanych akcji',
convert_int)
apply_on_column(dataFrame, 'Wartość rynkowa (mln zł)',
convert_float)
apply_on_column(dataFrame, 'Wartość księgowa (mln zł)',
convert_float)
apply_on_column(dataFrame, 'C/WK', convert_float)
apply_on_column(dataFrame, 'C/Z', convert_float)
apply_on_column(dataFrame, 'Stopa dywidendy (%)', convert_float)
return dataFrame
def _parseDataFromFile(self, dataFile: str) -> DataFrame:
# _LOGGER.debug( "opening workbook: %s", dataFile )
dataFrameList = pandas.read_html(dataFile,
thousands='',
decimal=',')
dataFrame = dataFrameList[0]
## flatten multi level header (column names)
dataFrame.columns = dataFrame.columns.get_level_values(0)
## drop last row containing summary
dataFrame.drop(dataFrame.tail(1).index, inplace=True)
## remove trash from column
cleanup_column(dataFrame, 'Nazwa')
apply_on_column(dataFrame, 'Kurs odn.', convert_float)
apply_on_column(dataFrame, 'Kurs otw.', convert_float)
apply_on_column(dataFrame, 'Kurs min.', convert_float)
apply_on_column(dataFrame, 'Kurs maks.', convert_float)
apply_on_column(dataFrame, 'Kurs ost. trans. / zamk.',
convert_float)
apply_on_column(dataFrame, 'Zm.do k.odn.(%)', convert_float)
try:
apply_on_column(dataFrame, 'Wol. obr. - skumul.', convert_int)
except KeyError:
_LOGGER.exception("unable to get values by key")
try:
apply_on_column(dataFrame, 'Wart. obr. - skumul.(tys.)',
convert_float)
except KeyError:
_LOGGER.exception("unable to get values by key")
append_stock_isin(dataFrame, dataFile)
return dataFrame
def convert_indexes_data(dataFrame: DataFrame):
apply_on_column(dataFrame, 'Kurs otw.', convert_float)
apply_on_column(dataFrame, 'Kurs min.', convert_float)
apply_on_column(dataFrame, 'Kurs maks.', convert_float)
apply_on_column(dataFrame, 'Wart. ost.', convert_float)
apply_on_column(dataFrame, 'Wartość obrotu skum. (w tys. zł)',
convert_float)
apply_on_column(dataFrame, 'Liczba spółek', convert_int)
apply_on_column(dataFrame, '% otw. portfela', convert_int)