def solveCurrentExpression(self, CIK, ticker, session):
rs = FileDataDao().getLastFileDataByCIK(CIK=CIK, session=session)
row = rs.fetchone()
tickerObject = CompanyDao().getTicker(ticker, session)
fileData = FileDataDao().getFileData(row[0], session)
expressionDict = self.getExpressionDict2(isCurrent=True,
session=session)
cfvDict = self.getValuesForExpression(fileDataOID=fileData.OID,
isCurrent=True,
session=session)
priceValue = PricingInterface().getMarketPriceByAssetName(
tickerObject.onlinePricingSymbol, PricingInterface.TRADIER)
cfvDict['PRICE'] = {'value': priceValue, 'periodOID': None}
# print('PRICE', priceValue)
return self.solveExpression(expressionDict, fileData, cfvDict)
def executeReport(fileStatus, companyStatus, entityFactStatus, priceStatus,
factStatus, copyStatus, calculateStatus, expressionStatus):
rs2 = FileDataDao().getFileDataForReport(fileStatus=fileStatus,
companyStatus=companyStatus,
entityFactStatus=entityFactStatus,
priceStatus=priceStatus,
factStatus=factStatus,
copyStatus=copyStatus,
calculateStatus=calculateStatus,
expressionStatus=expressionStatus,
limit=100)
df2 = DataFrame(rs2)
dt2 = dt.DataTable(id='dt-fileDataReport',
columns=[{
"name": i,
"id": i,
"deletable": False
} for i in df2.columns],
data=df2.to_dict("rows"),
filter_action="native",
sort_action="native",
sort_mode="multi",
row_selectable="multi",
style_cell_conditional=[{
'textAlign': 'left'
}])
return dt2
def importeFileData(rows, selected_rows, errorKey, action, statusKey):
if (selected_rows != None and len(selected_rows) >= 1):
for selected_row in selected_rows:
errorMessage = rows[selected_row]["errorMessage"]
fileDataList = FileDataDao().getFileDataByError(errorKey=errorKey, errorMessage=errorMessage)
importerClass = ConstantStatus.getStatusDict()[statusKey]['importerClass']
FileDataEngine().processFileData2(action=action, fileDataList=fileDataList, importerClass=importerClass)
return refreshDT(statusKey=statusKey, errorKey=errorKey)
def __init__(self,
errorKey,
filename,
replace,
previousStatus=None,
actualStatus=None,
isNullPool=False):
self.initLogger()
self.errorKey = errorKey
self.filename = filename
self.fileDataDao = FileDataDao()
self.session = DBConnector(isNullPool=isNullPool).getNewSession()
self.replace = replace
self.previousStatus = previousStatus
self.actualStatus = actualStatus
self.fileData = FileDataDao.getFileData(self.filename, self.session)
if (not AbstractImporter.cacheDict):
self.initCache()
def setFileDataStatus(self, voList):
if(voList is not None):
missingObjects = self.getMissingObjects()
if(len(missingObjects) > 0):
setattr(self.fileData, self.actualStatus, Constant.STATUS_WARNING)
FileDataDao().setErrorMessage(errorMessage=str(missingObjects)[0:149], errorKey=self.errorKey, fileData=self.fileData)
else:
setattr(self.fileData, self.actualStatus, Constant.STATUS_OK)
else:
setattr(self.fileData, self.actualStatus, Constant.STATUS_OK)
Dao().addObject(objectToAdd=self.fileData, session=self.session, doCommit=True)
def doSubmit(filename, tickerRows):
if (filename != '' or (len(tickerRows) != 0 and len(tickerRows[0]) != 0)):
for row in tickerRows:
if (len(tickerRows[0]) != 0):
rs2 = FileDataDao().getFileDataList4(ticker=row["ticker"])
else:
rs2 = FileDataDao().getFileDataList3(filename=filename)
if (len(rs2) != 0):
df2 = DataFrame(rs2)
dt2 = dt.DataTable(id='dt-fileData',
columns=[{
"name": i,
"id": i,
"deletable": False
} for i in df2.columns],
data=df2.to_dict("rows"),
filter_action="native",
sort_action="native",
sort_mode="multi",
row_selectable="multi")
return dt2
def execute(self, session=None):
dbconnector = DBConnector()
statusDict = ConstantStatus().getStatusDict()
for status in statusDict:
session = dbconnector.getNewSession()
prevStatus = statusDict[status].get('prevStatus')
importerClass = statusDict[status].get('importerClass')
print('status ' + status)
if(status == ConstantStatus.ENTITY_FACT_STATUS):
fileDataList = FileDataDao().getFileData3(statusAttr=status, statusValue=Constant.STATUS_PENDING, statusAttr2=prevStatus, statusValue2=Constant.STATUS_OK, session=session, listed=True)
elif(prevStatus is not None):
fileDataList = FileDataDao().getFileData4(statusAttr=status, statusValue=Constant.STATUS_PENDING, statusAttr2=prevStatus, statusValue2=Constant.STATUS_OK, session=session)
else:
fileDataList = FileDataDao().getFileData6(statusAttr=status, statusValue=Constant.STATUS_PENDING, session=session)
self.logger.info("READY TO IMPORT " + status + " FILEDATA - count " + str(len(fileDataList)))
if (self.isSequential):
for fileData in fileDataList:
try:
fi = importerClass(filename=fileData.fileName, replace=self.replace)
fi.doImport()
except Exception as e:
self.logger.exception(e)
def doSubmitProcessStatus3(n_clicks, quarterPeriodOID, autoMassiveImporter):
buttonID = tools.getButtonID()
if (n_clicks > 0):
if (buttonID == 'btn-submit-processStatus3'):
if (quarterPeriodOID is not None):
quarterPeriod = FileDataDao().getQuarterPeriod(
quarterPeriodOID=quarterPeriodOID)
ImportFileEngine().importMasterIndexFor(period=quarterPeriod,
replaceMasterFile=True,
threadNumber=3)
if (1 in autoMassiveImporter):
ImporterMassiveExecutor().execute()
raise PreventUpdate
def doImport(self):
try:
logging.info("START")
time1 = datetime.now()
if (self.skipOrProcess()):
self.addOrModifyInit()
if (self.replace):
self.deleteImportedObject()
self.logger.debug("**********START - Processing filename " +
self.filename)
voList = self.doImport2()
persistentList = self.getPersistentList(voList)
Dao().addObjectList(persistentList, self.session)
self.setFileDataStatus(voList)
if (voList is None):
self.logger.info("***********FINISH AT " +
str(datetime.now() - time1) + " " +
self.filename)
else:
self.logger.info("***********FINISH AT " +
str(datetime.now() - time1) + " " +
self.filename + " objects added " +
str(len(voList)))
else:
self.logger.info("Skipped " + self.filename)
except (CustomException) as e:
self.logger.error(self.filename + " " + e.status + " " + str(e))
self.addOrModifyFDError1(e)
except MemoryError as e:
self.logger.error(self.filename + " " + str(e))
FileDataDao().addOrModifyFileData(
statusKey=self.actualStatus,
statusValue=Constant.STATUS_ERROR,
filename=self.filename,
errorMessage='MemoryError',
errorKey=self.errorKey)
except ExpatError as e:
self.logger.error(self.filename + " " + str(e))
self.session.rollback()
self.session.close()
self.session = DBConnector().getNewSession()
self.addOrModifyFDError2('not well-formed')
except Exception as e:
self.logger.error(self.filename + " " + str(e))
self.session.rollback()
self.session.close()
self.session = DBConnector().getNewSession()
self.addOrModifyFDError2(str(e)[0:149])
finally:
self.session.commit()
self.session.close()
def showErrors(n_clicks, rows, selected_rows):
if (n_clicks > 0):
if (selected_rows is not None and len(selected_rows) != 0):
fileName = rows[selected_rows[0]]["fileName"]
rs2 = FileDataDao().getErrorList(fileName)
if (len(rs2) != 0):
df2 = DataFrame(rs2)
dt2 = dt.DataTable(
id='dt-errorMessage',
columns=[{
"name": i,
"id": i,
"deletable": False
} for i in df2.columns],
data=df2.to_dict("rows"),
)
return dt2
def importMasterIndexFor(self,
period,
replaceMasterFile,
session=None,
threadNumber=1):
dbconnector = DBConnector()
if (session is None):
session = dbconnector.getNewSession()
localURL = Constant.CACHE_FOLDER + 'master' + str(
period.year) + "-Q" + str(period.quarter) + '.gz'
secURL = "https://www.sec.gov/Archives/edgar/full-index/" + str(
period.year) + "/QTR" + str(period.quarter) + "/master.gz"
print(localURL, secURL)
file = getBinaryFileFromCache(localURL, secURL, replaceMasterFile)
with gzip.open(BytesIO(file), 'rb') as f:
file_content = f.read()
text = file_content.decode("ISO-8859-1")
text = text[text.find("CIK", 0, len(text)):len(text)]
point1 = text.find("\n")
point2 = text.find("\n", point1 + 1)
text2 = text[0:point1] + text[point2:len(text)]
df = pandas.read_csv(StringIO(text2), sep="|")
df.set_index("CIK", inplace=True)
df.head()
print("STARTED FOR PERIOD " + str(period.year) + "-" +
str(period.quarter))
for row in df.iterrows():
CIK = row[0]
filename = row[1]["Filename"]
formType = row[1]["Form Type"]
if (formType == "10-Q" or formType == "10-K"):
fd = FileDataDao.getFileData(filename, session)
if (fd is None):
company = CompanyEngine().getOrCreateCompany(
CIK=CIK, session=session)
fd = FileData()
fd.fileName = filename
fd.company = company
Dao().addObject(objectToAdd=fd,
session=session,
doCommit=True)
print("FD Added " + filename)
print("FINISHED")
def refreshDT(statusKey, errorKey):
rs2 = FileDataDao().getErrorMessageGroup(errorKey=errorKey, statusKey=statusKey)
df2 = DataFrame(rs2)
dt2 = dt.DataTable(
id='dt-errorMessage',
columns=[
{"name": i, "id": i, "deletable": False} for i in df2.columns
],
data=df2.to_dict("rows"),
filter_action="native",
sort_action="native",
sort_mode="multi",
row_selectable="multi",
style_cell_conditional=[
{
'if': {'column_id': 'errorMessage'},
'textAlign': 'left'
}]
)
return dt2
'''
Created on 22 ago. 2018
@author: afunes
'''
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerEntityFact import ImporterEntityFact
if __name__ == "__main__":
Initializer()
session = DBConnector().getNewSession()
fileDataList = FileDataDao().getFileData6(
statusAttr='fileName',
statusValue='edgar/data/1564408/0001564590-20-017775.txt',
session=session)
#fileDataList = FileDataDao().getFileData5( statusAttr=ConstantStatus.ENTITY_FACT_STATUS, statusValue='ERROR', session=session, errorMessage2='%uncon%')
importerExecutor = ImporterExecutor(threadNumber=1,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterEntityFact)
importerExecutor.execute(fileDataList)
def getQuarterPeriodDict(self):
resultList = FileDataDao().getQuarterPeriodList()
return tools.convertListToDDDict(resultList=resultList,
indexLabel=1,
indexValue=0)
def getExpressionStatusDict(self):
resultList = FileDataDao().getStatusList(
statusAttr=ConstantStatus.EXPRESSION_STATUS)
return tools.convertListToDDDict(resultList)
def getCalculateStatusDict(self):
resultList = FileDataDao().getStatusList(
statusAttr=ConstantStatus.CALCULATE_STATUS)
return tools.convertListToDDDict(resultList)
'''
Created on 22 ago. 2018
@author: afunes
'''
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerCompany import ImporterCompany
if __name__ == "__main__":
Initializer()
session = DBConnector().getNewSession()
fileDataList = FileDataDao().getFileData6(
statusAttr='fileName',
statusValue=
'ImporterCompany:edgar/data/788784/0000788784-20-000010.txt',
session=session)
# fileDataList = FileDataDao().getLastFileData(session)
# fileDataList = FileDataDao().getFileDataByError(errorKey=Constant.ERROR_KEY_COMPANY, errorMessage='%not well-formed (invalid token)%', session=session)
importerExecutor = ImporterExecutor(threadNumber=1,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=ImporterCompany)
importerExecutor.execute(fileDataList)
'''
Created on 22 ago. 2018
@author: afunes
'''
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerExpression import ImporterExpression
if __name__ == "__main__":
Initializer()
session = DBConnector().getNewSession()
fileDataList = FileDataDao().getFileData6(
statusAttr='fileName',
statusValue='edgar/data/804328/0001728949-19-000055.txt',
session=session)
importerExecutor = ImporterExecutor(threadNumber=1,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=ImporterExpression)
importerExecutor.execute(fileDataList)
def addOrModifyFDError2(self, errorMessage):
FileDataDao().addOrModifyFileData(statusKey=self.actualStatus,
statusValue=Constant.STATUS_ERROR,
filename=self.filename,
errorMessage=errorMessage,
errorKey=self.errorKey)
class AbstractImporter(object):
cacheDict = {}
logger = None
def __init__(self,
errorKey,
filename,
replace,
previousStatus=None,
actualStatus=None,
isNullPool=False):
self.initLogger()
self.errorKey = errorKey
self.filename = filename
self.fileDataDao = FileDataDao()
self.session = DBConnector(isNullPool=isNullPool).getNewSession()
self.replace = replace
self.previousStatus = previousStatus
self.actualStatus = actualStatus
self.fileData = FileDataDao.getFileData(self.filename, self.session)
if (not AbstractImporter.cacheDict):
self.initCache()
def doImport(self):
try:
logging.info("START")
time1 = datetime.now()
if (self.skipOrProcess()):
self.addOrModifyInit()
if (self.replace):
self.deleteImportedObject()
self.logger.debug("**********START - Processing filename " +
self.filename)
voList = self.doImport2()
persistentList = self.getPersistentList(voList)
Dao().addObjectList(persistentList, self.session)
self.setFileDataStatus(voList)
if (voList is None):
self.logger.info("***********FINISH AT " +
str(datetime.now() - time1) + " " +
self.filename)
else:
self.logger.info("***********FINISH AT " +
str(datetime.now() - time1) + " " +
self.filename + " objects added " +
str(len(voList)))
else:
self.logger.info("Skipped " + self.filename)
except (CustomException) as e:
self.logger.error(self.filename + " " + e.status + " " + str(e))
self.addOrModifyFDError1(e)
except MemoryError as e:
self.logger.error(self.filename + " " + str(e))
FileDataDao().addOrModifyFileData(
statusKey=self.actualStatus,
statusValue=Constant.STATUS_ERROR,
filename=self.filename,
errorMessage='MemoryError',
errorKey=self.errorKey)
except ExpatError as e:
self.logger.error(self.filename + " " + str(e))
self.session.rollback()
self.session.close()
self.session = DBConnector().getNewSession()
self.addOrModifyFDError2('not well-formed')
except Exception as e:
self.logger.error(self.filename + " " + str(e))
self.session.rollback()
self.session.close()
self.session = DBConnector().getNewSession()
self.addOrModifyFDError2(str(e)[0:149])
finally:
self.session.commit()
self.session.close()
@abstractmethod
def setFileDataStatus(self, voList):
if (voList is not None):
missingObjects = self.getMissingObjects()
if (len(voList) == 0):
setattr(self.fileData, self.actualStatus,
Constant.STATUS_NO_DATA)
elif (len(missingObjects) > 0):
setattr(self.fileData, self.actualStatus,
Constant.STATUS_WARNING)
FileDataDao().setErrorMessage(
errorMessage=str(missingObjects)[0:149],
errorKey=self.errorKey,
fileData=self.fileData)
else:
setattr(self.fileData, self.actualStatus, Constant.STATUS_OK)
else:
setattr(self.fileData, self.actualStatus, Constant.STATUS_OK)
Dao().addObject(objectToAdd=self.fileData,
session=self.session,
doCommit=True)
@abstractmethod
def addOrModifyInit(self):
self.fileDataDao.addOrModifyFileData(statusKey=self.actualStatus,
statusValue=Constant.STATUS_INIT,
filename=self.filename,
errorKey=self.errorKey,
externalSession=self.session)
@abstractmethod
def addOrModifyFDError1(self, e):
self.fileDataDao.addOrModifyFileData(statusKey=self.actualStatus,
statusValue=e.status,
filename=self.filename,
errorMessage=e.message,
errorKey=self.errorKey,
externalSession=self.session,
extraData=e.extraData)
@abstractmethod
def addOrModifyFDError2(self, errorMessage):
FileDataDao().addOrModifyFileData(statusKey=self.actualStatus,
statusValue=Constant.STATUS_ERROR,
filename=self.filename,
errorMessage=errorMessage,
errorKey=self.errorKey)
@abstractmethod
def doImport2(self):
pass
@abstractmethod
def getMissingObjects(self):
return []
@abstractmethod
def skipOrProcess(self):
#self.previousStatus is None for fileStatus
if (self.previousStatus is None
or (getattr(self.fileData, self.previousStatus) in [
Constant.STATUS_OK, Constant.STATUS_WARNING,
Constant.STATUS_NO_DATA
])):
if (getattr(self.fileData, self.actualStatus) != Constant.STATUS_OK
or self.replace == True):
return True
else:
self.logger.info("File Data Skipped " + self.filename)
return False
else:
self.logger.info("File Data Skipped " + self.filename)
return False
@abstractmethod
def getPersistent(self, vo):
pass
@abstractmethod
def getPersistentList(self, voList):
if (voList is not None):
persistentList = []
for vo in voList:
persistentList.append(self.getPersistent(vo))
return persistentList
else:
return []
@abstractmethod
def initCache(self):
pass
@abstractmethod
def deleteImportedObject(self):
pass
@abstractmethod
def initLogger(self):
if (AbstractImporter.logger is None
or self.__class__.__name__ != AbstractImporter.logger.name):
AbstractImporter.logger = createLog(self.__class__.__name__,
logging.INFO)
def doCommit(self):
self.session.commit()
def addOrModifyFDPending(self):
setattr(self.fileData, self.actualStatus, Constant.STATUS_PENDING)
Dao().addObject(objectToAdd=self.fileData,
session=self.session,
doCommit=True)
def doSubmitProcessStatus2(n_clicks, copyStatus, calculateStatus,
expressionStatus):
if (n_clicks > 0):
logging.info("START")
print("Start")
createLog(Constant.LOGGER_IMPORT_GENERAL, logging.DEBUG)
session = DBConnector().getNewSession()
if (expressionStatus is not None):
fileDataList = FileDataDao().getFileData3(
statusAttr=ConstantStatus.EXPRESSION_STATUS,
statusValue=expressionStatus,
statusAttr2=ConstantStatus.CALCULATE_STATUS,
statusValue2=calculateStatus,
session=session)
importerExecutor = ImporterExecutor(
threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=False,
importerClass=ImporterExpression)
importerExecutor.execute(fileDataList)
elif (calculateStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.CALCULATE_STATUS,
statusValue=calculateStatus,
statusAttr2=ConstantStatus.COPY_STATUS,
statusValue2=copyStatus,
session=session)
importerExecutor = ImporterExecutor(
threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterCalculate)
importerExecutor.execute(fileDataList)
elif (copyStatus is not None):
fileDataList = FileDataDao().getFileData3(
statusAttr=ConstantStatus.COPY_STATUS,
statusValue=copyStatus,
statusAttr2=ConstantStatus.FACT_STATUS,
statusValue2=Constant.STATUS_OK,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterCopy)
importerExecutor.execute(fileDataList)
rs = FileDataDao().getStatusCount3()
df = DataFrame(rs,
columns=[
ConstantStatus.FACT_STATUS, ConstantStatus.COPY_STATUS,
ConstantStatus.CALCULATE_STATUS,
ConstantStatus.EXPRESSION_STATUS, 'value_'
])
df_all_trees = build_hierarchical_dataframe(df, levels2, value_column)
s = go.Sunburst(labels=df_all_trees['label'],
parents=df_all_trees['parent'],
ids=df_all_trees['id'],
customdata=df_all_trees['level'],
text=df_all_trees['value'],
name="",
marker=dict(colors=df_all_trees['color'],
colorscale='RdBu'),
hovertemplate=hoverTemplate)
sunburstStatus2 = go.Figure(s, layout=layout2)
sunburstStatus2.update_layout(margin=dict(t=0, l=0, r=0, b=0))
listCopyStatus = getUniqueValues(df_all_trees, ConstantStatus.COPY_STATUS)
listCalculateStatus = getUniqueValues(df_all_trees,
ConstantStatus.CALCULATE_STATUS)
listExpressionStatus = getUniqueValues(df_all_trees,
ConstantStatus.EXPRESSION_STATUS)
return sunburstStatus2, listCopyStatus, listCalculateStatus, listExpressionStatus
def doSubmitProcessStatus1(n_clicks, fileStatus, companyStatus, listedStatus,
entityStatus, priceStatus, factStatus):
if (n_clicks > 0):
logging.info("START")
print("Start")
createLog(Constant.LOGGER_IMPORT_GENERAL, logging.DEBUG)
session = DBConnector().getNewSession()
if (factStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.PRICE_STATUS,
statusValue=priceStatus,
statusAttr2=ConstantStatus.FACT_STATUS,
statusValue2=factStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterFact)
importerExecutor.execute(fileDataList)
elif (priceStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.ENTITY_FACT_STATUS,
statusValue=entityStatus,
statusAttr2=ConstantStatus.PRICE_STATUS,
statusValue2=priceStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=False,
importerClass=ImporterPrice)
importerExecutor.execute(fileDataList)
elif (entityStatus is not None):
if (listedStatus == 'LISTED'):
listed = 1
else:
listed = 0
fileDataList = FileDataDao().getFileData3(
statusAttr=ConstantStatus.COMPANY_STATUS,
statusValue=companyStatus,
statusAttr2=ConstantStatus.ENTITY_FACT_STATUS,
statusValue2=entityStatus,
session=session,
listed=listed)
importerExecutor = ImporterExecutor(
threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterEntityFact)
importerExecutor.execute(fileDataList)
elif (companyStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.FILE_STATUS,
statusValue=fileStatus,
statusAttr2=ConstantStatus.COMPANY_STATUS,
statusValue2=companyStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterCompany)
importerExecutor.execute(fileDataList)
elif (fileStatus is not None):
fileDataList = FileDataDao().getFileData6(
statusAttr=ConstantStatus.FILE_STATUS,
statusValue=fileStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterFile)
importerExecutor.execute(fileDataList)
rs = FileDataDao().getStatusCount2()
df = DataFrame(rs,
columns=[
ConstantStatus.FILE_STATUS,
ConstantStatus.COMPANY_STATUS, 'listedStatus',
ConstantStatus.ENTITY_FACT_STATUS,
ConstantStatus.PRICE_STATUS, ConstantStatus.FACT_STATUS,
'value_'
])
levels = [
ConstantStatus.FACT_STATUS, ConstantStatus.PRICE_STATUS,
ConstantStatus.ENTITY_FACT_STATUS, 'listedStatus',
ConstantStatus.COMPANY_STATUS, ConstantStatus.FILE_STATUS
]
df_all_trees = build_hierarchical_dataframe(df, levels, value_column)
sunburstImportStatus = go.Figure(go.Sunburst(
labels=df_all_trees['label'],
parents=df_all_trees['parent'],
ids=df_all_trees['id'],
customdata=df_all_trees['level'],
text=df_all_trees['value'],
name="",
marker=dict(colors=df_all_trees['color'], colorscale='RdBu'),
hovertemplate=hoverTemplate),
layout=layout2)
sunburstImportStatus.update_layout(margin=dict(t=0, l=0, r=0, b=0))
listFileStatus = getUniqueValues(df_all_trees, ConstantStatus.FILE_STATUS)
listCompanyStatus = getUniqueValues(df_all_trees,
ConstantStatus.COMPANY_STATUS)
listListedStatus = getUniqueValues(df_all_trees, 'listedStatus')
listEntityStatus = getUniqueValues(df_all_trees,
ConstantStatus.ENTITY_FACT_STATUS)
listPriceStatus = getUniqueValues(df_all_trees,
ConstantStatus.PRICE_STATUS)
listFactStatus = getUniqueValues(df_all_trees, ConstantStatus.FACT_STATUS)
return sunburstImportStatus, listFileStatus, listCompanyStatus, listListedStatus, listEntityStatus, listPriceStatus, listFactStatus
from dao.fileDataDao import FileDataDao
fdList = FileDataDao.getFileDataYearPeriodList()
prevRow = None
periodList = ["FY", "Q1", "Q2", "Q3"]
foundPeriodList = []
for row in fdList:
if (row[0] == 50863):
if (prevRow is not None):
if (prevRow[0] == row[0]
and prevRow[1] == row[1]): # compare CIK and Year
foundPeriodList.append(row[2])
else:
missingPeriodList = [
x for x in periodList
if x not in (y for y in foundPeriodList)
]
for missingPeriod in missingPeriodList:
print("PERIOD NOT FOUND: CIK: " + str(prevRow[0]) +
" YEAR: " + str(prevRow[1]) + " PERIOD " +
str(missingPeriod))
foundPeriodList = []
foundPeriodList.append(row[2])
else:
foundPeriodList.append(row[2])
prevRow = row
'''
Created on 22 ago. 2018
@author: afunes
'''
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerCalculate import ImporterCalculate
from valueobject.constantStatus import ConstantStatus
if __name__ == "__main__":
threadNumber = 1
maxProcessInQueue = 5
Initializer()
session = DBConnector().getNewSession()
fileDataList = FileDataDao().getFileData6(
statusAttr=ConstantStatus.CALCULATE_STATUS,
statusValue='INIT',
session=session)
importerExecutor = ImporterExecutor(threadNumber=1,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=ImporterCalculate)
importerExecutor.execute(fileDataList)
def getPriceStatusDict(self):
resultList = FileDataDao().getStatusList(
statusAttr=ConstantStatus.PRICE_STATUS)
return tools.convertListToDDDict(resultList)
'''
Created on 22 ago. 2018
@author: afunes
'''
import logging
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerFact import ImporterFact
from valueobject.constantStatus import ConstantStatus
if __name__ == "__main__":
#logging.basicConfig(level=logging.DEBUG)
Initializer()
session = DBConnector().getNewSession()
#statusAttr='documentFiscalYearFocus', statusValue='2018'
fileDataList = FileDataDao().getFileData6(
statusAttr=ConstantStatus.FACT_STATUS,
statusValue='PENDING',
session=session)
importerExecutor = ImporterExecutor(threadNumber=1,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=ImporterFact)
importerExecutor.execute(fileDataList)
def getEntityFactStatusDict(self):
resultList = FileDataDao().getStatusList(
statusAttr=ConstantStatus.ENTITY_FACT_STATUS)
return tools.convertListToDDDict(resultList)
import logging
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerPrice import ImporterPrice
if __name__ == "__main__":
#logging.basicConfig(level=logging.ERROR)
Initializer()
session = DBConnector().getNewSession()
fileDataList = FileDataDao().getFileData6(
statusAttr='fileName',
statusValue='edgar/data/1006269/0001558370-20-002499.txt',
session=session)
#fileDataList = FileDataDao().getFileData5( statusAttr='priceStatus', statusValue='ERROR', session=session, errorMessage2='%= 60%')
importerExecutor = ImporterExecutor(threadNumber=3,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=ImporterPrice)
importerExecutor.execute(fileDataList)
def getCopyStatusDict(self):
resultList = FileDataDao().getStatusList(
statusAttr=ConstantStatus.COPY_STATUS)
return tools.convertListToDDDict(resultList)