def processFileData2(self, action, fileDataList, importerClass):
if action == 'Reprocess':
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=importerClass)
importerExecutor.execute(fileDataList)
elif action == 'Process':
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=importerClass)
importerExecutor.execute(fileDataList)
elif action == 'Delete':
pass
# importer = importerClass(filename=fileName, replace=True)
# importer.deleteImportedObject()
# importer.addOrModifyFDPending()
'''
Created on 22 ago. 2018
@author: afunes
'''
from base.dbConnector import DBConnector
from base.initializer import Initializer
from dao.fileDataDao import FileDataDao
from dataImport.importerExecutor import ImporterExecutor
from importer.importerExpression import ImporterExpression
if __name__ == "__main__":
Initializer()
session = DBConnector().getNewSession()
fileDataList = FileDataDao().getFileData6(
statusAttr='fileName',
statusValue='edgar/data/804328/0001728949-19-000055.txt',
session=session)
importerExecutor = ImporterExecutor(threadNumber=1,
maxProcessInQueue=5,
replace=True,
isSequential=True,
importerClass=ImporterExpression)
importerExecutor.execute(fileDataList)
def doSubmitProcessStatus2(n_clicks, copyStatus, calculateStatus,
expressionStatus):
if (n_clicks > 0):
logging.info("START")
print("Start")
createLog(Constant.LOGGER_IMPORT_GENERAL, logging.DEBUG)
session = DBConnector().getNewSession()
if (expressionStatus is not None):
fileDataList = FileDataDao().getFileData3(
statusAttr=ConstantStatus.EXPRESSION_STATUS,
statusValue=expressionStatus,
statusAttr2=ConstantStatus.CALCULATE_STATUS,
statusValue2=calculateStatus,
session=session)
importerExecutor = ImporterExecutor(
threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=False,
importerClass=ImporterExpression)
importerExecutor.execute(fileDataList)
elif (calculateStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.CALCULATE_STATUS,
statusValue=calculateStatus,
statusAttr2=ConstantStatus.COPY_STATUS,
statusValue2=copyStatus,
session=session)
importerExecutor = ImporterExecutor(
threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterCalculate)
importerExecutor.execute(fileDataList)
elif (copyStatus is not None):
fileDataList = FileDataDao().getFileData3(
statusAttr=ConstantStatus.COPY_STATUS,
statusValue=copyStatus,
statusAttr2=ConstantStatus.FACT_STATUS,
statusValue2=Constant.STATUS_OK,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterCopy)
importerExecutor.execute(fileDataList)
rs = FileDataDao().getStatusCount3()
df = DataFrame(rs,
columns=[
ConstantStatus.FACT_STATUS, ConstantStatus.COPY_STATUS,
ConstantStatus.CALCULATE_STATUS,
ConstantStatus.EXPRESSION_STATUS, 'value_'
])
df_all_trees = build_hierarchical_dataframe(df, levels2, value_column)
s = go.Sunburst(labels=df_all_trees['label'],
parents=df_all_trees['parent'],
ids=df_all_trees['id'],
customdata=df_all_trees['level'],
text=df_all_trees['value'],
name="",
marker=dict(colors=df_all_trees['color'],
colorscale='RdBu'),
hovertemplate=hoverTemplate)
sunburstStatus2 = go.Figure(s, layout=layout2)
sunburstStatus2.update_layout(margin=dict(t=0, l=0, r=0, b=0))
listCopyStatus = getUniqueValues(df_all_trees, ConstantStatus.COPY_STATUS)
listCalculateStatus = getUniqueValues(df_all_trees,
ConstantStatus.CALCULATE_STATUS)
listExpressionStatus = getUniqueValues(df_all_trees,
ConstantStatus.EXPRESSION_STATUS)
return sunburstStatus2, listCopyStatus, listCalculateStatus, listExpressionStatus
def doSubmitProcessStatus1(n_clicks, fileStatus, companyStatus, listedStatus,
entityStatus, priceStatus, factStatus):
if (n_clicks > 0):
logging.info("START")
print("Start")
createLog(Constant.LOGGER_IMPORT_GENERAL, logging.DEBUG)
session = DBConnector().getNewSession()
if (factStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.PRICE_STATUS,
statusValue=priceStatus,
statusAttr2=ConstantStatus.FACT_STATUS,
statusValue2=factStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterFact)
importerExecutor.execute(fileDataList)
elif (priceStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.ENTITY_FACT_STATUS,
statusValue=entityStatus,
statusAttr2=ConstantStatus.PRICE_STATUS,
statusValue2=priceStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=False,
importerClass=ImporterPrice)
importerExecutor.execute(fileDataList)
elif (entityStatus is not None):
if (listedStatus == 'LISTED'):
listed = 1
else:
listed = 0
fileDataList = FileDataDao().getFileData3(
statusAttr=ConstantStatus.COMPANY_STATUS,
statusValue=companyStatus,
statusAttr2=ConstantStatus.ENTITY_FACT_STATUS,
statusValue2=entityStatus,
session=session,
listed=listed)
importerExecutor = ImporterExecutor(
threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterEntityFact)
importerExecutor.execute(fileDataList)
elif (companyStatus is not None):
fileDataList = FileDataDao().getFileData4(
statusAttr=ConstantStatus.FILE_STATUS,
statusValue=fileStatus,
statusAttr2=ConstantStatus.COMPANY_STATUS,
statusValue2=companyStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterCompany)
importerExecutor.execute(fileDataList)
elif (fileStatus is not None):
fileDataList = FileDataDao().getFileData6(
statusAttr=ConstantStatus.FILE_STATUS,
statusValue=fileStatus,
session=session)
importerExecutor = ImporterExecutor(threadNumber=4,
maxProcessInQueue=5,
replace=False,
isSequential=True,
importerClass=ImporterFile)
importerExecutor.execute(fileDataList)
rs = FileDataDao().getStatusCount2()
df = DataFrame(rs,
columns=[
ConstantStatus.FILE_STATUS,
ConstantStatus.COMPANY_STATUS, 'listedStatus',
ConstantStatus.ENTITY_FACT_STATUS,
ConstantStatus.PRICE_STATUS, ConstantStatus.FACT_STATUS,
'value_'
])
levels = [
ConstantStatus.FACT_STATUS, ConstantStatus.PRICE_STATUS,
ConstantStatus.ENTITY_FACT_STATUS, 'listedStatus',
ConstantStatus.COMPANY_STATUS, ConstantStatus.FILE_STATUS
]
df_all_trees = build_hierarchical_dataframe(df, levels, value_column)
sunburstImportStatus = go.Figure(go.Sunburst(
labels=df_all_trees['label'],
parents=df_all_trees['parent'],
ids=df_all_trees['id'],
customdata=df_all_trees['level'],
text=df_all_trees['value'],
name="",
marker=dict(colors=df_all_trees['color'], colorscale='RdBu'),
hovertemplate=hoverTemplate),
layout=layout2)
sunburstImportStatus.update_layout(margin=dict(t=0, l=0, r=0, b=0))
listFileStatus = getUniqueValues(df_all_trees, ConstantStatus.FILE_STATUS)
listCompanyStatus = getUniqueValues(df_all_trees,
ConstantStatus.COMPANY_STATUS)
listListedStatus = getUniqueValues(df_all_trees, 'listedStatus')
listEntityStatus = getUniqueValues(df_all_trees,
ConstantStatus.ENTITY_FACT_STATUS)
listPriceStatus = getUniqueValues(df_all_trees,
ConstantStatus.PRICE_STATUS)
listFactStatus = getUniqueValues(df_all_trees, ConstantStatus.FACT_STATUS)
return sunburstImportStatus, listFileStatus, listCompanyStatus, listListedStatus, listEntityStatus, listPriceStatus, listFactStatus