def RequestAllProjectsDataAndDefineCompleteness():
global JiraWrapper
global CacheData
kwargs={}
print('Request for all projects and completeness definition started')
CurrentPath=GetProjectRootFolder()
LogsFolder=os.path.join(CurrentPath,'Logs')
if not os.path.exists(LogsFolder):
os.mkdir(LogsFolder)
if not JiraWrapper:
JiraWrapper=JIRA_Wrapper(JIRAServer,"data/"+AuthorizationDataFile)
#Run request of all projects
kwargs.clear()
kwargs.update({'ProcessID':3})#Process identifier
kwargs.update({'Loggers':Loggers(1,'Process_'+str(kwargs['ProcessID']),LogsFolder)})#Loggers object
kwargs.update({'JiraWrapper':JiraWrapper})#Function parameter
kwargs.update({'CompletedProjects':[]})#Function parameter
RequestAllProjectProcess=ProcessWrapper(ModuleName='JiraPythonClientObj_gevent',FunctionName='RequestAllProjects',**kwargs)
RequestAllProjectProcess.start()
RequestAllProjectProcess.join()
print('LastOutputData',len(CacheData['Online']['LastOutputData']))
print('NumberOfCompletedProjects',len(kwargs['CompletedProjects']))
print('CompletedProjects',kwargs['CompletedProjects'])
print('Request for all projects and completeness definition ended')
return json.dumps({'NumberOfCompletedProjects':len(kwargs['CompletedProjects']),'CompletedProjects':kwargs['CompletedProjects']})
def RunOtherTasks(IssuesList):
global JiraWrapper
global ExecutiveProcesses
kwargs={}
CurrentPath=GetProjectRootFolder()
LogsFolder=os.path.join(CurrentPath,'Logs')
if not os.path.exists(LogsFolder):
os.mkdir(LogsFolder)
#Wait for termination of previous processes tasks
for ExeProcess in ExecutiveProcesses:
if isinstance(ExeProcess, Process):
ExeProcess.join()
ExecutiveProcesses.clear()
#Run Update Projects Information process
kwargs.clear()
kwargs.update({'ProcessID':0})#Process identifier
kwargs.update({'Loggers':Loggers(1,'Process_'+str(kwargs['ProcessID']),LogsFolder)})#Loggers object
kwargs.update({'JiraWrapper':JiraWrapper})#Function parameter
kwargs.update({'IssuesList':IssuesList})#Function parameter
kwargs.update({'ProjectsXMLDict':{}})#Function parameter
ExecutiveProcesses.append(ProcessWrapper(ModuleName='Project_Info',FunctionName='UpdateProjectsInfo',**kwargs))
#Run Save Projects Issues Data process
kwargs.clear()
kwargs.update({'ProcessID':1})#Process identifier
kwargs.update({'Loggers':Loggers(1,'Process_'+str(kwargs['ProcessID']),LogsFolder)})#Loggers object
kwargs.update({'IssuesList':IssuesList})#Function parameter
ExecutiveProcesses.append(ProcessWrapper(ModuleName='JiraPythonClientObj_gevent',FunctionName='SaveIssuesForAnalysis',**kwargs))
#Start tasks processes
for ExeProcess in ExecutiveProcesses:
ExeProcess.start()
def SaveIssuesForAnalysis(IssuesList,ProcessID,Loggers):
ProcessLogger=Loggers.Loggers[0]['Logger']
ProcessLogger.info('Start saving issues for analysis')
ProjectList=[]
#Get Folder with saved worklogs (Create this folder if it does not exist)
CurrentPath=GetProjectRootFolder()
WorklogsFolder=os.path.join(CurrentPath,'Worklogs')
if not os.path.exists(WorklogsFolder):
os.mkdir(WorklogsFolder)
#Get files with saved worklogs
FilesInFolder=GetFullFileNamesInFolder(WorklogsFolder)
#Get list of projects
ProjectsInfoList=GetProjectsInfo(CurrentPath)
ProjectList.extend(ProjectsInfoList[i]['BudgetLabel'] for i in range(len(ProjectsInfoList)))
#Prepare List of dictionaries for each file: {'File':'','Content':[]}}
DataForSave=[]
for Issues in IssuesList:
for issue in Issues["issues"]:
#Get file where issue has to be written
ChangedIssueLabels=ChangeIssueLabels(issue["fields"]["labels"])
IssueFile = GetIssueFile(WorklogsFolder,FilesInFolder,ChangedIssueLabels,ProjectList)
#Prepare Dictionary for issue
DataFound = False
DataID = 0
for i in range(len(DataForSave)):
if DataForSave[i]['File'] == IssueFile:
DataFound = True
DataID = i
break
if DataFound == False:
DataForSave.append({'File':IssueFile,'Content':[]})
DataID = len(DataForSave)-1
#Append issue to the dictionary with id = DataID in list of dictionaries
DataForSave[DataID]['Content'].append(issue)
#Extend DataForSave by different issues from from file (by issues which were not in Issues parameter)
for SaveData in DataForSave:
with open(SaveData['File'],'r') as IssueJSONFile:
JSONData = IssueJSONFile.read()
if JSONData != '':
JSONData=json.loads(JSONData)
for i in range(len(JSONData)-1,-1,-1):
for issue in SaveData['Content']:
if JSONData[i]['id'] == issue['id']:
del JSONData[i]
break
SaveData['Content'].extend(JSONData)
#Save Dictionary with issues
for data in DataForSave:
with open(data['File'],'w') as IssueJSONFile:
json.dump(data['Content'],IssueJSONFile)
ProcessLogger.info('End saving issues for analysis')
def GetSavedDictionaryWithWorklogs(InputParametersdict,
DataType='Offline',
StatusNames=[]):
#Get worklogs folder
CurrentPath = GetProjectRootFolder()
WorklogsFolder = os.path.join(CurrentPath, 'Worklogs')
if not os.path.exists(WorklogsFolder):
return
#Get files with saved worklogs
FilesInFolder = GetFullFileNamesInFolder(WorklogsFolder)
#Get issues from files
for f in FilesInFolder:
CacheData[DataType]['LastOutputData'].append(
GetIssuesFromFile(f, InputParametersdict, StatusNames))
def UpdateProjectList():
ProjectListUpdated = False
ExcludedProjects = GetExcludedProjects()
CurrentPath = GetProjectRootFolder()
ProjectsDataFileFullName = GetDataFile(CurrentPath,
ProjectsDataFile,
CreateIfNotExist=True)
ExtDataFileFullName = GetDataFile(CurrentPath, ExtDataFile)
ProjectsFromExt, ExtXMLDict = GetProjectsFromFile(ExtDataFileFullName,
ExcludedProjects,
'Entry')
ProjectsFromProjectsData, ProjectsXMLDict = GetProjectsFromFile(
ProjectsDataFileFullName, ExcludedProjects, 'Project')
if 'Root' not in ProjectsXMLDict:
ProjectsXMLDict.update({'Root': {'Project': []}})
else:
if 'Project' not in ProjectsXMLDict['Root']:
ProjectsXMLDict['Root'].update({'Project': []})
for Project in ProjectsXMLDict['Root']['Project']:
ProjectDict, Updated = UpdateProjectDict(
ProjectDict=Project, Info={'BudgetLabel': Project['BudgetLabel']})
if Updated:
ProjectListUpdated = True
for ExtProject in ProjectsFromExt:
if ExtProject not in ProjectsFromProjectsData:
ProjectDict, Updated = UpdateProjectDict(
ProjectDict={}, Info={'BudgetLabel': ExtProject})
ProjectsXMLDict['Root']['Project'].append(ProjectDict)
if Updated:
ProjectListUpdated = True
if ProjectListUpdated == True:
SaveProjectsDataDictionary(ProjectsXMLDict, ProjectsDataFileFullName)
print('List of projects was updated')
def RequestAllProjects(JiraWrapper,ProcessID,Loggers,CompletedProjects):
ProcessLogger=Loggers.Loggers[0]['Logger']
ProcessLogger.info('Request of All Projects Started')
ExcludedProjects=GetExcludedProjects()
CurrentPath=GetProjectRootFolder()
ProjectsDataFileFullName=GetDataFile(CurrentPath,ProjectsDataFile)
ProjectsFromProjectsData,ProjectsXMLDict = GetProjectsFromFile(ProjectsDataFileFullName,ExcludedProjects,'Project')
Counter=0
NumberOfProjects=len(ProjectsXMLDict['Root']['Project'])
for Project in ProjectsXMLDict['Root']['Project']:
NewIssuesStartID,NewIssuesEndID=GetDictionaryWithWorklogs({'Projects':[Project['BudgetLabel']]},DataType='Online',OtherJiraWrapper=JiraWrapper)
SaveIssuesForAnalysis(CacheData['Online']['LastOutputData'][NewIssuesStartID:NewIssuesEndID+1],ProcessID,Loggers)
ProcessLogger.info('Issues of project '+str(Counter+1)+' from '+str(NumberOfProjects)+' ('+ Project['BudgetLabel']+') were saved')
#Set Project's Completion information
if CheckIssuesCompleted(CacheData['Online']['LastOutputData'][NewIssuesStartID:NewIssuesEndID+1]) == True:
ProjectDict,Updated=UpdateProjectDict(ProjectDict=Project,Info={'BudgetLabel':Project['BudgetLabel']},IssuesCompletion={'AllIssuesCompleted':'X','CompletenessCheckDate':datetime.datetime.now().strftime('%d.%m.%Y')})
CompletedProjects.append(Project['BudgetLabel'])
ProcessLogger.info('All issues of project '+str(Counter+1)+' from '+str(NumberOfProjects)+' ('+ Project['BudgetLabel']+') are completed')
else:
ProjectDict,Updated=UpdateProjectDict(ProjectDict=Project,Info={'BudgetLabel':Project['BudgetLabel']},IssuesCompletion={'AllIssuesCompleted':'','CompletenessCheckDate':datetime.datetime.now().strftime('%d.%m.%Y')})
ProcessLogger.info('Some issues of project '+str(Counter+1)+' from '+str(NumberOfProjects)+' ('+ Project['BudgetLabel']+') are not completed')
SaveProjectsDataDictionary(ProjectsXMLDict,ProjectsDataFileFullName) #Save each change in ProjectsXMLDict
#Set Project's info
ProcessLogger.info('NewIssuesStartID '+str(NewIssuesStartID))
ProcessLogger.info('NewIssuesEndID '+str(NewIssuesEndID))
ProcessLogger.info('Length of CacheData[Online][LastOutputData] '+str(len(CacheData['Online']['LastOutputData'])))
for i in range(NewIssuesStartID,NewIssuesEndID+1):
ProcessLogger.info('Length of CacheData[Online][LastOutputData]['+str(i)+'][issues] '+str(len(CacheData['Online']['LastOutputData'][i]["issues"])))
if len(CacheData['Online']['LastOutputData'][i]["issues"]) > 0:
UpdateProjectsInfo(JiraWrapper,[{"issues":[CacheData['Online']['LastOutputData'][i]["issues"][0]]}],ProcessID,Loggers,ProjectsXMLDict)
Counter=Counter+1
time.sleep(20)
#GetSavedDictionaryWithWorklogs(NewInputParametersdict,DataType='Offline',StatusNames=[])
ProcessLogger.info('Request of All Projects Ended')
def UpdateProjectsInfo(JiraWrapper,
IssuesList,
ProcessID,
Loggers,
ProjectsXMLDict={}):
ProcessLogger = Loggers.Loggers[0]['Logger']
ProcessLogger.info('Start update of projects information')
global AllProjectsDataUpdated
global AlreadyRequestedProjects
ProjectInfoDict = {}
if AllProjectsDataUpdated:
ProcessLogger.info(
'All Projects data are updated. File with projects data will not be read'
)
ProcessLogger.info('End update of projects information')
return
CurrentPath = GetProjectRootFolder()
#Get projects which information was not read
ProjectsInfoList = GetProjectsInfo(CurrentPath)
ProjectsWithoutInfo = []
for i in range(len(ProjectsInfoList) - 1, -1, -1):
for k in ProjectsInfoList[i]:
if ProjectsInfoList[i][k] == '' or ProjectsInfoList[i][k] == None:
ProjectsWithoutInfo.append(ProjectsInfoList[i].copy())
del ProjectsInfoList[i]
break
if len(ProjectsWithoutInfo) == 0:
AllProjectsDataUpdated = True
ProcessLogger.info('All Projects data, read from file, are updated')
ProcessLogger.info('End update of projects information')
return
#Update projects with received data
NumberOfProjectsWithoutInfo = len(ProjectsWithoutInfo)
for Issues in IssuesList:
for issue in Issues["issues"]:
if len(ProjectsWithoutInfo) == 0:
break
IssueProject, ProjectNumber = GetIssueProject(
issue["fields"]["labels"], ProjectsWithoutInfo)
if IssueProject != '':
ProcessLogger.info('Start reading information for project ' +
IssueProject)
AlreadyRequestedProjects.append(IssueProject)
ProjectInfoDict.clear()
ProjectInfoDict = GetProjectInfo(JiraWrapper.jira, issue,
IssueProject, Loggers)
if len(ProjectInfoDict.keys()) > 0:
ProjectsInfoList.append(ProjectInfoDict.copy())
del ProjectsWithoutInfo[ProjectNumber]
ProcessLogger.info('End reading information for project ' +
IssueProject)
#Save projects back in file only in case of at least one project data were updated
if len(ProjectsWithoutInfo) != NumberOfProjectsWithoutInfo:
#Add into ProjectsInfoList data from ProjectsWithoutInfo which were not deleted (as they have no estimates in root jira item)
for ProjectInfo in ProjectsWithoutInfo:
ProjectsInfoList.append(ProjectInfo.copy())
#Save updated projects back in file
SaveProjectsInfo(CurrentPath, ProjectsInfoList, ProjectsXMLDict)
if len(ProjectsWithoutInfo) == 0:
AllProjectsDataUpdated = True
ProcessLogger.info('All Projects data were updated in current request')
ProcessLogger.info('End update of projects information')