コード例 #1
0
ファイル: tree.py プロジェクト: josegutab/scipion
 def getObjectInfo(self, obj):
     objId = obj.getObjId()
     self._objDict[objId] = obj
     info = {'key': obj.getObjId(), 'text': obj.getRunName(),
             'values': (obj.getStatusMessage(), prettyDelta(obj.getElapsedTime()))}
     objPid = obj.getObjParentId()
     if objPid in self._objDict:
         info['parent'] = self._objDict[objPid]
   
     return info
コード例 #2
0
    def fileModified(self, fileName, fileTimeout):
        """ Check if the fileName modification time is less
        than a given timeout.
        Params:
            fileName: input filename that will be checked.
            fileTimeout: timeout """
        self.debug('Checking file: %s' % fileName)
        mTime = datetime.fromtimestamp(os.path.getmtime(fileName))
        delta = datetime.now() - mTime
        self.debug('   Modification time: %s' % pwutils.prettyTime(mTime))
        self.debug('   Delta: %s' % pwutils.prettyDelta(delta))

        return delta < fileTimeout
コード例 #3
0
ファイル: base.py プロジェクト: I2PC/scipion
    def fileModified(self, fileName, fileTimeout):
        """ Check if the fileName modification time is less
        than a given timeout.
        Params:
            fileName: input filename that will be checked.
            fileTimeout: timeout """
        self.debug('Checking file: %s' % fileName)
        mTime = datetime.fromtimestamp(os.path.getmtime(fileName))
        delta = datetime.now() - mTime
        self.debug('   Modification time: %s' % pwutils.prettyTime(mTime))
        self.debug('   Delta: %s' % pwutils.prettyDelta(delta))

        return delta < fileTimeout
コード例 #4
0
ファイル: tree.py プロジェクト: kevinpetersavage/scipion
    def getObjectInfo(self, obj):
        objId = obj.getObjId()
        self._objDict[objId] = obj
        info = {
            'key': obj.getObjId(),
            'text': obj.getRunName(),
            'values':
            (obj.getStatusMessage(), prettyDelta(obj.getElapsedTime()))
        }
        objPid = obj.getObjParentId()
        if objPid in self._objDict:
            info['parent'] = self._objDict[objPid]

        return info
コード例 #5
0
ファイル: images.py プロジェクト: the-best-elephant/scipion
    def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
                               amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        imgSet = self._getOutputSet() if self.isContinued() else None

        self.importedFiles = set()
        if imgSet is None:
            createSetFunc = getattr(self, '_create' + self._outputClassName)
            imgSet = createSetFunc()
        elif imgSet.getSize() > 0:  # in case of continue
            imgSet.loadAllProperties()
            self._fillImportedFiles(imgSet)
            imgSet.enableAppend()

        pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
        if pointerExcludedMovs is not None:
            excludedMovs = pointerExcludedMovs.get()
            self._fillImportedFiles(excludedMovs)

        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()
        self.fillAcquisition(acquisition)
        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)
        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()
        outputName = self._getOutputName()

        finished = False
        # this is only used when creating stacks from frame files
        self.createdStacks = set()

        i = 0
        lastDetectedChange = datetime.now()

        # Ignore the timeout variables if we are not really in streaming mode
        if self.dataStreaming:
            timeout = timedelta(seconds=self.timeout.get())
            fileTimeout = timedelta(seconds=self.fileTimeout.get())
        else:
            timeout = timedelta(seconds=5)
            fileTimeout = timedelta(seconds=5)

        while not finished:
            time.sleep(3)  # wait 3 seconds before check for new files
            someNew = False
            someAdded = False

            for fileName, uniqueFn, fileId in self.iterNewInputFiles():
                someNew = True
                if self.fileModified(fileName, fileTimeout):
                    continue

                dst = self._getExtraPath(uniqueFn)
                self.importedFiles.add(uniqueFn)
                copyOrLink(fileName, dst)

                self.debug('Importing file: %s' % fileName)
                self.debug("uniqueFn: %s" % uniqueFn)
                self.debug("dst Fn: %s" % dst)

                if self._checkStacks:
                    _, _, _, n = imgh.getDimensions(dst)

                someAdded = True
                self.debug('Appending file to DB...')
                if self.importedFiles:  # enable append after first append
                    imgSet.enableAppend()

                if n > 1:
                    for index in range(1, n + 1):
                        img.cleanObjId()
                        img.setMicId(fileId)
                        img.setFileName(dst)
                        img.setIndex(index)
                        self._addImageToSet(img, imgSet)
                else:
                    img.setObjId(fileId)
                    img.setFileName(dst)
                    # Fill the micName if img is either a Micrograph or a Movie
                    self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
                    self._fillMicName(img, uniqueFn)
                    self._addImageToSet(img, imgSet)

                outFiles.append(dst)
                self.debug('After append. Files: %d' % len(outFiles))

            if someAdded:
                self.debug('Updating output...')
                self._updateOutputSet(outputName,
                                      imgSet,
                                      state=imgSet.STREAM_OPEN)
                self.debug('Update Done.')

            self.debug('Checking if finished...someNew: %s' % someNew)

            now = datetime.now()

            if not someNew:
                # If there are no new detected files, we should check the
                # inactivity time elapsed (from last event to now) and
                # if it is greater than the defined timeout, we conclude
                # the import and close the output set
                # Another option is to check if the protocol have some
                # special stop condition, this can be used to manually stop
                # some protocols such as import movies
                finished = (now - lastDetectedChange > timeout
                            or self.streamingHasFinished())
                self.debug("Checking if finished:")
                self.debug("   Now - Last Change: %s" %
                           pwutils.prettyDelta(now - lastDetectedChange))
                self.debug("Finished: %s" % finished)
            else:
                # If we have detected some files, we should update
                # the timestamp of the last event
                lastDetectedChange = now

        self._updateOutputSet(outputName, imgSet, state=imgSet.STREAM_CLOSED)

        self._cleanUp()

        return outFiles
コード例 #6
0
    def generate(self, finished):
        reportTemplate = self.getHTMLReportText()

        if not reportTemplate:
            raise Exception("HTML template file '%s' not found. " %
                            self.template)

        project = self.protocol.getProject()
        projName = project.getShortName()
        acquisitionLines = ''
        self.provider.refreshObjects()

        for item in self.provider.acquisition:
            if not acquisitionLines == '':
                acquisitionLines += ','

            acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item

        runLines = ''
        wasProtocol = None
        for obj in self.provider.getObjects():

            # If it's a protocol
            isProtocol = True if obj.name else False

            if isProtocol:
                if runLines != '': runLines += ']},'
                runLines += '{protocolName: "%s", output:[' % obj.name
            else:
                if not wasProtocol: runLines += ','
                runLines += '{name: "%s",  size:"%s"}' % (obj.output,
                                                          obj.outSize)

            wasProtocol = isProtocol

        # End the runLines JSON object
        runLines += ']}'

        # Ctf monitor chart data
        data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()

        if data:
            numMicsDone = len(self.thumbPaths[PSD_THUMBS])
            numMics = len(data[PSD_PATH])
            numMicsToDo = numMics - numMicsDone
            self.getThumbPaths(ctfData=data,
                               thumbsDone=numMicsDone,
                               micIdSet=data['idValues'])

            if len(data['defocusU']) < 100:
                data['defocusCoverage'] = self.processDefocusValues(
                    data['defocusU'])
            else:
                data['defocusCoverage'] = self.processDefocusValues(
                    data['defocusU'][:-50])
                data['defocusCoverageLast50'] = self.processDefocusValues(
                    data['defocusU'][-50:])

            data['resolutionHistogram'] = self.getResolutionHistogram(
                data['resolution'])

        else:
            # Thumbnails for Micrograph Table
            numMicsDone = len(self.thumbPaths[MIC_THUMBS])
            self.getThumbPaths(thumbsDone=numMicsDone)
            numMics = len(self.thumbPaths[MIC_PATH])
            numMicsToDo = numMics - numMicsDone

        if numMicsToDo <= 10:
            # we have few new images, eg streaming mode, generate thumbnails now
            self.generateReportImages(firstThumbIndex=numMicsDone)
        else:
            # we have many images, generate thumbs in a separate process
            process = multiprocessing.Process(target=self.generateReportImages,
                                              args=(numMicsDone, ))
            process.start()

        # send over only thumbnails of the mics that have been fully processed
        self.thumbsReady = self.checkNewThumbsReady()
        thumbsLoading = numMics - self.thumbsReady
        for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
            if k in self.thumbPaths:
                data[k] = self.thumbPaths[k][:self.thumbsReady] + [
                    ''
                ] * thumbsLoading

        data[MIC_ID] = self.thumbPaths[MIC_ID]

        reportFinished = self.thumbsReady == numMics

        ctfData = json.dumps(data)

        # Movie gain monitor chart data
        data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData(
        )

        movieGainData = json.dumps(data)

        # system monitor chart data
        data = self.sysMonitor.getData()
        systemData = json.dumps(data)
        tnow = datetime.now()
        args = {
            'projectName':
            projName,
            'startTime':
            pwutils.dateStr(project.getCreationTime(), secs=True),
            'dateStr':
            pwutils.prettyTime(dt=tnow, secs=True),
            'projectDuration':
            pwutils.prettyDelta(tnow - project.getCreationTime()),
            'projectStatus':
            "FINISHED" if finished else "RUNNING",
            'scipionVersion':
            os.environ['SCIPION_VERSION'],
            'acquisitionLines':
            acquisitionLines,
            'runLines':
            runLines,
            'ctfData':
            ctfData,
            'movieGainData':
            movieGainData,
            'systemData':
            systemData,
            'refresh':
            self.refreshSecs
        }

        self.info("Writing report html to: %s" % abspath(self.reportPath))
        pwutils.cleanPath(self.reportPath)
        reportFile = open(self.reportPath, 'w')
        reportTemplate = reportTemplate % args
        reportFile.write(reportTemplate.encode('utf-8'))
        reportFile.close()

        if self.publishCmd:
            self.info("Publishing the report:")
            cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
            self.info(cmd)
            os.system(cmd)

        return reportFinished
コード例 #7
0
ファイル: images.py プロジェクト: I2PC/scipion
    def importImagesStreamStep(self, pattern, voltage, sphericalAberration,
                         amplitudeContrast, magnification):
        """ Copy images matching the filename pattern
        Register other parameters.
        """
        self.info("Using pattern: '%s'" % pattern)

        imgSet = self._getOutputSet() if self.isContinued() else None
        
        self.importedFiles = set()
        if imgSet is None:
            createSetFunc = getattr(self, '_create' + self._outputClassName)
            imgSet = createSetFunc()
        elif imgSet.getSize() > 0: # in case of continue
            imgSet.loadAllProperties()
            self._fillImportedFiles(imgSet)
            imgSet.enableAppend()
        
        pointerExcludedMovs = getattr(self, 'moviesToExclude', None)
        if pointerExcludedMovs is not None:
            excludedMovs = pointerExcludedMovs.get()
            self._fillImportedFiles(excludedMovs)

        imgSet.setIsPhaseFlipped(self.haveDataBeenPhaseFlipped.get())
        acquisition = imgSet.getAcquisition()
        self.fillAcquisition(acquisition)
        # Call a function that should be implemented by each subclass
        self.setSamplingRate(imgSet)
        outFiles = [imgSet.getFileName()]
        imgh = ImageHandler()
        img = imgSet.ITEM_TYPE()
        img.setAcquisition(acquisition)
        n = 1
        copyOrLink = self.getCopyOrLink()
        outputName = self._getOutputName()
        alreadyWarned = False  # Use this flag to warn only once

        finished = False
        # this is only used when creating stacks from frame files
        self.createdStacks = set()

        i = 0
        lastDetectedChange = datetime.now()

        # Ignore the timeout variables if we are not really in streaming mode
        if self.dataStreaming:
            timeout = timedelta(seconds=self.timeout.get())
            fileTimeout = timedelta(seconds=self.fileTimeout.get())
        else:
            timeout = timedelta(seconds=5)
            fileTimeout = timedelta(seconds=5)

        while not finished:
            time.sleep(3) # wait 3 seconds before check for new files
            someNew = False
            someAdded = False

            for fileName, uniqueFn, fileId in self.iterNewInputFiles():
                someNew = True
                if self.fileModified(fileName, fileTimeout):
                    continue
                
                dst = self._getExtraPath(uniqueFn)
                self.importedFiles.add(uniqueFn)
                if ' ' in dst:
                    if not alreadyWarned:
                        self.warning('Warning: your file names have white spaces!')
                        self.warning('Removing white spaces from copies/symlinks.')
                        alreadyWarned = True
                    dst = dst.replace(' ', '')
                copyOrLink(fileName, dst)

                self.debug('Importing file: %s' % fileName)
                self.debug("uniqueFn: %s" % uniqueFn)
                self.debug("dst Fn: %s" % dst)

                if self._checkStacks:
                    _, _, _, n = imgh.getDimensions(dst)

                someAdded = True
                self.debug('Appending file to DB...')
                if self.importedFiles: # enable append after first append
                    imgSet.enableAppend()

                if n > 1:
                    for index in range(1, n+1):
                        img.cleanObjId()
                        img.setMicId(fileId)
                        img.setFileName(dst)
                        img.setIndex(index)
                        self._addImageToSet(img, imgSet)
                else:
                    img.setObjId(fileId)
                    img.setFileName(dst)
                    # Fill the micName if img is either a Micrograph or a Movie
                    uniqueFn = uniqueFn.replace(' ', '')
                    self.debug("FILENAME TO fillMicName: %s" % uniqueFn)
                    self._fillMicName(img, uniqueFn)
                    self._addImageToSet(img, imgSet)

                outFiles.append(dst)
                self.debug('After append. Files: %d' % len(outFiles))

            if someAdded:
                self.debug('Updating output...')
                self._updateOutputSet(outputName, imgSet,
                                      state=imgSet.STREAM_OPEN)
                self.debug('Update Done.')

            self.debug('Checking if finished...someNew: %s' % someNew)

            now = datetime.now()

            if not someNew:
                # If there are no new detected files, we should check the
                # inactivity time elapsed (from last event to now) and
                # if it is greater than the defined timeout, we conclude
                # the import and close the output set
                # Another option is to check if the protocol have some
                # special stop condition, this can be used to manually stop
                # some protocols such as import movies
                finished = (now - lastDetectedChange > timeout or
                            self.streamingHasFinished())
                self.debug("Checking if finished:")
                self.debug("   Now - Last Change: %s"
                           % pwutils.prettyDelta(now - lastDetectedChange))
                self.debug("Finished: %s" % finished)
            else:
                # If we have detected some files, we should update
                # the timestamp of the last event
                lastDetectedChange = now

        self._updateOutputSet(outputName, imgSet,
                              state=imgSet.STREAM_CLOSED)

        self._cleanUp()

        return outFiles
コード例 #8
0
ファイル: report_html.py プロジェクト: I2PC/scipion
    def generate(self, finished):
        reportTemplate = self.getHTMLReportText()

        if not reportTemplate:
            raise Exception("HTML template file '%s' not found. "
                            % self.template)

        project = self.protocol.getProject()
        projName = project.getShortName()
        acquisitionLines = ''
        self.provider.refreshObjects()

        for item in self.provider.acquisition:
            if not acquisitionLines == '':
                acquisitionLines += ','

            acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item

        runLines = ''
        wasProtocol = None
        for obj in self.provider.getObjects():

            # If it's a protocol
            isProtocol = True if obj.name else False

            if isProtocol:
                if runLines != '': runLines += ']},'
                runLines += '{protocolName: "%s", output:[' % obj.name
            else:
                if not wasProtocol: runLines += ','
                runLines += '{name: "%s",  size:"%s"}' % (obj.output,
                                                          obj.outSize)

            wasProtocol = isProtocol

        # End the runLines JSON object
        runLines += ']}'

        # Ctf monitor chart data
        data = {} if self.ctfMonitor is None else self.ctfMonitor.getData()

        if data:
            numMicsDone = len(self.thumbPaths[PSD_THUMBS])
            numMics = len(data[PSD_PATH])
            numMicsToDo = numMics - numMicsDone
            self.getThumbPaths(ctfData=data, thumbsDone=numMicsDone, micIdSet=data['idValues'])

            if len(data['defocusU']) < 100:
                data['defocusCoverage'] = self.processDefocusValues(data['defocusU'])
            else:
                data['defocusCoverage'] = self.processDefocusValues(data['defocusU'][:-50])
                data['defocusCoverageLast50'] = self.processDefocusValues(data['defocusU'][-50:])

            data['resolutionHistogram'] = self.getResolutionHistogram(data['resolution'])

        else:
            # Thumbnails for Micrograph Table
            numMicsDone = len(self.thumbPaths[MIC_THUMBS])
            self.getThumbPaths(thumbsDone=numMicsDone)
            numMics = len(self.thumbPaths[MIC_PATH])
            numMicsToDo = numMics - numMicsDone

        if numMicsToDo <= 10:
            # we have few new images, eg streaming mode, generate thumbnails now
            self.generateReportImages(firstThumbIndex=numMicsDone)
        else:
            # we have many images, generate thumbs in a separate process
            process = multiprocessing.Process(target=self.generateReportImages,
                                              args=(numMicsDone,))
            process.start()

        # send over only thumbnails of the mics that have been fully processed
        self.thumbsReady = self.checkNewThumbsReady()
        thumbsLoading = numMics - self.thumbsReady
        for k in [MIC_THUMBS, SHIFT_THUMBS, PSD_THUMBS]:
            if k in self.thumbPaths:
                data[k] = self.thumbPaths[k][:self.thumbsReady] + ['']*thumbsLoading

        data[MIC_ID] = self.thumbPaths[MIC_ID]

        reportFinished = self.thumbsReady == numMics

        ctfData = json.dumps(data)

        # Movie gain monitor chart data
        data = [] if self.movieGainMonitor is None else self.movieGainMonitor.getData()

        movieGainData = json.dumps(data)

        # system monitor chart data
        data = self.sysMonitor.getData()
        systemData = json.dumps(data)
        tnow = datetime.now()
        args = {'projectName': projName,
                'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
                'dateStr': pwutils.prettyTime(dt=tnow, secs=True),
                'projectDuration': pwutils.prettyDelta(tnow-project.getCreationTime()),
                'projectStatus': "FINISHED" if finished else "RUNNING",
                'scipionVersion': os.environ['SCIPION_VERSION'],
                'acquisitionLines': acquisitionLines,
                'runLines': runLines,
                'ctfData': ctfData,
                'movieGainData': movieGainData,
                'systemData': systemData,
                'refresh': self.refreshSecs
                }

        self.info("Writing report html to: %s" % abspath(self.reportPath))
        pwutils.cleanPath(self.reportPath)
        reportFile = open(self.reportPath, 'w')
        reportTemplate = reportTemplate % args
        reportFile.write(reportTemplate.encode('utf-8'))
        reportFile.close()

        if self.publishCmd:
            self.info("Publishing the report:")
            cmd = self.publishCmd % {'REPORT_FOLDER': self.reportDir}
            self.info(cmd)
            os.system(cmd)

        return reportFinished
コード例 #9
0
    def generate(self, finished):
        reportTemplate = self.getHTMLReportText()

        if not reportTemplate:
            raise Exception("HTML template file '%s' not found. "
                            % self.template)

        project = self.protocol.getProject()

        reportName = 'index.html'
        projName = project.getShortName()
        reportDir = abspath(self.protocol._getExtraPath(projName))

        self.info("Creating report directory: %s" % reportDir)
        pwutils.cleanPath(reportDir)
        pwutils.makePath(reportDir)

        reportPath = join(reportDir, reportName)

        acquisitionLines = ''
        self.provider.refreshObjects()

        for item in self.provider.acquisition:
            if not acquisitionLines == '':
                acquisitionLines += ','

            acquisitionLines += '{propertyName:"%s", propertyValue:"%s"}' % item

        runLines = ''
        wasProtocol = None
        for obj in self.provider.getObjects():

            # If it's a protocol
            isProtocol = True if obj.name else False

            if isProtocol:
                if runLines != '': runLines += ']},'
                runLines += '{protocolName: "%s", output:[' % obj.name
            else:
                if not wasProtocol: runLines += ','
                runLines += '{name: "%s",  size:"%s"}' % (obj.output,
                                                          obj.outSize)

            wasProtocol = isProtocol

        # End the runLines JSON object
        runLines += ']}'

        # Ctf monitor chart data
        data = [] if self.ctfMonitor is None else self.ctfMonitor.getData()

        ctfData = json.dumps(data)

        # system monitor chart data
        data = self.sysMonitor.getData()
        systemData = json.dumps(data)

        args = {'projectName': projName,
                'startTime': pwutils.dateStr(project.getCreationTime(), secs=True),
                'dateStr': pwutils.prettyTime(secs=True),
                'projectDuration': pwutils.prettyDelta(project.getElapsedTime()),
                'projectStatus': "FINISHED" if finished else "RUNNING",
                'scipionVersion': os.environ['SCIPION_VERSION'],
                'acquisitionLines': acquisitionLines,
                'runLines': runLines,
                'ctfData': ctfData,
                'systemData': systemData,
                'refresh': '<META http-equiv="refresh" content="%s" >' % self.refreshSecs if not finished else ''
                }

        self.info("Writing report html to: %s" % abspath(reportPath))
        reportFile = open(reportPath, 'w')
        reportTemplate = reportTemplate % args
        reportFile.write(reportTemplate.encode('utf-8'))
        reportFile.close()

        if self.publishCmd:
            self.info("Publishing the report:")
            cmd = self.publishCmd % {'REPORT_FOLDER': reportDir}
            self.info(cmd)
            os.system(cmd)

        return reportPath
コード例 #10
0
    def generate(self, finished):

        project = self.protocol.getProject()

        # Project Properties Section
        # Do not delete this variables. We are using them
        # in an eval command
        self.projectName = project.getShortName()
        startTime = pwutils.dateStr(project.getCreationTime(), secs=True),
        tnow = datetime.now()
        _now = project.getCreationTime()
        dateStr = pwutils.prettyTime(dt=tnow, secs=True),
        projectDuration = pwutils.prettyDelta(tnow -
                                              project.getCreationTime()),
        projectStatus = "FINISHED" if finished else "RUNNING",
        scipionVersion = os.environ['SCIPION_VERSION'],

        # create structure with information related with the properties section
        # the primmary key is time, projectionName, section
        # if we insert two points with these three values idential
        # the second updates the first
        pointsDict = {}  # dictionary for data points
        pointsDict['measurement'] = self.projectName
        tags = {}
        tags['section'] = 'properties'
        pointsDict['tags'] = tags
        fields = {}
        firstTime = self.confParser.getint("project", "properties")

        fieldKeys = {'dateStr': 4, 'projectDuration': 3, 'projectStatus': 2}
        fieldNames = {
            'dateStr': "<b>Last Update</b>",
            'projectDuration': '<b>Duration</b>',
            'projectStatus': '<b>Status</b>'
        }
        if firstTime:
            fieldKeys['startTime'] = 5
            fieldKeys['scipionVersion'] = 1
            fieldNames['startTime'] = '<b>Start Time</b>'
            fieldNames['scipionVersion'] = '<b>Scipion Version</b>'

        for metric, delta in fieldKeys.items():
            localNow = _now + timedelta(seconds=delta)
            pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
            fields['metric'] = fieldNames[metric]
            # str is need because all values must have the same type
            fields['valueStr'] = str(eval(metric)[0])
            pointsDict['fields'] = fields
            self.client.write_points([pointsDict])
        self.confParser.set("project", "properties", "0")
        with open(self.confFileName, 'w') as confFile:
            self.confParser.write(confFile)

        # acquisition section
        self.provider.refreshObjects()
        pointsDict = {}  # dictionary for data points
        pointsDict['measurement'] = self.projectName
        tags = {}
        tags['section'] = 'acquisition'
        pointsDict['tags'] = tags

        fields = {}
        firstTime = self.confParser.getint("project", "acquisition")

        if firstTime:
            delta = 0
            for metricName, value in self.provider.acquisition:
                localNow = _now + timedelta(seconds=delta)
                pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
                fields['metric'] = "<b>" + metricName + "</b>"
                fields['valueNum'] = float(value)
                pointsDict['fields'] = fields
                self.client.write_points([pointsDict])
                delta = delta - 1
                # update first time only if some date has been read.
                # do not place this upside the loop
                self.confParser.set("project", "acquisition", "0")
                with open(self.confFileName, 'w') as confFile:
                    self.confParser.write(confFile)

        # send summary section
        pointsDict = {}  # dictionary for data points
        pointsDict['measurement'] = self.projectName
        tags = {}
        tags['section'] = 'summary'
        pointsDict['tags'] = tags
        localNow = _now
        for obj in self.provider.getObjects():
            fields = {}
            localNow = localNow + timedelta(seconds=-1)
            pointsDict['time'] = localNow.strftime('%Y-%m-%dT%H:%M:%SZ')
            # If it's a protocol
            isProtocol = True if obj.name else False

            if isProtocol:
                fields['protocolName'] = '<b>' + obj.name + '</b>'
                fields['output'] = ""
                fields['size'] = ""
            else:
                fields['protocolName'] = ""
                fields['output'] = obj.output
                fields['size'] = str(obj.outSize)
            pointsDict['fields'] = fields
            self.client.write_points([pointsDict])

        # Ctf monitor chart data
        last_id = self.confParser.getint("ctf", "lastId")
        listDictionaryCTF = {} if self.ctfMonitor is None else \
            self.ctfMonitor.getData(lastId=last_id)
        if listDictionaryCTF:
            pointsDict = {}  # dictionary for data points
            pointsDict['measurement'] = self.projectName
            tags = {}
            tags['section'] = 'ctf'
            for ctf in listDictionaryCTF:
                fields = {}
                for key in ctf:
                    if key == 'timestamp':
                        pointsDict['time'] = ctf['timestamp']
                    # id must be a tag since tw CTF may have the same timestamp
                    # and time + tags must be unique
                    elif key == 'id':
                        tags['id'] = ctf['id']
                    # elif key == 'ctfID':
                    #     fields['ctfID'] = "<b>" + str(ctf[key]) + "</b>"
                    elif key == 'shiftPlotPath':
                        temp = os.path.join(self.apacheImgDir,
                                            self.projectName,
                                            basename(ctf[key]))
                        popUpStr = """<a href = "%s" target = "_blank"> <img src="%s"  alt="%s" width="128px" height="128px"> </a>""" % \
                                   (temp, temp, basename(ctf[key]))
                        fields[key] = popUpStr
                        fields[key + 'Local'] = ctf[key]
                    elif key == 'psdPath':
                        # convert to pǹg
                        baseName = basename(ctf[key])
                        baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
                        temp = os.path.join(self.apacheImgDir,
                                            self.projectName, baseName)
                        temp = pwutils.replaceExt(temp, "png")
                        popUpStr = """<a href = "%s" target = "_blank"> <img src="%s"  alt="%s" width="128px" height="128px"> </a>""" % \
                                   (temp, temp, baseNamePng)
                        fields[key] = popUpStr
                        fields[key + 'Local'] = ctf[key]
                        fields[key + 'LocalPng'] = pwutils.replaceExt(
                            ctf[key], "png")
                    elif key == 'micPath':
                        # convert to pǹg
                        baseName = basename(ctf[key])
                        baseNamePng = pwutils.replaceBaseExt(ctf[key], "png")
                        temp = os.path.join(self.apacheImgDir,
                                            self.projectName, baseName)
                        temp = pwutils.replaceExt(temp, "png")
                        popUpStr = """<a href = "%s" target = "_blank"> <img src="%s"  alt="%s" width="128px" height="128px"> </a>""" % \
                                   (temp, temp, baseNamePng)
                        fields[key] = popUpStr
                        fields[key + 'Local'] = ctf[key]
                        fields[key + 'LocalPng'] = pwutils.replaceExt(
                            ctf[key], "png")
                    else:
                        fields[key] = ctf[key]
                # while be use to control image creation
                fields["transferImage"] = False
                pointsDict['fields'] = fields
                pointsDict['tags'] = tags
                self.client.write_points([pointsDict])
                last_id += 1
            self.confParser.set("ctf", "lastId", str(last_id))
            with open(self.confFileName, 'w') as confFile:
                self.confParser.write(confFile)

        # GAIN Section
        last_id = self.confParser.getint("gain", "lastId")
        listDictionaryGain = {} if self.movieGainMonitor is None else \
            self.movieGainMonitor.getData(lastId=last_id)
        if listDictionaryGain:
            pointsDict = {}  # dictionary for data points
            pointsDict['measurement'] = self.projectName
            tags = {}
            tags['section'] = 'gain'
            pointsDict['tags'] = tags
            counter = 1
            tnow = datetime.now()
            for gain in listDictionaryGain:
                fields = {}
                for key in gain:
                    fields[key] = gain[key]
                pointsDict['fields'] = fields
                # gain has no time information so I just
                # put current time
                localNow = tnow + timedelta(seconds=counter)
                pointsDict[
                    'time'] = localNow  # .strftime('%Y-%m-%dT%H:%M:%SZ')
                self.client.write_points([pointsDict])
                last_id += 1
            self.confParser.set("gain", "lastId", str(last_id))
            with open(self.confFileName, 'w') as confFile:
                self.confParser.write(confFile)

        # SYSTEM data
        last_id = self.confParser.getint("system", "lastId")
        listDictionarySystem = {} if self.sysMonitor is None else \
            self.sysMonitor.getData(lastId=last_id)
        if listDictionarySystem:
            pointsDict = {}  # dictionary for data points
            pointsDict['measurement'] = self.projectName
            tags = {}
            tags['section'] = 'system'
            pointsDict['tags'] = tags
            for system in listDictionarySystem:
                fields = {}
                for key in system:
                    if key == 'timestamp':
                        pointsDict['time'] = system['timestamp']
                    elif key == 'id':
                        continue
                    else:
                        fields[key] = system[key]
                pointsDict['fields'] = fields
                self.client.write_points([pointsDict])
                last_id += 1
            self.confParser.set("system", "lastId", str(last_id))
            with open(self.confFileName, 'w') as confFile:
                self.confParser.write(confFile)
        self.transferFiles()
        return last_id  # reportFinished