Пример #1
0
def updateDatasetFromContext(context, datasetName, Session):
    """

    Update a persistent dataset with values from context (name/value dictionary). The context may have
    fields such as event fields, not associated with the project handler.

    context
      A property (name/value) dictionary.

    datasetName
      String dataset identifier.

    Session
      Database session factory.

    """

    dset = Dataset.lookup(datasetName, Session)
    if dset is None:
        raise ESGQueryError("Dataset not found: %s" % datasetName)
    projectName = dset.get_project(Session)
    handler = getHandlerByName(projectName, None, Session)
    basicHeaders, eventHeaders, categories, derivedHeaders = getQueryFields(
        handler, return_list=False)
    properties = context.copy()

    # Set basic and event properties
    session = Session()
    session.add(dset)
    for key, value in properties.items():
        if key in basicHeaders:
            if key != 'id':
                if key == 'name':
                    if len(handler.parseDatasetName(value, {})) == 0:
                        warning(
                            "Dataset name: %s does not match dataset_id pattern in config file."
                            % value)
                setattr(dset, key, value)
            else:
                warning("Cannot update id field")
            del properties[key]
        elif key in eventHeaders:
            event = dset.events[-1]
            setEvent(event, key, value)
            del properties[key]

    # Set attribute headers
    handler.setContext(properties)
    handler.saveContext(datasetName, Session)

    session.commit()
    session.close()
Пример #2
0
def updateDatasetFromContext(context, datasetName, Session):
    """

    Update a persistent dataset with values from context (name/value dictionary). The context may have
    fields such as event fields, not associated with the project handler.

    context
      A property (name/value) dictionary.

    datasetName
      String dataset identifier.

    Session
      Database session factory.

    """

    dset = Dataset.lookup(datasetName, Session)
    if dset is None:
        raise ESGQueryError("Dataset not found: %s" % datasetName)
    projectName = dset.get_project(Session)
    handler = getHandlerByName(projectName, None, Session)
    basicHeaders, eventHeaders, categories, derivedHeaders = getQueryFields(
        handler, return_list=False)
    properties = context.copy()

    # Set basic and event properties
    session = Session()
    session.add(dset)
    for key, value in properties.items():
        if key in basicHeaders:
            if key != 'id':
                if key == 'name':
                    if len(handler.parseDatasetName(value, {})) == 0:
                        warning(
                            "Dataset name: %s does not match dataset_id pattern in config file."
                            % value)
                setattr(dset, key, value)
            else:
                warning("Cannot update id field")
            del properties[key]
        elif key in eventHeaders:
            event = dset.events[-1]
            setEvent(event, key, value)
            del properties[key]

    # Set attribute headers
    handler.setContext(properties)
    handler.saveContext(datasetName, Session)

    session.commit()
    session.close()
Пример #3
0
def queryDatasetMap(datasetNames, Session, extra_fields=False):
    """Query the database for a dataset map.

    datasetNames is a list of (datasetName, version) tuples.

    Returns (dataset_map, offline_map) where dataset_map is a dictionary:

       (dataset_id, version) => [(path, size), (path, size), ...]

    and offline_map is a dictionary:

       dataset_id => True | False, where True iff the corresponding dataset is offline.

    If extra_fields = True. returns (dataset_map, offline_map, extraFields) where 
    
      extrafields[(dataset_id, absolute_file_path, *field_name*)] => field_value

      where *field_name* is one of:

      - ``mod_time``

    """

    dmap = {}
    offlineMap = {}
    extraFields = {}
    for versionId in datasetNames:
        name, useVersion = parseDatasetVersionId(versionId)
        dset = Dataset.lookup(name, Session)
        session = Session()
        if dset is None:
            raise ESGQueryError("Dataset not found: %s" % name)
        session.add(dset)
        if useVersion == -1:
            useVersion = dset.getVersion()

        versionObj = dset.getVersionObj(useVersion)
        if versionObj is None:
            raise ESGPublishError(
                "Version %d of dataset %s not found, cannot republish." %
                (useVersion, dset.name))
        filelist = versionObj.getFiles()  # file versions
        dmap[(name, useVersion)] = [(file.getLocation(), ` file.getSize() `)
                                    for file in filelist]
Пример #4
0
def queryDatasetMap(datasetNames, Session, extra_fields=False):
    """Query the database for a dataset map.

    datasetNames is a list of (datasetName, version) tuples.

    Returns (dataset_map, offline_map) where dataset_map is a dictionary:

       (dataset_id, version) => [(path, size), (path, size), ...]

    and offline_map is a dictionary:

       dataset_id => True | False, where True iff the corresponding dataset is offline.

    If extra_fields = True. returns (dataset_map, offline_map, extraFields) where 
    
      extrafields[(dataset_id, absolute_file_path, *field_name*)] => field_value

      where *field_name* is one of:

      - ``mod_time``

    """

    dmap = {}
    offlineMap = {}
    extraFields = {}
    for versionId in datasetNames:
        name, useVersion = parseDatasetVersionId(versionId)
        dset = Dataset.lookup(name, Session)
        session = Session()
        if dset is None:
            raise ESGQueryError("Dataset not found: %s" % name)
        session.add(dset)
        if useVersion == -1:
            useVersion = dset.getVersion()

        versionObj = dset.getVersionObj(useVersion)
        if versionObj is None:
            raise ESGPublishError(
                "Version %d of dataset %s not found, cannot republish." %
                (useVersion, dset.name))
        filelist = versionObj.getFiles()  # file versions
        dmap[(name, useVersion)] = [(file.getLocation(), ` file.getSize() `)
                                    for file in filelist]
    def evt_selected_dataset_text(self, datasetTuple, num_tag, genFrom):
        dataset, vers = datasetTuple
        datasetVersId = generateDatasetVersionId(datasetTuple)
        list_of_dataset_tabs = self.parent.parent.top_notebook.pagenames()
        tab_name = datasetVersId.replace("_", "-")
        dset = None
        if (
            self.parent.parent.main_frame.top_page_id2[self.parent.parent.main_frame.selected_top_page][num_tag].cget(
                "relief"
            )
            == "raised"
        ):
            if tab_name in list_of_dataset_tabs:
                self.parent.parent.top_notebook.selectpage(tab_name)
            else:
                self.parent.parent.main_frame.top_page_id2[self.parent.parent.main_frame.selected_top_page][
                    num_tag
                ].configure(bg=self.keycolor3, fg=self.keycolor2)

                from_tab = self.parent.parent.top_notebook.getcurselection()
                dset = Dataset.lookup(dataset, self.Session)
                if genFrom:
                    handler = self.parent.parent.handlerDictionary[dataset]
                else:
                    handler = getHandlerFromDataset(dset, self.Session)

                pub_editorviewer = self.parent.parent.create_publisher_editor_viewer(
                    self.parent.parent, tab_name, dataset, from_tab, self.Session
                )
                pub_editorviewer.dataset_page(dataset=dset, Session=self.Session, handler=handler)
                self.parent.parent.top_notebook.selectpage(tab_name)

                # Disable the "Data Publication" button
                self.parent.parent.pub_buttonexpansion.control_frame3.pack_forget()
                self.parent.parent.pub_buttonexpansion.control_toggle3 = 0
                self.parent.parent.pub_buttonexpansion.ControlButton3.configure(
                    image=self.parent.parent.pub_buttonexpansion.right, state="disabled"
                )
    def start_harvest(self, parent):
        from esgcet.publish import publishDatasetList
        from esgcet.model import Dataset, PUBLISH_FAILED_EVENT, ERROR_LEVEL

        dcolor1 = Pmw.Color.changebrightness(self.parent.parent, 'aliceblue',
                                             0.8)

        # Make sure the publisher is logged in
        # if not self.parent.parent.password_flg:
        #    self.parent.parent.menu.login_menu.evt_login( self.parent.parent )

        # Start the busy routine to indicate to the users something is happening
        self.parent.parent.busyCursor = 'watch'
        self.parent.parent.busyWidgets = [
            self.parent.parent.pane2.pane('EditPaneTop'),
            self.parent.parent.pane2.pane('EditPaneBottom'),
            self.parent.parent.pane2.pane('EditPaneStatus'),
            self.parent.parent.pane.pane('ControlPane')
        ]
        pub_busy.busyStart(self.parent.parent)
        try:
            # Generate the list of datasets to be published
            datasetNames = []
            GUI_line = {}
            tab_name = self.parent.parent.top_notebook.getcurselection()
            selected_page = self.parent.parent.main_frame.selected_top_page

            if (selected_page is None):
                warning(
                    "Must generate a list of datasets to scan before publishing can occur."
                )
                pub_busy.busyEnd(self.parent.parent)
                return

            for x in self.parent.parent.main_frame.top_page_id[selected_page]:

                if self.parent.parent.main_frame.top_page_id[selected_page][x].cget(
                        'bg'
                ) != 'salmon' and self.parent.parent.main_frame.top_page_id2[
                        selected_page][x].cget('bg') != 'salmon':
                    dset_name = self.parent.parent.main_frame.top_page_id2[
                        selected_page][x].cget('text')
                    #######################################
                    # ganz added this 1/18/11
                    versionNum = self.parent.parent.main_frame.version_label[
                        selected_page][x].cget('text')
                    dsetTuple = (dset_name, versionNum)
                    #dsetName = generateDatasetVersionId(dsetTuple)
                    #####################################################################################
                    # dsetTuple = parseDatasetVersionId(dset_name) # ganz no longer necessary
                    datasetNames.append(dsetTuple)
                    GUI_line[dset_name] = x
                else:
                    if self.parent.parent.main_frame.top_page_id2[
                            selected_page][x].cget('bg') == 'salmon':
                        self.parent.parent.main_frame.top_page_id[
                            selected_page][x].configure(relief='raised',
                                                        background='salmon',
                                                        image=self.off)

        # Publish collection of datasets
            testProgress = (self.parent.parent.statusbar.show, 0, 100)
            publishThredds = (quality_control_widgets.get_CheckBox3() == 1)
            publishGateway = (quality_control_widgets.get_CheckBox2() == 1)
            if (publishThredds):
                print 'publishing to Thredds'
            if (publishGateway):
                print 'publishing to Gateway'

            status_dict = publishDatasetList(datasetNames,
                                             self.Session,
                                             publish=publishGateway,
                                             thredds=publishThredds,
                                             progressCallback=testProgress)

            # Show the published status
            for x in status_dict.keys():
                status = status_dict[x]
                dsetName, versionNo = x
                dsetVersionName = generateDatasetVersionId(x)
                guiLine = GUI_line[dsetName]  # dsetVersionName]

                self.parent.parent.main_frame.status_label[selected_page][
                    guiLine].configure(
                        text=pub_controls.return_status_text(status))
                dset = Dataset.lookup(dsetName, self.Session)
                if dset.has_warnings(self.Session):
                    warningLevel = dset.get_max_warning_level(self.Session)
                    if warningLevel >= ERROR_LEVEL:
                        buttonColor = "pink"
                        buttonText = "Error"
                    else:
                        buttonColor = "yellow"
                        buttonText = "Warning"
                    self.parent.parent.main_frame.ok_err[selected_page][
                        guiLine].configure(
                            text=buttonText,
                            bg=buttonColor,
                            relief='raised',
                            command=pub_controls.Command(
                                self.parent.parent.pub_buttonexpansion.
                                extraction_widgets.error_extraction_button,
                                dset))
                else:
                    self.parent.parent.main_frame.ok_err[selected_page][
                        guiLine].configure(
                            text='Ok',
                            bg=dcolor1,
                            highlightcolor=dcolor1,
                            relief='sunken',
                        )
        except:
            pub_busy.busyEnd(
                self.parent.parent
            )  # catch here in order to turn off the busy cursor ganz
            raise
        finally:
            pub_busy.busyEnd(self.parent.parent)
            self.my_refresh()
    def error_extraction_button( self, dset ):
       from pkg_resources import resource_filename
       remove_gif = resource_filename('esgcet.ui', 'remove.gif')
       
       
       dset_name = dset.get_name(self.Session)
       
       print dset_name
       
       tab_name = dset_name.replace('_', '-')
       list_of_tabs = self.parent.parent.bottom_notebook.pagenames( )
       tabFont = tkFont.Font(self.parent.parent, family = pub_controls.tab_font_type, size=pub_controls.tab_font_size)

       dset = Dataset.lookup(dset_name, self.Session)
       if dset.has_warnings(self.Session):
          warningLevel = dset.get_max_warning_level(self.Session)
          if warningLevel>=ERROR_LEVEL:
             tab_color = "pink"
          else:
             tab_color = "yellow"
       else:
           tab_color = "lightgreen"

       if tab_name in list_of_tabs:
          self.parent.parent.bottom_notebook.selectpage( tab_name )
       else:
          dset_error_page = self.parent.parent.bottom_notebook.add( tab_name, tab_bg=tab_color, tab_font = tabFont )

          # Generate the group with the remove page icon
          group_error_page = Pmw.Group( dset_error_page,
                tag_pyclass = MyButton,
                tag_text='Remove',
                tagindent = 0,
                tag_command = pub_controls.Command( self.evt_remove_error_page, tab_name )
          )
          group_error_page.pack(fill = 'both', expand = 1, padx = 0, pady = 0)
          self.parent.parent.balloon.bind(group_error_page, "Use the button in the upper left to remove the page.")

          # Create the ScrolledFrame.
          dset_output_window = Pmw.ScrolledText(group_error_page.interior(),
                  text_state='disabled',
                  text_background=tab_color,
                  text_relief = 'sunken',
                  text_borderwidth = 2,
          )
          dset_output_window.pack(side = 'top', expand=1, fill='both' , pady = 2)

          dset_warnings = dset.get_warnings(self.Session)
          for x in dset_warnings:
              dset_output_window.appendtext( x+"\n" )


          #dset_output_window.appendtext( "This is a test from ganzberger \n" )
          self.file_display_button( dset, dset_output_window )
          
          self.parent.parent.bottom_notebook.selectpage( tab_name )

          # Button remove
          tab = self.parent.parent.bottom_notebook.tab(tab_name)
          remove_icon = Tkinter.PhotoImage(file=remove_gif)
          b = Tkinter.Button(tab,image=remove_icon,command=pub_controls.Command(self.evt_remove_error_page, tab_name) )
          b.pack(side='right')
          b.image = remove_icon
    def return_content2(self, appendOpt=False):
        from esgcet.publish import iterateOverDatasets, processIterator
        from esgcet.config import getHandlerByName
        from esgcet.model import eventName
        from esgcet.config import loadConfig

        # Initialize parameters for interating over datasets
        initcontext = {}
        aggregateOnly = False
        # appendOpt = False
        initcontext = {}
        properties = {}
        publish = False
        publishOnly = False
        thredds = False
        testProgress1 = [self.parent.parent.statusbar.show, 0, 50]
        testProgress2 = [self.parent.parent.statusbar.show, 50, 100]
        handlerDictionary = {}

        # Get the currently selected tab and the selected datasets
        tab_name = self.parent.parent.top_notebook.getcurselection()
        selected_page = self.parent.parent.main_frame.selected_top_page
        datasetNames = []
       # datasetNames2 = []
        if (selected_page is None):
           warning("Must generate a list of datasets to scan before data extraction can occur.")
           return

        if (selected_page is not None) or (self.parent.parent.hold_offline[selected_page] == True):
           extraFields = None 
           if (self.parent.parent.hold_offline[selected_page] == False) or (isinstance(self.parent.parent.hold_offline[selected_page], types.DictType)):
              for x in self.parent.parent.main_frame.top_page_id[selected_page]:
                dsetVersionName = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text') # GANZ TODO version_label
                
                   # ganz added this 1/21/11
                if (self.parent.parent.main_frame.version_label[selected_page] ):
                    dset_name = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text')               
                    dsetVersion = self.parent.parent.main_frame.version_label[selected_page][x].cget('text')                 
                  #####################################################################################               
                else:
                    dset_name, dsetVersion = parseDatasetVersionId(dsetVersionName)

                # Retrieve all the datasets in the collection for display
                """ ganz test code
                status = pollDatasetPublicationStatus(dset_name, self.Session)
                status_text = pub_controls.return_status_text( status )
                if status_text != 'Error':
                   dsetTuple = parseDatasetVersionId(self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text'))
                   datasetNames2.append(dsetTuple)
                """
                # Retrieve only the datasets that have been selected
                if self.parent.parent.main_frame.top_page_id[selected_page][x].cget('bg') != 'salmon':
                   dsetTuple =  parseDatasetVersionId(self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text'))
                   datasetNames.append(dsetTuple)

              dmap = self.parent.parent.main_frame.dmap[selected_page]
              extraFields = self.parent.parent.main_frame.extraFields[selected_page]
              datasetMapfile = self.parent.parent.main_frame.datasetMapfile[selected_page]
              projectName = self.parent.parent.main_frame.projectName[selected_page]
              directoryMap = self.parent.parent.directoryMap[selected_page]

              if dmap is not None:
                 for x in datasetNames:
                    dsetId = x[0] 
                    datasetName = x
                    try:
                        dmapentry = dmap[datasetName]
                    except:

                        # Check if the dataset map key was changed from (dsetname,-1) to (dsetname,version).
                        # If so, replace the entry with the new key.
                        trykey = (datasetName[0], -1)
                        dmapentry = dmap[trykey]
                        del dmap[trykey]
                        dmap[datasetName] = dmapentry
                    firstFile = dmapentry[0][0]
  
                    self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, firstFile, self.Session)
                    handler = self.parent.parent.handlerDictionary[dsetId]
                 # Copy the defaultGlobalValues into initcontext
                 initcontext = self.parent.parent.main_frame.defaultGlobalValues[selected_page]
              else:
                  # more test code
                 myholdDirectoryMap = self.parent.parent.directoryMap[selected_page] 
                 #mydatasetNames = [(item,-1) for item in myholdDirectoryMap.keys()]
                 mydatasetNames = [(item) for item in myholdDirectoryMap.keys()]
                 #end
                 for x in mydatasetNames:
                    dsetId = x[0] 
                    datasetName = x
                    # ganz this is test code
                    try:
                        dmapentry = myholdDirectoryMap[datasetName]
                    except:

                        # Check if the dataset map key was changed from (dsetname,-1) to (dsetname,version).
                        # If so, replace the entry with the new key.
                        
                        trykey = (datasetName[0], -1)
                        dmapentry = myholdDirectoryMap[trykey]
                        del myholdDirectoryMap[trykey]
                        myholdDirectoryMap[datasetName] = dmapentry
                        
                    firstFile = dmapentry[0][1]
                    #end of test code
                    
                    #firstFile = self.parent.parent.main_frame.dirp_firstfile[selected_page]
 
                    self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, firstFile, self.Session)
                    handler = self.parent.parent.handlerDictionary[dsetId]
           else:      # working off-line
              projectName = self.parent.parent.main_frame.projectName[selected_page]
              if self.parent.parent.offline_file_directory[selected_page] == "directory":
                 if self.parent.parent.config is None:
                    extraction_controls.call_sessionmaker( self.parent.parent )
                 datasetPaths = []
                 dmap = {self.parent.parent.offline_datasetName : datasetPaths}
                 listerSection = getOfflineLister(self.parent.parent.config, "project:%s"%projectName, None)
                 offlineLister = self.parent.parent.config.get(listerSection, 'offline_lister_executable')
                 lastargs = self.parent.parent.offline_directories
                 commandArgs = "--config-section %s "%listerSection
                 commandArgs += " ".join(lastargs)
                 for filepath, size in processIterator(offlineLister, commandArgs, filefilt=self.parent.parent.filefilt):
                   datasetPaths.append((filepath, str(size)))
                 datasetNames = self.parent.parent.datasetNames
                 directoryMap = None

                 # get the handler
                 for x in datasetNames:
                    dsetId = x[0] 
                    self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, None, self.Session, offline=True)

              elif self.parent.parent.offline_file_directory[selected_page] == "file":
                 dmap = self.parent.parent.main_frame.dmap[selected_page]
                 extraFields = self.parent.parent.main_frame.extraFields[selected_page]
                 datasetMapfile = self.parent.parent.main_frame.datasetMapfile[selected_page]
                 projectName = self.parent.parent.main_frame.projectName[selected_page]
                 directoryMap = None
                 if datasetMapfile is not None:
                     dmap, extraFields = readDatasetMap(datasetMapfile, parse_extra_fields=True)
                     datasetNames = dmap.keys()

                 # get the handlers
                 for x in datasetNames:
                    dsetId = x[0] 
                    self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, None, self.Session, offline=True)


           # Iterate over datasets
           if appendOpt:
               operation = UPDATE_OP
           else:
               operation = CREATE_OP
        
           datasets = iterateOverDatasets(projectName, dmap, directoryMap, datasetNames, self.Session, self.parent.parent.aggregateDimension, operation, self.parent.parent.filefilt, initcontext, self.parent.parent.hold_offline[selected_page], properties, comment=self.comments, testProgress1=testProgress1, testProgress2=testProgress2 , handlerDictionary=self.parent.parent.handlerDictionary, extraFields=extraFields, readFiles=True)

           # If working on-line then replace the scanned list of datasets with 
           # the complete list of datasets
           #test
           """
           print 'datasetNames:'
           for t1 in datasetNames:
               print t1
           print 'datasetNames2:'    
           for t2 in datasetNames2:
               print t2
           """   
           if not self.parent.parent.hold_offline[selected_page]:
              datasets = []
              versionObjs = []
              # ganz finally, tested datasetNames2 here
              for dsetName, version in datasetNames:
                  result = Dataset.lookup(dsetName, self.Session, version=version)
                  if result is not None:
                      entry, versionObj = result
                      datasets.append(entry)
                      versionObjs.append(versionObj)

           # Get the summary of errors after doing a data extraction
           dset_error = []
           for dset in datasets:
               status = dset.get_publication_status(self.Session)
               status_name = eventName[status]
               if dset.has_warnings(self.Session):
                   dset_error.append(dset.get_name(self.Session))

           try:
              list_fields = getQueryFields( handler )
           except:
              handler = getHandlerByName(projectName, None, self.Session)
              list_fields = getQueryFields( handler )

           # Display the datasets in the "Collection" page
#           if self.parent.parent.hold_offline[selected_page] == True:
#              tab_name = "Collection_Offline"
#              from_tab = "Collection"
#              pub_editorviewer = self.parent.parent.create_publisher_editor_viewer( self.parent.parent, tab_name, dataset, from_tab, self.Session)

           # Show the extracted datasets
           self.set_column_labels( len(datasets), list_fields )
           self.show_extracted_info(datasets, dset_error, list_fields, versionObjs)

        # Enable the "Data Publication" button
        self.parent.ControlButton3.configure( state = 'normal' )
    def addQueryPageRow(self, num_tag, query_result, list_fields):

        # set the color for each item in the row
        dcolor1 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.8)
        dcolor2 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.7)
        dcolor3 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.5)
        dcolor4 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.6)
        dcolor5 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.7)
        dcolor6 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.8)
        dcolor7 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.9)

        frame_ct = self.parent.parent.top_ct
        labelFont = tkFont.Font(
            self.parent.parent, family=pub_controls.collection_font_type, size=pub_controls.collection_font_size
        )
        self.add_row_frame[num_tag] = self.pageFrame[frame_ct].interior()

        self.select_button[num_tag] = Tkinter.Button(
            self.add_row_frame[num_tag],
            #                compound = Tkinter.LEFT,
            #                justify = Tkinter.LEFT,
            #                anchor = Tkinter.W,
            text=num_tag,
            #                font = labelFont,
            image=self.on,
            bg=self.keycolor1,
            relief="raised",
            #                width = 70,
            height=1,
            command=pub_controls.Command(self.evt_selected_dataset, num_tag),
        )
        self.select_button[num_tag].image = self.on  # save the image from garbage collection
        self.select_button[num_tag].grid(row=self.row, column=0, sticky="nsew")

        dset = Dataset.lookup(query_result[1], self.Session)
        if dset.has_warnings(self.Session):
            warningLevel = dset.get_max_warning_level(self.Session)
            if warningLevel >= ERROR_LEVEL:
                buttonColor = "pink"
                buttonText = "Error"
            else:
                buttonColor = "yellow"
                buttonText = "Warning"
            self.ok_err[num_tag] = Tkinter.Button(
                self.add_row_frame[num_tag],
                text=buttonText,
                bg=buttonColor,
                width=4,
                relief="raised",
                command=pub_controls.Command(
                    self.parent.parent.pub_buttonexpansion.extraction_widgets.error_extraction_button, dset
                ),
            )
            self.ok_err[num_tag].grid(row=self.row, column=1, sticky="nsew")
        else:
            self.ok_err[num_tag] = Tkinter.Button(
                self.add_row_frame[num_tag], text="Ok", bg=dcolor1, highlightcolor=dcolor1, width=4, relief="sunken"
            )
            self.ok_err[num_tag].grid(row=self.row, column=1, sticky="nsew")

        status = pollDatasetPublicationStatus(query_result[1], self.Session)
        status_text = pub_controls.return_status_text(status)
        self.status[num_tag] = Tkinter.Label(
            self.add_row_frame[num_tag], text=status_text, bg=dcolor1, highlightcolor=dcolor1, width=10, relief="sunken"
        )
        self.status[num_tag].grid(row=self.row, column=2, sticky="nsew")

        if "id" in list_fields:
            id = Tkinter.Label(self.add_row_frame[num_tag], text=query_result[0], bg=dcolor2, width=6, relief="sunken")
            id.grid(row=self.row, column=3, sticky="nsew")

        dsetName = query_result[1]
        versionNum = -1
        try:
            versionIndex = list_fields.index("version")
            versionNum = string.atoi(query_result[versionIndex])
        except:
            pass

        dsetTuple = (dsetName, versionNum)
        dsetName = generateDatasetVersionId(dsetTuple)
        # ganz adding rows here...need to add versions
        datasetName = dsetTuple[0]  # dataset[0]  #parseDatasetVersionId(dataset)
        versionno = dsetTuple[1]  # dataset[1]

        ver_1 = versionno
        if ver_1 == -1:
            ver_1 = "N/A"

        self.select_labelV[num_tag] = Tkinter.Label(
            self.add_row_frame[num_tag], text=ver_1, bg=dcolor2, width=11, relief="sunken"
        )
        self.select_labelV[num_tag].grid(row=self.row, column=4, sticky="nsew")

        self.select_label[num_tag] = Tkinter.Button(
            self.add_row_frame[num_tag],
            text=datasetName,  # dsetName,
            font=labelFont,
            bg=self.keycolor2,
            relief="raised",
            borderwidth=2,
            anchor="w",
            justify="left",
            width=71,
            command=pub_controls.Command(self.evt_selected_dataset_text, dsetTuple, num_tag, 0),
        )
        self.select_label[num_tag].grid(row=self.row, column=5, columnspan=2, sticky="nsew")

        if "project" in list_fields:
            project = Tkinter.Label(
                self.add_row_frame[num_tag], text=query_result[2], bg=dcolor3, width=20, relief="sunken", borderwidth=2
            )
            project.grid(row=self.row, column=7, sticky="nsew")

        if "model" in list_fields:
            model = Tkinter.Label(
                self.add_row_frame[num_tag], text=query_result[3], bg=dcolor4, width=20, relief="sunken", borderwidth=2
            )
            model.grid(row=self.row, column=8, sticky="nsew")

        if "experiment" in list_fields:
            experiment = Tkinter.Label(
                self.add_row_frame[num_tag], text=query_result[4], bg=dcolor5, width=20, relief="sunken", borderwidth=2
            )
            experiment.grid(row=self.row, column=9, sticky="nsew")

        if "run_name" in list_fields:
            run_name = Tkinter.Label(
                self.add_row_frame[num_tag], text=query_result[5], bg=dcolor6, width=20, relief="sunken", borderwidth=2
            )
            run_name.grid(row=self.row, column=10, sticky="nsew")

        self.add_row_frame[num_tag].grid_rowconfigure(self.row, weight=1)
        self.add_row_frame[num_tag].grid_columnconfigure(1, weight=1)
        if (
            self.pageFrame[frame_ct].cget("horizflex") == "expand"
            or self.pageFrame[frame_ct].cget("vertflex") == "expand"
        ):
            self.pageFrame[frame_ct].reposition()

        self.row = self.row + 1  # increment to the next dataset row
Пример #10
0
    def evt_refresh_list_of_datasets(self, selected_page):

        # Start the busy routine to indicate to the users something is happening
        self.parent.parent.busyCursor = "watch"
        self.parent.parent.busyWidgets = [
            self.parent.parent.pane2.pane("EditPaneTop"),
            self.parent.parent.pane2.pane("EditPaneBottom"),
            self.parent.parent.pane2.pane("EditPaneStatus"),
            self.parent.parent.pane.pane("ControlPane"),
        ]
        pub_busy.busyStart(self.parent.parent)

        try:
            if self.parent.parent.refreshButton[selected_page].cget("relief") == "raised":
                for x in self.parent.parent.main_frame.top_page_id[selected_page]:
                    if self.parent.parent.main_frame.top_page_id[selected_page][x].cget("relief") == "raised":
                        dsetVersionName = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget("text")

                        # ganz added this 1/18/11
                        query_name = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget("text")
                        versionNum = self.parent.parent.main_frame.version_label[selected_page][x].cget("text")
                        #####################################################################################

                        query_name, versionNum = parseDatasetVersionId(dsetVersionName)
                        # ganz TODO test only remove
                        #                  print query_name
                        #                  print versionNum

                        status = pollDatasetPublicationStatus(query_name, self.Session)
                        # ganz catch non selected Red entries to skip 3/20/2011
                        try:
                            self.parent.parent.main_frame.status_label[selected_page][x].configure(
                                text=pub_controls.return_status_text(status)
                            )
                        except:
                            continue

                        # Make sure you update the Ok/Err button
                        # ganz added this (1/18/11) here to catch the case when dset=None (e.g. no local db entry exists)
                        dset = Dataset.lookup(query_name, self.Session)
                        if dset == None:
                            buttonColor = "yellow"
                            buttonText = "Warning"
                            self.parent.parent.main_frame.ok_err[selected_page][x].configure(
                                bg=buttonColor, text=buttonText
                            )
                        elif dset.has_warnings(self.Session):
                            warningLevel = dset.get_max_warning_level(self.Session)
                            if warningLevel >= ERROR_LEVEL:
                                buttonColor = "pink"
                                buttonText = "Error"
                            else:
                                buttonColor = "yellow"
                                buttonText = "Warning"
                            self.parent.parent.main_frame.ok_err[selected_page][x].configure(
                                bg=buttonColor, text=buttonText
                            )
        except:
            pub_busy.busyEnd(self.parent.parent)  # catch here in order to turn off the busy cursor ganz
            raise
        finally:
            pub_busy.busyEnd(self.parent.parent)
        #  pub_busy.busyEnd( self.parent.parent )
        info("Completed refreshing the display.")
Пример #11
0
def main(argv):

    try:
        args, lastargs = getopt.getopt(argv, "a:cdehi:m:p:ru", ['append', 'create', 'dataset=', 'delete-files', 'echo-sql', 'experiment=', 'filter=', 'help', 'keep-version', 'log=', 'map=', 'message=', 'model=', 'offline',  'parent=', 'per-time', 'per-variable', 'project=', 'property=', 'publish', 'new-version=', 'no-thredds-reinit', 'noscan', 'read-directories', 'read-files', 'rename-files', 'replace', 'replica=', 'rest-api', 'service=', 'set-replica', 'summarize-errors', 'thredds', 'thredds-reinit', 'update', 'use-existing=', 'use-list=', 'validate=', 'version-list=', 'nodbwrite'])
    except getopt.error:
        print sys.exc_value
        return

    aggregateDimension = "time"
    datasetMapfile = None
    datasetName = None
    echoSql = False
    filefilt = '.*\.nc$'
    init_file = None
    initcontext = {}
    keepVersion = False
    las = False
    log_filename = None
    masterGateway = None
    message = None
    offline = False
    parent = None
    perVariable = None
    projectName = None
    properties = {}
    publish = False
    publishOnly = False
    publishOp = CREATE_OP
    readFiles = False
    rescan = False
    rescanDatasetName = []
    restApi = None
    schema = None
    service = None
    summarizeErrors = False
    testProgress1 = testProgress2 = None
    thredds = False
    threddsReinit = None
    version = None
    versionList = None
    nodbwrite = False

    for flag, arg in args:
        if flag=='-a':
            aggregateDimension = arg
        elif flag=='--append':
            publishOp = UPDATE_OP
        elif flag in ['-c', '--create']:
            publishOp = CREATE_OP
        elif flag=='--dataset':
            datasetName = arg
        elif flag in ['-d', '--delete-files']:
            publishOp = DELETE_OP
        elif flag=='--echo-sql':
            echoSql = True
        elif flag=='--experiment':
            initcontext['experiment'] = arg
        elif flag=='--filter':
            filefilt = arg
        elif flag in ['-h', '--help']:
            print usage
            sys.exit(0)
        elif flag=='-i':
            init_file = arg
        elif flag=='--keep-version':
            keepVersion = True
        elif flag=='--log':
            log_filename = arg
        elif flag=='--map':
            datasetMapfile = arg
        elif flag in ['-m', '--message']:
            message = arg
        elif flag=='--model':
            initcontext['model'] = arg
        elif flag=='--nodbwrite':
            nodbwrite = True
        elif flag=='--new-version':
            try:
                version = string.atoi(arg)
                if version <=0:
                    raise ValueError
            except ValueError:
                raise ESGPublishError("Version number must be a positive integer: %s"%arg)
        elif flag=='--no-thredds-reinit':
            threddsReinit = False
        elif flag=='--noscan':
            publishOnly = True
        elif flag=='--offline':
            offline = True
        elif flag=='--parent':
            parent = arg
        elif flag=='--per-time':
            perVariable = False
        elif flag=='--per-variable':
            perVariable = True
        elif flag=='--project':
            projectName = arg
        elif flag in ['-p', '--property']:
            name, value = arg.split('=')
            properties[name] = value
        elif flag=='--publish':
            publish = True
        elif flag in ['-e', '--read-directories']:
            readFiles = False
        elif flag=='--read-files':
            readFiles = True
        elif flag=='--rename-files':
            publishOp = RENAME_OP
        elif flag in ['-r', '--replace']:
            publishOp = REPLACE_OP
        elif flag=='--replica':
            masterGateway = arg
            warning("The --replica option is deprecated. Use --set-replica instead")
        elif flag=='--rest-api':
            restApi = True
        elif flag=='--service':
            service = arg
        elif flag=='--set-replica':
            masterGateway = 'DEFAULT'
        elif flag=='--summarize-errors':
            summarizeErrors = True
        elif flag=='--thredds':
            thredds = True
        elif flag=='--thredds-reinit':
            threddsReinit = True
        elif flag in ['-u', '--update']:
            publishOp = UPDATE_OP
        elif flag=='--use-existing':
            rescan = True
            rescanDatasetName.append(arg)
        elif flag=='--use-list':
            rescan = True
            if arg=='-':
                namelist=sys.stdin
            else:
                namelist = open(arg)
            for line in namelist.readlines():
                line = line.strip()
                if line[0]!='#':
                    rescanDatasetName.append(line)
        elif flag=='--validate':
            schema = arg
            restApi = True
        elif flag=='--version-list':
            versionList = arg

    # If offline, the project must be specified
    if offline and (projectName is None):
        raise ESGPublishError("Must specify project with --project for offline datasets")

    if version is not None and versionList is not None:
        raise ESGPublishError("Cannot specify both --new-version and --version-list")

    if versionList is not None:
        version = {}
        f = open(versionList)
        lines = f.readlines()
        f.close()
        for line in lines:
            line = line.strip()
            dsid, vers = line.split('|')
            dsid = dsid.strip()
            vers = int(vers.strip())
            version[dsid] = vers

    # Load the configuration and set up a database connection
    config = loadConfig(init_file)
    engine = create_engine(config.getdburl('extract'), echo=echoSql, pool_recycle=3600)
    initLogging('extract', override_sa=engine, log_filename=log_filename)
    Session = sessionmaker(bind=engine, autoflush=True, autocommit=False)

    # Register project handlers
    registerHandlers()

    # Get the default publication interface (REST or Hessian)
    if restApi is None:
        restApi = config.getboolean('DEFAULT', 'use_rest_api', default=False)

    # If the dataset map is input, just read it ...
    dmap = None
    directoryMap = None
    extraFields = None
    if datasetMapfile is not None:
        dmap, extraFields = readDatasetMap(datasetMapfile, parse_extra_fields=True)
        datasetNames = dmap.keys()

    elif rescan:
        # Note: No need to get the extra fields, such as mod_time, since
        # they are already in the database, and will be used for file comparison if necessary.
        dmap, offline = queryDatasetMap(rescanDatasetName, Session)
        datasetNames = dmap.keys()

    # ... otherwise generate the directory map.
    else:
        # Online dataset(s)
        if not offline:
            if len(lastargs)==0:
                print "No directories specified."
                return

            if projectName is not None:
                handler = getHandlerByName(projectName, None, Session)
            else:
                multiIter = multiDirectoryIterator(lastargs, filefilt=filefilt)
                firstFile, size = multiIter.next()
                listIter = list(multiIter)
                handler = getHandler(firstFile, Session, validate=True)
                if handler is None:
                    raise ESGPublishError("No project found in file %s, specify with --project."%firstFile)
                projectName = handler.name

            props = properties.copy()
            props.update(initcontext)
            if not readFiles:
                directoryMap = handler.generateDirectoryMap(lastargs, filefilt, initContext=props, datasetName=datasetName)
            else:
                directoryMap = handler.generateDirectoryMapFromFiles(lastargs, filefilt, initContext=props, datasetName=datasetName)
            datasetNames = [(item,-1) for item in directoryMap.keys()]

        # Offline dataset. Format the spec as a dataset map : dataset_name => [(path, size), (path, size), ...]
        else:
            handler = getHandlerByName(projectName, None, Session, offline=True)
            dmap = {}
            listerSection = getOfflineLister(config, "project:%s"%projectName, service)
            offlineLister = config.get(listerSection, 'offline_lister_executable')
            commandArgs = "--config-section %s "%listerSection
            commandArgs += " ".join(lastargs)
            for dsetName, filepath, sizet in processNodeMatchIterator(offlineLister, commandArgs, handler, filefilt=filefilt, datasetName=datasetName, offline=True):
                size, mtime = sizet
                if dmap.has_key((dsetName,-1)):
                    dmap[(dsetName,-1)].append((filepath, str(size)))
                else:
                    dmap[(dsetName,-1)] = [(filepath, str(size))]

            datasetNames = dmap.keys()

    datasetNames.sort()
    if len(datasetNames)==0:
        warning("No datasets found.")
        min_version = -1
    else:
        min_version = sorted(datasetNames, key=lambda x: x[1])[0][1]

    # Must specify version for replications
    if min_version == -1 and masterGateway is not None and version is None and versionList is None:
        raise ESGPublishError("Must specify version with --new-version (or --version-list) for replicated datasets")
    
    # Iterate over datasets
    if not publishOnly:

#        pdb.set_trace()

        datasets = iterateOverDatasets(projectName, dmap, directoryMap, datasetNames, Session, aggregateDimension, publishOp, filefilt, initcontext, offline, properties, keepVersion=keepVersion, newVersion=version, extraFields=extraFields, masterGateway=masterGateway, comment=message, readFiles=readFiles, nodbwrite=nodbwrite)


    if (not nodbwrite):
        result = publishDatasetList(datasetNames, Session, publish=publish, thredds=thredds, las=las, parentId=parent, service=service, perVariable=perVariable, reinitThredds=threddsReinit, restInterface=restApi, schema=schema)
    # print `result`

    if summarizeErrors:
        print 'Summary of errors:'
        for name,versionno in datasetNames:
            dset = Dataset.lookup(name, Session)
            print dset.get_name(Session), dset.get_project(Session), dset.get_model(Session), dset.get_experiment(Session), dset.get_run_name(Session)
            if dset.has_warnings(Session):
                print '=== Dataset: %s ==='%dset.name
                for line in dset.get_warnings(Session):
                    print line
    def start_harvest( self, parent ):
        from esgcet.publish import publishDatasetList
        from esgcet.model import Dataset, PUBLISH_FAILED_EVENT, ERROR_LEVEL

        dcolor1 = Pmw.Color.changebrightness(self.parent.parent, 'aliceblue', 0.8 )

        # Make sure the publisher is logged in
       # if not self.parent.parent.password_flg:
       #    self.parent.parent.menu.login_menu.evt_login( self.parent.parent )

        # Start the busy routine to indicate to the users something is happening
        self.parent.parent.busyCursor = 'watch'
        self.parent.parent.busyWidgets = [self.parent.parent.pane2.pane( 'EditPaneTop' ), self.parent.parent.pane2.pane( 'EditPaneBottom' ), self.parent.parent.pane2.pane( 'EditPaneStatus' ), self.parent.parent.pane.pane( 'ControlPane' )]
        pub_busy.busyStart( self.parent.parent )
        try:
        # Generate the list of datasets to be published
           datasetNames=[]
           GUI_line = {}
           tab_name = self.parent.parent.top_notebook.getcurselection()
           selected_page = self.parent.parent.main_frame.selected_top_page

           if (selected_page is None):
              warning("Must generate a list of datasets to scan before publishing can occur.")
              pub_busy.busyEnd( self.parent.parent )
              return

           for x in self.parent.parent.main_frame.top_page_id[selected_page]:

               if self.parent.parent.main_frame.top_page_id[selected_page][x].cget('bg') != 'salmon' and self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('bg') != 'salmon':
                   dset_name = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text')
                #######################################
                                  # ganz added this 1/18/11    
                   versionNum = self.parent.parent.main_frame.version_label[selected_page][x].cget('text') 
                   dsetTuple = (dset_name, versionNum)
                #dsetName = generateDatasetVersionId(dsetTuple)                
                  #####################################################################################
                # dsetTuple = parseDatasetVersionId(dset_name) # ganz no longer necessary
                   datasetNames.append(dsetTuple)
                   GUI_line[ dset_name ] = x
               else:
                   if self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('bg') == 'salmon':
                      self.parent.parent.main_frame.top_page_id[selected_page][x].configure(relief = 'raised', background = 'salmon', image = self.off)

        # Publish collection of datasets
           testProgress = (self.parent.parent.statusbar.show, 0, 100)
           publishThredds = (quality_control_widgets.get_CheckBox3()==1)
           publishGateway = (quality_control_widgets.get_CheckBox2()==1)
           if (publishThredds):
               print 'publishing to Thredds'
           if (publishGateway):
               print 'publishing to Gateway'  
                      
           status_dict = publishDatasetList(datasetNames, self.Session, publish=publishGateway, thredds=publishThredds, progressCallback=testProgress)

        # Show the published status
           for x in status_dict.keys():
                status = status_dict[ x ]
                dsetName, versionNo = x
                dsetVersionName = generateDatasetVersionId(x)
                guiLine = GUI_line[dsetName] # dsetVersionName]
            
                self.parent.parent.main_frame.status_label[selected_page][guiLine].configure(text=pub_controls.return_status_text( status) )
                dset = Dataset.lookup(dsetName, self.Session)
                if dset.has_warnings(self.Session):
                    warningLevel = dset.get_max_warning_level(self.Session)
                    if warningLevel>=ERROR_LEVEL:
                        buttonColor = "pink"
                        buttonText = "Error"
                    else:
                        buttonColor = "yellow"
                        buttonText = "Warning"
                    self.parent.parent.main_frame.ok_err[selected_page][guiLine].configure(
                        text = buttonText,
                        bg = buttonColor,
                        relief = 'raised',
                        command = pub_controls.Command( self.parent.parent.pub_buttonexpansion.extraction_widgets.error_extraction_button, dset ) )
                else:
                    self.parent.parent.main_frame.ok_err[selected_page][guiLine].configure(
                        text = 'Ok',
                        bg = dcolor1,
                        highlightcolor = dcolor1,
                        relief = 'sunken',
                        )
        except:
            pub_busy.busyEnd( self.parent.parent )  # catch here in order to turn off the busy cursor ganz
            raise
        finally:
           pub_busy.busyEnd( self.parent.parent )
           self.my_refresh()