def updateDatasetFromContext(context, datasetName, Session):
"""
Update a persistent dataset with values from context (name/value dictionary). The context may have
fields such as event fields, not associated with the project handler.
context
A property (name/value) dictionary.
datasetName
String dataset identifier.
Session
Database session factory.
"""
dset = Dataset.lookup(datasetName, Session)
if dset is None:
raise ESGQueryError("Dataset not found: %s" % datasetName)
projectName = dset.get_project(Session)
handler = getHandlerByName(projectName, None, Session)
basicHeaders, eventHeaders, categories, derivedHeaders = getQueryFields(
handler, return_list=False)
properties = context.copy()
# Set basic and event properties
session = Session()
session.add(dset)
for key, value in properties.items():
if key in basicHeaders:
if key != 'id':
if key == 'name':
if len(handler.parseDatasetName(value, {})) == 0:
warning(
"Dataset name: %s does not match dataset_id pattern in config file."
% value)
setattr(dset, key, value)
else:
warning("Cannot update id field")
del properties[key]
elif key in eventHeaders:
event = dset.events[-1]
setEvent(event, key, value)
del properties[key]
# Set attribute headers
handler.setContext(properties)
handler.saveContext(datasetName, Session)
session.commit()
session.close()
def updateDatasetFromContext(context, datasetName, Session):
"""
Update a persistent dataset with values from context (name/value dictionary). The context may have
fields such as event fields, not associated with the project handler.
context
A property (name/value) dictionary.
datasetName
String dataset identifier.
Session
Database session factory.
"""
dset = Dataset.lookup(datasetName, Session)
if dset is None:
raise ESGQueryError("Dataset not found: %s" % datasetName)
projectName = dset.get_project(Session)
handler = getHandlerByName(projectName, None, Session)
basicHeaders, eventHeaders, categories, derivedHeaders = getQueryFields(
handler, return_list=False)
properties = context.copy()
# Set basic and event properties
session = Session()
session.add(dset)
for key, value in properties.items():
if key in basicHeaders:
if key != 'id':
if key == 'name':
if len(handler.parseDatasetName(value, {})) == 0:
warning(
"Dataset name: %s does not match dataset_id pattern in config file."
% value)
setattr(dset, key, value)
else:
warning("Cannot update id field")
del properties[key]
elif key in eventHeaders:
event = dset.events[-1]
setEvent(event, key, value)
del properties[key]
# Set attribute headers
handler.setContext(properties)
handler.saveContext(datasetName, Session)
session.commit()
session.close()
def queryDatasetMap(datasetNames, Session, extra_fields=False):
"""Query the database for a dataset map.
datasetNames is a list of (datasetName, version) tuples.
Returns (dataset_map, offline_map) where dataset_map is a dictionary:
(dataset_id, version) => [(path, size), (path, size), ...]
and offline_map is a dictionary:
dataset_id => True | False, where True iff the corresponding dataset is offline.
If extra_fields = True. returns (dataset_map, offline_map, extraFields) where
extrafields[(dataset_id, absolute_file_path, *field_name*)] => field_value
where *field_name* is one of:
- ``mod_time``
"""
dmap = {}
offlineMap = {}
extraFields = {}
for versionId in datasetNames:
name, useVersion = parseDatasetVersionId(versionId)
dset = Dataset.lookup(name, Session)
session = Session()
if dset is None:
raise ESGQueryError("Dataset not found: %s" % name)
session.add(dset)
if useVersion == -1:
useVersion = dset.getVersion()
versionObj = dset.getVersionObj(useVersion)
if versionObj is None:
raise ESGPublishError(
"Version %d of dataset %s not found, cannot republish." %
(useVersion, dset.name))
filelist = versionObj.getFiles() # file versions
dmap[(name, useVersion)] = [(file.getLocation(), ` file.getSize() `)
for file in filelist]
def queryDatasetMap(datasetNames, Session, extra_fields=False):
"""Query the database for a dataset map.
datasetNames is a list of (datasetName, version) tuples.
Returns (dataset_map, offline_map) where dataset_map is a dictionary:
(dataset_id, version) => [(path, size), (path, size), ...]
and offline_map is a dictionary:
dataset_id => True | False, where True iff the corresponding dataset is offline.
If extra_fields = True. returns (dataset_map, offline_map, extraFields) where
extrafields[(dataset_id, absolute_file_path, *field_name*)] => field_value
where *field_name* is one of:
- ``mod_time``
"""
dmap = {}
offlineMap = {}
extraFields = {}
for versionId in datasetNames:
name, useVersion = parseDatasetVersionId(versionId)
dset = Dataset.lookup(name, Session)
session = Session()
if dset is None:
raise ESGQueryError("Dataset not found: %s" % name)
session.add(dset)
if useVersion == -1:
useVersion = dset.getVersion()
versionObj = dset.getVersionObj(useVersion)
if versionObj is None:
raise ESGPublishError(
"Version %d of dataset %s not found, cannot republish." %
(useVersion, dset.name))
filelist = versionObj.getFiles() # file versions
dmap[(name, useVersion)] = [(file.getLocation(), ` file.getSize() `)
for file in filelist]
def evt_selected_dataset_text(self, datasetTuple, num_tag, genFrom):
dataset, vers = datasetTuple
datasetVersId = generateDatasetVersionId(datasetTuple)
list_of_dataset_tabs = self.parent.parent.top_notebook.pagenames()
tab_name = datasetVersId.replace("_", "-")
dset = None
if (
self.parent.parent.main_frame.top_page_id2[self.parent.parent.main_frame.selected_top_page][num_tag].cget(
"relief"
)
== "raised"
):
if tab_name in list_of_dataset_tabs:
self.parent.parent.top_notebook.selectpage(tab_name)
else:
self.parent.parent.main_frame.top_page_id2[self.parent.parent.main_frame.selected_top_page][
num_tag
].configure(bg=self.keycolor3, fg=self.keycolor2)
from_tab = self.parent.parent.top_notebook.getcurselection()
dset = Dataset.lookup(dataset, self.Session)
if genFrom:
handler = self.parent.parent.handlerDictionary[dataset]
else:
handler = getHandlerFromDataset(dset, self.Session)
pub_editorviewer = self.parent.parent.create_publisher_editor_viewer(
self.parent.parent, tab_name, dataset, from_tab, self.Session
)
pub_editorviewer.dataset_page(dataset=dset, Session=self.Session, handler=handler)
self.parent.parent.top_notebook.selectpage(tab_name)
# Disable the "Data Publication" button
self.parent.parent.pub_buttonexpansion.control_frame3.pack_forget()
self.parent.parent.pub_buttonexpansion.control_toggle3 = 0
self.parent.parent.pub_buttonexpansion.ControlButton3.configure(
image=self.parent.parent.pub_buttonexpansion.right, state="disabled"
)
def start_harvest(self, parent):
from esgcet.publish import publishDatasetList
from esgcet.model import Dataset, PUBLISH_FAILED_EVENT, ERROR_LEVEL
dcolor1 = Pmw.Color.changebrightness(self.parent.parent, 'aliceblue',
0.8)
# Make sure the publisher is logged in
# if not self.parent.parent.password_flg:
# self.parent.parent.menu.login_menu.evt_login( self.parent.parent )
# Start the busy routine to indicate to the users something is happening
self.parent.parent.busyCursor = 'watch'
self.parent.parent.busyWidgets = [
self.parent.parent.pane2.pane('EditPaneTop'),
self.parent.parent.pane2.pane('EditPaneBottom'),
self.parent.parent.pane2.pane('EditPaneStatus'),
self.parent.parent.pane.pane('ControlPane')
]
pub_busy.busyStart(self.parent.parent)
try:
# Generate the list of datasets to be published
datasetNames = []
GUI_line = {}
tab_name = self.parent.parent.top_notebook.getcurselection()
selected_page = self.parent.parent.main_frame.selected_top_page
if (selected_page is None):
warning(
"Must generate a list of datasets to scan before publishing can occur."
)
pub_busy.busyEnd(self.parent.parent)
return
for x in self.parent.parent.main_frame.top_page_id[selected_page]:
if self.parent.parent.main_frame.top_page_id[selected_page][x].cget(
'bg'
) != 'salmon' and self.parent.parent.main_frame.top_page_id2[
selected_page][x].cget('bg') != 'salmon':
dset_name = self.parent.parent.main_frame.top_page_id2[
selected_page][x].cget('text')
#######################################
# ganz added this 1/18/11
versionNum = self.parent.parent.main_frame.version_label[
selected_page][x].cget('text')
dsetTuple = (dset_name, versionNum)
#dsetName = generateDatasetVersionId(dsetTuple)
#####################################################################################
# dsetTuple = parseDatasetVersionId(dset_name) # ganz no longer necessary
datasetNames.append(dsetTuple)
GUI_line[dset_name] = x
else:
if self.parent.parent.main_frame.top_page_id2[
selected_page][x].cget('bg') == 'salmon':
self.parent.parent.main_frame.top_page_id[
selected_page][x].configure(relief='raised',
background='salmon',
image=self.off)
# Publish collection of datasets
testProgress = (self.parent.parent.statusbar.show, 0, 100)
publishThredds = (quality_control_widgets.get_CheckBox3() == 1)
publishGateway = (quality_control_widgets.get_CheckBox2() == 1)
if (publishThredds):
print 'publishing to Thredds'
if (publishGateway):
print 'publishing to Gateway'
status_dict = publishDatasetList(datasetNames,
self.Session,
publish=publishGateway,
thredds=publishThredds,
progressCallback=testProgress)
# Show the published status
for x in status_dict.keys():
status = status_dict[x]
dsetName, versionNo = x
dsetVersionName = generateDatasetVersionId(x)
guiLine = GUI_line[dsetName] # dsetVersionName]
self.parent.parent.main_frame.status_label[selected_page][
guiLine].configure(
text=pub_controls.return_status_text(status))
dset = Dataset.lookup(dsetName, self.Session)
if dset.has_warnings(self.Session):
warningLevel = dset.get_max_warning_level(self.Session)
if warningLevel >= ERROR_LEVEL:
buttonColor = "pink"
buttonText = "Error"
else:
buttonColor = "yellow"
buttonText = "Warning"
self.parent.parent.main_frame.ok_err[selected_page][
guiLine].configure(
text=buttonText,
bg=buttonColor,
relief='raised',
command=pub_controls.Command(
self.parent.parent.pub_buttonexpansion.
extraction_widgets.error_extraction_button,
dset))
else:
self.parent.parent.main_frame.ok_err[selected_page][
guiLine].configure(
text='Ok',
bg=dcolor1,
highlightcolor=dcolor1,
relief='sunken',
)
except:
pub_busy.busyEnd(
self.parent.parent
) # catch here in order to turn off the busy cursor ganz
raise
finally:
pub_busy.busyEnd(self.parent.parent)
self.my_refresh()
def error_extraction_button( self, dset ):
from pkg_resources import resource_filename
remove_gif = resource_filename('esgcet.ui', 'remove.gif')
dset_name = dset.get_name(self.Session)
print dset_name
tab_name = dset_name.replace('_', '-')
list_of_tabs = self.parent.parent.bottom_notebook.pagenames( )
tabFont = tkFont.Font(self.parent.parent, family = pub_controls.tab_font_type, size=pub_controls.tab_font_size)
dset = Dataset.lookup(dset_name, self.Session)
if dset.has_warnings(self.Session):
warningLevel = dset.get_max_warning_level(self.Session)
if warningLevel>=ERROR_LEVEL:
tab_color = "pink"
else:
tab_color = "yellow"
else:
tab_color = "lightgreen"
if tab_name in list_of_tabs:
self.parent.parent.bottom_notebook.selectpage( tab_name )
else:
dset_error_page = self.parent.parent.bottom_notebook.add( tab_name, tab_bg=tab_color, tab_font = tabFont )
# Generate the group with the remove page icon
group_error_page = Pmw.Group( dset_error_page,
tag_pyclass = MyButton,
tag_text='Remove',
tagindent = 0,
tag_command = pub_controls.Command( self.evt_remove_error_page, tab_name )
)
group_error_page.pack(fill = 'both', expand = 1, padx = 0, pady = 0)
self.parent.parent.balloon.bind(group_error_page, "Use the button in the upper left to remove the page.")
# Create the ScrolledFrame.
dset_output_window = Pmw.ScrolledText(group_error_page.interior(),
text_state='disabled',
text_background=tab_color,
text_relief = 'sunken',
text_borderwidth = 2,
)
dset_output_window.pack(side = 'top', expand=1, fill='both' , pady = 2)
dset_warnings = dset.get_warnings(self.Session)
for x in dset_warnings:
dset_output_window.appendtext( x+"\n" )
#dset_output_window.appendtext( "This is a test from ganzberger \n" )
self.file_display_button( dset, dset_output_window )
self.parent.parent.bottom_notebook.selectpage( tab_name )
# Button remove
tab = self.parent.parent.bottom_notebook.tab(tab_name)
remove_icon = Tkinter.PhotoImage(file=remove_gif)
b = Tkinter.Button(tab,image=remove_icon,command=pub_controls.Command(self.evt_remove_error_page, tab_name) )
b.pack(side='right')
b.image = remove_icon
def return_content2(self, appendOpt=False):
from esgcet.publish import iterateOverDatasets, processIterator
from esgcet.config import getHandlerByName
from esgcet.model import eventName
from esgcet.config import loadConfig
# Initialize parameters for interating over datasets
initcontext = {}
aggregateOnly = False
# appendOpt = False
initcontext = {}
properties = {}
publish = False
publishOnly = False
thredds = False
testProgress1 = [self.parent.parent.statusbar.show, 0, 50]
testProgress2 = [self.parent.parent.statusbar.show, 50, 100]
handlerDictionary = {}
# Get the currently selected tab and the selected datasets
tab_name = self.parent.parent.top_notebook.getcurselection()
selected_page = self.parent.parent.main_frame.selected_top_page
datasetNames = []
# datasetNames2 = []
if (selected_page is None):
warning("Must generate a list of datasets to scan before data extraction can occur.")
return
if (selected_page is not None) or (self.parent.parent.hold_offline[selected_page] == True):
extraFields = None
if (self.parent.parent.hold_offline[selected_page] == False) or (isinstance(self.parent.parent.hold_offline[selected_page], types.DictType)):
for x in self.parent.parent.main_frame.top_page_id[selected_page]:
dsetVersionName = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text') # GANZ TODO version_label
# ganz added this 1/21/11
if (self.parent.parent.main_frame.version_label[selected_page] ):
dset_name = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text')
dsetVersion = self.parent.parent.main_frame.version_label[selected_page][x].cget('text')
#####################################################################################
else:
dset_name, dsetVersion = parseDatasetVersionId(dsetVersionName)
# Retrieve all the datasets in the collection for display
""" ganz test code
status = pollDatasetPublicationStatus(dset_name, self.Session)
status_text = pub_controls.return_status_text( status )
if status_text != 'Error':
dsetTuple = parseDatasetVersionId(self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text'))
datasetNames2.append(dsetTuple)
"""
# Retrieve only the datasets that have been selected
if self.parent.parent.main_frame.top_page_id[selected_page][x].cget('bg') != 'salmon':
dsetTuple = parseDatasetVersionId(self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text'))
datasetNames.append(dsetTuple)
dmap = self.parent.parent.main_frame.dmap[selected_page]
extraFields = self.parent.parent.main_frame.extraFields[selected_page]
datasetMapfile = self.parent.parent.main_frame.datasetMapfile[selected_page]
projectName = self.parent.parent.main_frame.projectName[selected_page]
directoryMap = self.parent.parent.directoryMap[selected_page]
if dmap is not None:
for x in datasetNames:
dsetId = x[0]
datasetName = x
try:
dmapentry = dmap[datasetName]
except:
# Check if the dataset map key was changed from (dsetname,-1) to (dsetname,version).
# If so, replace the entry with the new key.
trykey = (datasetName[0], -1)
dmapentry = dmap[trykey]
del dmap[trykey]
dmap[datasetName] = dmapentry
firstFile = dmapentry[0][0]
self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, firstFile, self.Session)
handler = self.parent.parent.handlerDictionary[dsetId]
# Copy the defaultGlobalValues into initcontext
initcontext = self.parent.parent.main_frame.defaultGlobalValues[selected_page]
else:
# more test code
myholdDirectoryMap = self.parent.parent.directoryMap[selected_page]
#mydatasetNames = [(item,-1) for item in myholdDirectoryMap.keys()]
mydatasetNames = [(item) for item in myholdDirectoryMap.keys()]
#end
for x in mydatasetNames:
dsetId = x[0]
datasetName = x
# ganz this is test code
try:
dmapentry = myholdDirectoryMap[datasetName]
except:
# Check if the dataset map key was changed from (dsetname,-1) to (dsetname,version).
# If so, replace the entry with the new key.
trykey = (datasetName[0], -1)
dmapentry = myholdDirectoryMap[trykey]
del myholdDirectoryMap[trykey]
myholdDirectoryMap[datasetName] = dmapentry
firstFile = dmapentry[0][1]
#end of test code
#firstFile = self.parent.parent.main_frame.dirp_firstfile[selected_page]
self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, firstFile, self.Session)
handler = self.parent.parent.handlerDictionary[dsetId]
else: # working off-line
projectName = self.parent.parent.main_frame.projectName[selected_page]
if self.parent.parent.offline_file_directory[selected_page] == "directory":
if self.parent.parent.config is None:
extraction_controls.call_sessionmaker( self.parent.parent )
datasetPaths = []
dmap = {self.parent.parent.offline_datasetName : datasetPaths}
listerSection = getOfflineLister(self.parent.parent.config, "project:%s"%projectName, None)
offlineLister = self.parent.parent.config.get(listerSection, 'offline_lister_executable')
lastargs = self.parent.parent.offline_directories
commandArgs = "--config-section %s "%listerSection
commandArgs += " ".join(lastargs)
for filepath, size in processIterator(offlineLister, commandArgs, filefilt=self.parent.parent.filefilt):
datasetPaths.append((filepath, str(size)))
datasetNames = self.parent.parent.datasetNames
directoryMap = None
# get the handler
for x in datasetNames:
dsetId = x[0]
self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, None, self.Session, offline=True)
elif self.parent.parent.offline_file_directory[selected_page] == "file":
dmap = self.parent.parent.main_frame.dmap[selected_page]
extraFields = self.parent.parent.main_frame.extraFields[selected_page]
datasetMapfile = self.parent.parent.main_frame.datasetMapfile[selected_page]
projectName = self.parent.parent.main_frame.projectName[selected_page]
directoryMap = None
if datasetMapfile is not None:
dmap, extraFields = readDatasetMap(datasetMapfile, parse_extra_fields=True)
datasetNames = dmap.keys()
# get the handlers
for x in datasetNames:
dsetId = x[0]
self.parent.parent.handlerDictionary[dsetId] = getHandlerByName(projectName, None, self.Session, offline=True)
# Iterate over datasets
if appendOpt:
operation = UPDATE_OP
else:
operation = CREATE_OP
datasets = iterateOverDatasets(projectName, dmap, directoryMap, datasetNames, self.Session, self.parent.parent.aggregateDimension, operation, self.parent.parent.filefilt, initcontext, self.parent.parent.hold_offline[selected_page], properties, comment=self.comments, testProgress1=testProgress1, testProgress2=testProgress2 , handlerDictionary=self.parent.parent.handlerDictionary, extraFields=extraFields, readFiles=True)
# If working on-line then replace the scanned list of datasets with
# the complete list of datasets
#test
"""
print 'datasetNames:'
for t1 in datasetNames:
print t1
print 'datasetNames2:'
for t2 in datasetNames2:
print t2
"""
if not self.parent.parent.hold_offline[selected_page]:
datasets = []
versionObjs = []
# ganz finally, tested datasetNames2 here
for dsetName, version in datasetNames:
result = Dataset.lookup(dsetName, self.Session, version=version)
if result is not None:
entry, versionObj = result
datasets.append(entry)
versionObjs.append(versionObj)
# Get the summary of errors after doing a data extraction
dset_error = []
for dset in datasets:
status = dset.get_publication_status(self.Session)
status_name = eventName[status]
if dset.has_warnings(self.Session):
dset_error.append(dset.get_name(self.Session))
try:
list_fields = getQueryFields( handler )
except:
handler = getHandlerByName(projectName, None, self.Session)
list_fields = getQueryFields( handler )
# Display the datasets in the "Collection" page
# if self.parent.parent.hold_offline[selected_page] == True:
# tab_name = "Collection_Offline"
# from_tab = "Collection"
# pub_editorviewer = self.parent.parent.create_publisher_editor_viewer( self.parent.parent, tab_name, dataset, from_tab, self.Session)
# Show the extracted datasets
self.set_column_labels( len(datasets), list_fields )
self.show_extracted_info(datasets, dset_error, list_fields, versionObjs)
# Enable the "Data Publication" button
self.parent.ControlButton3.configure( state = 'normal' )
def addQueryPageRow(self, num_tag, query_result, list_fields):
# set the color for each item in the row
dcolor1 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.8)
dcolor2 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.7)
dcolor3 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.5)
dcolor4 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.6)
dcolor5 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.7)
dcolor6 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.8)
dcolor7 = Pmw.Color.changebrightness(self.parent.parent, "aliceblue", 0.9)
frame_ct = self.parent.parent.top_ct
labelFont = tkFont.Font(
self.parent.parent, family=pub_controls.collection_font_type, size=pub_controls.collection_font_size
)
self.add_row_frame[num_tag] = self.pageFrame[frame_ct].interior()
self.select_button[num_tag] = Tkinter.Button(
self.add_row_frame[num_tag],
# compound = Tkinter.LEFT,
# justify = Tkinter.LEFT,
# anchor = Tkinter.W,
text=num_tag,
# font = labelFont,
image=self.on,
bg=self.keycolor1,
relief="raised",
# width = 70,
height=1,
command=pub_controls.Command(self.evt_selected_dataset, num_tag),
)
self.select_button[num_tag].image = self.on # save the image from garbage collection
self.select_button[num_tag].grid(row=self.row, column=0, sticky="nsew")
dset = Dataset.lookup(query_result[1], self.Session)
if dset.has_warnings(self.Session):
warningLevel = dset.get_max_warning_level(self.Session)
if warningLevel >= ERROR_LEVEL:
buttonColor = "pink"
buttonText = "Error"
else:
buttonColor = "yellow"
buttonText = "Warning"
self.ok_err[num_tag] = Tkinter.Button(
self.add_row_frame[num_tag],
text=buttonText,
bg=buttonColor,
width=4,
relief="raised",
command=pub_controls.Command(
self.parent.parent.pub_buttonexpansion.extraction_widgets.error_extraction_button, dset
),
)
self.ok_err[num_tag].grid(row=self.row, column=1, sticky="nsew")
else:
self.ok_err[num_tag] = Tkinter.Button(
self.add_row_frame[num_tag], text="Ok", bg=dcolor1, highlightcolor=dcolor1, width=4, relief="sunken"
)
self.ok_err[num_tag].grid(row=self.row, column=1, sticky="nsew")
status = pollDatasetPublicationStatus(query_result[1], self.Session)
status_text = pub_controls.return_status_text(status)
self.status[num_tag] = Tkinter.Label(
self.add_row_frame[num_tag], text=status_text, bg=dcolor1, highlightcolor=dcolor1, width=10, relief="sunken"
)
self.status[num_tag].grid(row=self.row, column=2, sticky="nsew")
if "id" in list_fields:
id = Tkinter.Label(self.add_row_frame[num_tag], text=query_result[0], bg=dcolor2, width=6, relief="sunken")
id.grid(row=self.row, column=3, sticky="nsew")
dsetName = query_result[1]
versionNum = -1
try:
versionIndex = list_fields.index("version")
versionNum = string.atoi(query_result[versionIndex])
except:
pass
dsetTuple = (dsetName, versionNum)
dsetName = generateDatasetVersionId(dsetTuple)
# ganz adding rows here...need to add versions
datasetName = dsetTuple[0] # dataset[0] #parseDatasetVersionId(dataset)
versionno = dsetTuple[1] # dataset[1]
ver_1 = versionno
if ver_1 == -1:
ver_1 = "N/A"
self.select_labelV[num_tag] = Tkinter.Label(
self.add_row_frame[num_tag], text=ver_1, bg=dcolor2, width=11, relief="sunken"
)
self.select_labelV[num_tag].grid(row=self.row, column=4, sticky="nsew")
self.select_label[num_tag] = Tkinter.Button(
self.add_row_frame[num_tag],
text=datasetName, # dsetName,
font=labelFont,
bg=self.keycolor2,
relief="raised",
borderwidth=2,
anchor="w",
justify="left",
width=71,
command=pub_controls.Command(self.evt_selected_dataset_text, dsetTuple, num_tag, 0),
)
self.select_label[num_tag].grid(row=self.row, column=5, columnspan=2, sticky="nsew")
if "project" in list_fields:
project = Tkinter.Label(
self.add_row_frame[num_tag], text=query_result[2], bg=dcolor3, width=20, relief="sunken", borderwidth=2
)
project.grid(row=self.row, column=7, sticky="nsew")
if "model" in list_fields:
model = Tkinter.Label(
self.add_row_frame[num_tag], text=query_result[3], bg=dcolor4, width=20, relief="sunken", borderwidth=2
)
model.grid(row=self.row, column=8, sticky="nsew")
if "experiment" in list_fields:
experiment = Tkinter.Label(
self.add_row_frame[num_tag], text=query_result[4], bg=dcolor5, width=20, relief="sunken", borderwidth=2
)
experiment.grid(row=self.row, column=9, sticky="nsew")
if "run_name" in list_fields:
run_name = Tkinter.Label(
self.add_row_frame[num_tag], text=query_result[5], bg=dcolor6, width=20, relief="sunken", borderwidth=2
)
run_name.grid(row=self.row, column=10, sticky="nsew")
self.add_row_frame[num_tag].grid_rowconfigure(self.row, weight=1)
self.add_row_frame[num_tag].grid_columnconfigure(1, weight=1)
if (
self.pageFrame[frame_ct].cget("horizflex") == "expand"
or self.pageFrame[frame_ct].cget("vertflex") == "expand"
):
self.pageFrame[frame_ct].reposition()
self.row = self.row + 1 # increment to the next dataset row
def evt_refresh_list_of_datasets(self, selected_page):
# Start the busy routine to indicate to the users something is happening
self.parent.parent.busyCursor = "watch"
self.parent.parent.busyWidgets = [
self.parent.parent.pane2.pane("EditPaneTop"),
self.parent.parent.pane2.pane("EditPaneBottom"),
self.parent.parent.pane2.pane("EditPaneStatus"),
self.parent.parent.pane.pane("ControlPane"),
]
pub_busy.busyStart(self.parent.parent)
try:
if self.parent.parent.refreshButton[selected_page].cget("relief") == "raised":
for x in self.parent.parent.main_frame.top_page_id[selected_page]:
if self.parent.parent.main_frame.top_page_id[selected_page][x].cget("relief") == "raised":
dsetVersionName = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget("text")
# ganz added this 1/18/11
query_name = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget("text")
versionNum = self.parent.parent.main_frame.version_label[selected_page][x].cget("text")
#####################################################################################
query_name, versionNum = parseDatasetVersionId(dsetVersionName)
# ganz TODO test only remove
# print query_name
# print versionNum
status = pollDatasetPublicationStatus(query_name, self.Session)
# ganz catch non selected Red entries to skip 3/20/2011
try:
self.parent.parent.main_frame.status_label[selected_page][x].configure(
text=pub_controls.return_status_text(status)
)
except:
continue
# Make sure you update the Ok/Err button
# ganz added this (1/18/11) here to catch the case when dset=None (e.g. no local db entry exists)
dset = Dataset.lookup(query_name, self.Session)
if dset == None:
buttonColor = "yellow"
buttonText = "Warning"
self.parent.parent.main_frame.ok_err[selected_page][x].configure(
bg=buttonColor, text=buttonText
)
elif dset.has_warnings(self.Session):
warningLevel = dset.get_max_warning_level(self.Session)
if warningLevel >= ERROR_LEVEL:
buttonColor = "pink"
buttonText = "Error"
else:
buttonColor = "yellow"
buttonText = "Warning"
self.parent.parent.main_frame.ok_err[selected_page][x].configure(
bg=buttonColor, text=buttonText
)
except:
pub_busy.busyEnd(self.parent.parent) # catch here in order to turn off the busy cursor ganz
raise
finally:
pub_busy.busyEnd(self.parent.parent)
# pub_busy.busyEnd( self.parent.parent )
info("Completed refreshing the display.")
def main(argv):
try:
args, lastargs = getopt.getopt(argv, "a:cdehi:m:p:ru", ['append', 'create', 'dataset=', 'delete-files', 'echo-sql', 'experiment=', 'filter=', 'help', 'keep-version', 'log=', 'map=', 'message=', 'model=', 'offline', 'parent=', 'per-time', 'per-variable', 'project=', 'property=', 'publish', 'new-version=', 'no-thredds-reinit', 'noscan', 'read-directories', 'read-files', 'rename-files', 'replace', 'replica=', 'rest-api', 'service=', 'set-replica', 'summarize-errors', 'thredds', 'thredds-reinit', 'update', 'use-existing=', 'use-list=', 'validate=', 'version-list=', 'nodbwrite'])
except getopt.error:
print sys.exc_value
return
aggregateDimension = "time"
datasetMapfile = None
datasetName = None
echoSql = False
filefilt = '.*\.nc$'
init_file = None
initcontext = {}
keepVersion = False
las = False
log_filename = None
masterGateway = None
message = None
offline = False
parent = None
perVariable = None
projectName = None
properties = {}
publish = False
publishOnly = False
publishOp = CREATE_OP
readFiles = False
rescan = False
rescanDatasetName = []
restApi = None
schema = None
service = None
summarizeErrors = False
testProgress1 = testProgress2 = None
thredds = False
threddsReinit = None
version = None
versionList = None
nodbwrite = False
for flag, arg in args:
if flag=='-a':
aggregateDimension = arg
elif flag=='--append':
publishOp = UPDATE_OP
elif flag in ['-c', '--create']:
publishOp = CREATE_OP
elif flag=='--dataset':
datasetName = arg
elif flag in ['-d', '--delete-files']:
publishOp = DELETE_OP
elif flag=='--echo-sql':
echoSql = True
elif flag=='--experiment':
initcontext['experiment'] = arg
elif flag=='--filter':
filefilt = arg
elif flag in ['-h', '--help']:
print usage
sys.exit(0)
elif flag=='-i':
init_file = arg
elif flag=='--keep-version':
keepVersion = True
elif flag=='--log':
log_filename = arg
elif flag=='--map':
datasetMapfile = arg
elif flag in ['-m', '--message']:
message = arg
elif flag=='--model':
initcontext['model'] = arg
elif flag=='--nodbwrite':
nodbwrite = True
elif flag=='--new-version':
try:
version = string.atoi(arg)
if version <=0:
raise ValueError
except ValueError:
raise ESGPublishError("Version number must be a positive integer: %s"%arg)
elif flag=='--no-thredds-reinit':
threddsReinit = False
elif flag=='--noscan':
publishOnly = True
elif flag=='--offline':
offline = True
elif flag=='--parent':
parent = arg
elif flag=='--per-time':
perVariable = False
elif flag=='--per-variable':
perVariable = True
elif flag=='--project':
projectName = arg
elif flag in ['-p', '--property']:
name, value = arg.split('=')
properties[name] = value
elif flag=='--publish':
publish = True
elif flag in ['-e', '--read-directories']:
readFiles = False
elif flag=='--read-files':
readFiles = True
elif flag=='--rename-files':
publishOp = RENAME_OP
elif flag in ['-r', '--replace']:
publishOp = REPLACE_OP
elif flag=='--replica':
masterGateway = arg
warning("The --replica option is deprecated. Use --set-replica instead")
elif flag=='--rest-api':
restApi = True
elif flag=='--service':
service = arg
elif flag=='--set-replica':
masterGateway = 'DEFAULT'
elif flag=='--summarize-errors':
summarizeErrors = True
elif flag=='--thredds':
thredds = True
elif flag=='--thredds-reinit':
threddsReinit = True
elif flag in ['-u', '--update']:
publishOp = UPDATE_OP
elif flag=='--use-existing':
rescan = True
rescanDatasetName.append(arg)
elif flag=='--use-list':
rescan = True
if arg=='-':
namelist=sys.stdin
else:
namelist = open(arg)
for line in namelist.readlines():
line = line.strip()
if line[0]!='#':
rescanDatasetName.append(line)
elif flag=='--validate':
schema = arg
restApi = True
elif flag=='--version-list':
versionList = arg
# If offline, the project must be specified
if offline and (projectName is None):
raise ESGPublishError("Must specify project with --project for offline datasets")
if version is not None and versionList is not None:
raise ESGPublishError("Cannot specify both --new-version and --version-list")
if versionList is not None:
version = {}
f = open(versionList)
lines = f.readlines()
f.close()
for line in lines:
line = line.strip()
dsid, vers = line.split('|')
dsid = dsid.strip()
vers = int(vers.strip())
version[dsid] = vers
# Load the configuration and set up a database connection
config = loadConfig(init_file)
engine = create_engine(config.getdburl('extract'), echo=echoSql, pool_recycle=3600)
initLogging('extract', override_sa=engine, log_filename=log_filename)
Session = sessionmaker(bind=engine, autoflush=True, autocommit=False)
# Register project handlers
registerHandlers()
# Get the default publication interface (REST or Hessian)
if restApi is None:
restApi = config.getboolean('DEFAULT', 'use_rest_api', default=False)
# If the dataset map is input, just read it ...
dmap = None
directoryMap = None
extraFields = None
if datasetMapfile is not None:
dmap, extraFields = readDatasetMap(datasetMapfile, parse_extra_fields=True)
datasetNames = dmap.keys()
elif rescan:
# Note: No need to get the extra fields, such as mod_time, since
# they are already in the database, and will be used for file comparison if necessary.
dmap, offline = queryDatasetMap(rescanDatasetName, Session)
datasetNames = dmap.keys()
# ... otherwise generate the directory map.
else:
# Online dataset(s)
if not offline:
if len(lastargs)==0:
print "No directories specified."
return
if projectName is not None:
handler = getHandlerByName(projectName, None, Session)
else:
multiIter = multiDirectoryIterator(lastargs, filefilt=filefilt)
firstFile, size = multiIter.next()
listIter = list(multiIter)
handler = getHandler(firstFile, Session, validate=True)
if handler is None:
raise ESGPublishError("No project found in file %s, specify with --project."%firstFile)
projectName = handler.name
props = properties.copy()
props.update(initcontext)
if not readFiles:
directoryMap = handler.generateDirectoryMap(lastargs, filefilt, initContext=props, datasetName=datasetName)
else:
directoryMap = handler.generateDirectoryMapFromFiles(lastargs, filefilt, initContext=props, datasetName=datasetName)
datasetNames = [(item,-1) for item in directoryMap.keys()]
# Offline dataset. Format the spec as a dataset map : dataset_name => [(path, size), (path, size), ...]
else:
handler = getHandlerByName(projectName, None, Session, offline=True)
dmap = {}
listerSection = getOfflineLister(config, "project:%s"%projectName, service)
offlineLister = config.get(listerSection, 'offline_lister_executable')
commandArgs = "--config-section %s "%listerSection
commandArgs += " ".join(lastargs)
for dsetName, filepath, sizet in processNodeMatchIterator(offlineLister, commandArgs, handler, filefilt=filefilt, datasetName=datasetName, offline=True):
size, mtime = sizet
if dmap.has_key((dsetName,-1)):
dmap[(dsetName,-1)].append((filepath, str(size)))
else:
dmap[(dsetName,-1)] = [(filepath, str(size))]
datasetNames = dmap.keys()
datasetNames.sort()
if len(datasetNames)==0:
warning("No datasets found.")
min_version = -1
else:
min_version = sorted(datasetNames, key=lambda x: x[1])[0][1]
# Must specify version for replications
if min_version == -1 and masterGateway is not None and version is None and versionList is None:
raise ESGPublishError("Must specify version with --new-version (or --version-list) for replicated datasets")
# Iterate over datasets
if not publishOnly:
# pdb.set_trace()
datasets = iterateOverDatasets(projectName, dmap, directoryMap, datasetNames, Session, aggregateDimension, publishOp, filefilt, initcontext, offline, properties, keepVersion=keepVersion, newVersion=version, extraFields=extraFields, masterGateway=masterGateway, comment=message, readFiles=readFiles, nodbwrite=nodbwrite)
if (not nodbwrite):
result = publishDatasetList(datasetNames, Session, publish=publish, thredds=thredds, las=las, parentId=parent, service=service, perVariable=perVariable, reinitThredds=threddsReinit, restInterface=restApi, schema=schema)
# print `result`
if summarizeErrors:
print 'Summary of errors:'
for name,versionno in datasetNames:
dset = Dataset.lookup(name, Session)
print dset.get_name(Session), dset.get_project(Session), dset.get_model(Session), dset.get_experiment(Session), dset.get_run_name(Session)
if dset.has_warnings(Session):
print '=== Dataset: %s ==='%dset.name
for line in dset.get_warnings(Session):
print line
def start_harvest( self, parent ):
from esgcet.publish import publishDatasetList
from esgcet.model import Dataset, PUBLISH_FAILED_EVENT, ERROR_LEVEL
dcolor1 = Pmw.Color.changebrightness(self.parent.parent, 'aliceblue', 0.8 )
# Make sure the publisher is logged in
# if not self.parent.parent.password_flg:
# self.parent.parent.menu.login_menu.evt_login( self.parent.parent )
# Start the busy routine to indicate to the users something is happening
self.parent.parent.busyCursor = 'watch'
self.parent.parent.busyWidgets = [self.parent.parent.pane2.pane( 'EditPaneTop' ), self.parent.parent.pane2.pane( 'EditPaneBottom' ), self.parent.parent.pane2.pane( 'EditPaneStatus' ), self.parent.parent.pane.pane( 'ControlPane' )]
pub_busy.busyStart( self.parent.parent )
try:
# Generate the list of datasets to be published
datasetNames=[]
GUI_line = {}
tab_name = self.parent.parent.top_notebook.getcurselection()
selected_page = self.parent.parent.main_frame.selected_top_page
if (selected_page is None):
warning("Must generate a list of datasets to scan before publishing can occur.")
pub_busy.busyEnd( self.parent.parent )
return
for x in self.parent.parent.main_frame.top_page_id[selected_page]:
if self.parent.parent.main_frame.top_page_id[selected_page][x].cget('bg') != 'salmon' and self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('bg') != 'salmon':
dset_name = self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('text')
#######################################
# ganz added this 1/18/11
versionNum = self.parent.parent.main_frame.version_label[selected_page][x].cget('text')
dsetTuple = (dset_name, versionNum)
#dsetName = generateDatasetVersionId(dsetTuple)
#####################################################################################
# dsetTuple = parseDatasetVersionId(dset_name) # ganz no longer necessary
datasetNames.append(dsetTuple)
GUI_line[ dset_name ] = x
else:
if self.parent.parent.main_frame.top_page_id2[selected_page][x].cget('bg') == 'salmon':
self.parent.parent.main_frame.top_page_id[selected_page][x].configure(relief = 'raised', background = 'salmon', image = self.off)
# Publish collection of datasets
testProgress = (self.parent.parent.statusbar.show, 0, 100)
publishThredds = (quality_control_widgets.get_CheckBox3()==1)
publishGateway = (quality_control_widgets.get_CheckBox2()==1)
if (publishThredds):
print 'publishing to Thredds'
if (publishGateway):
print 'publishing to Gateway'
status_dict = publishDatasetList(datasetNames, self.Session, publish=publishGateway, thredds=publishThredds, progressCallback=testProgress)
# Show the published status
for x in status_dict.keys():
status = status_dict[ x ]
dsetName, versionNo = x
dsetVersionName = generateDatasetVersionId(x)
guiLine = GUI_line[dsetName] # dsetVersionName]
self.parent.parent.main_frame.status_label[selected_page][guiLine].configure(text=pub_controls.return_status_text( status) )
dset = Dataset.lookup(dsetName, self.Session)
if dset.has_warnings(self.Session):
warningLevel = dset.get_max_warning_level(self.Session)
if warningLevel>=ERROR_LEVEL:
buttonColor = "pink"
buttonText = "Error"
else:
buttonColor = "yellow"
buttonText = "Warning"
self.parent.parent.main_frame.ok_err[selected_page][guiLine].configure(
text = buttonText,
bg = buttonColor,
relief = 'raised',
command = pub_controls.Command( self.parent.parent.pub_buttonexpansion.extraction_widgets.error_extraction_button, dset ) )
else:
self.parent.parent.main_frame.ok_err[selected_page][guiLine].configure(
text = 'Ok',
bg = dcolor1,
highlightcolor = dcolor1,
relief = 'sunken',
)
except:
pub_busy.busyEnd( self.parent.parent ) # catch here in order to turn off the busy cursor ganz
raise
finally:
pub_busy.busyEnd( self.parent.parent )
self.my_refresh()
