def set_entry(self, summary, storage, temporary=False):
"""Sets the meta data in the database according to the
summary dictionary. If the according entry already exists,
the database is not changed since the same emd5s should always
reference the same (meta)data.
Arguments:
- summary: dictionary with meta data
- storage: string type (e.g. path in local file system)
- temporary: Flag that marks data to be deleted upon next
call to setup_dbase().
"""
if self.has_entry(summary['id']):
return
exe = self.cursor.execute
if temporary:
exe("INSERT INTO km_temporary VALUES (?)",
(summary['id'],))
insert_dict = dict([(key, value) for key, value in \
summary.iteritems() if key in \
SQLiteWrapper.common_keys])
insert_dict['storage'] = storage
insert_dict['date'] = date2dbase(insert_dict['date'])
type = emd52type(summary['id'])
attr_query = "INSERT INTO km_attributes VALUES (?, ?, ?)"
for key, value in summary['attributes'].iteritems():
assert isinstance(key, StringTypes)
if isinstance(value, StringTypes):
value = utf82uc(value)
exe(attr_query, (summary['id'], key, value.__repr__()))
if type == 'fc':
self._set_fc_keys(insert_dict, summary)
else:
insert_dict['sc_id'] = summary['id']
column_query = "INSERT INTO km_sc_columns VALUES (?, ?, ?)"
for fc_id, fc_index in zip(summary['columns'],
range(len(summary['columns']))):
exe(column_query, (summary['id'], fc_id, fc_index))
insert_query = "INSERT INTO km_%s %s VALUES %s"
value_list = []
key_query = "("
value_query = "("
for key, value in insert_dict.iteritems():
value_query += "?, "
key_query += key + ", "
value_list.append(value)
key_query = key_query[:-2] + ")"
value_query = value_query[:-2] + ")"
insert_query = insert_query % (type, key_query, value_query)
exe(insert_query, tuple(value_list))
def set_entry(self, summary, storage, temporary=False):
"""Sets the meta data in the database according to the
summary dictionary. If the according entry already exists,
the database is not changed since the same emd5s should always
reference the same (meta)data.
Arguments:
- summary: dictionary with meta data
- storage: string type (e.g. path in local file system)
- temporary: Flag that marks data to be deleted upon next
call to setup_dbase().
"""
if self.has_entry(summary['id']):
return
exe = self.cursor.execute
if temporary:
exe("INSERT INTO km_temporary VALUES (?)", (summary['id'], ))
insert_dict = dict([(key, value) for key, value in \
summary.iteritems() if key in \
SQLiteWrapper.common_keys])
insert_dict['storage'] = storage
insert_dict['date'] = date2dbase(insert_dict['date'])
type = emd52type(summary['id'])
attr_query = "INSERT INTO km_attributes VALUES (?, ?, ?)"
for key, value in summary['attributes'].iteritems():
assert isinstance(key, StringTypes)
if isinstance(value, StringTypes):
value = utf82uc(value)
exe(attr_query, (summary['id'], key, value.__repr__()))
if type == 'fc':
self._set_fc_keys(insert_dict, summary)
else:
insert_dict['sc_id'] = summary['id']
column_query = "INSERT INTO km_sc_columns VALUES (?, ?, ?)"
for fc_id, fc_index in zip(summary['columns'],
range(len(summary['columns']))):
exe(column_query, (summary['id'], fc_id, fc_index))
insert_query = "INSERT INTO km_%s %s VALUES %s"
value_list = []
key_query = "("
value_query = "("
for key, value in insert_dict.iteritems():
value_query += "?, "
key_query += key + ", "
value_list.append(value)
key_query = key_query[:-2] + ")"
value_query = value_query[:-2] + ")"
insert_query = insert_query % (type, key_query, value_query)
exe(insert_query, tuple(value_list))
def translate_attr_search(self, value, type):
if value == {}:
return ('1', [], True)
expr = '('
new_value = []
for attr_key, attr_value in value.iteritems():
if isinstance(attr_value, AnyValue):
value_expr = ''
new_value.append(attr_key)
else:
value_expr = ' AND value=?'
if isinstance(attr_value, StringTypes):
attr_value = utf82uc(attr_value)
new_value.extend([attr_key, attr_value.__repr__()])
expr += '(%s IN (SELECT dc_id FROM km_attributes '\
'WHERE key=?%s))' \
% (replace_type('%s_id', type), value_expr)
expr += ' AND '
expr = expr[:-5] + ')'
return (expr, new_value, True)
def translate_attr_search(self, value, type):
if value == {}:
return ('1', [], True)
expr = '('
new_value = []
for attr_key, attr_value in value.iteritems():
if isinstance(attr_value, AnyValue):
value_expr = ''
new_value.append(attr_key)
else:
value_expr = ' AND value=?'
if isinstance(attr_value, StringTypes):
attr_value = utf82uc(attr_value)
new_value.extend([attr_key, attr_value.__repr__()])
expr += '(%s IN (SELECT dc_id FROM km_attributes '\
'WHERE key=?%s))' \
% (replace_type('%s_id', type), value_expr)
expr += ' AND '
expr = expr[:-5] + ')'
return (expr, new_value, True)
def testany(str1):
str2 = utf82uc(str1)
if str2 == u"":
str2 = self.kn.km.any_value
return str2
def loadSummary(self, dcId=None):
"""
Extracts meta data about a given DataContainer and returns it
as a dictionary.
dcId -- emd5 of the DC to summarize. If the emd5 belongs to an
IndexMarker object, u'IndexMarker' is returned.
If dcId == None, a dictionary that maps emd5s to summaries
is returned, where IndexMarker objects are ignored.
"""
if dcId == None:
summary = {}
for group in self.handle.walkGroups(where="/results"):
currDcId = group._v_attrs.TITLE
if len(currDcId) > 0:
tmp = self.loadSummary(currDcId)
if tmp == 'IndexMarker':
summary[im_id] = im_summary
else:
summary[currDcId] = tmp
elif self.isIndexMarker(dcId):
return u'IndexMarker'
else:
summary = {}
summary['id'] = dcId
resNode, uriType = self.getNodeAndTypeFromId(dcId)
summary['longname'] = utf82uc(
self.handle.getNodeAttr(resNode, "longname"))
summary['shortname'] = utf82uc(
self.handle.getNodeAttr(resNode, "shortname"))
summary.update(emd52dict(dcId))
try:
summary['machine'] = utf82uc(
self.handle.getNodeAttr(resNode, "machine"))
summary['creator'] = utf82uc(
self.handle.getNodeAttr(resNode, "creator"))
except:
pass # machine, creator set by emd52dict(dcId) before
attributes = {}
if uriType == 'field':
for key in resNode.data._v_attrs._v_attrnamesuser:
attributes[key] = self.handle.getNodeAttr(
resNode.data, key)
unit = eval(utf82uc(self.handle.getNodeAttr(resNode, "unit")))
summary['unit'] = unit
dimTable = resNode.dimensions
def filterIndexMarker(emd5):
if self.isIndexMarker(emd5):
return im_id
else:
return emd5
dimensions = [filterIndexMarker(row['id']) \
for row in dimTable.iterrows()]
summary['dimensions'] = dimensions
elif uriType == 'sample':
for key in resNode._v_attrs._v_attrnamesuser:
if key not in PyTablesPersister._reservedAttributes:
attributes[key] = self.handle.getNodeAttr(resNode, key)
columns = []
for resId in self.handle.getNodeAttr(resNode, "columns"):
nodename = "/results/" + resId
columnId = self.handle.getNodeAttr(nodename, "TITLE")
columns.append(columnId)
summary['columns'] = columns
summary['attributes'] = attributes
return summary
__version__ = "$Revision$"
# $Source$:
import tables
from pyphant.core import DataContainer
from pyphant.quantities import Quantity
PhysicalQuantity = Quantity
import logging
import os
from pyphant.core import PyTablesPersister
from pyphant.core.DataContainer import IndexMarker
from pyphant.core.Helpers import (utf82uc, emd52dict)
_logger = logging.getLogger("pyphant")
im = IndexMarker()
im_id = u"emd5://pyphant/pyphant/0001-01-01_00:00:00.000000/%s.field" \
% utf82uc(im.hash)
im_summary = {
'id': im_id,
'longname': utf82uc(im.longname),
'shortname': utf82uc(im.shortname),
'hash': utf82uc(im.hash),
'creator': u'pyphant',
'machine': u'pyphant',
'date': u'0001-01-01_00:00:00.000000',
'unit': 1,
'dimensions': [im_id],
'attributes': {}
}
class H5FileHandler(object):
def loadSummary(self, dcId=None):
"""
Extracts meta data about a given DataContainer and returns it
as a dictionary.
dcId -- emd5 of the DC to summarize. If the emd5 belongs to an
IndexMarker object, u'IndexMarker' is returned.
If dcId == None, a dictionary that maps emd5s to summaries
is returned, where IndexMarker objects are ignored.
"""
if dcId is None:
summary = {}
for group in self.handle.walkGroups(where="/results"):
currDcId = group._v_attrs.TITLE
if len(currDcId) > 0:
tmp = self.loadSummary(currDcId)
if tmp == 'IndexMarker':
summary[im_id] = im_summary
else:
summary[currDcId] = tmp
elif self.isIndexMarker(dcId):
return u'IndexMarker'
else:
summary = {}
summary['id'] = dcId
resNode, uriType = self.getNodeAndTypeFromId(dcId)
summary['longname'] = utf82uc(self.handle.getNodeAttr(resNode,
"longname"))
summary['shortname'] = utf82uc(
self.handle.getNodeAttr(resNode, "shortname")
)
summary.update(emd52dict(dcId))
try:
summary['machine'] = utf82uc(
self.handle.getNodeAttr(resNode, "machine")
)
summary['creator'] = utf82uc(
self.handle.getNodeAttr(resNode, "creator")
)
except:
pass # machine, creator set by emd52dict(dcId) before
attributes = {}
if uriType == 'field':
for key in resNode.data._v_attrs._v_attrnamesuser:
attributes[key] = self.handle.getNodeAttr(
resNode.data, key
)
unit = eval(utf82uc(self.handle.getNodeAttr(resNode, "unit")))
summary['unit'] = unit
dimTable = resNode.dimensions
def filterIndexMarker(emd5):
if self.isIndexMarker(emd5):
return im_id
else:
return emd5
dimensions = [filterIndexMarker(row['id']) \
for row in dimTable.iterrows()]
summary['dimensions'] = dimensions
elif uriType == 'sample':
for key in resNode._v_attrs._v_attrnamesuser:
if key not in PyTablesPersister._reservedAttributes:
attributes[key] = self.handle.getNodeAttr(resNode, key)
columns = []
for resId in self.handle.getNodeAttr(resNode, "columns"):
nodename = "/results/" + resId
columnId = self.handle.getNodeAttr(nodename, "TITLE")
columns.append(columnId)
summary['columns'] = columns
summary['attributes'] = attributes
return summary
This module provides the H5FileHandler class.
"""
import tables
from pyphant.core import DataContainer
from pyphant.quantities import Quantity
PhysicalQuantity = Quantity
import logging
import os
from pyphant.core import PyTablesPersister
from pyphant.core.DataContainer import IndexMarker
from pyphant.core.Helpers import (utf82uc, emd52dict)
_logger = logging.getLogger("pyphant")
im = IndexMarker()
im_id = u"emd5://pyphant/pyphant/0001-01-01_00:00:00.000000/%s.field" \
% utf82uc(im.hash)
im_summary = {'id': im_id, 'longname': utf82uc(im.longname),
'shortname': utf82uc(im.shortname), 'hash': utf82uc(im.hash),
'creator': u'pyphant', 'machine': u'pyphant',
'date': u'0001-01-01_00:00:00.000000',
'unit': 1, 'dimensions': [im_id], 'attributes': {}}
class H5FileHandler(object):
"""
This class is used to handle IO operations on HDF5 files.
"""
def __init__(self, filename, mode='a'):
"""
Opens an HDF5 file.
filename -- path to the file that should be opened