def fetch(Klass, filters=(), query=(), session='default', sa=None, exists=True):
"""
Return exactly one or none for filtered query.
"""
if not sa:
sa = Klass.get_session(session)
rs = None
if query:
q = sa.query(*query)
else:
q = sa.query(Klass)
if filters:
q = q.filter(*filters)
try:
rs = q.one()
except NoResultFound as e:
if exists:
log.err("No results for %s.fetch(%r)", Klass.__name__, filters)
raise e
return rs
def update(self):
now = datetime.datetime.now()
if 'X-First-Seen' not in self.data:
self.data['X-First-Seen'] = util.iso8601_datetime_format(now.timetuple())
envelope = (
#('X-Meta-Checksum', lambda x: self.get_meta_hash()),
('X-Last-Modified', util.last_modified_header),
('X-Last-Update', lambda x: util.iso8601_datetime_format(now.timetuple())),
('X-Last-Seen', lambda x: util.iso8601_datetime_format(now.timetuple())),
)
for handlers in self.handlers, envelope:
for header, handler in handlers:
value = None
try:
value = handler(self.path)
except Exception, e:
traceback.print_exc()
log.err("%s: %s", header, e)
continue
#print header, value
if header in self.allow_multiple:
if header not in self.data:
self.data[header] = []
elif not isinstance(self.data[header], list):
self.data[header] = [ self.data[header] ]
self.data[header].append(value)
else:
self.data[header] = value
def run(self, mp_class, path, opts):
mp = mp_class(self)
# self.props[mp.cname] = mp.applies(self.data, path, opts)
# if self.propstat[mp.cname] == 0:
# return
try:
self.data[mp.cname] = mp.extract(self.data, path, opts)
except Exception, e:
log.err('Failed getting %s for %s: %s', mp.cname, path, e)
def add(self, name, prog, opts):
"""
Start with MP default(s), continue to discover.
Put data into STAGE index.
"""
data = {}
path = self.volume.pathname(name, prog.pwd)
if not os.path.exists(path):
log.err("Ignored %s", path)
return
if path in self.stage:
data = self.stage.get(path)
resolver = MetaResolver(self, data)
resolver.discover(path, opts)
print 'todo add', resolver.data
def rsr_tree(self, sa=None, *nodes):
"Print a tree of nodes as nested lists"
if not nodes:
roots = sa.query(GroupNode)\
.filter( GroupNode.root == True, ).all()
if not roots:
log.err("No roots")
else:
roots = []
for node in nodes:
group = GroupNode.find(( Node.name == node, ))
if not group:
log.warn(group)
continue
roots.append(group)
for group in roots:
self.execute( 'rsr_node_recurse', dict( group=group ) )
def all(Klass, filters=None, session='default', sa=None):
"""
Return all for filtered query.
"""
if not sa:
sa = Klass.get_session(session)
q = sa.query(Klass)
if not filters and isinstance(filters, type(None)):
if hasattr(Klass, 'default_filters'):
filters = Klass.default_filters()
if filters:
for f in filters:
q = q.filter(f)
try:
return q.all()
except Exception as e:
log.err("Error executing .all: %s", e)
return []
def current_hostname(initialize=False, interactive=False):
"""
"""
host = dict( name = get_hostname() )
hostnameId = host['name'].lower()
return hostnameId;
# FIXME: current_hostname
hostname = None
hostname_file = os.path.expanduser('~/.cllct/host')
if os.path.exists(hostname_file):
hostname = open(hostname_file).read().strip()
elif initialize:
hostname = socket.gethostname()
assert not isinstance(hostname, (tuple, list)), hostname
log.debug(hostname)
hostnames = socket.gethostbyaddr(hostname)
while True:
if socket.getfqdn() != hostname:
hostname = hostnames[0] +"."
else:
log.err("FQDN is same as hostname")
# cannot figure out what host to use
while interactive:
hostname = prompt_choice_with_input("Which? ", hostnames[1])
if hostname: break
#if not interactive:
# raise ValueError("")
if hostname:
try:
nameinfo((hostname, 80))
except Exception as e:
print('Warning: Cannot resolve FQDN', e)
open(hostname_file, 'w+').write(hostname)
print("Stored %s in %s" % (hostname, hostname_file))
break
return hostname
def lnd_tag(opts=None, sa=None, ur=None, pwd=None):
"""
Experiment, interactive interface.
Tagging.
"""
log.debug("{bblack}lnd{bwhite}:tag{default}")
if not pwd:
log.err("Not initialized")
yield 1
tags = {}
if '' not in tags:
tags[''] = 'Root'
FS_Path_split = re.compile('[\/\.\+,]+').split
log.info("{bblack}Tagging paths in {green}%s{default}",
os.path.realpath('.') + os.sep)
try:
for root, dirs, files in os.walk(pwd.local_path):
for name in files + dirs:
log.info("{bblack}Typing tags for {green}%s{default}",
name)
path = FS_Path_split(os.path.join(root, name))
for tag in path:
yield
# Ask about each new tag, TODO: or rename, fuzzy match.
if tag not in tags:
type = raw_input('%s%s%s:?' % (
log.palette['yellow'], tag,
log.palette['default']) )
if not type: type = 'Tag'
tags[tag] = type
log.info(''.join( [ "{bwhite} %s:{green}%s{default}" % (tag, name)
for tag in path if tag in tags] ))
except KeyboardInterrupt as e:
log.err(e)
yield 1
def walk(self, path, max_depth=-1):
# XXX: maybe rewrite to Dir.walk
"""
Walk all files that may have a metafile, and notice any metafile(-like)
neighbors?
"""
for root, nodes, leafs in os.walk(path):
for node in list(nodes):
dirpath = os.path.join(root, node)
if not os.path.exists(dirpath):
log.err("Error: reported non existant node %s", dirpath)
nodes.remove(node)
continue
depth = dirpath.replace(path,'').strip('/').count('/')
if fs.Dir.ignored(dirpath):
log.warn("Ignored directory %r", dirpath)
nodes.remove(node)
elif max_depth != -1:
if depth >= max_depth:
nodes.remove(node)
for leaf in leafs:
cleaf = os.path.join(root, leaf)
if not os.path.exists(dirpath):
log.err("Error: non existant leaf %s", cleaf)
continue
if not os.path.isfile(cleaf) or os.path.islink(cleaf):
#log.warn("Ignored non-regular file %r", cleaf)
continue
if fs.File.ignored(cleaf):
#log.warn("Ignored file %r", cleaf)
continue
if Metafile.is_metafile(cleaf, strict=False):
if not Metafile(cleaf).path:
log.err("Metafile without resource file")
else:
yield cleaf
def walk(Klass, path, opts=walk_opts, filters=(None,None)):
"""
Build on os.walk, this goes over all directories and other paths
non-recursively.
It returns all full paths according to walk-opts.
FIXME: could, but does not, yield INode subtype instances.
XXX: filters, see dev_treemap
"""
# if not opts.descend:
# return self.walkRoot( path, opts=opts, filters=filters )
if not isinstance(opts, confparse.Values):
opts_ = confparse.Values(Klass.walk_opts)
opts_.update(opts)
opts = opts_
else:
opts = confparse.Values(opts.copy())
# FIXME: validate/process opts or put filter somewhere
if opts.max_depth > 0:
assert opts.recurse
exclusive( opts, 'dirs files symlinks links pipes blockdevs' )
assert isinstance(path, basestring), (path, path.__class__)
dirpath = None
file_filters, dir_filters = filters
if not os.path.isdir( path ):
if opts.exists > -1:
log.err("Cannot walk non-dir path with opt.exists. ")
else:
yield path
else:
if opts.dirs and opts.include_root:
yield unicode( path, 'utf-8' )
for root, dirs, files in os.walk(path):
for node in list(dirs):
if not opts.recurse and not opts.interactive:
dirs.remove(node)
if not opts.dirs:
continue
dirpath = join(root, node)
if dir_filters:
if not Dir.filter(dirpath, *dir_filters):
dirs.remove(node)
continue
#dirpath = os.path.join(root, node).replace(path,'').lstrip('/') +'/'
depth = pathdepth(dirpath.replace(path, ''))
if not os.path.exists(dirpath):
log.err("Error: reported non existant node %s", dirpath)
if node in dirs: dirs.remove(node)
continue
elif Klass.check_ignored(dirpath, opts):
if node in dirs: dirs.remove(node)
continue
elif not Klass.check_recurse(dirpath, opts):
if node in dirs:
dirs.remove(node)
# continue # exception to rule excluded == no yield
# caller can sort out wether they want entries to subpaths at this level
assert isinstance(dirpath, basestring)
try:
dirpath = unicode(dirpath)
except UnicodeDecodeError, e:
log.err("Ignored non-ascii/illegal filename %s", dirpath)
continue
assert isinstance(dirpath, unicode)
try:
dirpath.encode('ascii')
except UnicodeDecodeError, e:
log.err("Ignored non-ascii filename %s", dirpath)
continue
dirpath = Klass.decode_path(dirpath, opts)
yield dirpath
for leaf in list(files):
filepath = join(root, leaf)
if file_filters:
if not File.filter(filepath, *file_filters):
files.remove(leaf)
continue
if not os.path.exists(filepath):
log.err("Error: non existant leaf %s", filepath)
if opts.exists != None and not opts.exists:
if opts.files:
yield filepath
else:
files.remove(leaf)
continue
elif Klass.check_ignored(filepath, opts):
log.info("Ignored file %r", filepath)
files.remove(leaf)
continue
filepath = Klass.decode_path(filepath, opts)
if not opts.files: # XXX other types
continue
#try:
# filepath.encode('ascii')
#except UnicodeEncodeError, e:
# log.err("Ignored non-ascii/illegal filename %s", filepath)
# continue
yield filepath
import ujson
loads = ujson.loads
dumps = ujson.dumps
load = ujson.load
dump = ujson.dump
except Exception, e:
pass#log.warn("Failed loading ujson %r", e)
try:
import json as json_
log.debug("Using json")
loads = json_.loads
dumps = json_.dumps
load = json_.load
dump = json_.dump
except Exception, e:
pass#log.warn("Failed loading json %r", e)
if not loads:
log.err("No known json library installed. Plain Python printing.")
def require(self):
if not loads:
import sys
sys.exit(1)
