def fetch(Klass, filters=(), query=(), session='default', sa=None, exists=True): """ Return exactly one or none for filtered query. """ if not sa: sa = Klass.get_session(session) rs = None if query: q = sa.query(*query) else: q = sa.query(Klass) if filters: q = q.filter(*filters) try: rs = q.one() except NoResultFound as e: if exists: log.err("No results for %s.fetch(%r)", Klass.__name__, filters) raise e return rs
def update(self): now = datetime.datetime.now() if 'X-First-Seen' not in self.data: self.data['X-First-Seen'] = util.iso8601_datetime_format(now.timetuple()) envelope = ( #('X-Meta-Checksum', lambda x: self.get_meta_hash()), ('X-Last-Modified', util.last_modified_header), ('X-Last-Update', lambda x: util.iso8601_datetime_format(now.timetuple())), ('X-Last-Seen', lambda x: util.iso8601_datetime_format(now.timetuple())), ) for handlers in self.handlers, envelope: for header, handler in handlers: value = None try: value = handler(self.path) except Exception, e: traceback.print_exc() log.err("%s: %s", header, e) continue #print header, value if header in self.allow_multiple: if header not in self.data: self.data[header] = [] elif not isinstance(self.data[header], list): self.data[header] = [ self.data[header] ] self.data[header].append(value) else: self.data[header] = value
def run(self, mp_class, path, opts): mp = mp_class(self) # self.props[mp.cname] = mp.applies(self.data, path, opts) # if self.propstat[mp.cname] == 0: # return try: self.data[mp.cname] = mp.extract(self.data, path, opts) except Exception, e: log.err('Failed getting %s for %s: %s', mp.cname, path, e)
def add(self, name, prog, opts): """ Start with MP default(s), continue to discover. Put data into STAGE index. """ data = {} path = self.volume.pathname(name, prog.pwd) if not os.path.exists(path): log.err("Ignored %s", path) return if path in self.stage: data = self.stage.get(path) resolver = MetaResolver(self, data) resolver.discover(path, opts) print 'todo add', resolver.data
def rsr_tree(self, sa=None, *nodes): "Print a tree of nodes as nested lists" if not nodes: roots = sa.query(GroupNode)\ .filter( GroupNode.root == True, ).all() if not roots: log.err("No roots") else: roots = [] for node in nodes: group = GroupNode.find(( Node.name == node, )) if not group: log.warn(group) continue roots.append(group) for group in roots: self.execute( 'rsr_node_recurse', dict( group=group ) )
def all(Klass, filters=None, session='default', sa=None): """ Return all for filtered query. """ if not sa: sa = Klass.get_session(session) q = sa.query(Klass) if not filters and isinstance(filters, type(None)): if hasattr(Klass, 'default_filters'): filters = Klass.default_filters() if filters: for f in filters: q = q.filter(f) try: return q.all() except Exception as e: log.err("Error executing .all: %s", e) return []
def current_hostname(initialize=False, interactive=False): """ """ host = dict( name = get_hostname() ) hostnameId = host['name'].lower() return hostnameId; # FIXME: current_hostname hostname = None hostname_file = os.path.expanduser('~/.cllct/host') if os.path.exists(hostname_file): hostname = open(hostname_file).read().strip() elif initialize: hostname = socket.gethostname() assert not isinstance(hostname, (tuple, list)), hostname log.debug(hostname) hostnames = socket.gethostbyaddr(hostname) while True: if socket.getfqdn() != hostname: hostname = hostnames[0] +"." else: log.err("FQDN is same as hostname") # cannot figure out what host to use while interactive: hostname = prompt_choice_with_input("Which? ", hostnames[1]) if hostname: break #if not interactive: # raise ValueError("") if hostname: try: nameinfo((hostname, 80)) except Exception as e: print('Warning: Cannot resolve FQDN', e) open(hostname_file, 'w+').write(hostname) print("Stored %s in %s" % (hostname, hostname_file)) break return hostname
def lnd_tag(opts=None, sa=None, ur=None, pwd=None): """ Experiment, interactive interface. Tagging. """ log.debug("{bblack}lnd{bwhite}:tag{default}") if not pwd: log.err("Not initialized") yield 1 tags = {} if '' not in tags: tags[''] = 'Root' FS_Path_split = re.compile('[\/\.\+,]+').split log.info("{bblack}Tagging paths in {green}%s{default}", os.path.realpath('.') + os.sep) try: for root, dirs, files in os.walk(pwd.local_path): for name in files + dirs: log.info("{bblack}Typing tags for {green}%s{default}", name) path = FS_Path_split(os.path.join(root, name)) for tag in path: yield # Ask about each new tag, TODO: or rename, fuzzy match. if tag not in tags: type = raw_input('%s%s%s:?' % ( log.palette['yellow'], tag, log.palette['default']) ) if not type: type = 'Tag' tags[tag] = type log.info(''.join( [ "{bwhite} %s:{green}%s{default}" % (tag, name) for tag in path if tag in tags] )) except KeyboardInterrupt as e: log.err(e) yield 1
def walk(self, path, max_depth=-1): # XXX: maybe rewrite to Dir.walk """ Walk all files that may have a metafile, and notice any metafile(-like) neighbors? """ for root, nodes, leafs in os.walk(path): for node in list(nodes): dirpath = os.path.join(root, node) if not os.path.exists(dirpath): log.err("Error: reported non existant node %s", dirpath) nodes.remove(node) continue depth = dirpath.replace(path,'').strip('/').count('/') if fs.Dir.ignored(dirpath): log.warn("Ignored directory %r", dirpath) nodes.remove(node) elif max_depth != -1: if depth >= max_depth: nodes.remove(node) for leaf in leafs: cleaf = os.path.join(root, leaf) if not os.path.exists(dirpath): log.err("Error: non existant leaf %s", cleaf) continue if not os.path.isfile(cleaf) or os.path.islink(cleaf): #log.warn("Ignored non-regular file %r", cleaf) continue if fs.File.ignored(cleaf): #log.warn("Ignored file %r", cleaf) continue if Metafile.is_metafile(cleaf, strict=False): if not Metafile(cleaf).path: log.err("Metafile without resource file") else: yield cleaf
def walk(Klass, path, opts=walk_opts, filters=(None,None)): """ Build on os.walk, this goes over all directories and other paths non-recursively. It returns all full paths according to walk-opts. FIXME: could, but does not, yield INode subtype instances. XXX: filters, see dev_treemap """ # if not opts.descend: # return self.walkRoot( path, opts=opts, filters=filters ) if not isinstance(opts, confparse.Values): opts_ = confparse.Values(Klass.walk_opts) opts_.update(opts) opts = opts_ else: opts = confparse.Values(opts.copy()) # FIXME: validate/process opts or put filter somewhere if opts.max_depth > 0: assert opts.recurse exclusive( opts, 'dirs files symlinks links pipes blockdevs' ) assert isinstance(path, basestring), (path, path.__class__) dirpath = None file_filters, dir_filters = filters if not os.path.isdir( path ): if opts.exists > -1: log.err("Cannot walk non-dir path with opt.exists. ") else: yield path else: if opts.dirs and opts.include_root: yield unicode( path, 'utf-8' ) for root, dirs, files in os.walk(path): for node in list(dirs): if not opts.recurse and not opts.interactive: dirs.remove(node) if not opts.dirs: continue dirpath = join(root, node) if dir_filters: if not Dir.filter(dirpath, *dir_filters): dirs.remove(node) continue #dirpath = os.path.join(root, node).replace(path,'').lstrip('/') +'/' depth = pathdepth(dirpath.replace(path, '')) if not os.path.exists(dirpath): log.err("Error: reported non existant node %s", dirpath) if node in dirs: dirs.remove(node) continue elif Klass.check_ignored(dirpath, opts): if node in dirs: dirs.remove(node) continue elif not Klass.check_recurse(dirpath, opts): if node in dirs: dirs.remove(node) # continue # exception to rule excluded == no yield # caller can sort out wether they want entries to subpaths at this level assert isinstance(dirpath, basestring) try: dirpath = unicode(dirpath) except UnicodeDecodeError, e: log.err("Ignored non-ascii/illegal filename %s", dirpath) continue assert isinstance(dirpath, unicode) try: dirpath.encode('ascii') except UnicodeDecodeError, e: log.err("Ignored non-ascii filename %s", dirpath) continue dirpath = Klass.decode_path(dirpath, opts) yield dirpath for leaf in list(files): filepath = join(root, leaf) if file_filters: if not File.filter(filepath, *file_filters): files.remove(leaf) continue if not os.path.exists(filepath): log.err("Error: non existant leaf %s", filepath) if opts.exists != None and not opts.exists: if opts.files: yield filepath else: files.remove(leaf) continue elif Klass.check_ignored(filepath, opts): log.info("Ignored file %r", filepath) files.remove(leaf) continue filepath = Klass.decode_path(filepath, opts) if not opts.files: # XXX other types continue #try: # filepath.encode('ascii') #except UnicodeEncodeError, e: # log.err("Ignored non-ascii/illegal filename %s", filepath) # continue yield filepath
import ujson loads = ujson.loads dumps = ujson.dumps load = ujson.load dump = ujson.dump except Exception, e: pass#log.warn("Failed loading ujson %r", e) try: import json as json_ log.debug("Using json") loads = json_.loads dumps = json_.dumps load = json_.load dump = json_.dump except Exception, e: pass#log.warn("Failed loading json %r", e) if not loads: log.err("No known json library installed. Plain Python printing.") def require(self): if not loads: import sys sys.exit(1)