def ready(self): from indigo.models import initialise, Collection, User, Group logging.basicConfig(level=logging.WARNING) logging.getLogger("models").setLevel(logging.WARNING) logging.getLogger("dse.policies").setLevel(logging.WARNING) logging.getLogger("dse.cluster").setLevel(logging.WARNING) logging.getLogger("dse.cqlengine.management").setLevel(logging.WARNING) cfg = get_config(None) initialise(keyspace=cfg.get('KEYSPACE', 'indigo'), hosts=cfg.get('CASSANDRA_HOSTS', ('127.0.0.1', ))) # Try to get the root. It will create it if it doesn't exist root = Collection.get_root() # TODO: Review that at some point # Check that the graph vertices for users are still there User.check_graph_users()
def do_work(self): # if self.include_pattern: # include_pattern = self.include_pattern.lower() # matcher = lambda x: include_pattern in x.lower() # else: # matcher = lambda x: True timer = TimerCounter() root_collection = Collection.get_root() self.collection_cache["/"] = root_collection t0 = time.time() if self.include_pattern: start_dir = None for (path, dirs, files) in os.walk(self.folder, topdown=True, followlinks=True): for k in dirs: if self.include_pattern.lower() in k.lower(): start_dir = os.path.join(path, k) break if start_dir: break else: start_dir = self.folder if start_dir: print "STARTING AT ", start_dir else: print "No start dir matching {0} found ... giving up ".format( self.include_pattern) return None for (path, dirs, files) in os.walk(start_dir, topdown=True, followlinks=True): if '/.' in path: continue # Ignore .paths path = path.replace(self.folder, '') parent_path, name = split(path) t1 = time.time() msg = "Processing {0} - '{1}' Previous={2:02f}s , this={3} files".format( path, name, t1 - t0, len(files)) t0 = t1 logger.info("Processing {} - '{}'".format(path, name)) print(msg) if name: timer.enter('get-collection') # parent = self.get_collection(parent_path) current_collection = self.get_collection(path) if not current_collection: current_collection = self.create_collection( parent_path, # parent.path, name, path) timer.exit('get-collection') else: current_collection = root_collection # Now we can add the resources from self.folder + path for filename in files: fullpath = self.folder + path + '/' + filename if filename.startswith("."): continue if filename.endswith(SKIP): continue if not os.path.isfile(fullpath): continue # Extract information needed to create the new resources resc_dict = local_file_dict(fullpath) resc_dict["read_access"] = self.groups resc_dict["write_access"] = self.groups resc_dict["container"] = current_collection.path resc_dict['compress'] = self.compress context = { "fullpath": fullpath, "local_ip": self.local_ip, "path": path, "filename": filename, "user": self.user.name } timer.enter('push') self.create_entry(resc_dict, context, self.is_reference) timer.exit('push') timer.summary()