def ready(self):
from indigo.models import initialise, Collection, User, Group
logging.basicConfig(level=logging.WARNING)
logging.getLogger("models").setLevel(logging.WARNING)
logging.getLogger("dse.policies").setLevel(logging.WARNING)
logging.getLogger("dse.cluster").setLevel(logging.WARNING)
logging.getLogger("dse.cqlengine.management").setLevel(logging.WARNING)
cfg = get_config(None)
initialise(keyspace=cfg.get('KEYSPACE', 'indigo'),
hosts=cfg.get('CASSANDRA_HOSTS', ('127.0.0.1', )))
# Try to get the root. It will create it if it doesn't exist
root = Collection.get_root()
# TODO: Review that at some point
# Check that the graph vertices for users are still there
User.check_graph_users()
def do_work(self):
# if self.include_pattern:
# include_pattern = self.include_pattern.lower()
# matcher = lambda x: include_pattern in x.lower()
# else:
# matcher = lambda x: True
timer = TimerCounter()
root_collection = Collection.get_root()
self.collection_cache["/"] = root_collection
t0 = time.time()
if self.include_pattern:
start_dir = None
for (path, dirs, files) in os.walk(self.folder,
topdown=True,
followlinks=True):
for k in dirs:
if self.include_pattern.lower() in k.lower():
start_dir = os.path.join(path, k)
break
if start_dir:
break
else:
start_dir = self.folder
if start_dir:
print "STARTING AT ", start_dir
else:
print "No start dir matching {0} found ... giving up ".format(
self.include_pattern)
return None
for (path, dirs, files) in os.walk(start_dir,
topdown=True,
followlinks=True):
if '/.' in path:
continue # Ignore .paths
path = path.replace(self.folder, '')
parent_path, name = split(path)
t1 = time.time()
msg = "Processing {0} - '{1}' Previous={2:02f}s , this={3} files".format(
path, name, t1 - t0, len(files))
t0 = t1
logger.info("Processing {} - '{}'".format(path, name))
print(msg)
if name:
timer.enter('get-collection')
# parent = self.get_collection(parent_path)
current_collection = self.get_collection(path)
if not current_collection:
current_collection = self.create_collection(
parent_path, # parent.path,
name,
path)
timer.exit('get-collection')
else:
current_collection = root_collection
# Now we can add the resources from self.folder + path
for filename in files:
fullpath = self.folder + path + '/' + filename
if filename.startswith("."):
continue
if filename.endswith(SKIP):
continue
if not os.path.isfile(fullpath):
continue
# Extract information needed to create the new resources
resc_dict = local_file_dict(fullpath)
resc_dict["read_access"] = self.groups
resc_dict["write_access"] = self.groups
resc_dict["container"] = current_collection.path
resc_dict['compress'] = self.compress
context = {
"fullpath": fullpath,
"local_ip": self.local_ip,
"path": path,
"filename": filename,
"user": self.user.name
}
timer.enter('push')
self.create_entry(resc_dict, context, self.is_reference)
timer.exit('push')
timer.summary()
