def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle
store, which is actually a symlink to the target directory, which the
user has configured as the mountpoint for some desired partition.
If `target` is None, then make the `new_partition_name` the actual directory.
"""
if target is not None:
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions, new_partition_name)
print("Adding new partition as %s..." % new_partition_location, file=sys.stderr)
if target is None:
path_util.make_directory(new_partition_location)
else:
path_util.soft_link(target, new_partition_location)
# Where the bundles are stored
mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY)
try:
path_util.make_directory(mdata)
except Exception as e:
print(e, file=sys.stderr)
print(
"Could not make directory %s on partition %s, aborting" % (mdata, target),
file=sys.stderr,
)
sys.exit(1)
self.refresh_partitions()
print(
"Successfully added partition '%s' to the pool." % new_partition_name, file=sys.stderr
)
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions, new_partition_name)
print >>sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY)
try:
path_util.make_directory(mdata)
except Exception as e:
print >>sys.stderr, e
print >>sys.stderr, "Could not make directory %s on partition %s, aborting" % (
mdata,
target,
)
sys.exit(1)
self.nodes.append(new_partition_name)
print >>sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions,
new_partition_name)
print >> sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
mdata = os.path.join(new_partition_location,
MultiDiskBundleStore.DATA_SUBDIRECTORY)
try:
path_util.make_directory(mdata)
except Exception as e:
print >> sys.stderr, e
print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (
mdata,
target,
)
sys.exit(1)
self.nodes.append(new_partition_name)
print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
First, all bundles that are to be relocated onto the new partition are copied to a temp location to be resilient
against failures. After the copy is performed, the bundles are subsequently moved to the new partition, and finally
the original copy of the bundles are deleted from their old locations
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions, new_partition_name)
mtemp = os.path.join(target, MultiDiskBundleStore.TEMP_SUBDIRECTORY)
try:
path_util.make_directory(mtemp)
except:
print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (mtemp, target)
sys.exit(1)
self.ring.add_node(new_partition_name) # Add the node to the partition locations
delete_on_success = [] # Paths to bundles that will be deleted after the copy finishes successfully
print >> sys.stderr, "Marking bundles for placement on new partition %s (might take a while)" % new_partition_name
# For each bundle in the bundle store, check to see if any hash to the new partition. If so move them over
partitions, _ = path_util.ls(self.partitions)
for partition in partitions:
partition_abs_path = os.path.join(self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY)
bundles = reduce(lambda dirs, files: dirs + files, path_util.ls(partition_abs_path))
for bundle in bundles:
correct_partition = self.ring.get_node(bundle)
if correct_partition != partition:
# Reposition the node to the correct partition
from_path = os.path.join(self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY, bundle)
to_path = os.path.join(mtemp, bundle)
print >> sys.stderr, "copying %s to %s" % (from_path, to_path)
path_util.copy(from_path, to_path)
delete_on_success += [from_path]
print >> sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
# Atomically move the temp location to the new partition's mdata
new_mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY)
new_mtemp = os.path.join(new_partition_location, MultiDiskBundleStore.TEMP_SUBDIRECTORY)
path_util.rename(new_mtemp, new_mdata)
path_util.make_directory(new_mtemp)
# Go through and purge all of the originals at this time
print >> sys.stderr, "Cleaning up drives..."
for to_delete in delete_on_success:
path_util.remove(to_delete)
print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def initialize_store(self):
"""
Initializes the multi-disk bundle store.
"""
path_util.make_directory(self.partitions)
path_util.make_directory(self.mtemp)
# Create the default partition, if there are no partitions currently
if self.__get_num_partitions() == 0:
# Create a default partition that links to the codalab_home
path_util.make_directory(os.path.join(self.codalab_home, MultiDiskBundleStore.DATA_SUBDIRECTORY))
path_util.make_directory(os.path.join(self.codalab_home, MultiDiskBundleStore.TEMP_SUBDIRECTORY))
default_partition = os.path.join(self.partitions, 'default')
path_util.soft_link(self.codalab_home, default_partition)
def initialize_store(self):
"""
Initializes the multi-disk bundle store.
"""
path_util.make_directory(self.partitions)
path_util.make_directory(self.mtemp)
# Create the default partition, if there are no partitions currently
if self.__get_num_partitions() == 0:
# Create a default partition that links to the codalab_home
path_util.make_directory(
os.path.join(self.codalab_home,
MultiDiskBundleStore.DATA_SUBDIRECTORY))
default_partition = os.path.join(self.partitions, 'default')
path_util.soft_link(self.codalab_home, default_partition)
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
First, all bundles that are to be relocated onto the new partition are copied to a temp location to be resilient
against failures. After the copy is performed, the bundles are subsequently moved to the new partition, and finally
the original copy of the bundles are deleted from their old locations
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions,
new_partition_name)
mtemp = os.path.join(target, MultiDiskBundleStore.TEMP_SUBDIRECTORY)
try:
path_util.make_directory(mtemp)
except:
print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (
mtemp, target)
sys.exit(1)
self.ring.add_node(
new_partition_name) # Add the node to the partition locations
delete_on_success = [
] # Paths to bundles that will be deleted after the copy finishes successfully
print >> sys.stderr, "Marking bundles for placement on new partition %s (might take a while)" % new_partition_name
# For each bundle in the bundle store, check to see if any hash to the new partition. If so move them over
partitions, _ = path_util.ls(self.partitions)
for partition in partitions:
partition_abs_path = os.path.join(
self.partitions, partition,
MultiDiskBundleStore.DATA_SUBDIRECTORY)
bundles = reduce(lambda dirs, files: dirs + files,
path_util.ls(partition_abs_path))
for bundle in bundles:
correct_partition = self.ring.get_node(bundle)
if correct_partition != partition:
# Reposition the node to the correct partition
from_path = os.path.join(
self.partitions, partition,
MultiDiskBundleStore.DATA_SUBDIRECTORY, bundle)
to_path = os.path.join(mtemp, bundle)
print >> sys.stderr, "copying %s to %s" % (from_path,
to_path)
path_util.copy(from_path, to_path)
delete_on_success += [from_path]
print >> sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
# Atomically move the temp location to the new partition's mdata
new_mdata = os.path.join(new_partition_location,
MultiDiskBundleStore.DATA_SUBDIRECTORY)
new_mtemp = os.path.join(new_partition_location,
MultiDiskBundleStore.TEMP_SUBDIRECTORY)
path_util.rename(new_mtemp, new_mdata)
path_util.make_directory(new_mtemp)
# Go through and purge all of the originals at this time
print >> sys.stderr, "Cleaning up drives..."
for to_delete in delete_on_success:
path_util.remove(to_delete)
print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name