def test_make_directory(self, mock_os):
'''
Check that make_directory calls normalize and then creates the directory.
'''
mock_os.path.join = os.path.join
path_util.make_directory(self.test_path)
self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls)
def start_bundle(self, bundle, bundle_store, parent_dict, username):
"""
Start a bundle in the background.
"""
if self.bundle != None:
return None
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
path_util.make_directory(temp_dir)
# We don't follow symlinks (for consistency with remote
# machine, where it is more secure, so people can't make us
# copy random files on the system). Of course in local mode,
# if some of those symlinks are absolute, the run can
# read/write those locations. But we're not sandboxed, so
# anything could happen. The dependencies are copied, so in
# practice, this is not a bit worry.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir)
print >> sys.stderr, "LocalMachine.start_bundle: copying dependencies of %s to %s" % (bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
script_file = temp_dir + ".sh"
with open(script_file, "w") as f:
f.write("cd %s &&\n" % temp_dir)
f.write("(%s) > stdout 2>stderr\n" % bundle.command)
# Use stdbuf (if it exists) to turn off buffering so we get real-time feedback.
if os.path.exists("/usr/bin/stdbuf"):
process = subprocess.Popen("/usr/bin/stdbuf -o0 bash " + script_file, shell=True)
else:
process = subprocess.Popen("bash " + script_file, shell=True)
self.bundle = bundle
self.temp_dir = temp_dir
self.process = process
return {"bundle": bundle, "temp_dir": temp_dir, "job_handle": str(process.pid)}
def codalab_home(self):
from codalab.lib import path_util
# Default to this directory in the user's home directory.
# In the future, allow customization based on.
result = path_util.normalize("~/.codalab")
path_util.make_directory(result)
return result
def start_bundle(self, bundle):
"""
Run the given bundle using an available Machine.
Return whether something was started.
"""
# Check that we're running a bundle in the QUEUED state.
state_message = "Unexpected bundle state: %s" % (bundle.state,)
precondition(bundle.state == State.QUEUED, state_message)
data_hash_message = "Unexpected bundle data_hash: %s" % (bundle.data_hash,)
precondition(bundle.data_hash is None, data_hash_message)
# Run the bundle.
with self.profile("Running bundle..."):
started = False
if isinstance(bundle, RunBundle):
try:
# Get the username of the bundle
results = self.auth_handler.get_users("ids", [bundle.owner_id])
if results.get(bundle.owner_id):
username = results[bundle.owner_id].name
else:
username = str(bundle.owner_id)
status = self.machine.start_bundle(
bundle, self.bundle_store, self.get_parent_dict(bundle), username
)
if status != None:
status["started"] = int(time.time())
started = True
except Exception as e:
# If there's an exception, we just make the bundle fail
# (even if it's not the bundle's fault).
real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(real_path)
status = {
"bundle": bundle,
"success": False,
"failure_message": "Internal error: " + str(e),
"temp_dir": real_path,
}
print "=== INTERNAL ERROR: %s" % e
started = True # Force failing
traceback.print_exc()
else: # MakeBundle
started = True
if started:
print "-- START BUNDLE: %s" % (bundle,)
self._update_events_log("start_bundle", bundle, (bundle.uuid,))
# If we have a MakeBundle, then just process it immediately.
if isinstance(bundle, MakeBundle):
real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(real_path)
status = {"bundle": bundle, "success": True, "temp_dir": real_path}
# Update database
if started:
self.update_running_bundle(status)
return started
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle
store, which is actually a symlink to the target directory, which the
user has configured as the mountpoint for some desired partition.
If `target` is None, then make the `new_partition_name` the actual directory.
"""
if target is not None:
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions, new_partition_name)
print("Adding new partition as %s..." % new_partition_location, file=sys.stderr)
if target is None:
path_util.make_directory(new_partition_location)
else:
path_util.soft_link(target, new_partition_location)
# Where the bundles are stored
mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY)
try:
path_util.make_directory(mdata)
except Exception as e:
print(e, file=sys.stderr)
print(
"Could not make directory %s on partition %s, aborting" % (mdata, target),
file=sys.stderr,
)
sys.exit(1)
self.refresh_partitions()
print(
"Successfully added partition '%s' to the pool." % new_partition_name, file=sys.stderr
)
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions, new_partition_name)
print >>sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY)
try:
path_util.make_directory(mdata)
except Exception as e:
print >>sys.stderr, e
print >>sys.stderr, "Could not make directory %s on partition %s, aborting" % (
mdata,
target,
)
sys.exit(1)
self.nodes.append(new_partition_name)
print >>sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def test_make_directory(self, mock_os):
'''
Check that make_directory calls normalize and then creates the directory.
'''
mock_os.path.join = os.path.join
path_util.make_directory(self.test_path)
self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls)
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions,
new_partition_name)
print >> sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
mdata = os.path.join(new_partition_location,
MultiDiskBundleStore.DATA_SUBDIRECTORY)
try:
path_util.make_directory(mdata)
except Exception as e:
print >> sys.stderr, e
print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (
mdata,
target,
)
sys.exit(1)
self.nodes.append(new_partition_name)
print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def codalab_home(self):
from codalab.lib import path_util
# Default to this directory in the user's home directory.
# In the future, allow customization based on.
home = os.getenv('CODALAB_HOME', '~/.codalab')
home = path_util.normalize(home)
path_util.make_directory(home)
return home
def codalab_home(self):
from codalab.lib import path_util
# Default to this directory in the user's home directory.
# In the future, allow customization based on.
home = os.getenv('CODALAB_HOME', '~/.codalab')
home = path_util.normalize(home)
path_util.make_directory(home)
return home
def setUpClass(cls):
cls.test_root = path_util.normalize("~/.codalab_tests")
path_util.make_directory(cls.test_root)
cls.bundle_store = BundleStore(cls.test_root)
cls.model = SQLiteModel(cls.test_root)
users = [User('root', 0), User('user1', 1), User('user2', 2), User('user4', 4)]
cls.auth_handler = MockAuthHandler(users)
cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, cls.auth_handler)
def setUpClass(cls):
cls.test_root = path_util.normalize("~/.codalab_tests")
path_util.make_directory(cls.test_root)
cls.bundle_store = BundleStore(cls.test_root)
cls.model = SQLiteModel("sqlite:///{}".format(os.path.join(cls.test_root, 'bundle.db')), {})
cls.model.root_user_id = '0'
users = [User('root', '0'), User('user1', '1'), User('user2', '2'), User('user4', '4')]
cls.auth_handler = MockAuthHandler(users)
cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, cls.auth_handler, verbose=1)
def __init__(self, bundle_model, codalab_home):
BundleStore.__init__(self, bundle_model, codalab_home)
self.partitions = os.path.join(self.codalab_home, 'partitions')
path_util.make_directory(self.partitions)
self.refresh_partitions()
if self.__get_num_partitions() == 0: # Ensure at least one partition exists.
self.add_partition(None, 'default')
self.lru_cache = OrderedDict()
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
First, all bundles that are to be relocated onto the new partition are copied to a temp location to be resilient
against failures. After the copy is performed, the bundles are subsequently moved to the new partition, and finally
the original copy of the bundles are deleted from their old locations
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions, new_partition_name)
mtemp = os.path.join(target, MultiDiskBundleStore.TEMP_SUBDIRECTORY)
try:
path_util.make_directory(mtemp)
except:
print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (mtemp, target)
sys.exit(1)
self.ring.add_node(new_partition_name) # Add the node to the partition locations
delete_on_success = [] # Paths to bundles that will be deleted after the copy finishes successfully
print >> sys.stderr, "Marking bundles for placement on new partition %s (might take a while)" % new_partition_name
# For each bundle in the bundle store, check to see if any hash to the new partition. If so move them over
partitions, _ = path_util.ls(self.partitions)
for partition in partitions:
partition_abs_path = os.path.join(self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY)
bundles = reduce(lambda dirs, files: dirs + files, path_util.ls(partition_abs_path))
for bundle in bundles:
correct_partition = self.ring.get_node(bundle)
if correct_partition != partition:
# Reposition the node to the correct partition
from_path = os.path.join(self.partitions, partition, MultiDiskBundleStore.DATA_SUBDIRECTORY, bundle)
to_path = os.path.join(mtemp, bundle)
print >> sys.stderr, "copying %s to %s" % (from_path, to_path)
path_util.copy(from_path, to_path)
delete_on_success += [from_path]
print >> sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
# Atomically move the temp location to the new partition's mdata
new_mdata = os.path.join(new_partition_location, MultiDiskBundleStore.DATA_SUBDIRECTORY)
new_mtemp = os.path.join(new_partition_location, MultiDiskBundleStore.TEMP_SUBDIRECTORY)
path_util.rename(new_mtemp, new_mdata)
path_util.make_directory(new_mtemp)
# Go through and purge all of the originals at this time
print >> sys.stderr, "Cleaning up drives..."
for to_delete in delete_on_success:
path_util.remove(to_delete)
print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def codalab_home(self):
from codalab.lib import path_util
# Default to this directory in the user's home directory.
# In the future, allow customization based on.
home = os.getenv('CODALAB_HOME', '~/.codalab')
home = path_util.normalize(home)
path_util.make_directory(home)
# Global setting! Make temp directory the same as the bundle store
# temporary directory. The default /tmp generally doesn't have enough
# space.
tempfile.tempdir = os.path.join(home, BundleStore.TEMP_SUBDIRECTORY)
return home
def codalab_home(self):
from codalab.lib import path_util
# Default to this directory in the user's home directory.
# In the future, allow customization based on.
home = os.getenv('CODALAB_HOME', '~/.codalab')
home = path_util.normalize(home)
path_util.make_directory(home)
# Global setting! Make temp directory the same as the bundle store
# temporary directory. The default /tmp generally doesn't have enough
# space.
tempfile.tempdir = os.path.join(home, BundleStore.TEMP_SUBDIRECTORY)
return home
def start_bundle(self, bundle):
'''
Run the given bundle using an available Machine.
Return whether something was started.
'''
# Check that we're running a bundle in the QUEUED state.
state_message = 'Unexpected bundle state: %s' % (bundle.state,)
precondition(bundle.state == State.QUEUED, state_message)
data_hash_message = 'Unexpected bundle data_hash: %s' % (bundle.data_hash,)
precondition(bundle.data_hash is None, data_hash_message)
# Run the bundle.
with self.profile('Running bundle...'):
started = False
if isinstance(bundle, RunBundle):
try:
# Get the username of the bundle
results = self.auth_handler.get_users('ids', [bundle.owner_id])
if results.get(bundle.owner_id):
username = results[bundle.owner_id].name
else:
username = str(bundle.owner_id)
status = self.machine.start_bundle(bundle, self.bundle_store, self.get_parent_dict(bundle), username)
if status != None:
status['started'] = int(time.time())
started = True
except Exception as e:
# If there's an exception, we just make the bundle fail
# (even if it's not the bundle's fault).
real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(real_path)
status = {'bundle': bundle, 'success': False, 'failure_message': 'Internal error: ' + str(e), 'temp_dir': real_path}
print '=== INTERNAL ERROR: %s' % e
started = True # Force failing
traceback.print_exc()
else: # MakeBundle
started = True
if started:
print '-- START BUNDLE: %s' % (bundle,)
self._update_events_log('start_bundle', bundle, (bundle.uuid,))
# If we have a MakeBundle, then just process it immediately.
if isinstance(bundle, MakeBundle):
real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(real_path)
status = {'bundle': bundle, 'success': True, 'temp_dir': real_path}
# Update database
if started:
self.update_running_bundle(status)
return started
def setUpClass(cls):
cls.test_root = path_util.normalize("~/.codalab_tests")
path_util.make_directory(cls.test_root)
cls.bundle_store = MultiDiskBundleStore(cls.test_root)
cls.model = SQLiteModel("sqlite:///{}".format(os.path.join(cls.test_root, 'bundle.db')),
{'time_quota': 1e12, 'disk_quota': 1e12})
cls.model.root_user_id = '0'
users = [User('root', '0'), User('user1', '1'), User('user2', '2'), User('user4', '4')]
cls.auth_handler = MockAuthHandler(users)
for user in users:
cls.model.add_user(user.name, user.name + '@codalab.org', '',
user_id=user.unique_id, is_verified=True)
cls.client = LocalBundleClient('local', cls.bundle_store, cls.model, None, None, None, cls.auth_handler, verbose=1)
def __init__(self, codalab_manager, torque_config):
assert(codalab_manager.worker_model().shared_file_system)
self._torque_ssh_host = torque_config['ssh_host']
self._torque_bundle_service_url = torque_config['bundle_service_url']
self._torque_password_file = torque_config['password_file']
self._torque_log_dir = torque_config['log_dir']
path_util.make_directory(self._torque_log_dir)
if 'worker_code_dir' in torque_config:
self._torque_worker_code_dir = torque_config['worker_code_dir']
else:
codalab_cli = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
self._torque_worker_code_dir = os.path.join(codalab_cli, 'worker')
self._last_delete_attempt = {}
def start_bundle(self, bundle):
'''
Run the given bundle using an available Machine.
Return whether something was started.
'''
# Check that we're running a bundle in the RUNNING state.
state_message = 'Unexpected bundle state: %s' % (bundle.state,)
precondition(bundle.state == State.RUNNING, state_message)
data_hash_message = 'Unexpected bundle data_hash: %s' % (bundle.data_hash,)
precondition(bundle.data_hash is None, data_hash_message)
# Run the bundle.
with self.profile('Running bundle...'):
started = False
if isinstance(bundle, RunBundle):
try:
# Get the username of the bundle
results = self.auth_handler.get_users('ids', [bundle.owner_id])
if results.get(bundle.owner_id):
username = results[bundle.owner_id].name
else:
username = str(bundle.owner_id)
status = self.machine.start_bundle(bundle, self.bundle_store, self.get_parent_dict(bundle), username)
if status != None:
started = True
except Exception as e:
# If there's an exception, we just make the bundle fail
# (even if it's not the bundle's fault).
temp_dir = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(temp_dir)
status = {'bundle': bundle, 'success': False, 'failure_message': str(e), 'temp_dir': temp_dir}
print '=== INTERNAL ERROR: %s' % e
started = True # Force failing
traceback.print_exc()
else: # MakeBundle
started = True
if started: print '-- START BUNDLE: %s' % (bundle,)
# If we have a MakeBundle, then just process it immediately.
if isinstance(bundle, MakeBundle):
temp_dir = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(temp_dir)
status = {'bundle': bundle, 'success': True, 'temp_dir': temp_dir}
# Update database
if started:
self.update_running_bundle(status)
return started
def test_make_directory_if_exists(self, mock_os):
'''
Check that make_directory still works if the directory exists.
'''
mock_os.path.join = os.path.join
failures = [0]
def mkdir_when_directory_exists(path):
failures[0] += 1
error = OSError()
error.errno = errno.EEXIST
raise error
mock_os.mkdir.side_effect = mkdir_when_directory_exists
path_util.make_directory(self.test_path)
self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls)
self.assertEqual(failures[0], 1)
def __init__(self, codalab_manager, torque_config):
assert(codalab_manager.worker_model().shared_file_system)
self._torque_ssh_host = torque_config['ssh_host']
self._torque_bundle_service_url = torque_config['bundle_service_url']
self._torque_password_file = torque_config['password_file']
self._torque_log_dir = torque_config['log_dir']
self._torque_min_seconds_between_qsub = torque_config.get('min_seconds_between_qsub', 0)
path_util.make_directory(self._torque_log_dir)
if 'worker_code_dir' in torque_config:
self._torque_worker_code_dir = torque_config['worker_code_dir']
else:
codalab_cli = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
self._torque_worker_code_dir = os.path.join(codalab_cli, 'worker')
self._last_delete_attempt = {}
self._last_qsub_time = 0
def setUpClass(cls):
cls.test_root = path_util.normalize("~/.codalab_tests")
path_util.make_directory(cls.test_root)
cls.bundle_store = BundleStore(cls.test_root, [])
cls.model = SQLiteModel(cls.test_root)
cls.model.root_user_id = '0'
users = [
User('root', '0'),
User('user1', '1'),
User('user2', '2'),
User('user4', '4')
]
cls.auth_handler = MockAuthHandler(users)
cls.client = LocalBundleClient('local',
cls.bundle_store,
cls.model,
cls.auth_handler,
verbose=1)
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Start a bundle in the background.
'''
if self.bundle != None: return None
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
path_util.make_directory(temp_dir)
# We don't follow symlinks (for consistency with remote
# machine, where it is more secure, so people can't make us
# copy random files on the system). Of course in local mode,
# if some of those symlinks are absolute, the run can
# read/write those locations. But we're not sandboxed, so
# anything could happen. The dependencies are copied, so in
# practice, this is not a bit worry.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict,
temp_dir)
print >> sys.stderr, 'LocalMachine.start_bundle: copying dependencies of %s to %s' % (
bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
script_file = temp_dir + '.sh'
with open(script_file, 'w') as f:
f.write("cd %s &&\n" % temp_dir)
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
# Use stdbuf (if it exists) to turn off buffering so we get real-time feedback.
if os.path.exists('/usr/bin/stdbuf'):
process = subprocess.Popen("/usr/bin/stdbuf -o0 bash " +
script_file,
shell=True)
else:
process = subprocess.Popen("bash " + script_file, shell=True)
self.bundle = bundle
self.temp_dir = temp_dir
self.process = process
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': str(process.pid)
}
def test_make_directory_with_failures(self, mock_os):
'''
Check that make_directory still works if the directory exists.
'''
mock_os.path.join = os.path.join
def mkdir_with_other_failure(path):
raise OSError()
mock_os.mkdir.reset_mock()
mock_os.mkdir.side_effect = mkdir_with_other_failure
self.assertRaises(OSError, lambda: path_util.make_directory(self.test_path))
self.assertEqual(mock_os.mkdir.call_args_list, self.mkdir_calls)
def setUpClass(cls):
cls.test_root = path_util.normalize("~/.codalab_tests")
path_util.make_directory(cls.test_root)
cls.bundle_store = MultiDiskBundleStore(cls.test_root)
cls.model = SQLiteModel(
"sqlite:///{}".format(os.path.join(cls.test_root, 'bundle.db')),
{})
cls.model.root_user_id = '0'
users = [
User('root', '0'),
User('user1', '1'),
User('user2', '2'),
User('user4', '4')
]
cls.auth_handler = MockAuthHandler(users)
cls.client = LocalBundleClient('local',
cls.bundle_store,
cls.model,
None,
cls.auth_handler,
verbose=1)
def initialize_store(self):
"""
Initializes the multi-disk bundle store.
"""
path_util.make_directory(self.partitions)
path_util.make_directory(self.mtemp)
# Create the default partition, if there are no partitions currently
if self.__get_num_partitions() == 0:
# Create a default partition that links to the codalab_home
path_util.make_directory(os.path.join(self.codalab_home, MultiDiskBundleStore.DATA_SUBDIRECTORY))
path_util.make_directory(os.path.join(self.codalab_home, MultiDiskBundleStore.TEMP_SUBDIRECTORY))
default_partition = os.path.join(self.partitions, 'default')
path_util.soft_link(self.codalab_home, default_partition)
def initialize_store(self):
"""
Initializes the multi-disk bundle store.
"""
path_util.make_directory(self.partitions)
path_util.make_directory(self.mtemp)
# Create the default partition, if there are no partitions currently
if self.__get_num_partitions() == 0:
# Create a default partition that links to the codalab_home
path_util.make_directory(
os.path.join(self.codalab_home,
MultiDiskBundleStore.DATA_SUBDIRECTORY))
path_util.make_directory(
os.path.join(self.codalab_home,
MultiDiskBundleStore.TEMP_SUBDIRECTORY))
default_partition = os.path.join(self.partitions, 'default')
path_util.soft_link(self.codalab_home, default_partition)
def make_directories(self):
'''
Create the data, and temp directories for this BundleStore.
'''
for path in (self.data, self.temp):
path_util.make_directory(path)
def upload_to_bundle_store(self, bundle, sources, follow_symlinks, exclude_patterns, remove_sources, git, unpack, simplify_archives):
"""
Uploads contents for the given bundle to the bundle store.
|sources|: specifies the locations of the contents to upload. Each element is
either a URL, a local path or a tuple (filename, file-like object).
|follow_symlinks|: for local path(s), whether to follow (resolve) symlinks,
but only if remove_sources is False.
|exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o),
but only if remove_sources is False.
|remove_sources|: for local path(s), whether |sources| should be removed
|git|: for URLs, whether |source| is a git repo to clone.
|unpack|: for each source in |sources|, whether to unpack it if it's an archive.
|simplify_archives|: whether to simplify unpacked archives so that if they
contain a single file, the final path is just that file,
not a directory containing that file.
If |sources| contains one source, then the bundle contents will be that source.
Otherwise, the bundle contents will be a directory with each of the sources.
Exceptions:
- If |git|, then each source is replaced with the result of running 'git clone |source|'
- If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source.
"""
bundle_path = self._bundle_store.get_bundle_location(bundle.uuid)
try:
path_util.make_directory(bundle_path)
# Note that for uploads with a single source, the directory
# structure is simplified at the end.
for source in sources:
is_url, is_local_path, is_fileobj, filename = self._interpret_source(source)
source_output_path = os.path.join(bundle_path, filename)
if is_url:
if git:
source_output_path = file_util.strip_git_ext(source_output_path)
file_util.git_clone(source, source_output_path)
else:
file_util.download_url(source, source_output_path)
if unpack and self._can_unpack_file(source_output_path):
self._unpack_file(
source_output_path, zip_util.strip_archive_ext(source_output_path),
remove_source=True, simplify_archive=simplify_archives)
elif is_local_path:
source_path = path_util.normalize(source)
path_util.check_isvalid(source_path, 'upload')
if unpack and self._can_unpack_file(source_path):
self._unpack_file(
source_path, zip_util.strip_archive_ext(source_output_path),
remove_source=remove_sources, simplify_archive=simplify_archives)
elif remove_sources:
path_util.rename(source_path, source_output_path)
else:
path_util.copy(source_path, source_output_path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
elif is_fileobj:
if unpack and zip_util.path_is_archive(filename):
self._unpack_fileobj(
source[0], source[1],
zip_util.strip_archive_ext(source_output_path),
simplify_archive=simplify_archives)
else:
with open(source_output_path, 'wb') as out:
shutil.copyfileobj(source[1], out)
if len(sources) == 1:
self._simplify_directory(bundle_path)
except:
if os.path.exists(bundle_path):
path_util.remove(bundle_path)
raise
from codalab.lib.codalab_manager import CodaLabManager
from codalab.lib import path_util
dry_run = False if len(sys.argv) > 1 and sys.argv[1] == '-f' else True
manager = CodaLabManager()
model = manager.model()
CODALAB_HOME = manager.codalab_home
"""Move data/ directory over to a temp area, and create a staging tree for uuid-based storage"""
DATA_DIR = os.path.join(CODALAB_HOME, 'data')
FINAL_LOCATION = os.path.join(CODALAB_HOME, 'bundles')
if not dry_run:
path_util.make_directory(FINAL_LOCATION)
"""For each data hash, get a list of all bundles that have that hash, and make a copy of the bundle in the staging
area under the UUID for the bundle."""
data_hashes = reduce(lambda x,y: x+y, path_util.ls(DATA_DIR))
for data_hash in data_hashes:
orig_location = os.path.join(DATA_DIR, data_hash)
bundles_with_hash = model.batch_get_bundles(data_hash=data_hash)
# We'd prefer renaming bundles to making copies, but because we are converting from deduplicated storage
# we need to make sure that we only perform renames if we map 1:1 UUID->Hash.
rename_allowed = len(bundles_with_hash) <= 1
for bundle in bundles_with_hash:
# Build the command to be executed in a subshell
uuid = bundle.uuid
copy_location = os.path.join(FINAL_LOCATION, uuid)
def upload_to_bundle_store(
self,
bundle,
sources,
follow_symlinks,
exclude_patterns,
remove_sources,
git,
unpack,
simplify_archives,
):
"""
Uploads contents for the given bundle to the bundle store.
|sources|: specifies the locations of the contents to upload. Each element is
either a URL, a local path or a tuple (filename, binary file-like object).
|follow_symlinks|: for local path(s), whether to follow (resolve) symlinks,
but only if remove_sources is False.
|exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o),
but only if remove_sources is False.
|remove_sources|: for local path(s), whether |sources| should be removed
|git|: for URLs, whether |source| is a git repo to clone.
|unpack|: for each source in |sources|, whether to unpack it if it's an archive.
|simplify_archives|: whether to simplify unpacked archives so that if they
contain a single file, the final path is just that file,
not a directory containing that file.
If |sources| contains one source, then the bundle contents will be that source.
Otherwise, the bundle contents will be a directory with each of the sources.
Exceptions:
- If |git|, then each source is replaced with the result of running 'git clone |source|'
- If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source.
"""
exclude_patterns = (self._default_exclude_patterns +
exclude_patterns if exclude_patterns else
self._default_exclude_patterns)
bundle_link_url = getattr(bundle.metadata, "link_url", None)
if bundle_link_url:
# Don't do anything for linked bundles.
return
bundle_path = self._bundle_store.get_bundle_location(bundle.uuid)
try:
path_util.make_directory(bundle_path)
# Note that for uploads with a single source, the directory
# structure is simplified at the end.
for source in sources:
is_url, is_local_path, is_fileobj, filename = self._interpret_source(
source)
source_output_path = os.path.join(bundle_path, filename)
if is_url:
if git:
source_output_path = file_util.strip_git_ext(
source_output_path)
file_util.git_clone(source, source_output_path)
else:
file_util.download_url(source, source_output_path)
if unpack and self._can_unpack_file(
source_output_path):
self._unpack_file(
source_output_path,
zip_util.strip_archive_ext(source_output_path),
remove_source=True,
simplify_archive=simplify_archives,
)
elif is_local_path:
source_path = path_util.normalize(source)
path_util.check_isvalid(source_path, 'upload')
if unpack and self._can_unpack_file(source_path):
self._unpack_file(
source_path,
zip_util.strip_archive_ext(source_output_path),
remove_source=remove_sources,
simplify_archive=simplify_archives,
)
elif remove_sources:
path_util.rename(source_path, source_output_path)
else:
path_util.copy(
source_path,
source_output_path,
follow_symlinks=follow_symlinks,
exclude_patterns=exclude_patterns,
)
elif is_fileobj:
if unpack and zip_util.path_is_archive(filename):
self._unpack_fileobj(
source[0],
source[1],
zip_util.strip_archive_ext(source_output_path),
simplify_archive=simplify_archives,
)
else:
with open(source_output_path, 'wb') as out:
shutil.copyfileobj(source[1], out)
if len(sources) == 1:
self._simplify_directory(bundle_path)
except:
if os.path.exists(bundle_path):
path_util.remove(bundle_path)
raise
def add_partition(self, target, new_partition_name):
"""
MultiDiskBundleStore specific method. Add a new partition to the bundle store. The "target" is actually a symlink to
the target directory, which the user has configured as the mountpoint for some desired partition.
First, all bundles that are to be relocated onto the new partition are copied to a temp location to be resilient
against failures. After the copy is performed, the bundles are subsequently moved to the new partition, and finally
the original copy of the bundles are deleted from their old locations
"""
target = os.path.abspath(target)
new_partition_location = os.path.join(self.partitions,
new_partition_name)
mtemp = os.path.join(target, MultiDiskBundleStore.TEMP_SUBDIRECTORY)
try:
path_util.make_directory(mtemp)
except:
print >> sys.stderr, "Could not make directory %s on partition %s, aborting" % (
mtemp, target)
sys.exit(1)
self.ring.add_node(
new_partition_name) # Add the node to the partition locations
delete_on_success = [
] # Paths to bundles that will be deleted after the copy finishes successfully
print >> sys.stderr, "Marking bundles for placement on new partition %s (might take a while)" % new_partition_name
# For each bundle in the bundle store, check to see if any hash to the new partition. If so move them over
partitions, _ = path_util.ls(self.partitions)
for partition in partitions:
partition_abs_path = os.path.join(
self.partitions, partition,
MultiDiskBundleStore.DATA_SUBDIRECTORY)
bundles = reduce(lambda dirs, files: dirs + files,
path_util.ls(partition_abs_path))
for bundle in bundles:
correct_partition = self.ring.get_node(bundle)
if correct_partition != partition:
# Reposition the node to the correct partition
from_path = os.path.join(
self.partitions, partition,
MultiDiskBundleStore.DATA_SUBDIRECTORY, bundle)
to_path = os.path.join(mtemp, bundle)
print >> sys.stderr, "copying %s to %s" % (from_path,
to_path)
path_util.copy(from_path, to_path)
delete_on_success += [from_path]
print >> sys.stderr, "Adding new partition as %s..." % new_partition_location
path_util.soft_link(target, new_partition_location)
# Atomically move the temp location to the new partition's mdata
new_mdata = os.path.join(new_partition_location,
MultiDiskBundleStore.DATA_SUBDIRECTORY)
new_mtemp = os.path.join(new_partition_location,
MultiDiskBundleStore.TEMP_SUBDIRECTORY)
path_util.rename(new_mtemp, new_mdata)
path_util.make_directory(new_mtemp)
# Go through and purge all of the originals at this time
print >> sys.stderr, "Cleaning up drives..."
for to_delete in delete_on_success:
path_util.remove(to_delete)
print >> sys.stderr, "Successfully added partition '%s' to the pool." % new_partition_name
def worker_socket_dir(self):
from codalab.lib import path_util
directory = os.path.join(self.codalab_home, 'worker_sockets')
path_util.make_directory(directory)
return directory
def make_temp_location(self, identifier):
'''
Creates directory with given name under TEMP_SUBDIRECTORY
'''
path_util.make_directory(self.get_temp_location(identifier))
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir)
print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set docker image
docker_image = self.default_docker_image
if bundle.metadata.request_docker_image:
docker_image = bundle.metadata.request_docker_image
# Write the command to be executed to a script.
if docker_image:
container_file = temp_dir + '.cid' # contains the docker container id
action_file = temp_dir + '.action' # send actions to the container (e.g., kill)
status_dir = temp_dir + '.status' # receive information from the container (e.g., memory)
script_file = temp_dir + '.sh' # main entry point
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = bundle.uuid
docker_internal_script_file = bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
# trap doesn't quite work reliably with Torque, so don't use it
#f.write('trap \'echo Killing docker container $(cat %s); docker kill $(cat %s); echo Killed: $?; exit 143\' TERM\n' % (container_file, container_file))
# Inspect doesn't tell us a lot, so don't use it
#f.write('while [ -e %s ]; do docker inspect $(cat %s) > %s; sleep 1; done &\n' % (temp_dir, container_file, status_dir))
# Monitor CPU/memory/disk
monitor_commands = [
# Report on status
'mkdir -p %s' % status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
'cp -f $cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat %s' % (container_file, status_dir),
'cp -f $cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes %s' % (container_file, status_dir),
'cp -f $cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes %s' % (container_file, status_dir),
# Respond to actions
'[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (action_file, action_file, container_file, action_file),
]
f.write('while [ -e %s ]; do %s; sleep 1; done &\n' % (temp_dir, '; '. join(monitor_commands)))
# Constrain resources
resource_args = ''
if bundle.metadata.request_memory:
resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % (
resource_args,
container_file, os.geteuid(),
temp_dir, docker_temp_dir,
internal_script_file, docker_internal_script_file,
docker_image, docker_internal_script_file))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (os.getlogin(), os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write(". .bashrc || exit 1\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
else:
# Just run the command regularly without docker
script_file = temp_dir + '.sh'
with open(script_file, 'w') as f:
f.write("cd %s &&\n" % temp_dir)
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if bundle.metadata.request_time:
resource_args.extend(['--request_time', formatting.parse_duration(bundle.metadata.request_time)])
if bundle.metadata.request_memory:
resource_args.extend(['--request_memory', formatting.parse_size(bundle.metadata.request_memory)])
if bundle.metadata.request_cpus:
resource_args.extend(['--request_cpus', bundle.metadata.request_cpus])
if bundle.metadata.request_gpus:
resource_args.extend(['--request_gpus', bundle.metadata.request_gpus])
if bundle.metadata.request_queue:
resource_args.extend(['--request_queue', bundle.metadata.request_queue])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
}
def upload(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, uuid):
"""
|sources|: specifies the locations of the contents to upload. Each element is either a URL or a local path.
|follow_symlinks|: for local path(s), whether to follow (resolve) symlinks
|exclude_patterns|: for local path(s), don't upload these patterns (e.g., *.o)
|git|: for URL, whether |source| is a git repo to clone.
|unpack|: for each source in |sources|, whether to unpack it if it's an archive.
|remove_sources|: remove |sources|.
If |sources| contains one source, then the bundle contents will be that source.
Otherwise, the bundle contents will be a directory with each of the sources.
Exceptions:
- If |git|, then each source is replaced with the result of running 'git clone |source|'
- If |unpack| is True or a source is an archive (zip, tar.gz, etc.), then unpack the source.
Install the contents of the directory at |source| into
DATA_SUBDIRECTORY in a subdirectory named by a hash of the contents.
Return a (data_hash, metadata) pair, where the metadata is a dict mapping
keys to precomputed statistics about the new data directory.
"""
to_delete = []
# If just a single file, set the final path to be equal to that file
single_path = len(sources) == 1
# Determine which disk this will go on
disk_choice = self.ring.get_node(uuid)
final_path = os.path.join(self.partitions, disk_choice, self.DATA_SUBDIRECTORY, uuid)
if os.path.exists(final_path):
raise UsageError('Path %s already present in bundle store' % final_path)
# Only make if not there
elif not single_path:
path_util.make_directory(final_path)
# Paths to resources
subpaths = []
for source in sources:
# Where to save |source| to (might change this value if we unpack).
if not single_path:
subpath = os.path.join(final_path, os.path.basename(source))
else:
subpath = final_path
if remove_sources:
to_delete.append(source)
source_unpack = unpack and zip_util.path_is_archive(source)
if source_unpack and single_path:
# Load the file into the bundle store under the given path
subpath += zip_util.get_archive_ext(source)
if path_util.path_is_url(source):
# Download the URL.
print_util.open_line('BundleStore.upload: downloading %s to %s' % (source, subpath))
if git:
file_util.git_clone(source, subpath)
else:
file_util.download_url(source, subpath, print_status=True)
if source_unpack:
zip_util.unpack(subpath, zip_util.strip_archive_ext(subpath))
path_util.remove(subpath)
subpath = zip_util.strip_archive_ext(subpath)
print_util.clear_line()
else:
# Copy the local path.
source_path = path_util.normalize(source)
path_util.check_isvalid(source_path, 'upload')
# Recursively copy the directory into the BundleStore
print_util.open_line('BundleStore.upload: %s => %s' % (source_path, subpath))
if source_unpack:
zip_util.unpack(source_path, zip_util.strip_archive_ext(subpath))
subpath = zip_util.strip_archive_ext(subpath)
else:
if remove_sources:
path_util.rename(source_path, subpath)
else:
path_util.copy(source_path, subpath, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns)
print_util.clear_line()
subpaths.append(subpath)
dirs_and_files = None
if os.path.isdir(final_path):
dirs_and_files = path_util.recursive_ls(final_path)
else:
dirs_and_files = [], [final_path]
# Hash the contents of the bundle directory. Update the data_hash attribute
# for the bundle
print_util.open_line('BundleStore.upload: hashing %s' % final_path)
data_hash = '0x%s' % (path_util.hash_directory(final_path, dirs_and_files))
print_util.clear_line()
print_util.open_line('BundleStore.upload: computing size of %s' % final_path)
data_size = path_util.get_size(final_path, dirs_and_files)
print_util.clear_line()
# Delete paths.
for path in to_delete:
if os.path.exists(path):
path_util.remove(path)
# After this operation there should always be a directory at the final path.
assert (os.path.lexists(final_path)), 'Uploaded to %s failed!' % (final_path,)
return (data_hash, {'data_size': data_size})
from codalab.lib.codalab_manager import CodaLabManager
from codalab.lib import path_util
dry_run = False if len(sys.argv) > 1 and sys.argv[1] == '-f' else True
manager = CodaLabManager()
model = manager.model()
CODALAB_HOME = manager.codalab_home
"""Move data/ directory over to a temp area, and create a staging tree for uuid-based storage"""
DATA_DIR = os.path.join(CODALAB_HOME, 'data')
FINAL_LOCATION = os.path.join(CODALAB_HOME, 'bundles')
if not dry_run:
path_util.make_directory(FINAL_LOCATION)
"""For each data hash, get a list of all bundles that have that hash, and make a copy of the bundle in the staging
area under the UUID for the bundle."""
data_hashes = reduce(lambda x, y: x + y, path_util.ls(DATA_DIR))
for data_hash in data_hashes:
orig_location = os.path.join(DATA_DIR, data_hash)
bundles_with_hash = model.batch_get_bundles(data_hash=data_hash)
# We'd prefer renaming bundles to making copies, but because we are converting from deduplicated storage
# we need to make sure that we only perform renames if we map 1:1 UUID->Hash.
rename_allowed = len(bundles_with_hash) <= 1
for bundle in bundles_with_hash:
# Build the command to be executed in a subshell
uuid = bundle.uuid
copy_location = os.path.join(FINAL_LOCATION, uuid)
command = '%s %s %s' % ('mv' if rename_allowed else 'cp -a',
def make_temp_location(self, identifier):
'''
Creates directory with given name under TEMP_SUBDIRECTORY
'''
path_util.make_directory(self.get_temp_location(identifier));
def make_directories(self):
'''
Create the data, and temp directories for this BundleStore.
'''
for path in (self.data, self.temp):
path_util.make_directory(path)
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir)
print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set defaults for the dispatcher.
docker_image = self.default_docker_image
if bundle.metadata.request_docker_image:
docker_image = bundle.metadata.request_docker_image
request_time = self.default_request_time
if bundle.metadata.request_time:
request_time = bundle.metadata.request_time
request_memory = self.default_request_memory
if bundle.metadata.request_memory:
request_memory = bundle.metadata.request_memory
request_cpus = self.default_request_cpus
if bundle.metadata.request_cpus:
request_cpus = bundle.metadata.request_cpus
request_gpus = self.default_request_gpus
if bundle.metadata.request_gpus:
request_gpus = bundle.metadata.request_gpus
request_queue = self.default_request_queue
if bundle.metadata.request_queue:
request_queue = bundle.metadata.request_queue
request_priority = self.default_request_priority
if bundle.metadata.request_priority:
request_priority = bundle.metadata.request_priority
script_file = temp_dir + '.sh' # main entry point
ptr_temp_dir = '$temp_dir'
# 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system).
# 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system).
set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n'
# Write the command to be executed to a script.
if docker_image:
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file)
ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id
ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill)
ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory)
ptr_script_file = ptr_temp_dir + '.sh' # main entry point
ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = bundle.uuid
docker_internal_script_file = bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
# Monitor CPU/memory/disk
def copy_if_exists(source_template, arg, target):
source = source_template % arg
# -f because target might be read-only
return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (arg, source, source, target)
monitor_commands = [
# Report on status (memory, cpu, etc.)
'mkdir -p %s' % ptr_status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
copy_if_exists('$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir),
copy_if_exists('$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir),
copy_if_exists('$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir),
# Respond to kill action
'[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s' % (ptr_action_file, ptr_action_file, ptr_container_file, ptr_action_file),
# Sleep
'sleep 1',
]
f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '. join(monitor_commands)))
# Tell docker to constrain resources (memory).
# Note: limiting memory is not always supported. See:
# http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html
resource_args = ''
if bundle.metadata.request_memory:
resource_args += ' -m %s' % int(formatting.parse_size(bundle.metadata.request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
f.write("docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s & wait $!\n" % (
resource_args,
ptr_container_file,
os.geteuid(),
ptr_temp_dir, docker_temp_dir,
ptr_internal_script_file, docker_internal_script_file,
docker_image,
docker_internal_script_file))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
username = pwd.getpwuid(os.getuid())[0] # do this because os.getlogin() doesn't always work
f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write(". .bashrc || exit 1\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
else:
# Just run the command regularly without docker
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
f.write("cd %s &&\n" % ptr_temp_dir)
f.write('(%s) > stdout 2>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if request_time:
resource_args.extend(['--request_time', formatting.parse_duration(request_time)])
if request_memory:
resource_args.extend(['--request_memory', formatting.parse_size(request_memory)])
if request_cpus:
resource_args.extend(['--request_cpus', request_cpus])
if request_gpus:
resource_args.extend(['--request_gpus', request_gpus])
if request_queue:
resource_args.extend(['--request_queue', request_queue])
if request_priority:
resource_args.extend(['--request_priority', request_priority])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
}
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict, temp_dir)
print >>sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set defaults for the dispatcher.
docker_image = bundle.metadata.request_docker_image or self.default_docker_image
# Parse |request_string| using |to_value|, but don't exceed |max_value|.
def parse_and_min(to_value, request_string, default_value, max_value):
# Use default if request value doesn't exist
if request_string:
request_value = to_value(request_string)
else:
request_value = default_value
if request_value and max_value:
return int(min(request_value, max_value))
elif request_value:
return int(request_value)
elif max_value:
return int(max_value)
else:
return None
request_time = parse_and_min(formatting.parse_duration, bundle.metadata.request_time, self.default_request_time, self.max_request_time)
request_memory = parse_and_min(formatting.parse_size, bundle.metadata.request_memory, self.default_request_memory, self.max_request_memory)
request_disk = parse_and_min(formatting.parse_size, bundle.metadata.request_disk, self.default_request_disk, self.max_request_disk)
request_cpus = bundle.metadata.request_cpus or self.default_request_cpus
request_gpus = bundle.metadata.request_gpus or self.default_request_gpus
request_queue = bundle.metadata.request_queue or self.default_request_queue
request_priority = bundle.metadata.request_priority or self.default_request_priority
request_network = bundle.metadata.request_network or self.default_request_network
script_file = temp_dir + '.sh' # main entry point
ptr_temp_dir = '$temp_dir'
# 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system).
# 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system).
set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n'
# Write the command to be executed to a script.
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file)
ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id
ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill)
ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory)
ptr_script_file = ptr_temp_dir + '.sh' # main entry point
ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = '/' + bundle.uuid
docker_internal_script_file = '/' + bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
# Monitor CPU/memory/disk
# Used to copy status about the docker container.
def copy_if_exists(source_template, arg, target):
source = source_template % arg
# -f because target might be read-only
return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (arg, source, source, target)
def get_field(path, col):
return 'cat %s | cut -f%s -d\'%s\'' % (path, col, BundleAction.SEPARATOR)
monitor_commands = [
# Report on status (memory, cpu, etc.)
'mkdir -p %s' % ptr_status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
copy_if_exists('$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat', ptr_container_file, ptr_status_dir),
copy_if_exists('$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes', ptr_container_file, ptr_status_dir),
copy_if_exists('$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes', ptr_container_file, ptr_status_dir),
# Enforce memory limits
'[ -e "%s/memory.usage_in_bytes" ] && mem=$(cat %s/memory.usage_in_bytes)' % (ptr_status_dir, ptr_status_dir),
'echo "memory: $mem (max %s)"' % request_memory,
'if [ -n "$mem" ] && [ "$mem" -gt "%s" ]; then echo "[CodaLab] Memory limit exceeded: $mem > %s, terminating." >> %s/stderr; docker kill $(cat %s); break; fi' % \
(request_memory, request_memory, ptr_temp_dir, ptr_container_file),
# Enforce disk limits
'disk=$(du -sb %s | cut -f1)' % ptr_temp_dir,
'echo "disk: $disk (max %s)"' % request_disk,
'if [ -n "$disk" ] && [ "$disk" -gt "%s" ]; then echo "[CodaLab] Disk limit exceeded: $disk > %s, terminating." >> %s/stderr; docker kill $(cat %s); break; fi' % \
(request_disk, request_disk, ptr_temp_dir, ptr_container_file),
# Execute "kill"
'if [ -e %s ] && [ "$(cat %s)" == "kill" ]; then echo "[CodaLab] Received kill command, terminating." >> %s/stderr; docker kill $(cat %s); rm %s; break; fi' % \
(ptr_action_file, ptr_action_file, ptr_temp_dir, ptr_container_file, ptr_action_file),
# Execute "write <subpath> <contents>"
'if [ -e %s ] && [ "$(%s)" == "write" ]; then echo Writing...; %s > %s/$(%s); rm %s; fi' % \
(ptr_action_file, get_field(ptr_action_file, 1),
get_field(ptr_action_file, '3-'), ptr_temp_dir, get_field(ptr_action_file, 2),
ptr_action_file),
# Sleep
'sleep 1',
]
f.write('while [ -e %s ]; do\n %s\ndone &\n' % (ptr_temp_dir, '\n '. join(monitor_commands)))
resource_args = ''
# Limiting memory in docker is not (always) supported. So we rely on bash (see above).
# http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html
#if request_memory:
# resource_args += ' -m %s' % int(formatting.parse_size(request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
# Attach all GPUs if any. Note that only the 64-bit version of
# libcuda.so is picked up.
f.write('devices=$(/bin/ls /dev/nvidia* 2>/dev/null)\n')
f.write('if [ -n "$devices" ]; then devices=$(for d in $devices; do echo --device $d:$d; done); fi\n')
f.write('libcuda=$(/sbin/ldconfig -p 2>/dev/null | grep "libcuda.so$" | grep "x86-64" | head -n 1 | cut -d " " -f 4)\n')
f.write('if [ -n "$libcuda" ]; then libcuda=" -v $libcuda:/usr/lib/x86_64-linux-gnu/libcuda.so:ro"; fi\n')
resource_args += ' $devices$libcuda'
# Enable network?
if not request_network:
resource_args += ' --net=none'
f.write("docker run%s --rm --cidfile %s -u %s -v %s:%s -v %s:%s -e HOME=%s %s bash %s >%s/stdout 2>%s/stderr & wait $!\n" % (
resource_args,
ptr_container_file,
os.geteuid(),
ptr_temp_dir, docker_temp_dir,
ptr_internal_script_file, docker_internal_script_file,
docker_temp_dir,
docker_image,
docker_internal_script_file,
ptr_temp_dir, ptr_temp_dir))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
username = pwd.getpwuid(os.getuid())[0] # do this because os.getlogin() doesn't always work
f.write("[ -w /etc/passwd ] && echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" % (username, os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write("[ -e .bashrc ] && . .bashrc\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) >>stdout 2>>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if request_time:
resource_args.extend(['--request-time', request_time])
if request_memory:
resource_args.extend(['--request-memory', request_memory])
if request_disk:
resource_args.extend(['--request-disk', request_disk])
if request_cpus:
resource_args.extend(['--request-cpus', request_cpus])
if request_gpus:
resource_args.extend(['--request-gpus', request_gpus])
if request_queue:
resource_args.extend(['--request-queue', request_queue])
if request_priority:
resource_args.extend(['--request-priority', request_priority])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
if not result['handle']:
raise SystemError('Starting bundle failed')
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
'request_time': str(request_time) if request_time else None,
'request_memory': str(request_memory) if request_memory else None,
'request_disk': str(request_disk) if request_disk else None,
'request_cpus': request_cpus,
'request_gpus': request_gpus,
'request_queue': request_queue,
'request_priority': request_priority,
'request_network': request_network,
}
def home(self):
from codalab.lib import path_util
result = path_util.normalize(self.config['home'])
path_util.make_directory(result)
return result
def worker_socket_dir(self):
from codalab.lib import path_util
directory = os.path.join(self.codalab_home, 'worker_sockets')
path_util.make_directory(directory)
return directory
def start_bundle(self, bundle, bundle_store, parent_dict, username):
'''
Sets up all the temporary files and then dispatches the job.
username: the username of the owner of the bundle
Returns the bundle information.
'''
# Create a temporary directory
temp_dir = canonicalize.get_current_location(bundle_store, bundle.uuid)
temp_dir = os.path.realpath(temp_dir) # Follow symlinks
path_util.make_directory(temp_dir)
# Copy all the dependencies to that temporary directory.
pairs = bundle.get_dependency_paths(bundle_store, parent_dict,
temp_dir)
print >> sys.stderr, 'RemoteMachine.start_bundle: copying dependencies of %s to %s' % (
bundle.uuid, temp_dir)
for (source, target) in pairs:
path_util.copy(source, target, follow_symlinks=False)
# Set defaults for the dispatcher.
docker_image = self.default_docker_image
if bundle.metadata.request_docker_image:
docker_image = bundle.metadata.request_docker_image
request_time = self.default_request_time
if bundle.metadata.request_time:
request_time = bundle.metadata.request_time
request_memory = self.default_request_memory
if bundle.metadata.request_memory:
request_memory = bundle.metadata.request_memory
request_cpus = self.default_request_cpus
if bundle.metadata.request_cpus:
request_cpus = bundle.metadata.request_cpus
request_gpus = self.default_request_gpus
if bundle.metadata.request_gpus:
request_gpus = bundle.metadata.request_gpus
request_queue = self.default_request_queue
if bundle.metadata.request_queue:
request_queue = bundle.metadata.request_queue
request_priority = self.default_request_priority
if bundle.metadata.request_priority:
request_priority = bundle.metadata.request_priority
script_file = temp_dir + '.sh' # main entry point
ptr_temp_dir = '$temp_dir'
# 1) If no argument to script_file, use the temp_dir (e.g., Torque, master/worker share file system).
# 2) If argument is 'use_script_for_temp_dir', use the script to determine temp_dir (e.g., qsub, no master/worker do not share file system).
set_temp_dir_header = 'if [ -z "$1" ]; then temp_dir=' + temp_dir + '; else temp_dir=`readlink -f $0 | sed -e \'s/\\.sh$//\'`; fi\n'
# Write the command to be executed to a script.
if docker_image:
internal_script_file = temp_dir + '-internal.sh' # run inside the docker container
# These paths depend on $temp_dir, an environment variable which will be set (referenced inside script_file)
ptr_container_file = ptr_temp_dir + '.cid' # contains the docker container id
ptr_action_file = ptr_temp_dir + '.action' # send actions to the container (e.g., kill)
ptr_status_dir = ptr_temp_dir + '.status' # receive information from the container (e.g., memory)
ptr_script_file = ptr_temp_dir + '.sh' # main entry point
ptr_internal_script_file = ptr_temp_dir + '-internal.sh' # run inside the docker container
# Names of file inside the docker container
docker_temp_dir = bundle.uuid
docker_internal_script_file = bundle.uuid + '-internal.sh'
# 1) script_file starts the docker container and runs internal_script_file in docker.
# --rm removes the docker container once the job terminates (note that this makes things slow)
# -v mounts the internal and user scripts and the temp directory
# Trap SIGTERM and forward it to docker.
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
# Monitor CPU/memory/disk
def copy_if_exists(source_template, arg, target):
source = source_template % arg
# -f because target might be read-only
return 'if [ -e %s ] && [ -e %s ]; then cp -f %s %s; fi' % (
arg, source, source, target)
monitor_commands = [
# Report on status (memory, cpu, etc.)
'mkdir -p %s' % ptr_status_dir,
'if [ -e /cgroup ]; then cgroup=/cgroup; else cgroup=/sys/fs/cgroup; fi', # find where cgroup is
copy_if_exists(
'$cgroup/cpuacct/docker/$(cat %s)/cpuacct.stat',
ptr_container_file, ptr_status_dir),
copy_if_exists(
'$cgroup/memory/docker/$(cat %s)/memory.usage_in_bytes',
ptr_container_file, ptr_status_dir),
copy_if_exists(
'$cgroup/blkio/docker/$(cat %s)/blkio.throttle.io_service_bytes',
ptr_container_file, ptr_status_dir),
# Respond to kill action
'[ -e %s ] && [ "$(cat %s)" == "kill" ] && docker kill $(cat %s) && rm %s'
% (ptr_action_file, ptr_action_file, ptr_container_file,
ptr_action_file),
# Sleep
'sleep 1',
]
f.write('while [ -e %s ]; do\n %s\ndone &\n' %
(ptr_temp_dir, '\n '.join(monitor_commands)))
# Tell docker to constrain resources (memory).
# Note: limiting memory is not always supported. See:
# http://programster.blogspot.com/2014/09/docker-implementing-container-memory.html
resource_args = ''
if bundle.metadata.request_memory:
resource_args += ' -m %s' % int(
formatting.parse_size(bundle.metadata.request_memory))
# TODO: would constrain --cpuset=0, but difficult because don't know the CPU ids
f.write(
"docker run%s --rm --cidfile %s -u %s -v %s:/%s -v %s:/%s %s bash %s >%s/stdout 2>%s/stderr & wait $!\n"
%
(resource_args, ptr_container_file, os.geteuid(),
ptr_temp_dir, docker_temp_dir, ptr_internal_script_file,
docker_internal_script_file, docker_image,
docker_internal_script_file, ptr_temp_dir, ptr_temp_dir))
# 2) internal_script_file runs the actual command inside the docker container
with open(internal_script_file, 'w') as f:
# Make sure I have a username
username = pwd.getpwuid(os.getuid())[
0] # do this because os.getlogin() doesn't always work
f.write("echo %s::%s:%s::/:/bin/bash >> /etc/passwd\n" %
(username, os.geteuid(), os.getgid()))
# Do this because .bashrc isn't sourced automatically (even with --login, though it works with docker -t -i, strange...)
f.write(". .bashrc || exit 1\n")
# Go into the temp directory
f.write("cd %s &&\n" % docker_temp_dir)
# Run the actual command
f.write('(%s) >>stdout 2>>stderr\n' % bundle.command)
else:
# Just run the command regularly without docker
with open(script_file, 'w') as f:
f.write(set_temp_dir_header)
f.write("cd %s &&\n" % ptr_temp_dir)
f.write('(%s) >stdout 2>stderr\n' % bundle.command)
# Determine resources to request
resource_args = []
if request_time:
resource_args.extend(
['--request_time',
formatting.parse_duration(request_time)])
if request_memory:
resource_args.extend(
['--request_memory',
formatting.parse_size(request_memory)])
if request_cpus:
resource_args.extend(['--request_cpus', request_cpus])
if request_gpus:
resource_args.extend(['--request_gpus', request_gpus])
if request_queue:
resource_args.extend(['--request_queue', request_queue])
if request_priority:
resource_args.extend(['--request_priority', request_priority])
if username:
resource_args.extend(['--username', username])
# Start the command
args = self.dispatch_command.split() + ['start'] + map(
str, resource_args) + [script_file]
if self.verbose >= 1: print '=== start_bundle(): running %s' % args
result = json.loads(self.run_command_get_stdout(args))
if self.verbose >= 1: print '=== start_bundle(): got %s' % result
# Return the information about the job.
return {
'bundle': bundle,
'temp_dir': temp_dir,
'job_handle': result['handle'],
'docker_image': docker_image,
}
