def install_dependencies(self, bundle_store, parent_dict, path, rel):
'''
Symlink this bundle's dependencies into the directory at path.
The caller is responsible for cleaning up this directory.
'''
precondition(os.path.isabs(path), '%s is a relative path!' % (path,))
for dep in self.dependencies:
parent = parent_dict[dep.parent_uuid]
# Compute an absolute target and check that the dependency exists.
target = path_util.safe_join(
bundle_store.get_location(parent.data_hash),
dep.parent_path,
)
if not os.path.exists(target):
parent_spec = getattr(parent.metadata, 'name', parent.uuid)
target_text = path_util.safe_join(parent_spec, dep.parent_path)
raise UsageError('Target not found: %s' % (target_text,))
if rel:
# Create a symlink that points to the dependency's relative target.
target = path_util.safe_join(
(os.pardir if dep.child_path else ''),
bundle_store.get_location(parent.data_hash, relative=True),
dep.parent_path,
)
link_path = path_util.safe_join(path, dep.child_path)
os.symlink(target, link_path)
def upload_bundle_zip(self, file_uuid, construct_args, worksheet_uuid, follow_symlinks, add_to_worksheet):
'''
|file_uuid| specifies a pointer to the temporary file X.
- If X is a non-zip file, then just upload X as an ordinary file.
- If X is a zip file containing one file/directory Y representing bundle, then upload Y.
- If X is a zip file containing multiple files/directories, then upload X.
Return the new bundle's uuid.
Note: delete the file_uuid file and X if needed (these are temporary files).
'''
if file_uuid:
orig_path = self.file_paths[file_uuid] # Note: cheat and look at file_server's data
precondition(orig_path, 'Unexpected file uuid: %s' % (file_uuid,))
if zip_util.is_zip_file(orig_path):
container_path = tempfile.mkdtemp() # Make temporary directory
zip_util.unzip(orig_path, container_path, file_name=None) # Unzip into a directory
# If the container path only has one item, then make that the final path
sub_files = os.listdir(container_path)
if len(sub_files) == 1:
final_path = os.path.join(container_path, sub_files[0])
else: # Otherwise, use the container path
final_path = container_path
container_path = None
else:
# Not a zip file! Just upload it normally as a file.
final_path = orig_path
container_path = None # Don't need to delete
else:
final_path = None
result = self.client.upload_bundle(final_path, construct_args, worksheet_uuid, follow_symlinks, exclude_patterns=[], add_to_worksheet=add_to_worksheet)
if file_uuid:
if container_path:
path_util.remove(container_path) # Remove temporary directory
self.finalize_file(file_uuid, final_path != orig_path) # Remove temporary file
return result
def start_bundle(self, bundle):
"""
Run the given bundle using an available Machine.
Return whether something was started.
"""
# Check that we're running a bundle in the QUEUED state.
state_message = "Unexpected bundle state: %s" % (bundle.state,)
precondition(bundle.state == State.QUEUED, state_message)
data_hash_message = "Unexpected bundle data_hash: %s" % (bundle.data_hash,)
precondition(bundle.data_hash is None, data_hash_message)
# Run the bundle.
with self.profile("Running bundle..."):
started = False
if isinstance(bundle, RunBundle):
try:
# Get the username of the bundle
results = self.auth_handler.get_users("ids", [bundle.owner_id])
if results.get(bundle.owner_id):
username = results[bundle.owner_id].name
else:
username = str(bundle.owner_id)
status = self.machine.start_bundle(
bundle, self.bundle_store, self.get_parent_dict(bundle), username
)
if status != None:
status["started"] = int(time.time())
started = True
except Exception as e:
# If there's an exception, we just make the bundle fail
# (even if it's not the bundle's fault).
real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(real_path)
status = {
"bundle": bundle,
"success": False,
"failure_message": "Internal error: " + str(e),
"temp_dir": real_path,
}
print "=== INTERNAL ERROR: %s" % e
started = True # Force failing
traceback.print_exc()
else: # MakeBundle
started = True
if started:
print "-- START BUNDLE: %s" % (bundle,)
self._update_events_log("start_bundle", bundle, (bundle.uuid,))
# If we have a MakeBundle, then just process it immediately.
if isinstance(bundle, MakeBundle):
real_path = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(real_path)
status = {"bundle": bundle, "success": True, "temp_dir": real_path}
# Update database
if started:
self.update_running_bundle(status)
return started
def nested_dict_get(obj, *args, **kwargs):
"""
Get a value from a nested dictionary.
Cleans up calls that look lke this:
bundle_info.get('owner', {}).get('user_name', None)
And turns them into:
safe_get(bundle_info, 'owner', 'user_name')
:param obj: dict-like object to 'get' value from
:param args: variable list of nested keys
:param kwargs: supports the kwarg 'default' to specify the default value to
return if any of the keys don't exist. (default is None)
Any other kwarg will raise an exception.
:return: retrieved value or default if it doesn't exist
"""
default = kwargs.pop('default', None)
precondition(not kwargs, 'unsupported kwargs %s' % kwargs.keys())
try:
for arg in args:
obj = obj[arg]
return obj
except (KeyError, TypeError):
return default
def batch_update_bundles(self, bundles, update, condition=None):
'''
Update a list of bundles given a dict mapping columns to new values and
return True if all updates succeed. This method does NOT update metadata.
If a condition is specified, only update bundles that satisfy the condition.
In general, this method should only be used for programmatic updates, as in
the bundle worker. It is provided as an efficient way to perform a simple
update on many, but these updates are not validated.
'''
message = 'Illegal update: %s' % (update,)
precondition('id' not in update and 'uuid' not in update, message)
if bundles:
bundle_ids = set(bundle.id for bundle in bundles)
clause = cl_bundle.c.id.in_(bundle_ids)
if condition:
clause = and_(clause, self.make_kwargs_clause(cl_bundle, condition))
with self.engine.begin() as connection:
result = connection.execute(
cl_bundle.update().where(clause).values(update)
)
success = result.rowcount == len(bundle_ids)
if success:
for bundle in bundles:
bundle.update_in_memory(update)
return success
return True
def get_target_info(self, uuid, path, depth):
"""
Returns information about an individual target inside the bundle, or
None if the target doesn't exist.
For information about the format of the return value, see
worker.download_util.get_target_info.
"""
if self._bundle_model.get_bundle_state(uuid) != State.RUNNING:
bundle_path = self._bundle_store.get_bundle_location(uuid)
try:
return download_util.get_target_info(bundle_path, uuid, path, depth)
except download_util.PathException as e:
raise UsageError(e.message)
else:
# get_target_info calls are sent to the worker even on a shared file
# system since 1) due to NFS caching the worker has more up to date
# information on directory contents, and 2) the logic of hiding
# the dependency paths doesn't need to be re-implemented here.
worker = self._worker_model.get_bundle_worker(uuid)
response_socket_id = self._worker_model.allocate_socket(worker['user_id'], worker['worker_id'])
try:
read_args = {
'type': 'get_target_info',
'depth': depth,
}
self._send_read_message(worker, response_socket_id, uuid, path, read_args)
with closing(self._worker_model.start_listening(response_socket_id)) as sock:
result = self._worker_model.get_json_message(sock, 60)
precondition(result is not None, 'Unable to reach worker')
if 'error_code' in result:
raise http_error_to_exception(result['error_code'], result['error_message'])
return result['target_info']
finally:
self._worker_model.deallocate_socket(response_socket_id)
def run(self, bundle_store, parent_dict, temp_dir):
if any(not dep.child_path for dep in self.dependencies):
message = '%s has keyed and anonymous targets!' % (self,),
precondition(len(self.dependencies) == 1, message)
temp_dir = os.path.join(temp_dir, 'anonymous_link')
self.install_dependencies(bundle_store, parent_dict, temp_dir, rel=True)
return bundle_store.upload(temp_dir, allow_symlinks=True)
def update_worksheet(self, worksheet_uuid, last_item_id, length, new_items):
'''
Updates the worksheet with the given uuid. If there were exactly
`last_length` items with database id less than `last_id`, replaces them all
with the items in new_items. Does NOT affect items in this worksheet with
database id greater than last_id.
Does NOT affect items that were added to the worksheet in between the
time it was retrieved and it was updated.
If this worksheet were updated between the time it was retrieved and
updated, this method will raise a UsageError.
'''
clause = and_(
cl_worksheet_item.c.worksheet_uuid == worksheet_uuid,
cl_worksheet_item.c.id <= last_item_id,
)
# See codalab.objects.worksheet for an explanation of the sort_key protocol.
# We need to produce sort keys here that are strictly upper-bounded by the
# last known item id in this worksheet, and which monotonically increase.
# The expression last_item_id + i - len(new_items) works. It can produce
# negative sort keys, but that's fine.
new_item_values = [{
'worksheet_uuid': worksheet_uuid,
'bundle_uuid': bundle_uuid,
'value': value,
'sort_key': (last_item_id + i - len(new_items)),
} for (i, (bundle_uuid, value)) in enumerate(new_items)]
with self.engine.begin() as connection:
result = connection.execute(cl_worksheet_item.delete().where(clause))
message = 'Found extra items for worksheet %s' % (worksheet_uuid,)
precondition(result.rowcount <= length, message)
if result.rowcount < length:
raise UsageError('Worksheet %s was updated concurrently!' % (worksheet_uuid,))
self.do_multirow_insert(connection, cl_worksheet_item, new_item_values)
def create_bundle_actions():
"""
Sends the message to the worker to do the bundle action, and adds the
action string to the bundle metadata.
"""
actions = BundleActionSchema(strict=True, many=True).load(request.json).data
check_bundles_have_all_permission(local.model, request.user, [a['uuid'] for a in actions])
for action in actions:
bundle = local.model.get_bundle(action['uuid'])
if bundle.state not in [State.RUNNING, State.PREPARING]:
raise UsageError('Cannot execute this action on a bundle that is not running.')
worker = local.worker_model.get_bundle_worker(action['uuid'])
precondition(
local.worker_model.send_json_message(worker['socket_id'], action, 60),
'Unable to reach worker.',
)
new_actions = getattr(bundle.metadata, 'actions', []) + [BundleAction.as_string(action)]
db_update = {'metadata': {'actions': new_actions}}
local.model.update_bundle(bundle, db_update)
return BundleActionSchema(many=True).dump(actions).data
def exit(self, message, error_code=1):
'''
Print the message to stderr and exit with the given error code.
'''
precondition(error_code, 'exit called with error_code == 0')
print >> sys.stderr, message
sys.exit(error_code)
def finish_upload_bundle(self, file_uuids, unpack, info, worksheet_uuid, add_to_worksheet):
'''
|file_uuids| specifies a pointer to temporary files.
Upload these and return the new bundle's uuid.
Note: delete the file_uuids as these are temporary files.
'''
if file_uuids is not None:
paths = []
for file_uuid in file_uuids:
# Note: cheat and look at file_server's data to get paths
precondition(file_uuid in self.file_paths, 'Invalid file_uuid: %s' % file_uuid)
paths.append(self.file_paths[file_uuid])
else:
paths = None
# Upload the paths
result = self.client.upload_bundle(
paths,
follow_symlinks=False,
exclude_patterns=None,
git=False,
unpack=unpack,
remove_sources=True,
info=info,
worksheet_uuid=worksheet_uuid,
add_to_worksheet=add_to_worksheet)
# Remove temporary file
if file_uuids is not None:
for file_uuid in file_uuids:
self.finalize_file(file_uuid)
return result
def derive_rest_address(self, address):
"""
Given bundle service address, return corresponding REST address
using manual translations (described in the comments below).
Temporary hack to ease transition to REST API.
"""
o = urlparse(address)
if is_local_address(address):
# local => http://localhost:<rest_port>
precondition('server' in self.config and
'rest_host' in self.config['server'] and
'rest_port' in self.config['server'],
'Working on local now requires running a local '
'server, please configure "rest_host" and '
'"rest_port" under "server" in your config.json.')
address = 'http://{rest_host}:{rest_port}'.format(**self.config['server'])
elif (o.hostname == 'localhost' and
'server' in self.config and
'port' in self.config['server'] and
'rest_port' in self.config['server'] and
o.port == self.config['server']['port']):
# http://localhost:<port> => http://localhost:<rest_port>
# Note that this does not affect the address of the NLP
# CodaLab instance behind an SSH tunnel
address = address.replace(str(self.config['server']['port']),
str(self.config['server']['rest_port']))
elif (o.netloc == 'worksheets.codalab.org' or
o.netloc == 'worksheets-test.codalab.org'):
# http://worksheets.codalab.org/bundleservice => http://worksheets.codalab.org
address = address.replace('/bundleservice', '')
return address
def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks):
bundle_type = info['bundle_type']
if 'uuid' in info:
existing = True
construct_args = self.bundle_info_to_construct_args(info)
else:
existing = False
construct_args = {'metadata': info['metadata']}
metadata = construct_args['metadata']
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
if not existing:
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
if not existing:
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
if path:
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path, follow_symlinks=follow_symlinks)
metadata.update(bundle_store_metadata)
precondition(construct_args.get('data_hash', data_hash) == data_hash, \
'Provided data_hash doesn\'t match: %s versus %s' % (construct_args.get('data_hash'), data_hash))
construct_args['data_hash'] = data_hash
# Set the owner
construct_args['owner_id'] = self._current_user_id()
bundle = bundle_subclass.construct(**construct_args)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, worksheet_util.bundle_item(bundle.uuid))
# TODO: don't fail if don't have permissions
return bundle.uuid
def send_json_message(self, socket_id, message, timeout_secs, autoretry=True):
"""
Sends a JSON message to the given socket, retrying until it is received
correctly.
If the message is not sent successfully after timeout_secs, return
False. Otherwise, returns True.
Note, only the worker should call this method with autoretry set to
False. See comments below.
"""
start_time = time.time()
while time.time() - start_time < timeout_secs:
with closing(socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) as sock:
sock.settimeout(timeout_secs)
success = False
try:
sock.connect(self._socket_path(socket_id))
if autoretry:
# This auto retry mechanisms helps ensure that messages
# sent to a worker are received more reliably. The
# socket API isn't particularly robust to our usage
# where we continuously start and stop listening on a
# socket, like the worker checkin mechanism does. In
# fact, it seems to spuriously accept connections
# just when a socket object is in the process of being
# destroyed. On the sending end, such a scenario results
# in a "Broken pipe" exception, which we catch here.
success = sock.recv(len(WorkerModel.ACK)) == WorkerModel.ACK
else:
success = True
except socket.error:
pass
if not success:
# Shouldn't be too expensive just to keep retrying.
# TODO: maybe exponential backoff
time.sleep(
0.3
) # changed from 0.003 to keep from rate-limiting due to dead workers
continue
if not autoretry:
# When messages are being sent from the worker, we don't
# have the problem with "Broken pipe" as above, since
# code waiting for a reply shouldn't just abruptly stop
# listening.
precondition(
sock.recv(len(WorkerModel.ACK)) == WorkerModel.ACK,
'Received invalid ack on socket.',
)
sock.sendall(json.dumps(message))
return True
return False
def check_isvalid(path, fn_name):
"""
Raise a PreconditionViolation if the path is not absolute or normalized.
Raise a UsageError if the file at that path does not exist.
"""
precondition(os.path.isabs(path), '%s got relative path: %s' % (fn_name, path))
# Broken symbolic links are valid paths, so we use lexists instead of exists.
if not os.path.lexists(path):
raise path_error('%s got non-existent path:' % (fn_name,), path)
def upload_zip(self, bundle_type, file_uuid, metadata, worksheet_uuid=None): ''' Unzip the zip in the temp file identified by the given file uuid and then upload the unzipped directory. Return the new bundle's id. ''' zip_path = self.temp_file_paths.pop(file_uuid, None) precondition(zip_path, 'Unexpected file uuid: %s' % (file_uuid,)) path = zip_util.unzip(zip_path) return self.client.upload(bundle_type, path, metadata, worksheet_uuid)
def _get_read_response_stream(self, response_socket_id):
with closing(self._worker_model.start_listening(response_socket_id)) as sock:
header_message = self._worker_model.get_json_message(sock, 60)
precondition(header_message is not None, 'Unable to reach worker')
if 'error_code' in header_message:
raise http_error_to_exception(header_message['error_code'], header_message['error_message'])
fileobj = self._worker_model.get_stream(sock, 60)
precondition(fileobj is not None, 'Unable to reach worker')
return fileobj
def validate(self):
'''
Check a number of basic conditions that would indicate serious errors if
they do not hold. Right now, validation only checks this worksheet's uuid
and its name.
'''
spec_util.check_uuid(self.uuid)
spec_util.check_name(self.name)
precondition(isinstance(self.owner_id, basestring), 'Invalid value: owner_id.')
precondition(isinstance(self.user_defined, bool), 'Invalid value: user_defined.')
def remove_dependencies(self, bundle_store, parent_dict, dest_path):
'''
Remove dependencies (for RunBundles).
'''
precondition(os.path.isabs(dest_path), '%s is a relative path!' % (dest_path,))
pairs = self.get_dependency_paths(bundle_store, parent_dict, dest_path, relative_symlinks=False)
for (target, link_path) in pairs:
# If the dependency already exists, remove it (this happens when we are reinstalling)
if os.path.exists(link_path):
path_util.remove(link_path)
def _send_read_message(self, worker, response_socket_id, uuid, path, read_args):
message = {
'type': 'read',
'socket_id': response_socket_id,
'uuid': uuid,
'path': path,
'read_args': read_args,
}
precondition(
self._worker_model.send_json_message(worker['socket_id'], message, 60),
'Unable to reach worker')
def check_isvalid(path, fn_name):
'''
Raise a PreconditionViolation if the path is not absolute or normalized.
Raise a UsageError if the file at that path does not exist.
'''
precondition(os.path.isabs(path), '%s got relative path: %s' % (fn_name, path))
# Broken symbolic links are valid paths, so we use lexists instead of exists.
# This case will come up when executing a make bundle with an anonymous target,
# because the symlink will be broken until it is moved into the bundle store.
if not os.path.lexists(path):
raise path_error('%s got non-existent path:' % (fn_name,), path)
def save_worksheet(self, worksheet):
'''
Save the given (empty) worksheet to the database. On success, set its id.
'''
message = 'save_worksheet called with non-empty worksheet: %s' % (worksheet,)
precondition(not worksheet.items, message)
worksheet.validate()
worksheet_value = worksheet.to_dict()
with self.engine.begin() as connection:
result = connection.execute(cl_worksheet.insert().values(worksheet_value))
worksheet.id = result.lastrowid
def update_in_memory(self, row, strict=False):
metadata = row.pop('metadata', None)
dependencies = row.pop('dependencies', None)
if strict:
precondition(metadata is not None, 'No metadata: %s' % (row,))
precondition(dependencies is not None, 'No dependencies: %s' % (row,))
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Bundle, self).update_in_memory(row)
if metadata is not None:
self.metadata = Metadata(self.METADATA_SPECS, metadata)
if dependencies is not None:
self.dependencies = [Dependency(dep) for dep in dependencies]
def upload(self, bundle_type, path, metadata, worksheet_uuid=None):
message = 'Invalid upload bundle_type: %s' % (bundle_type,)
precondition(bundle_type in UPLOADED_TYPES, message)
bundle_subclass = get_bundle_subclass(bundle_type)
self.validate_user_metadata(bundle_subclass, metadata)
# Upload the given path and record additional metadata from the upload.
(data_hash, bundle_store_metadata) = self.bundle_store.upload(path)
metadata.update(bundle_store_metadata)
bundle = bundle_subclass.construct(data_hash=data_hash, metadata=metadata)
self.model.save_bundle(bundle)
if worksheet_uuid:
self.add_worksheet_item(worksheet_uuid, bundle.uuid)
return bundle.uuid
def update_in_memory(self, row, strict=False):
items = row.pop('items', None)
if strict:
precondition(items is not None, 'No items: %s' % (row,))
item_sort_keys = [item_sort_key(item) for item in items]
message = 'Worksheet items were not distinct and sorted: %s' % (items,)
precondition(item_sort_keys == sorted(set(item_sort_keys)), message)
if 'uuid' not in row:
row['uuid'] = spec_util.generate_uuid()
super(Worksheet, self).update_in_memory(row)
if items is not None:
self.items = [(item['bundle_uuid'], item['value']) for item in items]
self.last_item_id = max(item['id'] for item in items) if items else -1
def start_bundle(self, bundle):
'''
Run the given bundle using an available Machine.
Return whether something was started.
'''
# Check that we're running a bundle in the RUNNING state.
state_message = 'Unexpected bundle state: %s' % (bundle.state,)
precondition(bundle.state == State.RUNNING, state_message)
data_hash_message = 'Unexpected bundle data_hash: %s' % (bundle.data_hash,)
precondition(bundle.data_hash is None, data_hash_message)
# Run the bundle.
with self.profile('Running bundle...'):
started = False
if isinstance(bundle, RunBundle):
try:
# Get the username of the bundle
results = self.auth_handler.get_users('ids', [bundle.owner_id])
if results.get(bundle.owner_id):
username = results[bundle.owner_id].name
else:
username = str(bundle.owner_id)
status = self.machine.start_bundle(bundle, self.bundle_store, self.get_parent_dict(bundle), username)
if status != None:
started = True
except Exception as e:
# If there's an exception, we just make the bundle fail
# (even if it's not the bundle's fault).
temp_dir = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(temp_dir)
status = {'bundle': bundle, 'success': False, 'failure_message': str(e), 'temp_dir': temp_dir}
print '=== INTERNAL ERROR: %s' % e
started = True # Force failing
traceback.print_exc()
else: # MakeBundle
started = True
if started: print '-- START BUNDLE: %s' % (bundle,)
# If we have a MakeBundle, then just process it immediately.
if isinstance(bundle, MakeBundle):
temp_dir = canonicalize.get_current_location(self.bundle_store, bundle.uuid)
path_util.make_directory(temp_dir)
status = {'bundle': bundle, 'success': True, 'temp_dir': temp_dir}
# Update database
if started:
self.update_running_bundle(status)
return started
def update_in_memory(self, row, strict=False):
'''
Initialize the attributes on this object from the data in the row.
The attributes of the row are inferred from the table columns.
If strict is True, checks that all columns are included in the row.
'''
if strict:
for column in self.COLUMNS:
precondition(column in row, 'Row %s missing column: %s' % (row, column))
for (key, value) in row.iteritems():
message = 'Row %s has extra column: %s' % (row, key)
precondition(key in self.COLUMNS or key == 'id', message)
setattr(self, key, value)
def fetch_one(self, resource_type, resource_id=None, params=None):
"""
Same as JsonApiClient.fetch, but always returns exactly one resource
dictionary, or throws a NotFoundError if the results contain any more
or less than exactly one.
"""
results = self.fetch(resource_type, resource_id=resource_id, params=params)
precondition(
not isinstance(results, list) or len(results) == 1,
"Got %d %s when expecting exactly 1." % (len(results), resource_type),
)
if not isinstance(results, list):
return results
else:
return results[0]
def validate(self):
'''
Check a number of basic conditions that would indicate serious errors if
they do not hold. Subclasses may override this method for further
validation, but they should always call the super's method.
'''
spec_util.check_uuid(self.uuid)
abstract_init = 'init-ed abstract bundle: %s' % (self.__class__.__name__,)
precondition(self.BUNDLE_TYPE, abstract_init)
type_mismatch = 'Mismatch: %s vs %s' % (self.bundle_type, self.BUNDLE_TYPE)
precondition(self.bundle_type == self.BUNDLE_TYPE, type_mismatch)
# Check that metadata conforms to specs and check each dependency.
self.metadata.validate(self.METADATA_SPECS)
for dep in self.dependencies:
dep.validate()
def install_dependencies(self, bundle_store, parent_dict, dest_path, copy):
'''
Symlink or copy this bundle's dependencies into the directory at dest_path.
The caller is responsible for cleaning up this directory.
'''
precondition(os.path.isabs(dest_path), '%s is a relative path!' % (dest_path,))
pairs = self.get_dependency_paths(bundle_store, parent_dict, dest_path, relative_symlinks=not copy)
for (target, link_path) in pairs:
# If the dependency already exists, remove it (this happens when we are reinstalling)
if os.path.exists(link_path):
path_util.remove(link_path)
# Either copy (but not follow further symlinks) or symlink.
if copy:
path_util.copy(target, link_path, follow_symlinks=False)
else:
os.symlink(target, link_path)
def send_json_message(self,
socket_id,
message,
timeout_secs,
autoretry=True):
"""
Sends a JSON message to the given socket, retrying until it is received
correctly.
If the message is not sent successfully after timeout_secs, return
False. Otherwise, returns True.
Note, only the worker should call this method with autoretry set to
False. See comments below.
"""
start_time = time.time()
while time.time() - start_time < timeout_secs:
with closing(socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)) as sock:
sock.settimeout(timeout_secs)
success = False
try:
sock.connect(self._socket_path(socket_id))
if autoretry:
# This auto retry mechanisms helps ensure that messages
# sent to a worker are received more reliably. The
# socket API isn't particularly robust to our usage
# where we continuously start and stop listening on a
# socket, like the worker checkin mechanism does. In
# fact, it seems to spuriously accept connections
# just when a socket object is in the process of being
# destroyed. On the sending end, such a scenario results
# in a "Broken pipe" exception, which we catch here.
success = sock.recv(len(
WorkerModel.ACK)) == WorkerModel.ACK
else:
success = True
except socket.error:
pass
if not success:
# Shouldn't be too expensive just to keep retrying.
# TODO: maybe exponential backoff
time.sleep(
0.3
) # changed from 0.003 to keep from rate-limiting due to dead workers
continue
if not autoretry:
# When messages are being sent from the worker, we don't
# have the problem with "Broken pipe" as above, since
# code waiting for a reply shouldn't just abruptly stop
# listening.
precondition(
sock.recv(len(WorkerModel.ACK)) == WorkerModel.ACK,
'Received invalid ack on socket.',
)
sock.sendall(json.dumps(message).encode())
return True
return False
def get_relative_path(root, path):
"""
Return the relative path from root to path, which should be nested under root.
"""
precondition(path.startswith(root), '%s is not under %s' % (path, root))
return path[len(root):]
def check_uuid(uuid_str):
"""
Raise a PreconditionViolation if the uuid does not conform to its regex.
"""
message = 'uuids must match %s, was %s' % (UUID_REGEX.pattern, uuid_str)
precondition(UUID_REGEX.match(uuid_str), message)
def json_api_meta(doc, meta_update):
precondition(isinstance(meta_update, dict), "Meta data must be dict")
meta = doc.setdefault('meta', {})
meta.update(meta_update)
return doc
