def __init__(self, args):
self.args = args
self.codalab_manager = CodaLabManager()
self.codalab_client = self.codalab_manager.client(args.server)
self.staged_uuids = []
self.last_worker_start_time = 0
logger.info('Started worker manager.')
def setUp(self):
self.codalab_manager = CodaLabManager()
self.codalab_manager.config['server']['class'] = 'SQLiteModel'
self.bundle_manager = BundleManager(self.codalab_manager)
self.download_manager = self.codalab_manager.download_manager()
self.upload_manager = self.codalab_manager.upload_manager()
# Create a standard user
self.user_id = generate_uuid()
self.bundle_manager._model.add_user(
"codalab_standard",
"*****@*****.**",
"Test",
"User",
"password",
"Stanford",
user_id=self.user_id,
)
# Create a root user
self.root_user_id = self.codalab_manager.root_user_id()
self.bundle_manager._model.add_user(
"codalab_root",
"*****@*****.**",
"Test",
"User",
"password",
"Stanford",
user_id=self.root_user_id,
)
def _create_cli(self, worksheet_uuid):
manager = CodaLabManager(
temporary=True,
clients={settings.BUNDLE_SERVICE_URL: self.client})
manager.set_current_worksheet_uuid(self.client, worksheet_uuid)
cli = bundle_cli.BundleCLI(manager, headless=True)
return cli
def main():
cli = BundleCLI(CodaLabManager())
try:
cli.do_command(sys.argv[1:])
except KeyboardInterrupt:
print('Terminated by Ctrl-C')
sys.exit(130)
def __init__(
self,
docker_image,
initial_command="",
manager=None,
dependencies=[],
bundle_locations={},
verbose=False,
stdout=sys.stdout,
stderr=sys.stderr,
):
# Instantiate a CodaLabManager if one is not passed in
self._manager = manager if manager else CodaLabManager()
self._docker_image = docker_image
self._initial_command = initial_command
InteractiveSession._validate_bundle_locations(bundle_locations,
dependencies)
self._dependencies = dependencies
self._bundle_locations = bundle_locations
self._docker_client = docker.from_env(
timeout=InteractiveSession._MAX_SESSION_TIMEOUT)
self._session_uuid = generate_uuid()
self._verbose = verbose
self._stdout = stdout
self._stderr = stderr
def test_temp_codalab_manager(self):
manager: CodaLabManager = CodaLabManager(temporary=True)
self.assertEqual(manager.state, {'auth': {}, 'sessions': {}})
manager.save_state()
self.assertFalse(
os.path.exists(manager.state_path),
msg=
'Assert that the current state is not written out to state_path for a temporary CodaLabManager',
)
def main(args):
manager = CodaLabManager()
model = manager.model()
# Get the the message
subject = args.subject
with open(args.body_file) as f:
body_template = f.read()
mime_type = 'html' if args.body_file.endswith('.html') else 'plain'
# Figure out who we want to send
to_send_list = get_to_send_list(model, args.threshold)
sent_list = get_sent_list(args.sent_file)
sent_emails = set(info['email'] for info in sent_list)
pending_to_send_list = [info for info in to_send_list if info['email'] not in sent_emails]
print 'Already sent %d emails, %d to go' % (len(sent_list), len(pending_to_send_list))
for i, info in enumerate(pending_to_send_list):
if args.only_email and args.only_email != info['email']:
continue
# Derived fields
info['greeting_name'] = info['first_name'] or info['last_name'] or info['user_name']
info['full_name'] = ' '.join([x for x in [info['first_name'], info['last_name']] if x])
info['email_description'] = '%s <%s>' % (info['full_name'], info['email']) if info['full_name'] else info['email']
info['sent_time'] = time.time()
print 'Sending %s/%s (%s>=%s, doit=%s): [%s] %s' % \
(i, len(pending_to_send_list), info['notifications'], args.threshold, args.doit, info['user_name'], info['email_description'])
# Apply template to get body of message
body = body_template
for field, value in info.items():
body = body.replace('{{' + field + '}}', unicode(value or ''))
if args.verbose >= 1:
print 'To : %s' % info['email_description']
print 'Subject : %s' % subject
print body
print '-------'
if not args.doit:
continue
# Send the actual email
manager.emailer.send_email(
recipient=info['email_description'],
subject=subject,
body=body,
mime_type=mime_type,
)
# Record that we sent
with open(args.sent_file, 'a') as f:
print >>f, json.dumps(info)
f.flush()
def _create_cli(self, worksheet_uuid):
output_buffer = StringIO()
manager = CodaLabManager(temporary=True,
clients={'local': self.client})
manager.set_current_worksheet_uuid(self.client, worksheet_uuid)
cli = bundle_cli.BundleCLI(manager,
headless=True,
stdout=output_buffer,
stderr=output_buffer)
return cli, output_buffer
def find_default_editor():
manager = CodaLabManager()
editor = os.getenv('EDITOR')
if editor:
return editor
# If not yet set, use a sane default.
if sys.platform == 'win32':
editor = 'notepad'
else:
editor = 'vi'
return editor
def test_temp_codalab_manager(self):
"""
A codalab manager with temporary state should initialize its state from an existing
state.json file if it is present.
"""
manager: CodaLabManager = CodaLabManager(temporary=True)
self.assertEqual(manager.state, {'auth': {}, 'sessions': {}})
manager.save_state()
self.assertFalse(
os.path.exists(manager.state_path),
msg=
'Assert that the current state is not written out to state_path for a temporary CodaLabManager',
)
def setUp(self):
self.codalab_manager = CodaLabManager()
self.codalab_manager.config['server']['class'] = 'SQLiteModel'
self.bundle_manager = BundleManager(self.codalab_manager)
self.user_id = generate_uuid()
self.bundle_manager._model.add_user(
"codalab",
"*****@*****.**",
"Test",
"User",
"password",
"Stanford",
user_id=self.user_id,
)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--sleep-time',
help='Number of seconds to wait between successive actions.',
type=int,
default=0.5,
)
args = parser.parse_args()
manager = BundleManager(CodaLabManager())
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: manager.signal())
manager.run(args.sleep_time)
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
manager = CodaLabManager()
engine = manager.model().engine
connection = engine.connect()
context.configure(connection=connection, target_metadata=target_metadata)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
manager = CodaLabManager()
url = manager.model().engine.url
context.configure(url=url, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
def _create_cli(self, worksheet_uuid):
"""
Create an instance of the CLI.
The CLI uses JsonApiClient to communicate back to the REST API.
This is admittedly not ideal since now the REST API is essentially
making HTTP requests back to itself. Future potential solutions might
include creating a subclass of JsonApiClient that can reroute HTTP
requests directly to the appropriate Bottle view functions.
"""
output_buffer = StringIO()
rest_client = JsonApiClient(self._rest_url(), lambda: get_user_token())
manager = CodaLabManager(
temporary=True,
config=local.config,
clients={
self._rest_url(): rest_client
})
manager.set_current_worksheet_uuid(self._rest_url(), worksheet_uuid)
cli = bundle_cli.BundleCLI(manager, headless=True, stdout=output_buffer, stderr=output_buffer)
return cli, output_buffer
def test_temp_codalab_manager_initialize_state(self):
initial_state: Dict = {
'auth': {
"https://worksheets.codalab.org": {
"token_info": {
"access_token": "secret"
}
}
},
'sessions': {},
}
cache_file = tempfile.NamedTemporaryFile(delete=False)
with open(cache_file.name, "w") as f:
json.dump(initial_state, f)
os.environ["CODALAB_STATE"] = cache_file.name
manager: CodaLabManager = CodaLabManager(temporary=True)
self.assertEqual(manager.state, initial_state)
os.remove(cache_file.name)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--sleep-time',
help='Number of seconds to wait between successive actions.',
type=int,
default=0.5,
)
parser.add_argument(
'--worker-timeout-seconds',
help=
'Number of seconds to wait after a worker check-in before determining a worker is offline',
type=int,
default=60,
)
args = parser.parse_args()
manager = BundleManager(CodaLabManager(), args.worker_timeout_seconds)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: manager.signal())
manager.run(args.sleep_time)
def create_rest_app(manager=CodaLabManager()):
"""Creates and returns a rest app."""
install(SaveEnvironmentPlugin(manager))
install(CheckJsonPlugin())
install(oauth2_provider.check_oauth())
install(CookieAuthenticationPlugin())
install(UserVerifiedPlugin())
install(PublicUserPlugin())
install(ErrorAdapter())
# Replace default JSON plugin with one that handles datetime objects
# Note: ErrorAdapter must come before JSONPlugin to catch serialization errors
uninstall(JSONPlugin())
install(JSONPlugin(json_dumps=DatetimeEncoder().encode))
# JsonApiPlugin must come after JSONPlugin, to inspect and modify response
# dicts before they are serialized into JSON
install(JsonApiPlugin())
for code in range(100, 600):
default_app().error(code)(error_handler)
root_app = Bottle()
root_app.mount('/rest', default_app())
# Look for templates in codalab-worksheets/views
bottle.TEMPLATE_PATH = [
os.path.join(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'views')
]
# Increase the request body size limit to 8 MiB
bottle.BaseRequest.MEMFILE_MAX = 8 * 1024 * 1024
return root_app
def main(args):
manager = CodaLabManager()
# Get the the message
subject = args.subject
with open(args.body_file) as f:
body_template = f.read()
if args.body_file.endswith('.md'):
body_template = f"""
<div style='margin:auto; width: 100%; max-width: 600px'>
<img src=https://worksheets.codalab.org/img/codalab-logo.png style='max-width: 100%;' />
<h1>CodaLab Worksheets</h1>
{markdown2.markdown(body_template)}<br><br>
<small>If you'd like stop receiving these emails, please <a href='https://worksheets.codalab.org/account/profile'>update your account settings on CodaLab</a>.</small>
</div>
"""
mime_type = ('html' if args.body_file.endswith('.html')
or args.body_file.endswith('.md') else 'plain')
# Figure out who we want to send
to_send_list = ([{
'email': e,
'first_name': '',
'last_name': '',
'user_name': '',
'notifications': 2
} for e in args.emails.split(",")] if args.emails else get_to_send_list(
manager.model(), args.threshold))
sent_list = get_sent_list(args.sent_file)
sent_emails = set(info['email'] for info in sent_list)
pending_to_send_list = [
info for info in to_send_list if info['email'] not in sent_emails
]
print('Already sent %d emails, %d to go' %
(len(sent_list), len(pending_to_send_list)))
for i, info in enumerate(pending_to_send_list):
if args.only_email and args.only_email != info['email']:
continue
# Derived fields
info['greeting_name'] = info['first_name'] or info[
'last_name'] or info['user_name']
info['full_name'] = ' '.join(
[x for x in [info['first_name'], info['last_name']] if x])
info['email_description'] = ('%s <%s>' %
(info['full_name'], info['email'])
if info['full_name'] else info['email'])
info['sent_time'] = time.time()
print(('Sending %s/%s (%s>=%s, doit=%s): [%s] %s' % (
i,
len(pending_to_send_list),
info['notifications'],
args.threshold,
args.doit,
info['user_name'],
info['email_description'],
)))
# Apply template to get body of message
body = body_template
for field, value in info.items():
body = body.replace('{{' + field + '}}', str(value or ''))
if args.verbose >= 1:
print('To : %s' % info['email_description'])
print('Subject : %s' % subject)
print(body)
print('-------')
if not args.doit:
continue
# Send the actual email
manager.emailer.send_email(recipient=info['email_description'],
subject=subject,
body=body,
mime_type=mime_type)
# Record that we sent
with open(args.sent_file, 'a') as f:
print(json.dumps(info), file=f)
f.flush()
@Commands.command(
'bundle-manager',
help='Start the bundle manager that executes run and make bundles.',
arguments=(Commands.Argument(
'--sleep-time',
help='Number of seconds to wait between successive actions.',
type=int,
default=0.5), ),
)
def do_bundle_manager_command(bundle_cli, args):
bundle_cli._fail_if_headless(args)
from codalab.worker.bundle_manager import BundleManager
manager = BundleManager.create(bundle_cli.manager)
# Register a signal handler to ensure safe shutdown.
for sig in [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]:
signal.signal(sig, lambda signup, frame: manager.signal())
manager.run(args.sleep_time)
if __name__ == '__main__':
cli = BundleCLI(CodaLabManager())
try:
cli.do_command(sys.argv[1:])
except KeyboardInterrupt:
print 'Terminated by Ctrl-C'
sys.exit(130)
observer = Observer()
observer.schedule(event_handler, path, recursive=True)
observer.start()
try:
while True:
time.sleep(10)
except KeyboardInterrupt:
observer.stop()
observer.join()
else:
from codalab.server.bundle_rpc_server import BundleRPCServer
rpc_server = BundleRPCServer(manager)
rpc_server.serve_forever()
def run_cli():
from codalab.lib.bundle_cli import BundleCLI
cli = BundleCLI(manager)
cli.do_command(sys.argv[1:])
if __name__ == '__main__':
manager = CodaLabManager()
# Either start the server or the client.
try:
if len(sys.argv) > 1 and sys.argv[1] == 'server':
run_server()
else:
run_cli()
except KeyboardInterrupt:
print 'Terminated by Ctrl-C'
sys.exit(130)
def run_command(
args,
expected_exit_code=0,
max_output_chars=1024,
env=None,
include_stderr=False,
binary=False,
force_subprocess=False,
cwd=None,
):
# We import the following imports here because codalab_service.py imports TestModule from
# this file. If we kept the imports at the top, then anyone who ran codalab_service.py
# would also have to install all the dependencies that BundleCLI and CodaLabManager use.
from codalab.lib.bundle_cli import BundleCLI
from codalab.lib.codalab_manager import CodaLabManager
def sanitize(string, max_chars=256):
# Sanitize and truncate output so it can be printed on the command line.
# Don't print out binary.
if isinstance(string, bytes):
string = '<binary>'
if len(string) > max_chars:
string = string[:max_chars] + ' (...more...)'
return string
# If we don't care about the exit code, set `expected_exit_code` to None.
print(">>", *map(str, args), sep=" ")
sys.stdout.flush()
try:
kwargs = dict(env=env)
if not binary:
kwargs = dict(kwargs, encoding="utf-8")
if include_stderr:
kwargs = dict(kwargs, stderr=subprocess.STDOUT)
if cwd:
kwargs = dict(kwargs, cwd=cwd)
if not force_subprocess:
# In this case, run the Codalab CLI directly, which is much faster
# than opening a new subprocess to do so.
stderr = io.StringIO() # Not used; we just don't want to redirect cli.stderr to stdout.
stdout = FakeStdout()
cli = BundleCLI(CodaLabManager(), stdout=stdout, stderr=stderr)
try:
cli.do_command(args[1:])
exitcode = 0
except SystemExit as e:
exitcode = e.code
output = stdout.getvalue()
else:
output = subprocess.check_output([a.encode() for a in args], **kwargs)
exitcode = 0
except subprocess.CalledProcessError as e:
output = e.output
exitcode = e.returncode
except Exception:
output = traceback.format_exc()
exitcode = 'test-cli exception'
if expected_exit_code is not None and exitcode != expected_exit_code:
colorize = Colorizer.red
extra = ' BAD'
else:
colorize = Colorizer.cyan
extra = ''
print(colorize(" (exit code %s, expected %s%s)" % (exitcode, expected_exit_code, extra)))
sys.stdout.flush()
print(sanitize(output, max_output_chars))
sys.stdout.flush()
assert expected_exit_code == exitcode, 'Exit codes don\'t match'
return output.rstrip()
