def main(arguments=None): """Command-line entry point. :param arguments: List of strings that contain the command-line arguments. When ``None``, the command-line arguments are looked up in ``sys.argv`` (``sys.argv[0]`` is ignored). :return: This function has no return value. :raise SystemExit: The command-line arguments are invalid. """ # Parse command-line arguments. if arguments is None: arguments = sys.argv[1:] arguments = cli.parse_args(arguments) # Read the procfile. try: process_types = procfile.loadfile(arguments.procfile) except FileNotFoundError: sys.stderr.write('Procfile not found at "%s".' % arguments.procfile) sys.exit(2) # Read the env file(s). env = {} if arguments.use_env: for path in arguments.envfiles: try: env.update(dotenvfile.loadfile(path)) except FileNotFoundError: sys.stderr.write( 'Warning: environment file "%s" not found.\n' % path ) # Determine how many processes of each type we need. effective_scale = {} requested_scale = dict(arguments.scale) for label in process_types: effective_scale[label] = requested_scale.get( label, requested_scale['*'], ) # Start the event loop. loop = asyncio.get_event_loop() # Register for shutdown events (idempotent, trap once only). shutdown = asyncio.Future() def stop_respawning(): if shutdown.done(): return shutdown.set_result(None) loop.remove_signal_handler(signal.SIGINT) loop.add_signal_handler(signal.SIGINT, stop_respawning) # Spawn tasks. tasks = [] for label, count in effective_scale.items(): process_type = process_types[label] the_cmd = shlex.split(process_type['cmd']) the_env = merge_envs(os.environ, env, process_type['env']) for i in range(count): task = loop.create_task(run_and_respawn( name='%s.%i' % (label, i), cmd=the_cmd, env=the_env, loop=loop, shutdown=shutdown, utc=arguments.use_utc, )) tasks.append(task) if not tasks: sys.stderr.write('Nothing to run.\n') sys.exit(2) # Wait for all tasks to complete. loop.run_until_complete(asyncio.wait(tasks)) loop.close()
def start_process(app, process, request_id, loop=None): loop = loop or asyncio.get_event_loop() # Download source archive. def is_archive(url, response): return response.headers['Content-Type'] in ( 'application/zip', 'application/x-gtar', ) client = app['smartmob.http-client'] archive_path = os.path.join( '.', '.smartmob', 'archives', process['slug'], ) process['state'] = 'downloading' try: content_type = yield from download( client, process['source_url'], archive_path, request_id=request_id, reject=negate(is_archive), ) except: process['state'] = 'download failure' raise process['state'] = 'unpacking' # Deduce archive format. archive_format = { 'application/zip': 'zip', 'application/x-gtar': 'tar', }[content_type] # Unpack source archive. source_path = os.path.join( '.', '.smartmob', 'sources', process['slug'], ) yield from loop.run_in_executor( None, unpack_archive, archive_format, archive_path, source_path, ) process['state'] = 'processing' # Load Procfile and lookup process type. try: process_types = procfile.loadfile( os.path.join(source_path, 'Procfile'), ) except FileNotFoundError: process['state'] = 'no procfile' return try: process_type = process_types[process['process_type']] except KeyError: process['state'] = 'unknown process type' return # Create virtual environment. venv_path = os.path.join( '.', '.smartmob', 'envs', process['slug'], ) try: yield from create_venv(venv_path) except: process['state'] = 'virtual environment failure' raise # Install dependencies. deps_path = os.path.join(source_path, 'requirements.txt') try: yield from pip_install(venv_path, deps_path) except: process['state'] = 'pip install failure' raise # Run it again and again until somebody requests to kill this process. # # TODO: figure out how to get status updates so REST API reflects actual # status. yield from run_and_respawn( name=process['slug'], cmd=process_type['cmd'], env=dict(process_type['env']), shutdown=process['stop'], )