def add(cmd, schedule='? * * * *', cmd_id=None):
"""
Schedule a command to run. This method is idempotent.
:param str cmd: The command to run.
:param str schedule: The schedule to run. Defaults to every hour with a random minute.
If '?' is used (default), it will be replaced with a random value from 0 to 59.
:param str cmd_id: Short version of cmd that we can use to uniquely identify the command for updating purpose.
Defaults to cmd without any redirect chars. It must a regex that matches cmd.
"""
_ensure_cron()
cmd = cmd.replace('"', r'\"')
if cmd_id:
cmd_id = cmd_id.replace('"', r'\"')
if not re.search(cmd_id.replace('\\', r'\\'), cmd):
raise ValueError(
f'cmd_id does not match cmd where:\n\tcmd_id = {cmd_id}\n\tcmd = {cmd}'
)
else:
cmd_id = re.sub('[ &12]*[>|<=].*', '', cmd)
if '?' in schedule:
schedule = schedule.replace('?', str(randint(0, 59)))
crontab_cmd = (
rf'( crontab -l | grep -vi "{cmd_id}"; echo "{schedule} PATH=/usr/local/bin:\$PATH {cmd}" )'
' | crontab -')
run(crontab_cmd, stderr=STDOUT, shell=True)
def remove(name):
""" Remove cmd with the given name """
_ensure_cron()
name = name.replace('"', r'\"')
run(f'( crontab -l | grep -vi "{name}" ) | crontab -',
stderr=STDOUT,
shell=True)
def test_install_python_2(autopip, mock_paths):
system_path, _, _ = mock_paths
assert autopip('install bumper --python 2.7 --update hourly') == """\
Installing bumper to /tmp/system/bumper/0.1.13
Hourly auto-update enabled via cron service
Updating script symlinks in /tmp/system/bin
+ bump
"""
version = run([str(system_path / 'bumper' / 'current' / 'bin' / 'python'), '--version'],
stderr=-2)
assert version.startswith('Python 2.7')
assert run([str(system_path / 'bin' / 'bump'), '-h']).startswith('usage: bump')
assert autopip('uninstall bumper')
def _ensure_cron():
""" Ensure cron is running and crontab is available """
try:
run('which crontab', stderr=STDOUT, shell=True)
except Exception:
raise MissingError(
'crontab is not available. Please install cron or ensure PATH is set correctly.'
)
try:
run('ps -ef | grep /usr/sbin/cron | grep -v grep',
stderr=STDOUT,
shell=True)
except Exception:
if platform.system() == 'Darwin':
return # macOS does not start cron until there is a crontab entry: https://apple.stackexchange.com/a/266836
raise MissingError(
f'cron service does not seem to be running. Try starting it: sudo service cron start'
)
def _pkg_info(self, path=None):
""" Get scripts and entry points from the app """
if not path:
if not self.current_path:
return
path = self.current_path
inspect_py = Path(__file__).parent / 'inspect_app.py'
try:
info = run(f"""set -e
source {path / 'bin/activate'}
python {inspect_py} {self.name}
""",
executable='/bin/bash',
stderr=STDOUT,
shell=True)
return json.loads(info)
except Exception as e:
debug('! Can not get package distribution info because: %s', e)
def install(self, version, app_spec, update=None, python_version=None):
"""
Install the version of the app if it is not already installed
:param str version: Version of the app to install
:param pkg_resources.Requirement app_spec: App version requirement from user
:param UpdateFreq|None update: How often to update. Choose from hourly, daily, weekly, monthly
:param str python_version: Python version to run app
:return: True if install or update happened, otherwise False when nothing happened (already installed / non-tty)
"""
version_path = self.path / version
prev_version_path = self.current_path and self.current_path.resolve()
important_paths = [
version_path, prev_version_path, self._current_symlink
]
if self.settings():
if not python_version:
python_version = self.settings().get('python_version')
if not update:
update = self.settings().get(
'update') and UpdateFreq.from_name(
self.settings()['update'])
if not python_version:
python_version = PYTHON_VERSION
if version_path.exists():
if self.current_version == version:
# Skip printing / ensuring symlinks / cronjob when running from cron
if not sys.stdout.isatty():
return False
pinned = str(app_spec).lstrip(self.name)
pin_info = f' [per spec: {pinned}]' if pinned else ''
info(f'{self.name} is up-to-date{pin_info}')
else:
info(
f'{self.name} {version} was previously installed and will be set as the current version'
)
else:
if not shutil.which('python' + python_version):
error(
f'! python{python_version} does not exist. '
'Please install it first, or ensure its path is in PATH.')
sys.exit(1)
if python_version.startswith('2'):
venv = f'virtualenv --python=python{python_version}'
else:
venv = f'python{python_version} -m venv'
old_venv_dir = None
old_path = None
no_compile = '--no-compile ' if os.getuid() else ''
info(f'Installing {self.name} to {version_path}')
os.environ.pop('PYTHONPATH', None)
if 'VIRTUAL_ENV' in os.environ:
old_venv_dir = os.environ.pop('VIRTUAL_ENV')
old_path = os.environ['PATH']
os.environ['PATH'] = os.pathsep.join([
p for p in os.environ['PATH'].split(os.pathsep)
if os.path.exists(p) and not p.startswith(old_venv_dir)
])
try:
run(f"""set -e
{venv} {version_path}
source {version_path / 'bin' / 'activate'}
pip install --upgrade pip wheel
pip install {no_compile}{self.name}=={version}
""",
executable='/bin/bash',
stderr=STDOUT,
shell=True)
except BaseException as e:
shutil.rmtree(version_path, ignore_errors=True)
if isinstance(e, CalledProcessError):
if e.output:
output = e.output.decode('utf-8')
if not python_version.startswith('2'):
python2 = shutil.which('python2') or shutil.which(
'python2.7')
if python2:
py2_version = Path(python2).name.lstrip(
'python')
if 'is a builtin module since Python 3' in output:
info(
f'Failed to install using Python {python_version} venv, '
f'let\'s try using Python {py2_version} virtualenv.'
)
return self.install(
version,
app_spec,
update=update,
python_version=py2_version)
info(
re.sub(r'(https?://)[^/]+:[^/]+@',
r'\1<xxx>:<xxx>@', output))
error(
f'! Failed to install using Python {python_version}.'
' If this app requires a different Python version, please specify it using --python option.'
)
raise
finally:
if old_venv_dir:
os.environ['VIRTUAL_ENV'] = old_venv_dir
os.environ['PATH'] = old_path
try:
shutil.rmtree(version_path / 'share' / 'python-wheels',
ignore_errors=True)
run(f"""set -e
source {version_path / 'bin' / 'activate'}
pip uninstall --yes pip
""",
executable='/bin/bash',
stderr=STDOUT,
shell=True)
except Exception as e:
debug('Could not remove unnecessary packages/files: %s', e)
# Update current symlink
if not self.current_path or self.current_path.resolve(
) != version_path:
atomic_symlink = self.path / f'atomic_symlink_for_{self.name}'
atomic_symlink.symlink_to(version_path)
atomic_symlink.replace(self._current_symlink)
# Remove older versions
for path in [
p for p in self.path.iterdir() if p not in important_paths
]:
shutil.rmtree(path, ignore_errors=True)
current_scripts = self.scripts()
if not (current_scripts or self.group_specs()):
self.uninstall()
raise exceptions.InvalidAction(
'Odd, there are no scripts included in the app, so there is no point installing it.\n'
' autopip is for installing apps with scripts. To install libraries, please use pip.\n'
' If you are the app owner, make sure to setup entry_points in setup.py.\n'
' See http://setuptools.readthedocs.io/en/latest/setuptools.html#automatic-script-creation'
)
self.settings(app_spec=str(app_spec), python_version=python_version)
# Install cronjobs
if 'update' not in sys.argv:
pinning = '==' in str(app_spec) and not str(app_spec).endswith('*')
if pinning and (self.settings().get('update') or update):
info(
'Auto-update will be disabled since we are pinning to a specific version.'
)
info(
'To enable, re-run without pinning to specific version with --update option'
)
if self.settings().get('update'):
self.settings(update=None)
try:
crontab.remove(self._crontab_id)
except exceptions.MissingError as e:
debug('Could not remove crontab for %s: %s',
self._crontab_id, e)
elif update:
try:
autopip_path = shutil.which('autopip')
if not autopip_path:
raise exceptions.MissingError(
'autopip is not available. Please make sure its bin folder is in PATH env var'
)
# Migrate old crontabs
try:
old_crons = [
c for c in crontab.list().split('\n')
if c and 'autopip update' not in c
]
if old_crons:
cron_re = re.compile('autopip install "(.+)"')
for cron in old_crons:
match = cron_re.search(cron)
if match:
old_app_spec = next(
iter(
pkg_resources.parse_requirements(
match.group(1))))
old_app = App(old_app_spec.name,
self.paths)
if old_app.is_installed:
old_app.settings(
app_spec=str(old_app_spec))
crontab.remove('autopip')
except Exception as e:
debug('Could not migrate old crontabs: %s', e)
crontab.add(
f'{autopip_path} update '
f'2>&1 >> {self.paths.log_root / "cron.log"}',
cmd_id='autopip update')
info(update.name.title() +
' auto-update enabled via cron service')
self.settings(update=update.name.lower())
except Exception as e:
error('! Auto-update was not enabled because: %s',
e,
exc_info=self.debug)
# Install script symlinks
prev_scripts = self.scripts(
prev_version_path) if prev_version_path else set()
old_scripts = prev_scripts - current_scripts
printed_updating = False
for script in sorted(current_scripts):
script_symlink = self.paths.symlink_root / script
script_path = self.current_path / 'bin' / script
if script_symlink.resolve() == script_path.resolve():
continue
if not printed_updating:
info('Updating script symlinks in {}'.format(
self.paths.symlink_root))
printed_updating = True
if script_symlink.exists():
if self.paths.covers(script_symlink) or self.name == 'autopip':
atomic_symlink = self.paths.symlink_root / f'atomic_symlink_for_{self.name}'
atomic_symlink.symlink_to(script_path)
atomic_symlink.replace(script_symlink)
info('* {} (updated)'.format(script_symlink.name))
else:
info('! {} (can not change / not managed by autopip)'.
format(script_symlink.name))
else:
script_symlink.symlink_to(script_path)
info('+ ' + str(script_symlink.name))
for script in sorted(old_scripts):
script_symlink = self.paths.symlink_root / script
if script_symlink.exists():
script_symlink.unlink()
info('- '.format(script_symlink.name))
if not printed_updating and sys.stdout.isatty(
) and current_scripts and 'update' not in sys.argv:
info('Scripts are in {}: {}'.format(
self.paths.symlink_root, ', '.join(sorted(current_scripts))))
# Remove pyc for non-root installs for all versions, not just current.
if os.getuid():
try:
run(f'find {self.path} -name *.pyc | xargs rm',
executable='/bin/bash',
stderr=STDOUT,
shell=True)
except Exception as e:
debug('Could not remove *.pyc files: %s', e)
return True
def list(name_filter='autopip'):
""" List current schedules """
_ensure_cron()
return run(f'crontab -l | grep {name_filter}', stderr=STDOUT, shell=True)
def test_autopip_common(monkeypatch, autopip, capsys, mock_paths):
system_root, _, _ = mock_paths
mock_run = MagicMock()
monkeypatch.setattr('autopip.crontab.run', mock_run)
monkeypatch.setattr('autopip.crontab.randint', Mock(return_value=10))
# Install latest
stdout = autopip('install bumper --update hourly')
assert 'Installing bumper to' in stdout
assert 'Updating script symlinks in' in stdout
assert '+ bump' in stdout
assert len(stdout.split('\n')) == 5
assert run([str(system_root / 'bin' / 'bump'), '-h']).startswith('usage: bump')
assert len(mock_run.call_args_list) == 6
assert mock_run.call_args_list[0:-1] == [
call('which crontab', shell=True, stderr=-2),
call('ps -ef | grep /usr/sbin/cron | grep -v grep', shell=True, stderr=-2),
call('crontab -l | grep autopip', shell=True, stderr=-2),
call('which crontab', shell=True, stderr=-2),
call('ps -ef | grep /usr/sbin/cron | grep -v grep', shell=True, stderr=-2)
]
update_call = re.sub('/tmp/.*/system/', '/tmp/system/',
re.sub('/home/.*virtualenvs/autopip[^/]*', '/home/venv/autopip',
mock_run.call_args_list[-1][0][0]))
assert update_call == (
r'( crontab -l | grep -vi "autopip update"; echo "10 * * * * PATH=/usr/local/bin:\$PATH '
r'/home/venv/autopip/bin/autopip update 2>&1 >> /tmp/system/log/cron.log" ) | crontab -')
assert 'system/bumper/0.1.13' in autopip('list')
assert autopip('list --scripts').split('\n')[1].strip().endswith('/bin/bump')
# Already installed
mock_run.reset_mock()
assert autopip('install bumper --update hourly') == """\
bumper is up-to-date
Hourly auto-update enabled via cron service
Scripts are in /tmp/system/bin: bump
"""
assert mock_run.call_count == 6
# Update manually
assert autopip('update') == 'bumper is up-to-date\n'
assert autopip('update blah') == 'No apps found matching: blah\nAvailable apps: bumper\n'
# Update via cron
assert autopip('update', isatty=False) == ''
bumper_root = system_root / 'bumper'
last_modified = bumper_root.stat().st_mtime
with monkeypatch.context() as m:
m.setattr('autopip.manager.time', Mock(return_value=time() + 3600))
assert autopip('update', isatty=False) == ''
current_modified = bumper_root.stat().st_mtime
assert current_modified > last_modified
# Wait for new version
mock_sleep = Mock(side_effect=[0, 0, 0, Exception('No new version')])
monkeypatch.setattr('autopip.manager.sleep', mock_sleep)
stdout, e = autopip('update bumper --wait', raises=SystemExit)
assert stdout.startswith('! No new version')
stdout, _ = capsys.readouterr()
lines = stdout.split('\n')
assert len(lines) == 5
assert lines[0] == 'Waiting for new version of bumper to be published...'.ljust(80)
assert lines[-2] == '\033[1AWaiting for new version of bumper to be published...'.ljust(80)
assert mock_sleep.call_count == 4
# Uninstall
mock_run.reset_mock()
assert autopip('uninstall bumper') == 'Uninstalling bumper\n'
assert mock_run.call_args_list == [
call('which crontab', shell=True, stderr=-2),
call('ps -ef | grep /usr/sbin/cron | grep -v grep', shell=True, stderr=-2),
call('( crontab -l | grep -vi "autopip install \\"bumper[^a-z]*\\"" ) | crontab -', shell=True, stderr=-2),
call('which crontab', shell=True, stderr=-2),
call('ps -ef | grep /usr/sbin/cron | grep -v grep', shell=True, stderr=-2),
call('( crontab -l | grep -vi "autopip" ) | crontab -', shell=True, stderr=-2)
]
assert autopip('list') == 'No apps are installed yet.\n'