def env_setup():
'''Initialize environment with requisite Python modules.'''
print green("Installing requisite modules...")
# Install our requistite modules for the website.
sh.pip("install", r="requirements.txt")
import platform
if platform.python_version_tuple() < (2,7):
sh.pip("install", "unittest2")
def add_package():
url = str(canopy.kvdb["pending-install"])
name = url.rpartition("/")[2].partition(".")[0]
bare_path = canopy.tree / "trunk/software/source" / (name + ".git")
sh.git("clone", "--bare", url, str(bare_path))
sh.git("-C", str(bare_path), "update-server-info")
shutil.move(str(bare_path / "hooks/post-update.sample"),
str(bare_path / "hooks/post-update"))
sh.pip("install", "-e", "git+{}#egg={}".format(bare_path, name))
del web.kvdb["reload-lock"]
def install_cocaine():
COCAINE_VERSION = "0.11.3.2"
# COCAINE_VERSION = "0.11.2.5"
# there is no libcocaine-plugin-docker2=0.11.3.2 package in precise repo,
# and no docker package in trusty repo at all
# ok, let's install cocaine 0.11.3.2 on trusty hoping that docker plugin is integrated into one of the packages
# Issue something like cocaine-runtime -c /cocaine-setup/conf/cocaine-local.conf -d to start the daemon
# apt-get install libcocaine-core2 cocaine-runtime msgpack-python build-essential python-dev cocaine-tools
install_packages(["libcocaine-core2=%s" % COCAINE_VERSION, "cocaine-runtime=%s" % COCAINE_VERSION,
# "libcocaine-plugin-docker2=%s" % COCAINE_VERSION,
"msgpack-python", "build-essential", "python-dev"])
sh.pip("install", "cocaine-tools")
sh.mkdir("/var/run/cocaine/") # cocaine-runtime can't do it itself
def test_remote_debug():
sh.pip('install tornado --user'.split())
enaml_native = sh.Command('enaml-native')
enaml_native('start', '--remote-debugging', _bg=True)
#: Add
sys.path.append('src/apps/')
sys.path.append('src/')
#: Init remote nativehooks implementation
from enamlnative.core import remotehooks
remotehooks.init()
main()
def resolve_deps(self, force=False):
if force:
self.dependencies = []
if len(self.dependencies) > 0:
return self.dependencies
pip_install_opts = ['--no-install', '--verbose', '-e']
rm('-r', '-f', "/tmp/pip_build_{0}".format(getuser()))
with pushd(self.path):
print('')
print('Gathering package requirements ...')
for line in pip('install', pip_install_opts, '.'):
string = line.rstrip()
match = re.search('Downloading/unpacking (.*?) \(from (.*?)\)',
string)
if match:
self._add_dependency(match.group(1),
dependents=match.group(2))
continue
match = re.search(
'Requirement already satisfied.*?: (.*?) in .*?\(from (.*?)\)',
string)
if match:
self._add_dependency(match.group(1),
dependents=match.group(2))
continue
print("... done. {0} records found.".format(len(self.dependencies)))
return self.dependencies
def execute(self):
with Authentication():
try:
sh.pip("show", self.package)
except sh.ErrorReturnCode_1:
try:
sh.pip("install", self.package)
except sh.ErrorReturnCode as err:
err_message = "\n\t" + err.stderr.replace("\n", "\n\t")
logging.error(
"Error with `sudo pip install %s`: %s",
self.package,
err_message
)
return False
return True
def execute(self):
with Authentication():
try:
sh.pip("show", self.package)
except sh.ErrorReturnCode_1:
try:
sh.pip("install", self.package)
except sh.ErrorReturnCode as err:
err_message = "\n\t" + err.stderr.replace("\n", "\n\t")
logging.error(
"Error with `sudo pip install %s`: %s",
self.package,
err_message
)
return False
return True
def build(self):
self.change_to_parent_dir()
self.init_virtualenv()
print('created {}'.format(self.virtualenv_dir()))
if self.virtualenv_dir_exists():
os.chdir(self.VIRTUAL_SUBDIR)
if IS_PY2:
self.link_supervisord_files()
print('linked supervisord')
# activate the virtualenv
with venv(dirname=self.virtualenv_dir()):
print('configuring virtualenv')
os.chdir('lib')
if IS_PY2:
sh.ln('-s', 'python2.7', 'python')
else:
sh.ln('-s', 'python3.4', 'python')
print(' installing requirements')
with open(self.get_requirements()) as f:
for line in f.readlines():
if len(line.strip()) > 0:
print(' {}'.format(line.strip()))
if IS_PY2:
sh.pip('install', line.strip())
else:
sh.pip3('install', line.strip())
print(' linking our packages')
os.chdir('python/site-packages')
self.link_packages()
self.finish_build()
print('Finished creating virtualenv')
else:
print('!!!Error: Unable to create virtualenv')
def download_depends(depends, type_, path):
logger.debug('Downloading dependencies for %s', path)
if type_ == 'python':
install_path = "%s/depends" % path
# pip install -b /tmp --src=/tmp --install-option="--install-lib=/home/inkvi/test" -v msgpack-python
output = sh.pip("install", "-v", "-I", "-b", path, "--src", path, "--install-option",
"--install-lib=%s" % install_path, *depends)
return os.listdir(install_path)
elif type_ == 'nodejs':
install_path = "%s/node_modules"%path
sh.mkdir("-p",install_path)
sh.npm("install","--production",_cwd=path)
return os.listdir(install_path)
def loop(config, jobloader):
# unique name for restart
redisclient = sparqueue.redis.client(config)
queue_manager = sparqueue.queue.QueueManager(config, redisclient)
queue_manager.add_list(config['queues'])
while RUN:
while RUN:
try:
(queue, job) = queue_manager.pop()
break
except sparqueue.queue.QueueException, e:
# TODO: want probably to spawn this in
# another cron process instead
requeued = queue_manager.requeue()
if len(requeued) > 1:
print 'Requeued %s' % ','.join(requeued)
# exit before continuing on when not running
if not RUN:
break
logger.info(job)
jobid = job['metadata']['jobid']
reporter = sparqueue.reporter.Reporter(logger, queue, jobid)
logger.info('Processing jobid %s' % jobid)
if 'install' in job:
if 'pip' in job['install']:
reporter.step('installing %s' % job['install']['pip'])
try:
logger.info(sh.pip('install', job['install']['pip']))
except sh.ErrorReturnCode, e:
logger.error('problem installing %s: %s' % (
job['metadata']['jobid'], traceback.format_exc(e)))
queue.failed(e)
continue
def resolve_from_dir(self, path):
self._pip_install_opts.append('-e')
if not os.path.exists(path):
raise Exception("Path not found '{0}'".format(path))
rm('-r', '-f', "/tmp/pip_build_{0}".format(getuser()))
with pushd(path):
print("")
print("'git status' in '{0}':".format(path))
print("------------")
print(git('status'))
print("------------")
self.package_name = tail(python("setup.py", "--name"), "-1").rstrip()
print("")
print("Gathering package requirements ...")
for line in pip('install', self._pip_install_opts, '.'):
string = line.rstrip()
match = re.search(
'Downloading/unpacking (.*?) \(from (.*?)\)',
string
)
if match:
self._add_pip_package(match.group(1), from_package=match.group(2))
continue
match = re.search(
'Requirement already satisfied.*?: (.*?) in .*?\(from (.*?)\)',
string
)
if match:
self._add_pip_package(match.group(1), from_package=match.group(2))
continue
print("Done. {0} records found.".format(len(self.entries)))
def _python_environment(self):
sh.mkdir('-p', hpath('.config'), **SHARG)
sh.cp(wpath('python/flake8'), hpath('.config/flake8'), **SHARG)
sh.pip('install', 'flake8', 'autopep8', 'pyyaml', '--user', **SHARG)
def get_packages_info(self, requirements_file=None):
for item in pip('freeze').splitlines():
yield self.extract_dependency(item)
def setup_pip_deps():
pip(['install', '--ignore-installed', 'azure-cli', 'homebrew-pypi-poet'], _out=sys.stdout, _err=sys.stdout)
def pip_install(package_name):
output = sh.pip("freeze", package_name)
if package_name in output:
return "%s already installed" % package_name
else:
return sh.pip("install", package_name)
def update_software():
logger('Updating CloudCompose stack...')
sh.apt_get('update')
sh.apt_get('-y', 'install', 'docker')
sh.pip('install', '-U', 'docker-compose')
def update_software():
logger('Updating CloudCompose stack...')
sh.apt_get('update')
sh.apt_get('-y', 'install', 'docker')
sh.pip('install', '-U', 'docker-compose')
def env_setup():
'''Initialize environment with requisite Python modules.'''
print(green("Installing requisite modules..."))
# Install our requistite modules for the website.
sh.pip("install", r="requirements.txt")
#!/usr/local/bin/python
import os
import sh
import sys
import time
import json
sys.path.insert(0, os.path.join(os.getcwd(), 'lib'))
try:
from finsymbols import symbols
from googlefinance import getQuotes
from yahoo_finance import Share
except ImportError:
print "pip bs4 and simplejson are needed"
sh.pip(['install', 'bs4', 'simplejson'])
sys.exit(2)
DEBUG = False
sp500 = symbols.get_sp500_symbols()
sp400 = symbols.get_sp400_symbols()
sp900 = sp500 + sp400
google_bad_list = ['APH', 'LMT']
sp900_all = {}
sp900_chg = {}
for stock in sp900:
sys.stdout.write(stock['symbol'])
try:
if stock['symbol'] not in google_bad_list and '.' in stock['symbol']: