def sync_database(root_path, dump_mysql, server=DEFAULT_SERVER,
sync_data=False, reset=False, remote=False):
appcfg = load_app_config(root_path)
devcfg = load_dev_config(root_path)
if 'mysql' not in devcfg:
logger.info("No MySQL configuration found in dev.yaml.")
return 'succeeded'
logger.info("Dumping database to %s...", dump_mysql)
with open(dump_mysql, 'w') as dumpfile:
devcfg['mysql'].setdefault('use_unicode', False)
devcfg['mysql'].setdefault('charset', 'utf8')
conn = MySQLdb.connect(**devcfg['mysql'])
try:
struct, data = dumps(dumpfile, conn, sync_data)
appname = appcfg['application']
finally:
conn.close()
if remote:
logger.info("Syncing database to servers...")
try:
result = verify(appname, struct, data, reset, server)
except:
logger.exception('Error occured')
return 'failed'
return result
def remote_main(args):
logger.warning("CAUTION! CAUTION! CAUTION! ")
logger.warning("You are connecting to the database in production environment!")
logger.warning("BE CAREFUL!")
root_path = args.root_path or find_app_root()
appcfg = load_app_config(root_path)
appname = appcfg['application']
try:
ws = create_connection("ws://%s.xiaom.co:5000/_sheep/mysql/" % appname)
read_thread = Thread(target=read_output, args=(ws, ))
read_thread.start()
except:
print 'Can\'t connect remote.'
return
try:
while True:
if not ws.connected:
break
else:
command = raw_input('')
if not command.endswith('\n'):
command += '\n'
ws.send(command)
if command == 'quit\n' or command == '\\q\n':
sys.stdout.write('Wait for connection closed.\t\n')
read_thread.join()
except:
ws.send('quit\n')
read_thread.join()
finally:
ws.close()
def mirror_statics(root_path, server):
appcfg = load_app_config(root_path)
statics = []
for handler_config in appcfg['handlers']:
if 'static_files' not in handler_config:
continue
statics.append(handler_config)
if not statics:
logger.info("No Statics configuration found.")
return 'Mirror succeeded.'
logger.info("Mirror static files to UpYun...")
post_data = json.dumps({'application': appcfg['application'], 'configs': statics})
post_url = '%s/statics/' % server
req = urllib2.Request(post_url, post_data)
f = urllib2.urlopen(req)
line = ''
for line in iter(f.readline, ''):
try:
loglevel, line = line.split(':', 1)
loglevel = int(loglevel)
except ValueError:
loglevel = logging.DEBUG
logger.log(loglevel, "%s", line.rstrip())
return line
def _main(args):
root_path = args.root_path or find_app_root()
appcfg = load_app_config(root_path)
ret = sync_database(root_path, args.dump_mysql, args.server,remote=True)
if 'succeeded' not in ret:
logger.info("Syncdb failed, deploy exit ...")
sys.exit(1)
logger.info("Pushing modifications...")
push_modifications(root_path)
vcs_url = get_vcs_url(root_path)
if not vcs_url:
logger.error("%s is not under version control. abort.", root_path)
sys.exit(1)
logger.debug("app url: %s", vcs_url)
logger.info("Deploying to servers...")
data = {'app_name': appcfg['application'],
'app_url': vcs_url}
if args.fast:
data['fast'] = '1'
ret = deploy_to_server(data, args.server)
if ret == 'Failed':
logger.warning("It seems that the deploy failed. Try again later. "
"If the failure persists, contact admin please.")
sys.exit(1)
ret = mirror_statics(root_path, args.server)
if 'succeeded' not in ret:
logger.info("Mirror failed, deploy exit ...")
sys.exit(1)
def run_server(approot, port=8080, pidfile=None, daemon=False):
appconf = load_app_config(approot)
sys.argv = ['sheep serve', '-b', '0.0.0.0:{0}'.format(port)]
if pidfile:
sys.argv += ['-p', pidfile]
if daemon:
sys.argv += ['-D']
sys.argv += ['-c', os.path.join(os.path.dirname(__file__),
'dev_appserver_config.py')]
worker = appconf.get('worker', 'async')
if worker == 'async':
worker = 'sheep.gworkers.ggevent.GeventWorker'
sys.argv += ['-k', worker]
nworkers = appconf.get('nworkers', '1')
sys.argv += ['-w', str(nworkers)]
sys.argv.append(approot) # must be the only positional parameter
def add_handler(app):
dev_handler = {'url': '/_sheep/.*', 'wsgi_app': 'sheep.dev:dispatcher'}
app.appconf['handlers'].insert(0, dev_handler)
app = SHEEPApplication(on_init=add_handler)
return app.run()
def main(args):
root_path = args.root_path or find_app_root()
appcfg = load_app_config(root_path)
appname = appcfg['application']
try:
ws = create_connection("ws://%s.xiaom.co:5000/_sheep/shell/" % appname)
read_thread = Thread(target=read_output, args=(ws, ))
read_thread.start()
except:
print 'Can\'t connect remote.'
return
try:
while True:
if not ws.connected:
break
else:
try:
command = raw_input('')
if not command.endswith('\n'):
command += '\n'
ws.send(command)
if command == 'quit()\n' or command == 'exit()\n':
sys.stdout.write('Wait for connection closed.\t\n')
read_thread.join()
except KeyboardInterrupt:
print
print 'KeyboardInterrupt'
ws.send('\n')
except:
ws.send('quit()\n')
read_thread.join()
finally:
ws.close()
def main(args):
root_path = args.root_path or find_app_root()
appcfg = load_app_config(root_path)
appname = appcfg['application']
try:
ws = create_connection("ws://%s.xiaom.co:5000/_sheep/log/" % appname)
except Exception, e:
print e
print 'Can\'t connect remote.'
return
def main(args):
approot = os.path.abspath(args.root_path or find_app_root())
appcfg = load_app_config(approot)
appname = appcfg['application']
venvdir = get_venvdir(approot)
vcs = get_vcs(approot)
if vcs == 'hg':
check_call(['hg', '-R', approot, 'pull', '-u'])
elif vcs == 'svn':
check_call(['svn', 'up', approot])
elif vcs == 'git':
try:
check_call(['git', '--git-dir', os.path.join(approot, '.git'),
'--work-tree', approot, 'pull'])
except:
call(['git', '--git-dir', os.path.join(approot, '.git'), 'pull'])
else:
logger.error("%s is not under version control", approot)
return 1
if not os.path.exists(os.path.join(approot, 'permdir')):
os.mkdir(os.path.join(approot, 'permdir'))
if not os.path.exists(venvdir):
pkgdir = os.path.join(os.path.dirname(os.path.dirname(here)),
'3rdparty')
logger.info('Creating virtualenv at %s...', venvdir)
check_call(['python', os.path.join(pkgdir, 'virtualenv.py'),
'--no-site-packages',
'--distribute',
'--extra-search-dir', pkgdir,
'--never-download',
'--prompt', '(%s)' % appname,
venvdir])
sitecustomize_path = os.path.join(venvdir, 'lib',
'python'+sys.version[:3], 'sitecustomize.py')
if not os.path.exists(sitecustomize_path):
logger.info("Create sitecustomize.py...")
with open(sitecustomize_path, 'w') as f:
f.write("""\
import os, sys, site
sdk_path = os.environ.get('SHEEP_SDK_PATH')
ignore_sdk_path = os.environ.get('SHEEP_IGN_SDKPATH')
if sdk_path and not ignore_sdk_path:
sdk_site_dir = os.path.join(sdk_path, 'venv', 'lib', 'python'+sys.version[:3],
'site-packages')
site.addsitedir(sdk_site_dir)
approot = os.environ['SHEEP_APPROOT']
from sheep.env import activate_app
activate_app(approot, chdir=False)
""")
os.environ['SHEEP_APPROOT'] = approot
os.environ['SHEEP_IGN_SDKPATH'] = 'True'
if not is_pip_compatible(os.path.join(venvdir, 'bin', 'pip')):
logger.info('Installing patched pip...')
check_call([os.path.join(venvdir, 'bin', 'pip'), 'install', '-U',
'hg+https://bitbucket.org/CMGS/pip'])
if os.path.exists(os.path.join(approot, 'pip-req.txt')):
logger.info('Installing requirements...')
check_call([os.path.join(venvdir, 'bin', 'pip'), 'install',
'-r', os.path.join(approot, 'pip-req.txt'),
'--save-download', os.path.join(approot, 'pip-download'),
'--no-index',
'--find-links', 'file://%s/pip-download/' % approot,
'--fallback-index-url', 'http://pypi.python.org/simple/',
])
clear_redundant_pkgs(venvdir)
if os.path.exists(os.path.join(approot, 'setup.py')):
logger.info("Running python setup.py develop")
check_call([os.path.join(venvdir, 'bin', 'python'),
os.path.join(approot, 'setup.py'),
'develop'])
logger.info('Sync success...')
#!/usr/bin/python
# encoding: UTF-8
"""
static_files url replace
"""
__all__ = ['static_files', 'public_files', 'upload_files']
import os
from sheep.util import load_app_config, find_app_root
approot = os.environ.get('SHEEP_APPROOT') or find_app_root(raises=False) or ''
appconf = load_app_config(approot)
appname = appconf['application']
upload_prefix = appconf.get('upload_prefix', '')
public_prefix = appconf.get('public_prefix', '')
static_files = lambda path: path
upload_files = lambda path: os.path.join(upload_prefix, appname) + path
public_files = lambda path: public_prefix + path
