def main(instance_map,
templates,
service_group=None,
with_heartbeat=False,
with_stats_worker=False,
config_spec=None):
"""Entry point"""
from optparse import OptionParser
parser = OptionParser()
parser.add_option("-n",
"--dry-run",
dest="dry_run",
action="store_true",
help="Show the generated config via stdout.")
options, args = parser.parse_args()
for server_name, env_service_map in instance_map.items():
config_content = generate_server_config(server_name,
env_service_map,
config_spec,
templates,
service_group,
with_heartbeat,
with_stats_worker,
with_header=True)
if options.dry_run:
print config_content
else:
filename = "configs/supervisor-%s.conf" % (server_name, )
with open(os.path.join(get_rootdir(), filename), 'w') as f:
f.write(config_content)
def main(instance_map, templates, service_group=None, with_heartbeat=False, with_stats_worker=False, config_spec=None):
"""Entry point"""
from optparse import OptionParser
parser = OptionParser()
parser.add_option(
"-n", "--dry-run", dest="dry_run", action="store_true", help="Show the generated config via stdout."
)
options, args = parser.parse_args()
for server_name, env_service_map in instance_map.items():
config_content = generate_server_config(
server_name,
env_service_map,
config_spec,
templates,
service_group,
with_heartbeat,
with_stats_worker,
with_header=True,
)
if options.dry_run:
print config_content
else:
filename = "configs/supervisor-%s.conf" % (server_name,)
with open(os.path.join(get_rootdir(), filename), "w") as f:
f.write(config_content)
#
# For further info, check http://launchpad.net/magicicada-server
"""Script to generate supervisor config for many dev services and workers."""
import os
import socket
import _pythonpath # NOQA
from ubuntuone.supervisor.config_helpers import generate_server_config
from utilities import utils
from supervisor_templates import TEMPLATES
ROOTDIR = utils.get_rootdir()
TMPDIR = os.path.join(ROOTDIR, 'tmp')
if not os.path.exists(TMPDIR):
os.mkdir(TMPDIR)
config_spec = {
"basepath": ROOTDIR,
"log_folder": TMPDIR,
"pid_folder": TMPDIR,
}
workers = {}
services = {
"development": {
"services": {}
"""
import os
import atexit
from twisted.internet import reactor, defer
reactor.suggestThreadPoolSize(30)
import _pythonpath # NOQA
from magicicada import settings
from magicicada.server import server
from utilities import utils
tmp_dir = os.path.join(utils.get_rootdir(), 'tmp')
api_port_filename = os.path.join(tmp_dir, "filesyncserver.port")
def cleanup():
"""Cleanup after ourselves"""
filename = api_port_filename
os.unlink(filename)
filename = filename + ".ssl"
os.unlink(filename)
@defer.inlineCallbacks
def main():
"""Start the server."""
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# For further info, check http://launchpad.net/magicicada-server
"""Start supervisord."""
import os
import sys
from utilities.utils import os_exec, is_running, get_tmpdir, get_rootdir
import utilities.localendpoints as local
ROOT_DIR = get_rootdir()
TMP_DIR = get_tmpdir()
def write_conf(tpl, inet_http_server_port):
"""Write out filled in conf template in tmp."""
template = open(os.path.join(ROOT_DIR, tpl)).read()
template_vars = {
'inet_http_server_port': inet_http_server_port,
'basepath': ROOT_DIR,
'tmp_dir': TMP_DIR,
'pythonpath': ':'.join(filter(None, sys.path)),
}
conf = template % template_vars
from ubuntuone.storage.tests.integration.helpers import debug, retryable
# to make dbus work
dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
# this should be done manually before:
DEP_STARTOAUTH = """
make start-oauth
"""
DEP_BUILDCLIENT = """
cd sourcecode/ubuntuone-client/
./autogen.sh --with-protocol=../ubuntuone-storage-protocol/
make
"""
ROOT = utils.get_rootdir()
TMP_DIR = os.path.join(ROOT, 'tmp')
PID_FILENAME = os.path.join(TMP_DIR, 'syncdaemon-process-%d.pid')
_dbus = find_executable("dbus-daemon")
if not _dbus:
raise RuntimeError("dbus-daemon was not found.")
def start_DBus():
"""Start our own session bus daemon for testing."""
config_file = os.path.join(os.getcwd(), "configs", "dbus-session.conf")
dbus_args = [
"--fork", "--config-file=" + config_file, "--print-address=1",
"--print-pid=2"
]
def launch(progname, username, params=None, environ=None, verbose=False,
homedir=None, machine_suffix=None, host=None, api_port=None,
auth_data=None):
"""Run a program in a developer context.
@param progname: the program name
@param username: the user name
@param params: extra parameters to pass to the program
@param environ: a dict of environment variables to set before calling
the program
"""
assert auth_data is not None
if host is None:
host = "localhost"
if api_port is None:
api_port = int(open('tmp/filesyncserver.port.ssl').read())
extra_args = ["--port", str(api_port), "--host", host,
"--auth", auth_data]
env = os.environ.copy()
if 'ubuntuone-syncdaemon' in progname:
if machine_suffix is not None:
username += machine_suffix
if homedir is None:
base = os.path.join(utils.get_rootdir(), 'tmp',
'syncdaemon_homes', username)
home = os.path.join(base, 'home')
else:
base = os.path.dirname(homedir)
home = homedir
data = os.path.join(home, '.local/share/ubuntuone/syncdaemon')
root = os.path.join(home, settings.ROOT_USERVOLUME_NAME)
shares = os.path.join(home, '.local/share/ubuntuone/shares')
env['HOME'] = home
env['XDG_DATA_HOME'] = os.path.join(home, '.local/share')
env['XDG_CONFIG_HOME'] = os.path.join(home, '.config')
env['XDG_CACHE_HOME'] = os.path.join(home, '.cache')
if not os.path.exists(base):
os.makedirs(base)
if not os.path.exists(data):
os.makedirs(data)
# get the progname root (two levels up)
progname_root = os.path.dirname(os.path.dirname(progname))
config_files_root = os.path.join(progname_root, 'data')
config_file = os.path.join(config_files_root, "syncdaemon.conf")
dev_config_file = os.path.join(config_files_root,
"syncdaemon-dev.conf")
log_config_file = os.path.join(config_files_root,
"logging.conf")
if not os.path.exists(log_config_file):
msg = "Please run: cd %s && ./setup.py build; cd -\n"
print msg % progname_root
return
extra_args.insert(0, config_file)
extra_args.insert(1, dev_config_file)
extra_args.insert(2, log_config_file)
extra_args += ["--data_dir", data, "--root_dir", root,
'--shares_dir', shares]
args = [progname] + extra_args
if params:
args += params
if environ:
env.update(environ)
if verbose:
print "Executing:", " ".join(args)
os.execvpe(progname, args, env)
import json
import os
import warnings
#enjoy the silence
warnings.simplefilter("ignore")
from backends.filesync.data.dbmanager import fsync_commit, get_shard_ids
from backends.db.db_admin_store import get_admin_store
from backends.db.schemas.fsync_main import create_schema as fsync_main
from backends.db.schemas.fsync_shard import create_schema as fsync_shard
from utilities import utils
TMP_DIR = os.path.join(utils.get_rootdir(), 'tmp')
AUTH_TOKENFILE = os.path.join(TMP_DIR, 'authkeys.json')
@fsync_commit
def delete_all_data():
"""Delete all data from the database."""
# they must be reversed to avoid dependency issues
fsync_main().delete(get_admin_store('storage'))
for shard_id in get_shard_ids():
fsync_shard().delete(get_admin_store(shard_id))
def add_auth_info_to_keyfile(username, auth_info, filename=AUTH_TOKENFILE):
"""Add auth info to the key file."""
token_dict = {}
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# For further info, check http://launchpad.net/filesync-server
"""Script to generate supervisor config for many dev services and workers."""
import os
import socket
import _pythonpath # NOQA
from ubuntuone.supervisor.config_helpers import generate_server_config
from utilities import utils
from supervisor_templates import TEMPLATES
ROOTDIR = utils.get_rootdir()
TMPDIR = os.path.join(ROOTDIR, 'tmp')
config_spec = {
"basepath": ROOTDIR,
"log_folder": TMPDIR,
"pid_folder": TMPDIR,
}
workers = {}
services = {"development": {"services": {}}}
if __name__ == '__main__':
hostname = socket.gethostname()
config_content = generate_server_config(hostname,
services,
# this should be done manually before:
DEP_STARTOAUTH = """
make start-oauth
"""
CLIENT_DIR = os.path.abspath(os.path.join('.sourcecode', 'magicicada-client'))
PROTOCOL_DIR = os.path.abspath(
os.path.join('.sourcecode', 'magicicada-protocol'))
DEP_BUILDCLIENT = """
cd %s
./autogen.sh --with-protocol=%s
make
""" % (CLIENT_DIR, PROTOCOL_DIR)
ROOT = utils.get_rootdir()
TMP_DIR = os.path.join(ROOT, 'tmp')
PID_FILENAME = os.path.join(TMP_DIR, 'syncdaemon-process-%d.pid')
_dbus = find_executable("dbus-daemon")
if not _dbus:
raise RuntimeError("dbus-daemon was not found.")
def start_DBus():
"""Start our own session bus daemon for testing."""
config_file = os.path.join(os.getcwd(), "configs", "dbus-session.conf")
dbus_args = ["--fork",
"--config-file=" + config_file,
"--print-address=1",
"--print-pid=2"]
def launch(progname, username, params=None, environ=None, verbose=False,
homedir=None, machine_suffix=None, host=None, api_port=None,
auth_data=None):
"""Run a program in a developer context.
@param progname: the program name
@param username: the user name
@param params: extra parameters to pass to the program
@param environ: a dict of environment variables to set before calling
the program
"""
if auth_data is None:
# import the function here, as it needs the DB up... so, if we
# already have auth_data, don't need the DB ;)
from utilities.userutils import get_auth_info_from_keyfile
data = get_auth_info_from_keyfile(username)
if not data:
raise ValueError()
# build the auth_data with the file info
auth_data = data['username'] + ":" + data['password']
if host is None:
host = "localhost"
if api_port is None:
api_port = int(open('tmp/filesyncserver.port.ssl').read())
extra_args = ["--port", str(api_port), "--host", host,
"--auth", auth_data]
env = os.environ.copy()
if 'ubuntuone-syncdaemon' in progname:
if machine_suffix is not None:
username += machine_suffix
if homedir is None:
base = os.path.join(utils.get_rootdir(), 'tmp',
'syncdaemon_homes', username)
home = os.path.join(base, 'home')
else:
base = os.path.dirname(homedir)
home = homedir
data = os.path.join(home, '.local/share/ubuntuone/syncdaemon')
root = os.path.join(home, 'Ubuntu One')
shares = os.path.join(home, '.local/share/ubuntuone/shares')
env['HOME'] = home
env['XDG_DATA_HOME'] = os.path.join(home, '.local/share')
env['XDG_CONFIG_HOME'] = os.path.join(home, '.config')
env['XDG_CACHE_HOME'] = os.path.join(home, '.cache')
if not os.path.exists(base):
os.makedirs(base)
if not os.path.exists(data):
os.makedirs(data)
# get the progname root (two levels up)
progname_root = os.path.dirname(os.path.dirname(progname))
config_files_root = os.path.join(progname_root, 'data')
config_file = os.path.join(config_files_root, "syncdaemon.conf")
dev_config_file = os.path.join(config_files_root,
"syncdaemon-dev.conf")
log_config_file = os.path.join(config_files_root,
"logging.conf")
if not os.path.exists(log_config_file):
msg = "Please run: cd %s && ./setup.py build; cd -\n"
print msg % progname_root
return
extra_args.insert(0, config_file)
extra_args.insert(1, dev_config_file)
extra_args.insert(2, log_config_file)
extra_args += ["--data_dir", data, "--root_dir", root,
'--shares_dir', shares]
args = [progname] + extra_args
if params:
args += params
if environ:
env.update(environ)
if verbose:
print "Executing:", " ".join(args)
os.execvpe(progname, args, env)
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# For further info, check http://launchpad.net/magicicada-server
"""Start supervisord."""
import os
import sys
from utilities.utils import os_exec, is_running, get_tmpdir, get_rootdir
import utilities.localendpoints as local
ROOT_DIR = get_rootdir()
TMP_DIR = get_tmpdir()
def write_conf(tpl, inet_http_server_port):
"""Write out filled in conf template in tmp."""
template = open(os.path.join(ROOT_DIR, tpl)).read()
template_vars = {}
template_vars['inet_http_server_port'] = inet_http_server_port
template_vars['basepath'] = ROOT_DIR
template_vars['tmp_dir'] = TMP_DIR
conf = template % template_vars
conf_file_path = os.path.join(TMP_DIR,
# For further info, check http://launchpad.net/magicicada-server
"""Utility functions to create users for testing."""
from __future__ import unicode_literals
import json
import os
from backends.filesync.data.dbmanager import fsync_commit
from backends.db.store import get_filesync_store
from backends.db.schemas.storage import create_schema as storage
from utilities import utils
TMP_DIR = os.path.join(utils.get_rootdir(), 'tmp')
AUTH_TOKENFILE = os.path.join(TMP_DIR, 'authkeys.json')
@fsync_commit
def delete_all_data():
"""Delete all data from the database."""
# they must be reversed to avoid dependency issues
storage().delete(get_filesync_store())
def add_auth_info_to_keyfile(username, auth_info, filename=AUTH_TOKENFILE):
"""Add auth info to the key file."""
token_dict = {}
if os.path.exists(filename):
with open(filename, 'r') as f:
"""
import os
import atexit
from twisted.internet import reactor, defer
reactor.suggestThreadPoolSize(30)
import _pythonpath # NOQA
from magicicada import settings
from magicicada.server import server
from utilities import utils
tmp_dir = os.path.join(utils.get_rootdir(), 'tmp')
api_port_filename = os.path.join(tmp_dir, "filesyncserver.port")
def cleanup():
"""Cleanup after ourselves"""
filename = api_port_filename
os.unlink(filename)
filename = filename + ".ssl"
os.unlink(filename)
@defer.inlineCallbacks
def main():
"""Start the server."""
def launch(progname, username, params=None, environ=None, verbose=False,
homedir=None, machine_suffix=None, host=None, api_port=None,
auth_data=None):
"""Run a program in a developer context.
@param progname: the program name
@param username: the user name
@param params: extra parameters to pass to the program
@param environ: a dict of environment variables to set before calling
the program
"""
assert auth_data is not None
if host is None:
host = "localhost"
if api_port is None:
api_port = int(open('tmp/filesyncserver.port.ssl').read())
extra_args = ["--port", str(api_port), "--host", host,
"--auth", auth_data]
env = os.environ.copy()
if 'ubuntuone-syncdaemon' in progname:
if machine_suffix is not None:
username += machine_suffix
if homedir is None:
base = os.path.join(utils.get_rootdir(), 'tmp',
'syncdaemon_homes', username)
home = os.path.join(base, 'home')
else:
base = os.path.dirname(homedir)
home = homedir
data = os.path.join(home, '.local/share/ubuntuone/syncdaemon')
root = os.path.join(home, settings.ROOT_USERVOLUME_NAME)
shares = os.path.join(home, '.local/share/ubuntuone/shares')
env['HOME'] = home
env['XDG_DATA_HOME'] = os.path.join(home, '.local/share')
env['XDG_CONFIG_HOME'] = os.path.join(home, '.config')
env['XDG_CACHE_HOME'] = os.path.join(home, '.cache')
if not os.path.exists(base):
os.makedirs(base)
if not os.path.exists(data):
os.makedirs(data)
# get the progname root (two levels up)
progname_root = os.path.dirname(os.path.dirname(progname))
config_files_root = os.path.join(progname_root, 'data')
config_file = os.path.join(config_files_root, "syncdaemon.conf")
dev_config_file = os.path.join(config_files_root,
"syncdaemon-dev.conf")
log_config_file = os.path.join(config_files_root,
"logging.conf")
if not os.path.exists(log_config_file):
msg = "Please run: cd %s && ./setup.py build; cd -\n"
print(msg % progname_root)
return
extra_args.insert(0, config_file)
extra_args.insert(1, dev_config_file)
extra_args.insert(2, log_config_file)
extra_args += ["--data_dir", data, "--root_dir", root,
'--shares_dir', shares]
args = [progname] + extra_args
if params:
args += params
if environ:
env.update(environ)
if verbose:
print("Executing:", " ".join(args))
os.execvpe(progname, args, env)
def launch(progname,
username,
params=None,
environ=None,
verbose=False,
homedir=None,
machine_suffix=None,
host=None,
api_port=None,
auth_data=None):
"""Run a program in a developer context.
@param progname: the program name
@param username: the user name
@param params: extra parameters to pass to the program
@param environ: a dict of environment variables to set before calling
the program
"""
if auth_data is None:
# import the function here, as it needs the DB up... so, if we
# already have auth_data, don't need the DB ;)
from utilities.userutils import get_auth_info_from_keyfile
data = get_auth_info_from_keyfile(username)
if not data:
raise ValueError()
# build the auth_data with the file info
auth_data = data['username'] + ":" + data['password']
if host is None:
host = "localhost"
if api_port is None:
api_port = int(open('tmp/filesyncserver.port.ssl').read())
extra_args = ["--port", str(api_port), "--host", host, "--auth", auth_data]
env = os.environ.copy()
if 'ubuntuone-syncdaemon' in progname:
if machine_suffix is not None:
username += machine_suffix
if homedir is None:
base = os.path.join(utils.get_rootdir(), 'tmp', 'syncdaemon_homes',
username)
home = os.path.join(base, 'home')
else:
base = os.path.dirname(homedir)
home = homedir
data = os.path.join(home, '.local/share/ubuntuone/syncdaemon')
root = os.path.join(home, 'Ubuntu One')
shares = os.path.join(home, '.local/share/ubuntuone/shares')
env['HOME'] = home
env['XDG_DATA_HOME'] = os.path.join(home, '.local/share')
env['XDG_CONFIG_HOME'] = os.path.join(home, '.config')
env['XDG_CACHE_HOME'] = os.path.join(home, '.cache')
if not os.path.exists(base):
os.makedirs(base)
if not os.path.exists(data):
os.makedirs(data)
# get the progname root (two levels up)
progname_root = os.path.dirname(os.path.dirname(progname))
config_files_root = os.path.join(progname_root, 'data')
config_file = os.path.join(config_files_root, "syncdaemon.conf")
dev_config_file = os.path.join(config_files_root,
"syncdaemon-dev.conf")
log_config_file = os.path.join(config_files_root, "logging.conf")
if not os.path.exists(log_config_file):
msg = "Please run: cd %s && ./setup.py build; cd -\n"
print msg % progname_root
return
extra_args.insert(0, config_file)
extra_args.insert(1, dev_config_file)
extra_args.insert(2, log_config_file)
extra_args += [
"--data_dir", data, "--root_dir", root, '--shares_dir', shares
]
args = [progname] + extra_args
if params:
args += params
if environ:
env.update(environ)
if verbose:
print "Executing:", " ".join(args)
os.execvpe(progname, args, env)
