def sync_server(self, path):
LOG.info('Syncing server ...')
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py syncdb --noinput'.format(
utils.to_env(self.base_path), utils.cd(path))
output = utils.run_command(command)
if 'Unknown command' in output[2]:
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py migrate --noinput'.format(
utils.to_env(self.base_path), utils.cd(path))
return utils.run_command(command)
def sync_server(self, path):
LOG.info('Syncing server ...')
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py syncdb --noinput'.format(
utils.to_env(self.base_path), utils.cd(path))
output = utils.run_command(command)
if 'Unknown command' in output[2]:
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py migrate --noinput'.format(
utils.to_env(self.base_path), utils.cd(path))
return utils.run_command(command)
def run_server(self, path, port):
self.configure_network()
LOG.info('Running server ...')
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py runserver 0.0.0.0:{}'.format(
utils.to_env(self.base_path),
utils.cd(path),
port)
return utils.run_command_async(command)
def run_server(self, path, port):
self.configure_network()
LOG.info('Running server ...')
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && python manage.py runserver 0.0.0.0:{}'.format(
utils.to_env(self.base_path),
utils.cd(path),
port)
return utils.run_command_async(command)
def load_fixtures(self, path):
LOG.info('Loading fixtures ...')
for file in os.listdir(os.path.join(path, 'fixtures')):
LOG.info('Loading fixtures: {}'.format(file))
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && {}'.format(
utils.to_env(self.base_path),
utils.cd(path),
"python manage.py loaddata {}".format(os.path.join(path, 'fixtures', file)))
utils.run_command(command)
def create_superuser(self, path):
LOG.info('Creating superuser ...')
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && {}'.format(
utils.to_env(self.base_path),
utils.cd(path),
"""
echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', '*****@*****.**', 'admin')" | python manage.py shell
""")
return utils.run_command(command)
def create_superuser(self, path):
LOG.info('Creating superuser ...')
command = '{} && {} && unset DJANGO_SETTINGS_MODULE && {}'.format(
utils.to_env(self.base_path),
utils.cd(path),
"""
echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', '*****@*****.**', 'admin')" | python manage.py shell
""")
return utils.run_command(command)
def deploy_repo_attempt(self, deploy_path):
LOG.info(utils.configure_env(self.base_path))
self.clear_database()
if self.repo.setup_scripts != None:
project_path = utils.search_dir(deploy_path, self.repo.repo_name())
LOG.info("Project Path: {}".format(project_path))
LOG.info("Setup Scripts: {} && {} && {} && {}".format(
utils.to_env(self.base_path),
"unset DJANGO_SETTINGS_MODULE",
"cd {}".format(project_path),
self.repo.setup_scripts))
code, stdout, stderr = utils.run_command("{} && {} && {} && {}".format(
utils.to_env(self.base_path),
"unset DJANGO_SETTINGS_MODULE",
"cd {}".format(project_path),
self.repo.setup_scripts))
LOG.info("Setup Return Code: {}".format(code))
LOG.debug("Setup Return STDOUT:{}".format(stdout))
LOG.debug("Setup Return STDERR:{}".format(stderr))
if 'myproject' in self.repo.setup_scripts:
deploy_path = os.path.join(deploy_path, 'myproject')
manage_files = utils.search_file(deploy_path, 'manage.py')
if not manage_files:
LOG.error("Can not find manage.py!")
return ATTEMPT_STATUS_MISSING_REQUIRED_FILES
manage_paths = [os.path.dirname(manage_file) for manage_file in manage_files]
base_dir = next(name for name in manage_paths if 'lib/python2.7/site-packages/' not in name)
manage_path = next(name for name in manage_paths if name.startswith(base_dir))
LOG.info('manage.py path: {}'.format(manage_path))
with open(os.path.join(manage_path, 'manage.py'), 'r') as manage_file:
s = re.search('os.environ.setdefault\("DJANGO_SETTINGS_MODULE", "(.*)"\)', manage_file.read())
if s:
setting_path = s.group(1)
else:
setting_paths = utils.search_file(deploy_path, 'settings.py')
if len(setting_paths) > 0:
setting_path = setting_paths[0][:-3]
else:
LOG.error("Can not find settings.py")
return ATTEMPT_STATUS_MISSING_REQUIRED_FILES
setting_path = setting_path.replace('.', '/')
if os.path.isdir(os.path.join(manage_path, setting_path)):
setting_path = os.path.join(manage_path, setting_path)
for setting_file in sorted(os.listdir(setting_path)):
if os.path.isfile(os.path.join(setting_path, setting_file)):
setting_path = os.path.join(setting_path, setting_file)
break
self.setting_path = setting_path
elif os.path.isfile(os.path.join(manage_path, setting_path + '.py')):
setting_path = os.path.join(manage_path, setting_path + '.py')
self.setting_path = setting_path
else:
for candidate_setting_files in utils.search_file_regex(deploy_path, '^settings.*\.py$'):
setting_path = os.path.join(manage_path, setting_path + '.py')
utils.copy_file(candidate_setting_files, setting_path)
self.setting_path = setting_path
break
if self.setting_path == None:
LOG.error("Can not find settings.py!")
return ATTEMPT_STATUS_MISSING_REQUIRED_FILES
LOG.info('setting.py path: {}'.format(setting_path))
requirement_files = utils.search_file(deploy_path, 'requirements.txt')
if requirement_files:
LOG.info('requirements.txt path: {}'.format(requirement_files))
return self.try_deploy(manage_path, requirement_files)
def try_deploy(self, deploy_path, requirement_files):
LOG.info('Configuring settings ...')
self.kill_server()
self.configure_settings()
self.runtime = self.get_runtime()
LOG.info(self.runtime)
self.attempt.database = self.get_database()
LOG.info('Database: ' + self.attempt.database.name)
if self.repo.setup_scripts != None:
project_path = utils.search_dir(self.base_path, self.repo.repo_name())
LOG.info("Project Path: {}".format(project_path))
LOG.info("Setup Scripts: {} && {} && {} && {}".format(
utils.to_env(self.base_path),
"unset DJANGO_SETTINGS_MODULE",
"cd {}".format(project_path),
self.repo.setup_scripts))
code, stdout, stderr = utils.run_command("{} && {} && {} && {}".format(
utils.to_env(self.base_path),
"unset DJANGO_SETTINGS_MODULE",
"cd {}".format(project_path),
self.repo.setup_scripts))
LOG.debug("Setup Return Code: {}".format(code))
LOG.debug("Setup Return STDOUT:{}".format(stdout))
LOG.debug("Setup Return STDERR:{}".format(stderr))
LOG.info('Installing requirements ...')
self.install_requirements(self.base_path, requirement_files)
threshold = 20
last_missing_module_name = ''
index = -1
dependencies = []
for _ in range(threshold):
output = self.sync_server(deploy_path)
if output[0] != 0:
LOG.debug(output)
if self.attempt_info != None:
LOG.error("Can not find dependencies!")
return
else:
LOG.debug(output)
break
match = re.search('pip install ([a-zA-Z\-_]+)', output[2].strip())
if match:
package_name = match.group(1)
output = utils.pip_install_text(self.base_path, package_name)
latest_package, created = Package.objects.get_or_create(name=package_name, version = '', project_type=self.repo.project_type)
dependencies.append((package_name, [latest_package], 0))
continue
# LOG.debug('pip install output: {}'.format(output))
split_output = output[2].strip().splitlines()
line = split_output[-1].strip()
match = re.search('(?<=No module named )\S+', line)
if match:
missing_module_name = match.group(0)
LOG.info('Missing module: ' + missing_module_name)
if missing_module_name == last_missing_module_name:
missing_module_name, candidate_packages, index = dependencies[-1]
index = index + 1
if index < len(candidate_packages):
dependencies[-1] = (missing_module_name, candidate_packages, index)
LOG.info('pip install {}'.format(candidate_packages[index]))
pip_output = utils.pip_install(self.base_path, [candidate_packages[index]], False)
LOG.debug('pip install output: {}'.format(pip_output))
else:
LOG.info('No more possible packages!')
return ATTEMPT_STATUS_MISSING_DEPENDENCIES
else:
last_missing_module_name = missing_module_name
candidate_package_ids = Module.objects.filter(name=missing_module_name).values_list('package_id', flat=True)
if not candidate_package_ids:
LOG.info('No possible packages!')
return ATTEMPT_STATUS_MISSING_DEPENDENCIES
candidate_packages = Package.objects.filter(id__in=candidate_package_ids).order_by('-count', '-version', 'name')
LOG.info('pip install {}'.format(candidate_packages[0]))
pip_output = utils.pip_install(self.base_path, [candidate_packages[0]], False, False)
LOG.debug('pip install output: {}'.format(pip_output))
try:
version = re.search('Successfully installed .*-(.*)', pip_output[1]).group(1)
if version and any(package.version == version for package in candidate_packages):
for package_index in range(len(candidate_packages)):
if candidate_packages[package_index].version == version:
candidate_packages = [candidate_packages[package_index]] + candidate_packages[:package_index] + candidate_packages[package_index + 1:]
break
else:
latest_package = Package()
latest_package.project_type = candidate_packages[0].project_type
latest_package.name = candidate_packages[0].name
latest_package.version = version
latest_package.save()
module = Module()
module.name = missing_module_name
module.package = latest_package
module.save()
candidate_packages = list([latest_package]) + list(candidate_packages)
except Exception, e:
LOG.exception(e)
dependencies.append((missing_module_name, candidate_packages, 0))
else:
return ATTEMPT_STATUS_DATABASE_ERROR