def requirements(ctx, base=False, addons=False, release=False, dev=False, metrics=False, quick=False):
"""Install python dependencies.
Examples:
inv requirements
inv requirements --quick
Quick requirements are, in order, addons, dev and the base requirements. You should be able to use --quick for
day to day development.
By default, base requirements will run. However, if any set of addons, release, dev, or metrics are chosen, base
will have to be mentioned explicitly in order to run. This is to remain compatible with previous usages. Release
requirements will prevent dev, metrics, and base from running.
"""
if quick:
base = True
addons = True
dev = True
if not(addons or dev or metrics):
base = True
if release or addons:
addon_requirements(ctx)
# "release" takes precedence
if release:
req_file = os.path.join(HERE, 'requirements', 'release.txt')
ctx.run(
pip_install(req_file, constraints_file=CONSTRAINTS_PATH),
echo=True
)
else:
if dev: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'dev.txt')
ctx.run(
pip_install(req_file, constraints_file=CONSTRAINTS_PATH),
echo=True
)
if metrics: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'metrics.txt')
ctx.run(
pip_install(req_file, constraints_file=CONSTRAINTS_PATH),
echo=True
)
if base: # then base requirements
req_file = os.path.join(HERE, 'requirements.txt')
ctx.run(
pip_install(req_file, constraints_file=CONSTRAINTS_PATH),
echo=True
)
# fix URITemplate name conflict h/t @github
ctx.run('pip uninstall uritemplate.py --yes || true')
ctx.run('pip install --no-cache-dir uritemplate.py==0.3.0')
def install_cloudify_packages():
utils.create_cloudify_venv()
ctx.logger.info('Installing cloudify packages...')
for package in CLOUDIFY_PACKAGES:
package_path = utils.clone(package_name=package['package_name'],
branch=package['branch'],
org=package['org'])
if package['package_name'] == 'cloudify-manager':
utils.pip_install_manager_deps(package_path)
else:
utils.pip_install(package_name=package['package_name'],
package_path=package_path)
for package, version in PYTHON_DEPENDENCIES:
utils.pip_install(package_name=package, version=version)
def install_cloudify_packages():
utils.create_cloudify_venv()
ctx.logger.info('Installing cloudify packages...')
for package in CLOUDIFY_PACKAGES:
package_path = utils.clone(package_name=package['package_name'],
branch=package['branch'],
org=package['org'])
if package['package_name'] == 'cloudify-manager':
utils.pip_install_manager_deps(package_path)
else:
utils.pip_install(package_name=package['package_name'],
package_path=package_path)
for package, version in PYTHON_DEPENDENCIES:
utils.pip_install(package_name=package,
version=version)
def install_requirements(self, deploy_path, requirement_files):
ret_packages = []
if self.attempt_info != None:
dependencies = self.attempt_info['dependencies']
for dependency in dependencies:
LOG.info('pip install {}'.format(dependency['package_info']['name']))
pip_output = utils.pip_install(self.base_path, [dependency['package_info']], False)
if pip_output != None and pip_output[0] != 0:
dependency['package_info']['version'] = ''
pip_output = utils.pip_install(self.base_path, [dependency['package_info']], False)
LOG.debug('pip install output: {}'.format(pip_output))
else:
latest_successful_attempt = self.get_latest_successful_attempt()
if latest_successful_attempt != None:
dependencies = Dependency.objects.filter(attempt = latest_successful_attempt)
for dependency in dependencies:
if dependency.source == PACKAGE_SOURCE_FILE:
self.packages_from_file.append(dependency.package)
else:
self.packages_from_database.append(dependency.package)
LOG.info('pip install {}'.format(dependency.package))
pip_output = utils.pip_install(self.base_path, [dependency.package], False)
LOG.debug('pip install output: {}'.format(pip_output))
else:
if requirement_files:
for requirement_file in requirement_files:
out = utils.pip_install(deploy_path, requirement_file, True)
with open(requirement_file, "r") as my_requirement_file:
ret_packages += my_requirement_file.readlines()
for package in ret_packages:
package = package.strip()
if len(package) == 0:
continue
if package[0] == '#':
continue
if len(package.split('==')) >= 2:
name, version = package.split('==')
elif len(package.split('>=')) >= 2:
name, version = package.split('>=')
else:
name, version = package, ''
try:
pkg, created = Package.objects.get_or_create(name=name, version=version, project_type=self.repo.project_type)
self.packages_from_file.append(pkg)
except Exception, e:
LOG.exception(e)
def addon_requirements(ctx):
"""Install all addon requirements."""
for directory in os.listdir(settings.ADDON_PATH):
path = os.path.join(settings.ADDON_PATH, directory)
requirements_file = os.path.join(path, "requirements.txt")
if os.path.isdir(path) and os.path.isfile(requirements_file):
print("Installing requirements for {0}".format(directory))
ctx.run(pip_install(requirements_file, constraints_file=CONSTRAINTS_PATH), echo=True)
print("Finished installing addon requirements")
def requirements(base=False,
addons=False,
release=False,
dev=False,
metrics=False,
quick=False):
"""Install python dependencies.
Examples:
inv requirements
inv requirements --quick
Quick requirements are, in order, addons, dev and the base requirements. You should be able to use --quick for
day to day development.
By default, base requirements will run. However, if any set of addons, release, dev, or metrics are chosen, base
will have to be mentioned explicitly in order to run. This is to remain compatible with previous usages. Release
requirements will prevent dev, metrics, and base from running.
"""
if quick:
base = True
addons = True
dev = True
if not (addons or dev or metrics):
base = True
if release or addons:
addon_requirements()
# "release" takes precedence
if release:
req_file = os.path.join(HERE, 'requirements', 'release.txt')
run(pip_install(req_file), echo=True)
else:
if dev: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'dev.txt')
run(pip_install(req_file), echo=True)
if metrics: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'metrics.txt')
run(pip_install(req_file), echo=True)
if base: # then base requirements
req_file = os.path.join(HERE, 'requirements.txt')
run(pip_install(req_file), echo=True)
def install_package(*args, **kargs):
'''Provide a string, an iterable or pass values to the
*args. Each a url or endoint used by pip
'''
v = True
for url in args:
log('installing %s' % url, color='yellow')
confirm('Would you like to install package \'%s\'' % url, True)
v = pip_install(url)
# To not have a horrible big in the future, if one package
# fails the entire procedure reports False
return v
def addon_requirements(ctx):
"""Install all addon requirements."""
for directory in os.listdir(settings.ADDON_PATH):
path = os.path.join(settings.ADDON_PATH, directory)
requirements_file = os.path.join(path, 'requirements.txt')
if os.path.isdir(path) and os.path.isfile(requirements_file):
print('Installing requirements for {0}'.format(directory))
ctx.run(pip_install(requirements_file,
constraints_file=CONSTRAINTS_PATH),
echo=True)
print('Finished installing addon requirements')
def install_requirements(self, deploy_path, requirement_files):
ret_packages = []
latest_successful_attempt = self.get_latest_successful_attempt()
if latest_successful_attempt != None:
dependencies = Dependency.objects.filter(attempt = latest_successful_attempt)
for dependency in dependencies:
if dependency.source == PACKAGE_SOURCE_FILE:
self.packages_from_file.append(dependency.package)
else:
self.packages_from_database.append(dependency.package)
LOG.info('pip install {}'.format(dependency.package))
pip_output = utils.pip_install(self.base_path, [dependency.package], False)
LOG.info('pip install output: {}'.format(pip_output))
else:
if requirement_files:
for requirement_file in requirement_files:
out = utils.pip_install(deploy_path, requirement_file, True)
with open(requirement_file, "r") as my_requirement_file:
ret_packages += my_requirement_file.readlines()
for package in ret_packages:
package = package.strip()
if len(package) == 0:
continue
if package[0] == '#':
continue
if len(package.split('==')) >= 2:
name, version = package.split('==')
elif len(package.split('>=')) >= 2:
name, version = package.split('>=')
else:
name, version = package, ''
try:
pkg, created = Package.objects.get_or_create(name=name, version=version, project_type=self.repo.project_type)
self.packages_from_file.append(pkg)
except Exception, e:
LOG.exception(e)
def requirements(base=False, addons=False, release=False, dev=False, metrics=False, quick=False):
"""Install python dependencies.
Examples:
inv requirements
inv requirements --quick
Quick requirements are, in order, addons, dev and the base requirements. You should be able to use --quick for
day to day development.
By default, base requirements will run. However, if any set of addons, release, dev, or metrics are chosen, base
will have to be mentioned explicitly in order to run. This is to remain compatible with previous usages. Release
requirements will prevent dev, metrics, and base from running.
"""
if quick:
base = True
addons = True
dev = True
if not(addons or dev or metrics):
base = True
if release or addons:
addon_requirements()
# "release" takes precedence
if release:
req_file = os.path.join(HERE, 'requirements', 'release.txt')
run(pip_install(req_file), echo=True)
else:
if dev: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'dev.txt')
run(pip_install(req_file), echo=True)
if metrics: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'metrics.txt')
run(pip_install(req_file), echo=True)
if base: # then base requirements
req_file = os.path.join(HERE, 'requirements.txt')
run(pip_install(req_file), echo=True)
def log(*args, **kargs):
a = [str(x) for x in args]
color = kargs.get('color', None)
s = ''. join(a)
dangle_print = True if s.endswith('\,') else False
s = s[:-2] if dangle_print else s
if color is not None:
try:
try: from termcolor import colored
except ImportError, ie: pip_install('termcolor')
print colored(s, color=color),
if not dangle_print:
print ''
except NameError, e:
print "Print Error: '%s'" % e
print s,
if not dangle_print:
print ''
def requirements(addons=False, release=False, dev=False, metrics=False):
"""Install python dependencies.
Examples:
inv requirements --dev
inv requirements --addons
inv requirements --release
inv requirements --metrics
"""
if release or addons:
addon_requirements()
# "release" takes precedence
if release:
req_file = os.path.join(HERE, 'requirements', 'release.txt')
elif dev: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'dev.txt')
elif metrics: # then dev requirements
req_file = os.path.join(HERE, 'requirements', 'metrics.txt')
else: # then base requirements
req_file = os.path.join(HERE, 'requirements.txt')
run(pip_install(req_file), echo=True)
def render(resource):
if resource is None:
return DEFAULT_LAYOUT
if resource.startswith(DASH_UPLOAD_RESULTS_FLAG):
loads = json.loads(str(homepage.layout[INVISIBLE_ID].children))
return render_layout(
add_dash(get_upload_dash(*loads), UPLOAD_RESULT_URL_PART))
dir_path = os.path.join(get_directory(), unquote(resource))
full_path = os.path.join(dir_path, unquote(resource) + ".py")
with open(full_path) as f:
if 'Dash' not in f.read():
return error_layout("Этот файл не содержит объект Dash")
try:
dash_module = SourceFileLoader(resource[:-3], full_path).load_module()
except ImportError as ie:
return render(resource) if pip_install(ie.__str__().split(
"'")[1]) == 0 else error_layout("Невозможно загрузить зависимости")
except:
error = traceback.format_exc().split("call_with_frames_removed\n",
1)[1].replace(
get_directory() + "/", "")
return error_layout(error)
else:
return render_layout(add_dash(dash_module, resource))
def try_deploy(self, deploy_path, requirement_files):
LOG.info('Configuring settings ...')
self.kill_server()
self.clear_database()
self.configure_settings()
self.runtime = self.get_runtime()
LOG.info(self.runtime)
self.attempt.database = self.get_database()
LOG.info('Database: ' + self.attempt.database.name)
LOG.info('Installing requirements ...')
self.install_requirements(self.base_path, requirement_files)
threshold = 20
last_missing_module_name = ''
index = -1
dependencies = []
for _ in range(threshold):
output = self.sync_server(deploy_path)
if output[0] != 0:
LOG.info(output)
else:
LOG.info(output)
break
match = re.search('pip install ([a-zA-Z\-_]+)', output[2].strip())
if match:
package_name = match.group(1)
output = utils.pip_install_text(self.base_path, package_name)
continue
# LOG.info('pip install output: {}'.format(output))
split_output = output[2].strip().splitlines()
line = split_output[-1].strip()
match = re.search('(?<=No module named )\S+', line)
if match:
missing_module_name = match.group(0)
LOG.info('Missing module: ' + missing_module_name)
if missing_module_name == last_missing_module_name:
missing_module_name, candidate_packages, index = dependencies[-1]
index = index + 1
if index < len(candidate_packages):
dependencies[-1] = (missing_module_name, candidate_packages, index)
LOG.info('pip install {}'.format(candidate_packages[index]))
pip_output = utils.pip_install(self.base_path, [candidate_packages[index]], False)
LOG.info('pip install output: {}'.format(pip_output))
else:
LOG.info('No more possible packages!')
return ATTEMPT_STATUS_MISSING_DEPENDENCIES
else:
last_missing_module_name = missing_module_name
candidate_package_ids = Module.objects.filter(name=missing_module_name).values_list('package_id', flat=True)
if not candidate_package_ids:
LOG.info('No possible packages!')
return ATTEMPT_STATUS_MISSING_DEPENDENCIES
candidate_packages = Package.objects.filter(id__in=candidate_package_ids).order_by('-count', '-version', 'name')
LOG.info('pip install {}'.format(candidate_packages[0]))
pip_output = utils.pip_install(self.base_path, [candidate_packages[0]], False, False)
LOG.info('pip install output: {}'.format(pip_output))
try:
version = re.search('Successfully installed .*-(.*)', pip_output[1]).group(1)
if version and any(package.version == version for package in candidate_packages):
for package_index in range(len(candidate_packages)):
if candidate_packages[package_index].version == version:
candidate_packages = [candidate_packages[package_index]] + candidate_packages[:package_index] + candidate_packages[package_index + 1:]
break
else:
latest_package = Package()
latest_package.project_type = candidate_packages[0].project_type
latest_package.name = candidate_packages[0].name
latest_package.version = version
latest_package.save()
module = Module()
module.name = missing_module_name
module.package = latest_package
module.save()
candidate_packages = list([latest_package]) + list(candidate_packages)
except Exception, e:
LOG.exception(e)
dependencies.append((missing_module_name, candidate_packages, 0))
else:
return ATTEMPT_STATUS_DATABASE_ERROR
import pickle
import numpy as np
import pandas as pd
from os.path import isfile
from utils import pip_install, extract, print_data_info, print_time_info, onehot_sparse
from constants import Algo
from data_processor import DataProcessor
from sampler import Sampler
pip_install('hyperopt')
pip_install('lightgbm')
pip_install('scipy')
from hyperopt import hp
from hyperparameters_tuner import HyperparametersTuner
from lightgbm import LGBMClassifier
from sklearn.linear_model import LogisticRegression
from stream_processor_old import *
params = {'algo': Algo.BASIC}
class Model:
def __init__(self, data_info, time_info):
# Print data information
info_dict = extract(data_info, time_info)
print_data_info(info_dict)
import numpy as np
import pandas as pd
from utils import pip_install
pip_install('category_encoders')
from category_encoders.binary import BinaryEncoder
from category_encoders.hashing import HashingEncoder
class CountWoeCipher:
def __init__(self):
self._count_woe_map = {}
def encode(self, incoming_data, incoming_labels=None):
print('\nencode')
print('incoming_data.shape: {}'.format(incoming_data.shape))
no_of_rows, no_of_cols = incoming_data.shape
result = np.array([])
if incoming_labels is None and self._count_woe_map: # predict
for i in range(no_of_cols):
d0 = pd.DataFrame({'X': incoming_data[:, i]})
d1 = d0.join(self._count_woe_map[i], on='X')[['COUNT',
'WOE']].values
result = d1 if len(result) == 0 else np.concatenate(
(result, d1), axis=1)
del d0
else: # fit
self._count_woe_map = {}
print('incoming_labels.shape: {}'.format(incoming_labels.shape))
def try_deploy(self, deploy_path, requirement_files):
LOG.info('Configuring settings ...')
self.kill_server()
self.configure_settings()
self.runtime = self.get_runtime()
LOG.info(self.runtime)
self.attempt.database = self.get_database()
LOG.info('Database: ' + self.attempt.database.name)
if self.repo.setup_scripts != None:
project_path = utils.search_dir(self.base_path, self.repo.repo_name())
LOG.info("Project Path: {}".format(project_path))
LOG.info("Setup Scripts: {} && {} && {} && {}".format(
utils.to_env(self.base_path),
"unset DJANGO_SETTINGS_MODULE",
"cd {}".format(project_path),
self.repo.setup_scripts))
code, stdout, stderr = utils.run_command("{} && {} && {} && {}".format(
utils.to_env(self.base_path),
"unset DJANGO_SETTINGS_MODULE",
"cd {}".format(project_path),
self.repo.setup_scripts))
LOG.debug("Setup Return Code: {}".format(code))
LOG.debug("Setup Return STDOUT:{}".format(stdout))
LOG.debug("Setup Return STDERR:{}".format(stderr))
LOG.info('Installing requirements ...')
self.install_requirements(self.base_path, requirement_files)
threshold = 20
last_missing_module_name = ''
index = -1
dependencies = []
for _ in range(threshold):
output = self.sync_server(deploy_path)
if output[0] != 0:
LOG.debug(output)
if self.attempt_info != None:
LOG.error("Can not find dependencies!")
return
else:
LOG.debug(output)
break
match = re.search('pip install ([a-zA-Z\-_]+)', output[2].strip())
if match:
package_name = match.group(1)
output = utils.pip_install_text(self.base_path, package_name)
latest_package, created = Package.objects.get_or_create(name=package_name, version = '', project_type=self.repo.project_type)
dependencies.append((package_name, [latest_package], 0))
continue
# LOG.debug('pip install output: {}'.format(output))
split_output = output[2].strip().splitlines()
line = split_output[-1].strip()
match = re.search('(?<=No module named )\S+', line)
if match:
missing_module_name = match.group(0)
LOG.info('Missing module: ' + missing_module_name)
if missing_module_name == last_missing_module_name:
missing_module_name, candidate_packages, index = dependencies[-1]
index = index + 1
if index < len(candidate_packages):
dependencies[-1] = (missing_module_name, candidate_packages, index)
LOG.info('pip install {}'.format(candidate_packages[index]))
pip_output = utils.pip_install(self.base_path, [candidate_packages[index]], False)
LOG.debug('pip install output: {}'.format(pip_output))
else:
LOG.info('No more possible packages!')
return ATTEMPT_STATUS_MISSING_DEPENDENCIES
else:
last_missing_module_name = missing_module_name
candidate_package_ids = Module.objects.filter(name=missing_module_name).values_list('package_id', flat=True)
if not candidate_package_ids:
LOG.info('No possible packages!')
return ATTEMPT_STATUS_MISSING_DEPENDENCIES
candidate_packages = Package.objects.filter(id__in=candidate_package_ids).order_by('-count', '-version', 'name')
LOG.info('pip install {}'.format(candidate_packages[0]))
pip_output = utils.pip_install(self.base_path, [candidate_packages[0]], False, False)
LOG.debug('pip install output: {}'.format(pip_output))
try:
version = re.search('Successfully installed .*-(.*)', pip_output[1]).group(1)
if version and any(package.version == version for package in candidate_packages):
for package_index in range(len(candidate_packages)):
if candidate_packages[package_index].version == version:
candidate_packages = [candidate_packages[package_index]] + candidate_packages[:package_index] + candidate_packages[package_index + 1:]
break
else:
latest_package = Package()
latest_package.project_type = candidate_packages[0].project_type
latest_package.name = candidate_packages[0].name
latest_package.version = version
latest_package.save()
module = Module()
module.name = missing_module_name
module.package = latest_package
module.save()
candidate_packages = list([latest_package]) + list(candidate_packages)
except Exception, e:
LOG.exception(e)
dependencies.append((missing_module_name, candidate_packages, 0))
else:
return ATTEMPT_STATUS_DATABASE_ERROR
