def requirements(ctx, base=False, addons=False, release=False, dev=False, metrics=False, quick=False): """Install python dependencies. Examples: inv requirements inv requirements --quick Quick requirements are, in order, addons, dev and the base requirements. You should be able to use --quick for day to day development. By default, base requirements will run. However, if any set of addons, release, dev, or metrics are chosen, base will have to be mentioned explicitly in order to run. This is to remain compatible with previous usages. Release requirements will prevent dev, metrics, and base from running. """ if quick: base = True addons = True dev = True if not(addons or dev or metrics): base = True if release or addons: addon_requirements(ctx) # "release" takes precedence if release: req_file = os.path.join(HERE, 'requirements', 'release.txt') ctx.run( pip_install(req_file, constraints_file=CONSTRAINTS_PATH), echo=True ) else: if dev: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'dev.txt') ctx.run( pip_install(req_file, constraints_file=CONSTRAINTS_PATH), echo=True ) if metrics: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'metrics.txt') ctx.run( pip_install(req_file, constraints_file=CONSTRAINTS_PATH), echo=True ) if base: # then base requirements req_file = os.path.join(HERE, 'requirements.txt') ctx.run( pip_install(req_file, constraints_file=CONSTRAINTS_PATH), echo=True ) # fix URITemplate name conflict h/t @github ctx.run('pip uninstall uritemplate.py --yes || true') ctx.run('pip install --no-cache-dir uritemplate.py==0.3.0')
def install_cloudify_packages(): utils.create_cloudify_venv() ctx.logger.info('Installing cloudify packages...') for package in CLOUDIFY_PACKAGES: package_path = utils.clone(package_name=package['package_name'], branch=package['branch'], org=package['org']) if package['package_name'] == 'cloudify-manager': utils.pip_install_manager_deps(package_path) else: utils.pip_install(package_name=package['package_name'], package_path=package_path) for package, version in PYTHON_DEPENDENCIES: utils.pip_install(package_name=package, version=version)
def install_requirements(self, deploy_path, requirement_files): ret_packages = [] if self.attempt_info != None: dependencies = self.attempt_info['dependencies'] for dependency in dependencies: LOG.info('pip install {}'.format(dependency['package_info']['name'])) pip_output = utils.pip_install(self.base_path, [dependency['package_info']], False) if pip_output != None and pip_output[0] != 0: dependency['package_info']['version'] = '' pip_output = utils.pip_install(self.base_path, [dependency['package_info']], False) LOG.debug('pip install output: {}'.format(pip_output)) else: latest_successful_attempt = self.get_latest_successful_attempt() if latest_successful_attempt != None: dependencies = Dependency.objects.filter(attempt = latest_successful_attempt) for dependency in dependencies: if dependency.source == PACKAGE_SOURCE_FILE: self.packages_from_file.append(dependency.package) else: self.packages_from_database.append(dependency.package) LOG.info('pip install {}'.format(dependency.package)) pip_output = utils.pip_install(self.base_path, [dependency.package], False) LOG.debug('pip install output: {}'.format(pip_output)) else: if requirement_files: for requirement_file in requirement_files: out = utils.pip_install(deploy_path, requirement_file, True) with open(requirement_file, "r") as my_requirement_file: ret_packages += my_requirement_file.readlines() for package in ret_packages: package = package.strip() if len(package) == 0: continue if package[0] == '#': continue if len(package.split('==')) >= 2: name, version = package.split('==') elif len(package.split('>=')) >= 2: name, version = package.split('>=') else: name, version = package, '' try: pkg, created = Package.objects.get_or_create(name=name, version=version, project_type=self.repo.project_type) self.packages_from_file.append(pkg) except Exception, e: LOG.exception(e)
def addon_requirements(ctx): """Install all addon requirements.""" for directory in os.listdir(settings.ADDON_PATH): path = os.path.join(settings.ADDON_PATH, directory) requirements_file = os.path.join(path, "requirements.txt") if os.path.isdir(path) and os.path.isfile(requirements_file): print("Installing requirements for {0}".format(directory)) ctx.run(pip_install(requirements_file, constraints_file=CONSTRAINTS_PATH), echo=True) print("Finished installing addon requirements")
def requirements(base=False, addons=False, release=False, dev=False, metrics=False, quick=False): """Install python dependencies. Examples: inv requirements inv requirements --quick Quick requirements are, in order, addons, dev and the base requirements. You should be able to use --quick for day to day development. By default, base requirements will run. However, if any set of addons, release, dev, or metrics are chosen, base will have to be mentioned explicitly in order to run. This is to remain compatible with previous usages. Release requirements will prevent dev, metrics, and base from running. """ if quick: base = True addons = True dev = True if not (addons or dev or metrics): base = True if release or addons: addon_requirements() # "release" takes precedence if release: req_file = os.path.join(HERE, 'requirements', 'release.txt') run(pip_install(req_file), echo=True) else: if dev: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'dev.txt') run(pip_install(req_file), echo=True) if metrics: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'metrics.txt') run(pip_install(req_file), echo=True) if base: # then base requirements req_file = os.path.join(HERE, 'requirements.txt') run(pip_install(req_file), echo=True)
def install_package(*args, **kargs): '''Provide a string, an iterable or pass values to the *args. Each a url or endoint used by pip ''' v = True for url in args: log('installing %s' % url, color='yellow') confirm('Would you like to install package \'%s\'' % url, True) v = pip_install(url) # To not have a horrible big in the future, if one package # fails the entire procedure reports False return v
def addon_requirements(ctx): """Install all addon requirements.""" for directory in os.listdir(settings.ADDON_PATH): path = os.path.join(settings.ADDON_PATH, directory) requirements_file = os.path.join(path, 'requirements.txt') if os.path.isdir(path) and os.path.isfile(requirements_file): print('Installing requirements for {0}'.format(directory)) ctx.run(pip_install(requirements_file, constraints_file=CONSTRAINTS_PATH), echo=True) print('Finished installing addon requirements')
def install_requirements(self, deploy_path, requirement_files): ret_packages = [] latest_successful_attempt = self.get_latest_successful_attempt() if latest_successful_attempt != None: dependencies = Dependency.objects.filter(attempt = latest_successful_attempt) for dependency in dependencies: if dependency.source == PACKAGE_SOURCE_FILE: self.packages_from_file.append(dependency.package) else: self.packages_from_database.append(dependency.package) LOG.info('pip install {}'.format(dependency.package)) pip_output = utils.pip_install(self.base_path, [dependency.package], False) LOG.info('pip install output: {}'.format(pip_output)) else: if requirement_files: for requirement_file in requirement_files: out = utils.pip_install(deploy_path, requirement_file, True) with open(requirement_file, "r") as my_requirement_file: ret_packages += my_requirement_file.readlines() for package in ret_packages: package = package.strip() if len(package) == 0: continue if package[0] == '#': continue if len(package.split('==')) >= 2: name, version = package.split('==') elif len(package.split('>=')) >= 2: name, version = package.split('>=') else: name, version = package, '' try: pkg, created = Package.objects.get_or_create(name=name, version=version, project_type=self.repo.project_type) self.packages_from_file.append(pkg) except Exception, e: LOG.exception(e)
def requirements(base=False, addons=False, release=False, dev=False, metrics=False, quick=False): """Install python dependencies. Examples: inv requirements inv requirements --quick Quick requirements are, in order, addons, dev and the base requirements. You should be able to use --quick for day to day development. By default, base requirements will run. However, if any set of addons, release, dev, or metrics are chosen, base will have to be mentioned explicitly in order to run. This is to remain compatible with previous usages. Release requirements will prevent dev, metrics, and base from running. """ if quick: base = True addons = True dev = True if not(addons or dev or metrics): base = True if release or addons: addon_requirements() # "release" takes precedence if release: req_file = os.path.join(HERE, 'requirements', 'release.txt') run(pip_install(req_file), echo=True) else: if dev: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'dev.txt') run(pip_install(req_file), echo=True) if metrics: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'metrics.txt') run(pip_install(req_file), echo=True) if base: # then base requirements req_file = os.path.join(HERE, 'requirements.txt') run(pip_install(req_file), echo=True)
def log(*args, **kargs): a = [str(x) for x in args] color = kargs.get('color', None) s = ''. join(a) dangle_print = True if s.endswith('\,') else False s = s[:-2] if dangle_print else s if color is not None: try: try: from termcolor import colored except ImportError, ie: pip_install('termcolor') print colored(s, color=color), if not dangle_print: print '' except NameError, e: print "Print Error: '%s'" % e print s, if not dangle_print: print ''
def requirements(addons=False, release=False, dev=False, metrics=False): """Install python dependencies. Examples: inv requirements --dev inv requirements --addons inv requirements --release inv requirements --metrics """ if release or addons: addon_requirements() # "release" takes precedence if release: req_file = os.path.join(HERE, 'requirements', 'release.txt') elif dev: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'dev.txt') elif metrics: # then dev requirements req_file = os.path.join(HERE, 'requirements', 'metrics.txt') else: # then base requirements req_file = os.path.join(HERE, 'requirements.txt') run(pip_install(req_file), echo=True)
def render(resource): if resource is None: return DEFAULT_LAYOUT if resource.startswith(DASH_UPLOAD_RESULTS_FLAG): loads = json.loads(str(homepage.layout[INVISIBLE_ID].children)) return render_layout( add_dash(get_upload_dash(*loads), UPLOAD_RESULT_URL_PART)) dir_path = os.path.join(get_directory(), unquote(resource)) full_path = os.path.join(dir_path, unquote(resource) + ".py") with open(full_path) as f: if 'Dash' not in f.read(): return error_layout("Этот файл не содержит объект Dash") try: dash_module = SourceFileLoader(resource[:-3], full_path).load_module() except ImportError as ie: return render(resource) if pip_install(ie.__str__().split( "'")[1]) == 0 else error_layout("Невозможно загрузить зависимости") except: error = traceback.format_exc().split("call_with_frames_removed\n", 1)[1].replace( get_directory() + "/", "") return error_layout(error) else: return render_layout(add_dash(dash_module, resource))
def try_deploy(self, deploy_path, requirement_files): LOG.info('Configuring settings ...') self.kill_server() self.clear_database() self.configure_settings() self.runtime = self.get_runtime() LOG.info(self.runtime) self.attempt.database = self.get_database() LOG.info('Database: ' + self.attempt.database.name) LOG.info('Installing requirements ...') self.install_requirements(self.base_path, requirement_files) threshold = 20 last_missing_module_name = '' index = -1 dependencies = [] for _ in range(threshold): output = self.sync_server(deploy_path) if output[0] != 0: LOG.info(output) else: LOG.info(output) break match = re.search('pip install ([a-zA-Z\-_]+)', output[2].strip()) if match: package_name = match.group(1) output = utils.pip_install_text(self.base_path, package_name) continue # LOG.info('pip install output: {}'.format(output)) split_output = output[2].strip().splitlines() line = split_output[-1].strip() match = re.search('(?<=No module named )\S+', line) if match: missing_module_name = match.group(0) LOG.info('Missing module: ' + missing_module_name) if missing_module_name == last_missing_module_name: missing_module_name, candidate_packages, index = dependencies[-1] index = index + 1 if index < len(candidate_packages): dependencies[-1] = (missing_module_name, candidate_packages, index) LOG.info('pip install {}'.format(candidate_packages[index])) pip_output = utils.pip_install(self.base_path, [candidate_packages[index]], False) LOG.info('pip install output: {}'.format(pip_output)) else: LOG.info('No more possible packages!') return ATTEMPT_STATUS_MISSING_DEPENDENCIES else: last_missing_module_name = missing_module_name candidate_package_ids = Module.objects.filter(name=missing_module_name).values_list('package_id', flat=True) if not candidate_package_ids: LOG.info('No possible packages!') return ATTEMPT_STATUS_MISSING_DEPENDENCIES candidate_packages = Package.objects.filter(id__in=candidate_package_ids).order_by('-count', '-version', 'name') LOG.info('pip install {}'.format(candidate_packages[0])) pip_output = utils.pip_install(self.base_path, [candidate_packages[0]], False, False) LOG.info('pip install output: {}'.format(pip_output)) try: version = re.search('Successfully installed .*-(.*)', pip_output[1]).group(1) if version and any(package.version == version for package in candidate_packages): for package_index in range(len(candidate_packages)): if candidate_packages[package_index].version == version: candidate_packages = [candidate_packages[package_index]] + candidate_packages[:package_index] + candidate_packages[package_index + 1:] break else: latest_package = Package() latest_package.project_type = candidate_packages[0].project_type latest_package.name = candidate_packages[0].name latest_package.version = version latest_package.save() module = Module() module.name = missing_module_name module.package = latest_package module.save() candidate_packages = list([latest_package]) + list(candidate_packages) except Exception, e: LOG.exception(e) dependencies.append((missing_module_name, candidate_packages, 0)) else: return ATTEMPT_STATUS_DATABASE_ERROR
import pickle import numpy as np import pandas as pd from os.path import isfile from utils import pip_install, extract, print_data_info, print_time_info, onehot_sparse from constants import Algo from data_processor import DataProcessor from sampler import Sampler pip_install('hyperopt') pip_install('lightgbm') pip_install('scipy') from hyperopt import hp from hyperparameters_tuner import HyperparametersTuner from lightgbm import LGBMClassifier from sklearn.linear_model import LogisticRegression from stream_processor_old import * params = {'algo': Algo.BASIC} class Model: def __init__(self, data_info, time_info): # Print data information info_dict = extract(data_info, time_info) print_data_info(info_dict)
import numpy as np import pandas as pd from utils import pip_install pip_install('category_encoders') from category_encoders.binary import BinaryEncoder from category_encoders.hashing import HashingEncoder class CountWoeCipher: def __init__(self): self._count_woe_map = {} def encode(self, incoming_data, incoming_labels=None): print('\nencode') print('incoming_data.shape: {}'.format(incoming_data.shape)) no_of_rows, no_of_cols = incoming_data.shape result = np.array([]) if incoming_labels is None and self._count_woe_map: # predict for i in range(no_of_cols): d0 = pd.DataFrame({'X': incoming_data[:, i]}) d1 = d0.join(self._count_woe_map[i], on='X')[['COUNT', 'WOE']].values result = d1 if len(result) == 0 else np.concatenate( (result, d1), axis=1) del d0 else: # fit self._count_woe_map = {} print('incoming_labels.shape: {}'.format(incoming_labels.shape))
def try_deploy(self, deploy_path, requirement_files): LOG.info('Configuring settings ...') self.kill_server() self.configure_settings() self.runtime = self.get_runtime() LOG.info(self.runtime) self.attempt.database = self.get_database() LOG.info('Database: ' + self.attempt.database.name) if self.repo.setup_scripts != None: project_path = utils.search_dir(self.base_path, self.repo.repo_name()) LOG.info("Project Path: {}".format(project_path)) LOG.info("Setup Scripts: {} && {} && {} && {}".format( utils.to_env(self.base_path), "unset DJANGO_SETTINGS_MODULE", "cd {}".format(project_path), self.repo.setup_scripts)) code, stdout, stderr = utils.run_command("{} && {} && {} && {}".format( utils.to_env(self.base_path), "unset DJANGO_SETTINGS_MODULE", "cd {}".format(project_path), self.repo.setup_scripts)) LOG.debug("Setup Return Code: {}".format(code)) LOG.debug("Setup Return STDOUT:{}".format(stdout)) LOG.debug("Setup Return STDERR:{}".format(stderr)) LOG.info('Installing requirements ...') self.install_requirements(self.base_path, requirement_files) threshold = 20 last_missing_module_name = '' index = -1 dependencies = [] for _ in range(threshold): output = self.sync_server(deploy_path) if output[0] != 0: LOG.debug(output) if self.attempt_info != None: LOG.error("Can not find dependencies!") return else: LOG.debug(output) break match = re.search('pip install ([a-zA-Z\-_]+)', output[2].strip()) if match: package_name = match.group(1) output = utils.pip_install_text(self.base_path, package_name) latest_package, created = Package.objects.get_or_create(name=package_name, version = '', project_type=self.repo.project_type) dependencies.append((package_name, [latest_package], 0)) continue # LOG.debug('pip install output: {}'.format(output)) split_output = output[2].strip().splitlines() line = split_output[-1].strip() match = re.search('(?<=No module named )\S+', line) if match: missing_module_name = match.group(0) LOG.info('Missing module: ' + missing_module_name) if missing_module_name == last_missing_module_name: missing_module_name, candidate_packages, index = dependencies[-1] index = index + 1 if index < len(candidate_packages): dependencies[-1] = (missing_module_name, candidate_packages, index) LOG.info('pip install {}'.format(candidate_packages[index])) pip_output = utils.pip_install(self.base_path, [candidate_packages[index]], False) LOG.debug('pip install output: {}'.format(pip_output)) else: LOG.info('No more possible packages!') return ATTEMPT_STATUS_MISSING_DEPENDENCIES else: last_missing_module_name = missing_module_name candidate_package_ids = Module.objects.filter(name=missing_module_name).values_list('package_id', flat=True) if not candidate_package_ids: LOG.info('No possible packages!') return ATTEMPT_STATUS_MISSING_DEPENDENCIES candidate_packages = Package.objects.filter(id__in=candidate_package_ids).order_by('-count', '-version', 'name') LOG.info('pip install {}'.format(candidate_packages[0])) pip_output = utils.pip_install(self.base_path, [candidate_packages[0]], False, False) LOG.debug('pip install output: {}'.format(pip_output)) try: version = re.search('Successfully installed .*-(.*)', pip_output[1]).group(1) if version and any(package.version == version for package in candidate_packages): for package_index in range(len(candidate_packages)): if candidate_packages[package_index].version == version: candidate_packages = [candidate_packages[package_index]] + candidate_packages[:package_index] + candidate_packages[package_index + 1:] break else: latest_package = Package() latest_package.project_type = candidate_packages[0].project_type latest_package.name = candidate_packages[0].name latest_package.version = version latest_package.save() module = Module() module.name = missing_module_name module.package = latest_package module.save() candidate_packages = list([latest_package]) + list(candidate_packages) except Exception, e: LOG.exception(e) dependencies.append((missing_module_name, candidate_packages, 0)) else: return ATTEMPT_STATUS_DATABASE_ERROR