def check_for_updates(): try: print(colors.green+"checking for updates..."+colors.end) r = requests.get("https://api.github.com/repos/4shadoww/hakkuframework/releases/latest") if(r.ok): items = json.loads(r.text or r.content) rver = items['tag_name'] if "beta" in rver and "alpha" in info.version: print(colors.green+"update found"+colors.end) return True elif "beta" not in rver and "alpha" not in rver: if "beta" in info.version or "alpha" in info.version: print(colors.green+"update found"+colors.end) return True elif version.parse(rver) > version.parse(info.version): print(colors.green+"update found"+colors.end) return True else: print(colors.yellow+"updates not found"+colors.end) return False else: print("error") except Exception as error: print(colors.red+"error: "+str(error)+colors.end)
def check_manifest(module_path): """ Verification de la version de geonature par rapport au manifest Retourne le code du module en majuscule """ log.info("checking manifest") configs_py = utilstoml.load_and_validate_toml( str(Path(module_path) / "manifest.toml"), ManifestSchemaConf ) gn_v = version.parse(GEONATURE_VERSION) if gn_v < version.parse( configs_py["min_geonature_version"] ) and gn_v > version.parse(configs_py["max_geonature_version"]): raise GeoNatureError( "Geonature version {} is imcompatible with module".format(GEONATURE_VERSION) ) for e_gn_v in configs_py["exclude_geonature_versions"]: if gn_v == version.parse(e_gn_v): raise GeoNatureError( "Geonature version {} is imcompatible with module".format( GEONATURE_VERSION ) ) log.info("...%s\n", MSG_OK) return configs_py["module_code"].upper()
def assert_and_infer_cfg(make_immutable=True): """Call this function in your script after you have finished setting all cfg values that are necessary (e.g., merging a config from a file, merging command line config options, etc.). By default, this function will also mark the global cfg as immutable to prevent changing the global cfg settings during script execution (which can lead to hard to debug errors or code that's harder to understand than is necessary). """ if __C.MODEL.RPN_ONLY or __C.MODEL.FASTER_RCNN: __C.RPN.RPN_ON = True if __C.RPN.RPN_ON or __C.RETINANET.RETINANET_ON: __C.TEST.PRECOMPUTED_PROPOSALS = False if __C.MODEL.LOAD_IMAGENET_PRETRAINED_WEIGHTS: assert __C.RESNETS.IMAGENET_PRETRAINED_WEIGHTS, \ "Path to the weight file must not be empty to load imagenet pertrained resnets." if set([__C.MRCNN.ROI_MASK_HEAD, __C.KRCNN.ROI_KEYPOINTS_HEAD]) & _SHARE_RES5_HEADS: __C.MODEL.SHARE_RES5 = True if version.parse(torch.__version__) < version.parse('0.4.0'): __C.PYTORCH_VERSION_LESS_THAN_040 = True # create alias for PyTorch version less than 0.4.0 init.uniform_ = init.uniform init.normal_ = init.normal init.constant_ = init.constant nn.GroupNorm = mynn.GroupNorm if make_immutable: cfg.immutable(True)
def updateCheck(self): ''' Check for update at pypi''' try: from packaging.version import parse except ImportError: print("Error importing packaging") from pip._vendor.packaging.version import parse req = requests.get(self.repo.format(package=self.pkg)) avail_version = parse('0') if req.status_code == requests.codes.ok: j = json.loads(req.text.encode(req.encoding)) if 'releases' in j: releases = j['releases'] for release in releases: ver = parse(release) if not ver.is_prerelease: avail_version = max(avail_version, ver) return str(avail_version)
def __init__(self, old, new): if isinstance(old, str): old = parse(old) if isinstance(new, str): new = parse(new) self.old = old self.new = new
def test_empty_specifier(self, version): spec = SpecifierSet(prereleases=True) assert version in spec assert spec.contains(version) assert parse(version) in spec assert spec.contains(parse(version))
def version_matrix(request): matrix = defaultdict(partial(defaultdict, LatestList)) latest = defaultdict(list) groups = defaultdict(list) judges = {judge.id: judge.name for judge in Judge.objects.filter(online=True)} languages = Language.objects.all() for runtime in RuntimeVersion.objects.filter(judge__online=True).order_by('priority'): if runtime.version: matrix[runtime.judge_id][runtime.language_id].append(runtime) for judge, data in six.iteritems(matrix): name_tuple = judges[judge].rpartition('.') groups[name_tuple[0] or name_tuple[-1]].append((judges[judge], data)) matrix = {} for group, data in six.iteritems(groups): if len(data) == 1: judge, data = data[0] matrix[judge] = data continue ds = list(range(len(data))) size = [1] * len(data) for i, (p, x) in enumerate(data): if ds[i] != i: continue for j, (q, y) in enumerate(data): if i != j and compare_version_list(x, y): ds[j] = i size[i] += 1 size[j] = 0 rep = max(range(len(data)), key=size.__getitem__) matrix[group] = data[rep][1] for i, (j, x) in enumerate(data): if ds[i] != rep: matrix[j] = x for data in six.itervalues(matrix): for language, versions in six.iteritems(data): versions.versions = [version.parse(runtime.version) for runtime in versions] if versions.versions > latest[language]: latest[language] = versions.versions for data in six.itervalues(matrix): for language, versions in six.iteritems(data): versions.is_latest = versions.versions == latest[language] languages = sorted(languages, key=lambda lang: version.parse(lang.name)) return render(request, 'status/versions.html', { 'title': _('Version matrix'), 'judges': sorted(matrix.keys()), 'languages': languages, 'matrix': matrix, })
def find_rpm_packaging_pkg_version(pkg_project_spec): """get a spec.j2 template and get the version""" if os.path.exists(pkg_project_spec): with open(pkg_project_spec) as f: for l in f: m = re.search('^Version:\s*(?P<version>.*)\s*$', l) if m: return version.parse(m.group('version')) # no version in spec found print('ERROR: no version in %s found' % pkg_project_spec) return version.parse('0') return version.parse('0')
def test_demo(preconfigured_testing_api, android_version): # Load necessary resources settings_ico = tm_engine.load_template("resources/settings_ico.png") # Create a dictionary with filters to find specific device device_spec = {"android_version": android_version} with preconfigured_testing_api(device_spec, device_manager_url) as api: api.adb("shell pm clear com.kms.free") # Launch an app without specifying Activity api.adb("shell monkey -p com.kms.free 1") # Use packaging.version to compare different versions if version.parse(android_version) >= version.parse("5.0"): api.tap( "next", config=dict(crop_x_range=(0.4, 0.6), crop_y_range=(0.7, 0.9))) # We need to allow permissions for i in range(0, 2): time.sleep(5) # Some special cases needs special configuration api.tap("allow", config=dict( ocr_postprocessing=filter_postprocessing, api_tap_index=-1)) # But most of cases are simple api.tap("accept and continue") api.wait_for("activation code") # Scroll down (move your "finger"" from the lowest point to the highest one) # Untill you find the text (maximum number of screens is adjustable) api.scroll_for("use free version", (0.5, 0.6), (0.5, 0.2)) api.tap("use free version") api.tap("run the scan") api.tap( "OK", config=dict( api_tap_timeout=60 * 5, crop_x_range=(0.1, 0.9), crop_y_range=(0.4, 0.8))) api.tap("rate later") api.tap(settings_ico) api.tap("additional") # Some checkboxes should change their state with api.assert_screen_change(): api.tap("get notifications about") api.tap("get sound")
def ensure_model_compatibility(metadata): from packaging import version model_version = metadata.get("rasa_core", "0.0.0") if version.parse(model_version) < version.parse("0.10.0a3"): raise UnsupportedDialogueModelError( "The model version is to old to be " "loaded by this Rasa Core instance. " "Either retrain the model, or run with" "an older version. " "Model version: {} Instance version: {}" "".format(model_version, rasa_core.__version__))
def supported_resampling(method): if method == Resampling.gauss: return False gdal110plus_only = ( Resampling.mode, Resampling.average) gdal2plus_only = ( Resampling.max, Resampling.min, Resampling.med, Resampling.q1, Resampling.q3) version = parse(rasterio.__gdal_version__) if version < parse('1.10'): return method not in gdal2plus_only and method not in gdal110plus_only if version < parse('2.0'): return method not in gdal2plus_only return True
def einsum(*args, **kwargs): """Call optimized einsum if possible Args: *args: Arguments for numpy.einsum. **kwargs: Keyword arguments for numpy.einsum. Returns: The output of numpy.einsum(*args, **kwargs). """ if version.parse(numpy.__version__) >= version.parse('1.12'): kwargs['optimize'] = True return numpy.einsum(*args, **kwargs) else: return numpy.einsum(*args, **kwargs)
def ensure_model_compatibility(metadata, version_to_check=None): from packaging import version if version_to_check is None: version_to_check = MINIMUM_COMPATIBLE_VERSION model_version = metadata.get("rasa_nlu_version", "0.0.0") if version.parse(model_version) < version.parse(version_to_check): raise UnsupportedModelError( "The model version is to old to be " "loaded by this Rasa NLU instance. " "Either retrain the model, or run with" "an older version. " "Model version: {} Instance version: {}" "".format(model_version, rasa_nlu.__version__))
def latest_version(pkg_name, prerelease=False, silent=False): try: info = get_pkg_info(pkg_name, silent=silent) except ValueError: if silent: return None, None else: raise if not info: return None, None try: versions = [ v for v in sorted( list(info['releases']), key=packaging_version.parse ) ] if not prerelease: versions = [v for v in versions if not packaging_version.parse(v).is_prerelease] version = versions[-1] except IndexError: return None, None return parse_version(version), version
def do_update_own_boot_file(self, arg): """ Update 'bootstrap_env/boot_bootstrap_env.py' via cookiecutter """ # https://packaging.pypa.io/en/latest/version/ parsed_bootstrap_env_version = parse(bootstrap_env_version) if parsed_bootstrap_env_version.is_prerelease: use_pre_release = "y" else: use_pre_release = "n" repro_path = Path(self.path_helper.base, "boot_source") # https://cookiecutter.readthedocs.io result = verbose_cookiecutter( template=str(repro_path), no_input=True, overwrite_if_exists=True, output_dir=str(self.path_helper.base.parent), extra_context={ "_version": bootstrap_env_version, "use_pre_release": use_pre_release, }, ) print("\nbootstrap file created here: %s" % result)
def _context_py2rpmversion(context): """get a python PEP0440 compatible version and translate it to an RPM version""" # the context needs a variable set via {% set upstream_version = 'ver' %} _context_check_variable(context, CONTEXT_VAR_UPSTREAM_VERSION, "py2rpmversion") version = context.vars[CONTEXT_VAR_UPSTREAM_VERSION] v_python = parse(version) # fedora does not allow '~' in versions but uses a combination of Version # and Release # https://fedoraproject.org/wiki/Packaging:Versioning\#Pre-Release_packages if context["spec_style"] == "fedora": if len(v_python._version.release) >= 4: return "%d.%d.%d" % (v_python._version.release[0:3]) else: return v_python.base_version else: v_rpm = v_python.public if v_python.is_prerelease: # we need to add the 'x' in front of alpha/beta releases because # in the python world, "1.1a10" > "1.1.dev10" # but in the rpm world, "1.1~a10" < "1.1~dev10" v_rpm = v_rpm.replace("a", "~xalpha") v_rpm = v_rpm.replace("b", "~xbeta") v_rpm = v_rpm.replace("rc", "~rc") v_rpm = v_rpm.replace(".dev", "~dev") return v_rpm
def simple_detail(project, request): # TODO: Handle files which are not hosted on PyPI # Make sure that we're using the normalized version of the URL. if project.normalized_name != request.matchdict.get( "name", project.normalized_name ): return HTTPMovedPermanently( request.current_route_path(name=project.normalized_name) ) # Get the latest serial number for this project. request.response.headers["X-PyPI-Last-Serial"] = str(project.last_serial) # Get all of the files for this project. files = sorted( request.db.query(File) .options(joinedload(File.release)) .join(Release) .filter(Release.project == project) .all(), key=lambda f: (parse(f.release.version), f.filename), ) return {"project": project, "files": files}
def _pretty_table(release, projects, include_obs): from prettytable import PrettyTable tb = PrettyTable() fn = ['name', 'release (%s)' % release, 'u-c (%s)' % release, 'rpm packaging (%s)' % release, 'reviews'] if include_obs: fn += ['obs'] fn += ['comment'] tb.field_names = fn for p_name, x in projects.items(): if x.rpm_packaging_pkg == version.parse('0'): comment = 'needs packaging' elif x.rpm_packaging_pkg < x.release: comment = 'needs upgrade' elif x.rpm_packaging_pkg == x.release: if x.release > x.upper_constraints: comment = 'needs downgrade (u-c)' comment = 'perfect' elif x.rpm_packaging_pkg > x.release: comment = 'needs downgrade' else: comment = '' row = [p_name, x.release, x.upper_constraints, x.rpm_packaging_pkg, x.reviews] if include_obs: row += [x.obs_published] row += [comment] tb.add_row(row) return tb
def _parse_version(version_str): # not sure how consistently these are installed, just try all of them try: from packaging import version return version.parse(version_str) except ImportError: pass try: from distutils.version import LooseVersion, StrictVersion try: return StrictVersion(version_str) except distutils.version.InvalidVersion: return LooseVersion(version_str) except ImportError: pass try: from setuptools import parse_version return parse_version(version_str) except ImportError: pass try: from pkg_resources import parse_version return parse_version(version_str) except ImportError: pass raise NotImplementedError(version_str)
def check_dependencies(): """Verify dependant libraries/binaries are present with correct versions.""" print('Checking library/binary dependencies') for (binary, binary_get_version, binary_present_re, binary_version_re, binary_minversion) in EXTERNAL_DEPENDENCIES: binary_args = [binary] + binary_get_version required_binary = 'required binary/library %s' % ( ' '.join(binary_args)) try: proc = subprocess.Popen( binary_args, stdin=mininet_test_util.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True) proc_out, proc_err = proc.communicate() binary_output = proc_out.decode() if proc_err is not None: binary_output += proc_err.decode() except subprocess.CalledProcessError: # Might have run successfully, need to parse output pass except OSError: print('could not run %s' % required_binary) return False present_match = re.search(binary_present_re, binary_output) if not present_match: print('%s not present or did not return expected string %s (%s)' % ( required_binary, binary_present_re, binary_output)) return False if binary_version_re: version_match = re.search(binary_version_re, binary_output) if version_match is None: print('could not get version from %s (%s)' % ( required_binary, binary_output)) return False try: binary_version = version_match.group(1) except ValueError: print('cannot parse version %s for %s' % ( version_match, required_binary)) return False if version.parse(binary_version) < version.parse(binary_minversion): print('%s version %s is less than required version %s' % ( required_binary, binary_version, binary_minversion)) return False return True
def _mac_version(): from packaging.version import parse from platform import mac_ver ver = parse(mac_ver()[0]) if parse("10.15") > ver >= parse("10.14"): return {"mac_version": "mojave"} if parse("10.14") > ver >= parse("10.13"): return {"mac_version": "highsierra"} if parse("10.13") > ver >= parse("10.12"): return {"mac_version": "sierra"} if parse("10.12") > ver >= parse("10.11"): return {"mac_version": "elcapitan"}
def test_runner_uses_source_from_filename(self): doc = Document() source = nbformat.v4.new_notebook() result = {} def load(filename): handler = bahn.NotebookHandler(filename=filename) handler.modify_document(doc) result['handler'] = handler result['filename'] = filename with_script_contents(source, load) assert result['handler']._runner.path == result['filename'] if version.parse(nbconvert.__version__) < version.parse("5.4"): assert result['handler']._runner.source == "\n# coding: utf-8\n" else: assert result['handler']._runner.source == "#!/usr/bin/env python\n# coding: utf-8\n" assert not doc.roots
def test_storage_celery_save(self): """ Make sure it actually works when using Celery as a task queue """ storage = QueuedStorage( local='django.core.files.storage.FileSystemStorage', remote='django.core.files.storage.FileSystemStorage', local_options=dict(location=self.local_dir), remote_options=dict(location=self.remote_dir)) field = models.TestModel._meta.get_field('testfile') field.storage = storage obj = models.TestModel() obj.testfile.save(self.test_file_name, File(self.test_file)) obj.save() self.assertTrue(obj.testfile.storage.result.get()) self.assertTrue(path.isfile(path.join(self.local_dir, obj.testfile.name))) self.assertTrue( path.isfile(path.join(self.remote_dir, obj.testfile.name)), "Remote file is not available.") self.assertFalse(storage.using_local(obj.testfile.name)) self.assertTrue(storage.using_remote(obj.testfile.name)) self.assertEqual(self.test_file_name, storage.get_valid_name(self.test_file_name)) self.assertEqual(self.test_file_name, storage.get_available_name(self.test_file_name)) subdir_path = os.path.join('test', self.test_file_name) self.assertTrue(storage.exists(subdir_path)) self.assertEqual(storage.path(self.test_file_name), path.join(self.local_dir, self.test_file_name)) self.assertEqual(storage.listdir('test')[1], [self.test_file_name]) self.assertEqual(storage.size(subdir_path), os.stat(self.test_file_path).st_size) self.assertEqual(storage.url(self.test_file_name), self.test_file_name) if version.parse(DJANGO_VERSION) in SpecifierSet("<=2.0"): self.assertIsInstance(storage.accessed_time(subdir_path), datetime) self.assertIsInstance(storage.created_time(subdir_path), datetime) self.assertIsInstance(storage.modified_time(subdir_path), datetime) else: self.assertIsInstance(storage.get_accessed_time(subdir_path), datetime) self.assertIsInstance(storage.get_created_time(subdir_path), datetime) self.assertIsInstance(storage.get_modified_time(subdir_path), datetime) subdir_name = 'queued_storage_2.txt' testfile = storage.open(subdir_name, 'w') try: testfile.write('test') finally: testfile.close() self.assertTrue(storage.exists(subdir_name)) storage.delete(subdir_name) self.assertFalse(storage.exists(subdir_name))
def check(version): try: url_pattern = 'https://pypi.python.org/pypi/deepctr/json' req = requests.get(url_pattern) latest_version = parse('0') version = parse(version) if req.status_code == requests.codes.ok: j = json.loads(req.text.encode('utf-8')) releases = j.get('releases', []) for release in releases: ver = parse(release) if not ver.is_prerelease: latest_version = max(latest_version, ver) if latest_version > version: logging.warning('\nDeepCTR version {0} detected. Your version is {1}.\nUse `pip install -U deepctr` to upgrade.Changelog: https://github.com/shenweichen/DeepCTR/releases/tag/v{0}'.format( latest_version, version)) except Exception: return
def tendermint_version_is_compatible(running_tm_ver): """ Check Tendermint compatability with BigchainDB server :param running_tm_ver: Version number of the connected Tendermint instance :type running_tm_ver: str :return: True/False depending on the compatability with BigchainDB server :rtype: bool """ # Splitting because version can look like this e.g. 0.22.8-40d6dc2e tm_ver = running_tm_ver.split('-') if not tm_ver: return False for ver in __tm_supported_versions__: if version.parse(ver) == version.parse(tm_ver[0]): return True return False
def show(self): ''' Show relevant suggestions based on the requested versions ''' sorted_keys = sorted([version.parse(suggestion) for suggestion in suggestions.keys() if version.parse(suggestion) > self.file_version]) for key in sorted_keys: print("\033[1m === {} === \033[0m".format(key)) for line in suggestions[str(key)]: print(line) print("")
def _acceptable_interpreter_constraints(self): default_constraints = PythonSetup.global_instance().interpreter_constraints whitelisted_constraints = self.get_options().interpreter_constraints_whitelist # The user wants to lint everything. if whitelisted_constraints == []: return [] # The user did not pass a whitelist option. elif whitelisted_constraints is None: whitelisted_constraints = () return [version.parse(v) for v in default_constraints + whitelisted_constraints]
def _constraints_are_whitelisted(self, constraint_tuple): """ Detect whether a tuple of compatibility constraints matches constraints imposed by the merged list of the global constraints from PythonSetup and a user-supplied whitelist. """ if self._acceptable_interpreter_constraints == []: # The user wants to lint everything. return True return all(version.parse(constraint) in self._acceptable_interpreter_constraints for constraint in constraint_tuple)
def find_openbuildservice_pkg_version(published_xml, pkg_name): """find the version in the openbuildservice published xml for the given pkg name""" import pymod2pkg import xml.etree.ElementTree as ET if published_xml and os.path.exists(published_xml): with open(published_xml) as f: tree = ET.fromstring(f.read()) distro_pkg_name = pymod2pkg.module2package(pkg_name, 'suse') for child in tree: if not child.attrib['name'].startswith('_') and \ child.attrib['name'].endswith('.rpm') and not \ child.attrib['name'].endswith('.src.rpm'): (name, ver, release, epoch, arch) = _rpm_split_filename( child.attrib['name']) if name == distro_pkg_name: return version.parse(ver) return version.parse('0')
def validate_community(specs, instance): validation_result = {} status_text = '' status = '' text_result = '' try: validator = jsonschema.validators.validator_for(specs[instance['api']]['schema']) validator.check_schema(specs[instance['api']]['schema']) v = validator(specs[instance['api']]['schema']) result = v.iter_errors(instance) has_error = False for error in sorted(result,key=str): if not has_error: text_result = '<ul>' has_error = True text_result = '%s<li>Error in %s: %s</li>' % (text_result, '->'.join(str(path) for path in error.path), error.message) if has_error: text_result = '%s</ul>' % (text_result) status = 'invalid' status_text = 'Invalid' elif version.parse(instance['api']) < version.parse('0.4.0'): status = 'warning' status_text = 'Warning' text_result = 'API version too old! You should upgrade your file' # TODO: Check lastchange date #elif instance['lastchange']: # status = 'warning' # status_text = 'Warning' # text_result = 'No Update on API file for more than 2 month!' else: status = 'valid' status_text = 'Valid' validation_result['status_text'] = status_text validation_result['status'] = status validation_result['result'] = text_result return validation_result except KeyError as e: print('Invalid or unknown API version %s: %s' % (instance['api'], url))
def __init__(self, dvc_dir): self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) self.lock = Lock(self.updater_file + ".lock", tmp_dir=os.path.join(dvc_dir, "tmp")) self.current = version.parse(__version__).base_version
from optuna.study import StudyDirection import optuna.trial with try_import() as _imports: from bokeh import __version__ as bokeh_version import bokeh.command.bootstrap import bokeh.document # NOQA import bokeh.layouts import bokeh.models import bokeh.models.widgets import bokeh.plotting import bokeh.themes import tornado.gen if version.parse(bokeh_version) >= version.parse("2.0.0"): raise ImportError( "Your version of bokeh is " + bokeh_version + " . " "Please install bokeh version earlier than 2.0.0. " "Bokeh can be installed by executing `$ pip install 'bokeh<2.0.0'`. " "For further information, please refer to the installation guide of bokeh. ", name="bokeh", ) _mode = None # type: Optional[str] _study = None # type: Optional[optuna.study.Study] _HEADER_FORMAT = """ <style> body {{
#!/usr/bin/env python import colored_traceback.always import os import sys import cv2 import scipy.io import numpy as np import itertools import shutil import threading import time import schedule import tensorflow as tf from packaging import version assert version.parse(tf.__version__) > version.parse("1.0.0"), \ "Tensorflow version >= 1.0.0 required" from drl.config import parse_flags FLAGS = parse_flags() import gym import gym_offroad_nav.envs from drl.monitor import Monitor # Show how each agent behaves in a seperate monitor thread monitor = Monitor() monitor.start() from drl.ac.estimators import get_estimator
def main(): """Create the model and start the evaluation process.""" args = get_arguments() gpu0 = args.gpu if not os.path.exists(args.save): os.makedirs(args.save) if args.model == 'DeeplabMulti': model = DeeplabMulti(num_classes=args.num_classes) elif args.model == 'Oracle': model = Res_Deeplab(num_classes=args.num_classes) if args.restore_from == RESTORE_FROM: args.restore_from = RESTORE_FROM_ORC elif args.model == 'DeeplabVGG': model = DeeplabVGG(num_classes=args.num_classes) if args.restore_from == RESTORE_FROM: args.restore_from = RESTORE_FROM_VGG if args.restore_from[:4] == 'http': saved_state_dict = model_zoo.load_url(args.restore_from) else: saved_state_dict = torch.load(args.restore_from) ### for running different versions of pytorch model_dict = model.state_dict() saved_state_dict = { k: v for k, v in saved_state_dict.items() if k in model_dict } model_dict.update(saved_state_dict) ### model.load_state_dict(saved_state_dict) model.eval() model.cuda(gpu0) testloader = data.DataLoader(cityscapesDataSet(args.data_dir, args.data_list, crop_size=(1024, 512), mean=IMG_MEAN, scale=False, mirror=False, set=args.set), batch_size=1, shuffle=False, pin_memory=True) if version.parse(torch.__version__) >= version.parse('0.4.0'): interp = nn.Upsample(size=(1024, 2048), mode='bilinear', align_corners=True) else: interp = nn.Upsample(size=(1024, 2048), mode='bilinear') for index, batch in enumerate(testloader): if index % 100 == 0: print('%d processd' % index) image, _, name = batch if args.model == 'DeeplabMulti': output1, output2 = model(Variable(image, volatile=True).cuda(gpu0)) output = interp(output2).cpu().data[0].numpy() elif args.model == 'DeeplabVGG' or args.model == 'Oracle': output = model(Variable(image, volatile=True).cuda(gpu0)) output = interp(output).cpu().data[0].numpy() output = output.transpose(1, 2, 0) output = np.asarray(np.argmax(output, axis=2), dtype=np.uint8) output_col = colorize_mask(output) output = Image.fromarray(output) name = name[0].split('/')[-1] output.save('%s/%s' % (args.save, name)) output_col.save('%s/%s_color.png' % (args.save, name.split('.')[0]))
logging.basicConfig(level=logging.WARN, format='[%(levelname)s] (%(threadName)-10s) %(message)s') # import PlotOptiX modules ############################################### from plotoptix.enums import * from plotoptix.npoptix import NpOptiX from plotoptix.tkoptix import TkOptiX # check PlotOptiX updates ################################################ import json import urllib.request from packaging import version try: url = "https://pypi.python.org/pypi/plotoptix/json" webURL = urllib.request.urlopen(url, timeout=3) data = webURL.read() encoding = webURL.info().get_content_charset('utf-8') data_dict = json.loads(data.decode(encoding)) versions = list(data_dict["releases"].keys()) versions.sort(key=version.parse) if version.parse(__version__) < version.parse(versions[-1]): print(80 * "*") print("PlotOptiX newer version is available:", versions[-1]) print("to update your release use:") print(" pip install plotoptix --upgrade") print(80 * "*") except: pass
# See the License for the specific language governing permissions and # limitations under the License. from cil.framework import ImageGeometry from cil.optimisation.functions import L2NormSquared, L1Norm import numpy as np import matplotlib.pyplot as plt from cil.utilities import dataexample from cil.utilities import noise import os import sys import unittest import warnings from cil.utilities.quality_measures import mse, mae, psnr from packaging import version if version.parse(np.version.version) >= version.parse("1.13"): try: from skimage import data, io, filters from skimage.metrics import mean_squared_error, peak_signal_noise_ratio, structural_similarity has_skimage = True except ImportError as ie: has_skimage = False else: has_skimage = False class CCPiTestClass(unittest.TestCase): def assertNumpyArrayEqual(self, first, second): res = True try: numpy.testing.assert_array_equal(first, second)
'scotland'] # Parametrized WINDOW_COUNT = 3 WINDOW_LENGTH = 10 SAMPLE_LENGTH = WINDOW_LENGTH * S_RATE ###################################################### os.environ['WANDB_PROJECT'] = 'w2v_did' os.environ['WANDB_LOG_MODEL'] = 'true' if is_apex_available(): pass if version.parse(torch.__version__) >= version.parse("1.6"): _is_native_amp_available = True logger = logging.getLogger(__name__) def list_field(default=None, metadata=None): return field(default_factory=lambda: default, metadata=metadata) @dataclass class ModelArguments: """ Arguments pertaining to which model/config/tokenizer we are going to fine-tune from. """
from __future__ import print_function import os import sys import io import datetime from packaging import version import numpy as np import matplotlib.pyplot as plt from math import pi import tensorflow as tf print(__doc__) # Display tensorflow version print("TensorFlow version: ", tf.version.VERSION) assert version.parse(tf.version.VERSION).release[0] >= 2, \ "This notebook requires TensorFlow 2.0 or above." def train_step(x_input, y_input, a, ratio_learn): with tf.GradientTape() as tape: loss = tf.square(tf.math.multiply(x_input, a) - y_input) dloss_da = tf.gradients(loss, a) optimizer = tf.keras.optimizers.SGD(learning_rate=ratio_learn) optimizer.apply_gradients(zip(dloss_da, a)) # Create data x_vals = np.random.normal(1, 0.1, 100) y_vals = np.repeat(10., 100)
#source_encoding = 'utf-8' # The master toctree document. master_doc = 'contents' # General information about the project. project = 'scikit-learn' copyright = '2007 - 2019, scikit-learn developers (BSD License)' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. import sklearn parsed_version = parse(sklearn.__version__) version = ".".join(parsed_version.base_version.split(".")[:2]) # The full version, including alpha/beta/rc tags. # Removes post from release name if parsed_version.is_postrelease: release = parsed_version.base_version else: release = sklearn.__version__ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. #language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: #today = ''
def __generate_custom_profile_devices_configuration(self, instance): ''' template = '[ {% for net in networks %}' \ '{"eth{{loop.index -1 }}": ' \ '{"name": "eth{{loop.index -1}}",' \ '"type" : "nic",' \ '"parent": "{{ net.intf_name }}",' \ '"nictype": "bridged" ,' \ '"hwaddr" : {{ net.mac }} }" '{% endfor %} ]' ''' devices = {} template_value_bridge = '{"name":"%s","type":"nic","parent":"%s","nictype":"bridged"}' template_value_bridge_mac = '{"name":"%s","type":"nic","parent":"%s","nictype":"bridged","hwaddr":"%s"}' template_value_phy = '{"name":"%s","type":"nic","parent":"%s","nictype":"physical"}' template_value_macvlan = '{"name":"%s","type":"nic","parent":"%s","nictype":"macvlan"}' ''' # Create tenant's storage pool # TODO: allow more storage backends lxc storage create $TENANT dir # Add a root disk to the tenant's profile lxc profile device add $TENANT root disk path=/ pool=$TENANT ''' template_disk = '{"path":"%s","type":"disk","pool":"%s"}' template_key = '%s' template_key2 = "eth%d" for i, n in enumerate(instance.networks): if n.get('network_uuid') is not None: nws = self.agent.get_network_plugin(None).get(list(self.agent.get_network_plugin(None).keys())[0]) # print(nws.getNetworkInfo(n.get('network_uuid'))) br_name = nws.get_network_info(n.get('network_uuid')).get('virtual_device') # print(br_name) n.update({'br_name': br_name}) if n.get('intf_name') is None: n.update({'intf_name': "eth"+str(i)}) #nw_k = str(template_key % n.get('intf_name')) nw_k = str(template_key2 % i) if n.get('mac') is not None: nw_v = json.loads(str(template_value_bridge_mac % (n.get('intf_name'), n.get('br_name'), n.get('mac')))) else: nw_v = json.loads(str(template_value_bridge % (n.get('intf_name'), n.get('br_name')))) elif self.agent.get_os_plugin().get_intf_type(n.get('br_name')) in ['ethernet']: #if n.get('') #cmd = "sudo ip link add name %s link %s type macvlan" #veth_name = str('veth-%s' % entity.uuid[:5]) #cmd = str(cmd % (veth_name, n.get('br_name'))) #self.agent.getOSPlugin().executeCommand(cmd, True) #nw_v = json.loads(str(template_value_phy % (n.get('intf_name'), veth_name))) nw_v = json.loads(str(template_value_macvlan % (n.get('intf_name'), n.get('br_name')))) #nw_k = str(template_key % n.get('intf_name')) nw_k = str(template_key2 % i) #self.agent.getOSPlugin().set_interface_unaviable(n.get('br_name')) elif self.agent.get_os_plugin().get_intf_type(n.get('br_name')) in ['wireless']: nw_v = json.loads(str(template_value_phy % (n.get('intf_name'), n.get('br_name')))) #nw_k = str(template_key % n.get('intf_name')) nw_k = str(template_key2 % i) self.agent.get_os_plugin().set_interface_unaviable(n.get('br_name')) else: if n.get('intf_name') is None: n.update({'intf_name': "eth" + str(i)}) #nw_k = str(template_key % n.get('intf_name')) nw_k = str(template_key2 % i) nw_v = json.loads(str(template_value_bridge % (n.get('intf_name'), n.get('br_name')))) devices.update({nw_k: nw_v}) lxd_version = self.conn.host_info['environment']['server_version'] if version.parse(lxd_version) >= version.parse("2.20"): if instance.storage is None or len(instance.storage) == 0: st_n = "root" st_v = json.loads(str(template_disk % ("/","default"))) devices.update({st_n: st_v}) else: for s in instance.storage: st_n = s.get("name") st_v = json.loads(str(template_disk % (s.get("path"), s.get("pool")))) devices.update({st_n: st_v}) #devices = Environment().from_string(template) #devices = devices.render(networks=entity.networks) return devices
class TestCreateUserTask(TestCase): """ Tests of UserTaskStatus creation for new UserTasks. """ def tearDown(self): super(TestCreateUserTask, self).tearDown() SIGNAL_DATA.clear() @classmethod def setUpTestData(cls): super(TestCreateUserTask, cls).setUpTestData() cls.user = User.objects.create_user('test_user', '*****@*****.**', 'password') def test_create_user_task(self): """The create_user_task signal handler should create a new UserTaskStatus record""" self._create_user_task(eager=False) @override_settings(CELERY_ALWAYS_EAGER=True) def test_create_user_task_eager(self): """Eager tasks should still have UserTaskStatus records created on execution.""" self._create_user_task(eager=True) assert SIGNAL_DATA['received_status'].state == UserTaskStatus.SUCCEEDED def test_create_group(self): """The create_user_task signal handler should correctly handle celery groups""" self._create_group(eager=False) @pytest.mark.skipif( CELERY_VERSION < version.parse('4.0'), reason= "celery 3.1 doesn't provide accurate group metadata to eager tasks") @override_settings(CELERY_ALWAYS_EAGER=True) def test_create_group_eager(self): """Eager groups should still have UserTaskStatus records created on execution.""" self._create_group(eager=True) def test_create_chain(self): """The create_user_task signal handler should correctly handle celery chains.""" self._create_chain(eager=False) @pytest.mark.skipif( CELERY_VERSION < version.parse('4.0'), reason= "celery 3.1 doesn't provide accurate chain metadata to eager tasks") @override_settings(CELERY_ALWAYS_EAGER=True) def test_create_chain_eager(self): """Eager chains should still have UserTaskStatus records created on execution.""" self._create_chain(eager=True) def test_create_chord(self): """The create_user_task signal handler should correctly handle celery chords""" self._create_chord(eager=False) @pytest.mark.skipif( CELERY_VERSION < version.parse('4.0'), reason= "celery 3.1 doesn't provide accurate chord metadata to eager tasks") @override_settings(CELERY_ALWAYS_EAGER=True) def test_create_chord_eager(self): """Eager chords should still have UserTaskStatus records created on execution.""" self._create_chord(eager=True) def test_create_chord_exclude_body(self): """If the body task of a chord is not a UserTask, it should be cleanly omitted from the status.""" chord([ sample_task.s(self.user.id, '1', user_task_name='Chord: 1 & 2'), sample_task.s(self.user.id, '2', user_task_name='I should be ignored') ])(normal_task.s('3')) assert UserTaskStatus.objects.count() == 4 chord_status = UserTaskStatus.objects.get(task_class='celery.chord') assert chord_status.task_id assert chord_status.parent is None assert chord_status.is_container assert chord_status.name == 'Chord: 1 & 2' assert chord_status.total_steps == 2 verify_state(chord_status, False) group_status = UserTaskStatus.objects.get(task_class='celery.group') assert group_status.task_id assert group_status.parent_id == chord_status.id assert group_status.is_container assert group_status.name == 'Chord: 1 & 2' assert group_status.total_steps == 2 verify_state(group_status, False) header_tasks = UserTaskStatus.objects.filter(parent=group_status) assert len(header_tasks) == 2 for status in header_tasks: assert status.task_id assert status.parent_id == group_status.id assert not status.is_container assert status.name in ['SampleTask: 1', 'SampleTask: 2'] assert status.total_steps == 1 verify_state(status, False) def test_missing_user_id(self): """Queueing of the task should fail if the user ID is not provided.""" with pytest.raises(TypeError) as exc_info: missing_user_id.delay() assert str( exc_info.value ) == 'Each invocation of a UserTaskMixin subclass must include the user_id' def test_invalid_user_id(self): """Queueing of the task should fail if an invalid user ID is given.""" with pytest.raises(TypeError) as exc_info: sample_task.delay('arg1', 'arg2') assert str(exc_info.value) == 'Invalid user_id: arg1' def test_non_user_task_publish(self): """Non-UserTask tasks should still pass through the before_task_publish handler cleanly.""" normal_task.delay('Argument') statuses = UserTaskStatus.objects.all() assert not statuses @override_settings(CELERY_ALWAYS_EAGER=True, CELERY_IGNORE_RESULT=False) def test_non_user_task_success(self): """Non-UserTask tasks should still pass through start and success handlers cleanly.""" result = normal_task.delay('Argument') assert result.get() == 'placeholder' statuses = UserTaskStatus.objects.all() assert not statuses def _create_user_task(self, eager): """Create a task based on UserTaskMixin and verify some assertions about its corresponding status.""" result = sample_task.delay(self.user.id, 'Argument') statuses = UserTaskStatus.objects.all() assert len(statuses) == 1 status = statuses[0] assert status.task_id == result.id assert status.task_class == 'test_signals.sample_task' assert status.user_id == self.user.id assert status.parent is None assert not status.is_container assert status.name == 'SampleTask: Argument' assert status.total_steps == 1 verify_state(status, eager) def _create_chain(self, eager): """Create a celery chain and verify some assertions about the corresponding status records""" chain( sample_task.si(self.user.id, '1'), sample_task.si(self.user.id, '2', user_task_name='Chain: 1, 2, 3'), sample_task.si(self.user.id, '3'), normal_task.si('Argument')).delay() assert UserTaskStatus.objects.count() == 4 chain_status = UserTaskStatus.objects.get(task_class='celery.chain') assert chain_status.task_id assert chain_status.parent is None assert chain_status.is_container assert chain_status.name == 'Chain: 1, 2, 3' assert chain_status.total_steps == 3 verify_state(chain_status, eager) children = UserTaskStatus.objects.filter(parent=chain_status) assert len(children) == 3 for status in children: assert not status.is_container assert status.name in [ 'SampleTask: 1', 'SampleTask: 2', 'SampleTask: 3' ] assert status.total_steps == 1 verify_state(status, eager) def _create_chord(self, eager): """Create a celery chord and verify some assertions about the corresponding status records""" chord([ sample_task.s(self.user.id, '1'), sample_task.s(self.user.id, '2', user_task_name='Chord: 1 & 2, then 3') ])(sample_task.s(self.user.id, '3')) assert UserTaskStatus.objects.count() == 5 chord_status = UserTaskStatus.objects.get(task_class='celery.chord') assert chord_status.task_id assert chord_status.parent is None assert chord_status.is_container assert chord_status.name == 'Chord: 1 & 2, then 3' assert chord_status.total_steps == 3 verify_state(chord_status, eager) group_status = UserTaskStatus.objects.get(task_class='celery.group') assert group_status.task_id assert group_status.parent_id == chord_status.id assert group_status.is_container assert group_status.name == 'Chord: 1 & 2, then 3' assert group_status.total_steps == 2 verify_state(group_status, eager) header_tasks = UserTaskStatus.objects.filter(parent=group_status) assert len(header_tasks) == 2 for status in header_tasks: assert status.task_id assert status.parent_id == group_status.id assert not status.is_container assert status.name in ['SampleTask: 1', 'SampleTask: 2'] assert status.total_steps == 1 verify_state(status, eager) body_status = UserTaskStatus.objects.get(parent=chord_status, is_container=False) assert body_status.task_id assert body_status.name == 'SampleTask: 3' assert body_status.total_steps == 1 verify_state(body_status, eager) def _create_group(self, eager): """Create a celery group and verify some assertions about the corresponding status records""" result = group( sample_task.s(self.user.id, '1'), sample_task.s(self.user.id, '2', user_task_name='Group: 1, 2')).delay() assert UserTaskStatus.objects.count() == 3 group_status = UserTaskStatus.objects.get(task_class='celery.group') assert group_status.task_id == result.id assert group_status.parent is None assert group_status.is_container assert group_status.name == 'Group: 1, 2' assert group_status.total_steps == 2 verify_state(group_status, eager) assert len(result.children) == 2 for result in result.children: task_id = result.id status = UserTaskStatus.objects.get(task_id=task_id) assert status.parent_id == group_status.id assert not status.is_container assert status.name in ['SampleTask: 1', 'SampleTask: 2'] assert status.total_steps == 1 verify_state(status, eager)
import mock import pytest from celery import __version__ as celery_version from celery import chain, chord, group, shared_task from packaging import version from django.contrib.auth.models import User from django.db import transaction from django.test import TestCase, TransactionTestCase, override_settings from user_tasks import user_task_stopped from user_tasks.models import UserTaskStatus from user_tasks.signals import start_user_task from user_tasks.tasks import UserTask CELERY_VERSION = version.parse(celery_version) LOGGER = logging.getLogger(__name__) USER_ID = 1 class SampleTask(UserTask): # pylint: disable=abstract-method """ Small UserTask subclass for use in test cases. """ @classmethod def generate_name(cls, arguments_dict): return 'SampleTask: {}'.format(arguments_dict['argument']) @staticmethod def calculate_total_steps(arguments_dict): return arguments_dict['kwargs'].get(
class TestQualityMeasures(CCPiTestClass): def setUp(self): print ("SETUP", np.version.version) if has_skimage: id_coins = dataexample.CAMERA.get() id_coins_noisy = noise.gaussian(id_coins, var=0.05, seed=10) ig = id_coins.geometry.copy() dc1 = ig.allocate('random') dc2 = ig.allocate('random') self.dc1 = dc1 self.dc2 = dc2 self.id_coins = id_coins self.id_coins_noisy = id_coins_noisy @unittest.skipIf(version.parse(np.version.version) < version.parse("1.13"), "Skip test with numpy < 1.13") def test_mse1(self): if has_skimage: #%% Check Mean Squared error for random image and images res1 = mse(self.id_coins, self.id_coins_noisy) res2 = mean_squared_error(self.id_coins.as_array(), self.id_coins_noisy.as_array()) print('Check MSE for CAMERA image gaussian noise') np.testing.assert_almost_equal(res1, res2, decimal=5) else: self.skipTest("scikit0-image not present ... skipping") @unittest.skipIf(version.parse(np.version.version) < version.parse("1.13"), "Skip test with numpy < 1.13") def test_mse2(self): if has_skimage: #%% Check Mean Squared error for random image and images res1 = mse(self.dc1, self.dc2) res2 = mean_squared_error(self.dc1.as_array(), self.dc2.as_array()) print('Check MSE for random ImageData') np.testing.assert_almost_equal(res1, res2, decimal=5) else: self.skipTest("scikit0-image not present ... skipping") @unittest.skipIf(version.parse(np.version.version) < version.parse("1.13"), "Skip test with numpy < 1.13") def test_psnr1(self): if has_skimage: res1 = psnr(self.id_coins, self.id_coins_noisy, data_range = self.dc1.max()) res2 = peak_signal_noise_ratio(self.id_coins.as_array(), self.id_coins_noisy.as_array()) print('Check PSNR for CAMERA image gaussian noise') np.testing.assert_almost_equal(res1, res2, decimal=3) else: self.skipTest("scikit0-image not present ... skipping") @unittest.skipIf(version.parse(np.version.version) < version.parse("1.13"), "Skip test with numpy < 1.13") def test_psnr2(self): if has_skimage: res1 = psnr(self.dc1, self.dc2, data_range = self.dc1.max()) res2 = peak_signal_noise_ratio(self.dc1.as_array(), self.dc2.as_array()) print('Check PSNR for random ImageData') np.testing.assert_almost_equal(res1, res2, decimal=3) else: self.skipTest("scikit0-image not present ... skipping")
async def upgrade(settings: dict): """Perform upgradation steps.""" context_builder = DefaultContextBuilder(settings) context = await context_builder.build_context() try: version_upgrade_config_inst = VersionUpgradeConfig( settings.get("upgrade.config_path")) upgrade_configs = version_upgrade_config_inst.upgrade_configs root_profile, public_did = await wallet_config(context) version_storage_record = None upgrade_to_version = f"v{__version__}" versions_found_in_config = upgrade_configs.keys() sorted_versions_found_in_config = sorted( versions_found_in_config, key=lambda x: package_version.parse(x)) async with root_profile.session() as session: storage = session.inject(BaseStorage) try: version_storage_record = await storage.find_record( type_filter=RECORD_TYPE_ACAPY_VERSION, tag_query={}) upgrade_from_version = version_storage_record.value if "upgrade.from_version" in settings: print((f"version {upgrade_from_version} found in storage" ", --from-version will be ignored.")) except StorageNotFoundError: if "upgrade.from_version" in settings: upgrade_from_version = settings.get("upgrade.from_version") else: upgrade_from_version = sorted_versions_found_in_config[-1] print("No ACA-Py version found in wallet storage and " "no --from-version specified. Selecting " f"{upgrade_from_version} as --from-version from " "the config.") if upgrade_from_version == upgrade_to_version: raise UpgradeError( f"Version {upgrade_from_version} to upgrade from and " f"current version to upgrade to {upgrade_to_version} " "are same.") if upgrade_from_version not in sorted_versions_found_in_config: raise UpgradeError( f"No upgrade configuration found for {upgrade_from_version}") upgrade_from_version_index = sorted_versions_found_in_config.index( upgrade_from_version) for config_from_version in sorted_versions_found_in_config[ upgrade_from_version_index:]: print(f"Running upgrade process for {config_from_version}") upgrade_config = upgrade_configs.get(config_from_version) # Step 1 re-saving all BaseRecord and BaseExchangeRecord if "resave_records" in upgrade_config: resave_record_paths = upgrade_config.get("resave_records") for record_path in resave_record_paths: try: record_type = ClassLoader.load_class(record_path) except ClassNotFoundError as err: raise UpgradeError( f"Unknown Record type {record_path}") from err if not issubclass(record_type, BaseRecord): raise UpgradeError( f"Only BaseRecord can be resaved, found: {str(record_type)}" ) async with root_profile.session() as session: all_records = await record_type.query(session) for record in all_records: await record.save( session, reason= "re-saving record during ACA-Py upgrade process", ) if len(all_records) == 0: print(f"No records of {str(record_type)} found") else: print( f"All records of {str(record_type)} successfully re-saved" ) # Step 2 Update existing records, if required if ("update_existing_records" in upgrade_config and upgrade_config.get("update_existing_records") is True): update_existing_recs_callable = ( version_upgrade_config_inst.get_update_existing_func( config_from_version)) if not update_existing_recs_callable: raise UpgradeError("No update_existing_records function " f"specified for {config_from_version}") await update_existing_recs_callable(root_profile) # Update storage version async with root_profile.session() as session: storage = session.inject(BaseStorage) if not version_storage_record: await storage.add_record( StorageRecord( RECORD_TYPE_ACAPY_VERSION, upgrade_to_version, )) else: await storage.update_record(version_storage_record, upgrade_to_version, {}) await root_profile.close() except BaseError as e: raise UpgradeError(f"Error during upgrade: {e}")
def __init__( self, device, *, shots=1000, wires=None, active_reset=True, load_qc=True, readout_error=None, symmetrize_readout="exhaustive", calibrate_readout="plus-eig", **kwargs, ): pl_version = pkg_resources.get_distribution("pennylane").version if version.parse(pl_version) >= version.parse("0.14.0.dev"): raise ValueError( "Using the QPU via PennyLane-Forest is being deprecated \ with PennyLane version 0.14.0 and higher.") if readout_error is not None and load_qc: raise ValueError("Readout error cannot be set on the physical QPU") self.readout_error = readout_error self._eigs = {} self._compiled_program = None """Union[None, pyquil.ExecutableDesignator]: the latest compiled program. If parametric compilation is turned on, this will be a parametric program.""" if kwargs.get("parametric_compilation", False): # Raise a warning if parametric compilation was explicitly turned on by the user # about turning the operator estimation off # TODO: Remove the warning and toggling once a migration to the new operator estimation # API has been executed. This new API provides compatibility between parametric # compilation and operator estimation. warnings.warn( "Parametric compilation is currently not supported with operator" "estimation. Operator estimation is being turned off.") self.parametric_compilation = kwargs.get("parametric_compilation", True) if self.parametric_compilation: self._compiled_program_dict = {} """dict[int, pyquil.ExecutableDesignator]: stores circuit hashes associated with the corresponding compiled programs.""" self._parameter_map = {} """dict[str, float]: stores the string of symbolic parameters associated with their numeric values. This map will be used to bind parameters in a parametric program using PyQuil.""" self._parameter_reference_map = {} """dict[str, pyquil.quilatom.MemoryReference]: stores the string of symbolic parameters associated with their PyQuil memory references.""" timeout = kwargs.pop("timeout", None) if shots <= 0: raise ValueError("Number of shots must be a positive integer.") self.connection = super()._get_connection(**kwargs) if load_qc: self.qc = get_qc(device, as_qvm=False, connection=self.connection) if timeout is not None: self.qc.compiler.quilc_client.timeout = timeout else: self.qc = get_qc(device, as_qvm=True, connection=self.connection) if timeout is not None: self.qc.compiler.client.timeout = timeout self.num_wires = len(self.qc.qubits()) if wires is None: # infer the number of modes from the device specs # and use consecutive integer wire labels wires = range(self.num_wires) if isinstance(wires, int): raise ValueError( "Device has a fixed number of {} qubits. The wires argument can only be used " "to specify an iterable of wire labels.".format( self.num_wires)) if self.num_wires != len(wires): raise ValueError("Device has a fixed number of {} qubits and " "cannot be created with {} wires.".format( self.num_wires, len(wires))) super(QVMDevice, self).__init__(wires, shots, **kwargs) self.active_reset = active_reset self.symmetrize_readout = symmetrize_readout self.calibrate_readout = calibrate_readout self.wiring = {i: q for i, q in enumerate(self.qc.qubits())}
def _is_outdated(self): return version.parse(self.current) < version.parse(self.latest)
def _parse_strict_version(ctx, tag): ver = parse(tag) if isinstance(ver, Version): ctx.state[Version] = ver return str(ver)
"""Unit tests for io module""" import platform import tempfile from pathlib import Path from packaging import version import pandas as pd import pytest import yaml import pandera as pa from pandera import io PYYAML_VERSION = version.parse(yaml.__version__) # type: ignore def _create_schema(index="single"): if index == "multi": index = pa.MultiIndex([ pa.Index(pa.Int, name="int_index0"), pa.Index(pa.Int, name="int_index1"), pa.Index(pa.Int, name="int_index2"), ]) elif index == "single": # make sure io modules can handle case when index name is None index = pa.Index(pa.Int, name=None) else: index = None return pa.DataFrameSchema(columns={
def torch_version() -> Tuple[int, ...]: result = version.parse(torch.__version__).release assert result return result
# -*- Blueprint setup -*- from packaging.version import parse from sanic import Blueprint from sanic.response import html, json from sanic.exceptions import NotFound from utils.SiteHelper import JailbreakMap as jMap from utils.SiteHelper import DeviceMapPG as dMap from utils.SiteHelper import MinVersionMap as minMap from utils.SiteHelper import MaxVersionMap as maxMap HomeBP = Blueprint("HomeBP") _tools14 = [ x for x in jMap if parse(x.get('minIOS')) <= parse("14.0") <= parse(x.get('maxIOS')) ] _tools13 = [ x for x in jMap if parse(x.get('minIOS')) <= parse("13.0") <= parse(x.get('maxIOS')) ] _tools12 = [ x for x in jMap if parse(x.get('minIOS')) <= parse("12.0") <= parse(x.get('maxIOS')) ] _tools11 = [ x for x in jMap if parse(x.get('minIOS')) <= parse("11.0") <= parse(x.get('maxIOS')) ] _tools10 = [ x for x in jMap if parse(x.get('minIOS')) <= parse("10.0") <= parse(x.get('maxIOS'))
KCONFIG_OUTPUT = os.path.abspath(os.environ["KCONFIG_OUTPUT"]) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. # # needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. # recommonmark didn't have a setup function before 0.5.0, so source_parsers # must be added manually recommonmark_version = pkg_resources.get_distribution("recommonmark").version if version.parse(recommonmark_version) < version.parse('0.5.0'): extensions = ['sphinx.ext.intersphinx', 'breathe', 'sphinx.ext.ifconfig'] else: extensions = [ 'sphinx.ext.intersphinx', 'breathe', 'sphinx.ext.ifconfig', 'recommonmark' ] # Add any paths that contain templates here, relative to this directory. #templates_path = ['../_templates'] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: source_suffix = ['.rst', '.md'] if version.parse(recommonmark_version) < version.parse('0.5.0'):
# third party from jinja2 import Template from packaging import version # this forces the import priority to use site-packages first and current dir last # this allows us to import torch when calling this file directly since there is a # subdir here also called torch del sys.path[0] sys.path.append("") # third party import torch as th # noqa: E402 # syft absolute from syft.lib.torch import allowlist # noqa: E402 TORCH_VERSION = version.parse(th.__version__.split("+")[0]) py_ver = sys.version_info PYTHON_VERSION = version.parse(f"{py_ver.major}.{py_ver.minor}") OS_NAME = platform.system().lower() # we need a file to keep all the errors in that makes it easy to debug failures TARGET_PLATFORM = f"{PYTHON_VERSION}_{OS_NAME}" REPORT_FILE_PATH = os.path.abspath( Path(__file__) / "../../../.." / f"allowlist_report_{TARGET_PLATFORM}.html") report_path = os.path.abspath((Path(__file__) / "../../../..")) support_files = glob.glob( os.path.join(report_path, "allowlist_test_support_*.jsonl")) if len(support_files) < 1:
def run_example(topdir, generator, ci_environment, buildflags, recipe, example): # extract global menu menu_file = topdir / 'testing' / 'menu.yml' skip_global, expect_failure_global, env_global, definitions_global, targets_global = extract_menu_file( menu_file, generator, ci_environment) sys.stdout.write('\n {}\n'.format(example)) # extract local menu menu_file = recipe / example / 'menu.yml' skip_local, expect_failure_local, env_local, definitions_local, targets_local = extract_menu_file( menu_file, generator, ci_environment) skip = skip_global or skip_local expect_failure = expect_failure_global or expect_failure_local # local env vars override global ones env = env_global.copy() for entry in env_local: env[entry] = env_local[entry] # local definitions override global ones definitions = definitions_global.copy() for entry in definitions_local: definitions[entry] = definitions_local[entry] # Decide configuration from CMAKE_BUILD_TYPE, by default it's Debug configuration = definitions[ 'CMAKE_BUILD_TYPE'] if 'CMAKE_BUILD_TYPE' in definitions else 'Debug' # local targets extend global targets targets = targets_global + targets_local for entry in env: os.environ[entry] = env[entry] definitions_string = ' '.join( r'-D{0}="{1}"'.format(entry, os.path.expandvars(definitions[entry])) for entry in definitions) # we append a time stamp to the build directory # to avoid it being re-used when running tests multiple times # when debugging on a laptop time_stamp = datetime.datetime.fromtimestamp( time.time()).strftime('%Y-%m-%d-%H-%M-%S') build_directory = recipe / example / 'build-{0}'.format(time_stamp) cmakelists_path = recipe / example min_cmake_version = get_min_cmake_version( cmakelists_path / 'CMakeLists.txt') system_cmake_version = get_system_cmake_version() if version.parse(system_cmake_version) < version.parse(min_cmake_version): sys.stdout.write( '\nSKIPPING (system cmake version < min. cmake version for this recipe)\n' ) return 0 if skip: sys.stdout.write('\nSKIPPING recipe (based on menu.yml)\n') return 0 return_code = 0 custom_script = 'custom.sh' custom_script_path = cmakelists_path / custom_script if custom_script_path.exists(): # if this directory contains a custom.sh script, we launch it step = custom_script command = 'bash "{0}" "{1}"'.format(custom_script_path, build_directory) return_code += run_command( step=step, command=command, expect_failure=expect_failure) else: # if there is no custom script, we run tests "normally" # configure step step = 'configuring' command = cmake_configuration_command(cmakelists_path, build_directory, generator, definitions_string) return_code += run_command( step=step, command=command, expect_failure=expect_failure) base_command = r'cmake --build "{0}"'.format(build_directory) # build step step = '{0} configuration {1}'.format('building', configuration) command = base_command + ' --config {0} -- {1}'.format( configuration, buildflags) return_code += run_command( step=step, command=command, expect_failure=expect_failure) # extra targets for target in targets: step = '{0} configuration {1}'.format(target, configuration) # on VS '--target test' fails but '--target RUN_TESTS' seems to work if generator.startswith('Visual Studio'): if target == 'test': target = 'RUN_TESTS' command = base_command + ' --config {0} --target {1}'.format( configuration, target) return_code += run_command( step=step, command=command, expect_failure=expect_failure) # execute dashboard script, if it exists dashboard_script = 'dashboard.cmake' dashboard_script_path = cmakelists_path / dashboard_script if dashboard_script_path.exists(): # if this directory contains a dashboard.cmake script, we launch it step = dashboard_script command = 'ctest -C {0} -S "{1}" -DCTEST_CMAKE_GENERATOR="{2}" {3}'.format( configuration, dashboard_script_path, generator, definitions_string) return_code += run_command( step=step, command=command, expect_failure=expect_failure) for entry in env: os.environ.pop(entry) return return_code
def get_release(releases_data): releases = list( filter(lambda x: x["prerelease"] is False and x["draft"] is False, releases_data)) return None if not releases else sorted( releases, key=lambda x: version.parse(x["tag_name"])).pop()
def main(): from transformers import __version__ as transformers_version if version.parse(transformers_version) < version.parse( "3.1.0"): # past_key_values name does not exist in 3.0.2 or older raise RuntimeError("This tool requires transformers 3.1.0 or later.") args = parse_arguments() setup_logger(args.verbose) if args.tolerance == 0: args.tolerance = DEFAULT_TOLERANCE[args.precision] logger.info(f"Arguments:{args}") cache_dir = args.cache_dir output_dir = args.output if not args.output.endswith( ".onnx") else os.path.dirname(args.output) prepare_environment(cache_dir, output_dir, args.use_gpu) if args.precision != Precision.FLOAT32: assert args.optimize_onnx, "fp16/int8 requires --optimize_onnx" if args.precision == Precision.FLOAT16: assert args.use_gpu, "fp16 requires --use_gpu" if args.precision == Precision.INT8: assert not args.use_gpu, "quantization only supports CPU" if args.use_external_data_format: assert not args.output.endswith( '.onnx' ), "output shall be a directory for --use_external_data_format" model_class = MODEL_CLASSES[args.model_class][0] model_type = "beam_search_step" if args.model_class == "GPT2LMHeadModel_BeamSearchStep" else "default" gpt2helper = Gpt2HelperFactory.create_helper(model_type) gpt2tester = Gpt2TesterFactory.create_tester(model_type) config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=cache_dir) if model_type == 'beam_search_step': model = model_class.from_pretrained(args.model_name_or_path, config=config, batch_size=args.batch_size, beam_size=args.beam_size, cache_dir=cache_dir) else: model = model_class.from_pretrained(args.model_name_or_path, config=config, cache_dir=cache_dir) device = torch.device("cuda:0" if args.use_gpu else "cpu") model.eval().to(device) if (not args.use_external_data_format) and (config.n_layer > 24): logger.info(f"Try --use_external_data_format when model size > 2GB") onnx_model_paths = gpt2helper.get_onnx_paths( output_dir, args.model_name_or_path, args.model_class, new_folder=args.use_external_data_format) raw_onnx_model = onnx_model_paths["raw"] logger.info(f"Exporting ONNX model to {raw_onnx_model}") use_padding = MODEL_CLASSES[args.model_class][2] gpt2helper.export_onnx(model, device, raw_onnx_model, args.verbose, args.use_external_data_format, has_position_ids=use_padding, has_attention_mask=use_padding) if args.optimize_onnx or args.precision != Precision.FLOAT32: output_path = onnx_model_paths[str(args.precision) if args. precision != Precision.INT8 else 'fp32'] logger.info(f"Optimizing model to {output_path}") gpt2helper.optimize_onnx(raw_onnx_model, output_path, args.precision == Precision.FLOAT16, model.config.num_attention_heads, model.config.hidden_size, args.use_external_data_format) else: output_path = raw_onnx_model if args.precision == Precision.INT8: logger.info("quantizing model...") QuantizeHelper.quantize_onnx_model(output_path, onnx_model_paths['int8'], args.use_external_data_format) model = QuantizeHelper.quantize_torch_model(model) logger.info("finished quantizing model") output_path = onnx_model_paths['int8'] if args.output.endswith( '.onnx' ) and output_path != args.output and not args.use_external_data_format: import shutil shutil.move(output_path, args.output) output_path = args.output logger.info(f"Output path: {output_path}") session = create_onnxruntime_session(output_path, args.use_gpu, enable_all_optimization=True, verbose=args.verbose) if session is not None: gpt2helper.test_parity(session, model, device, args.precision == Precision.FLOAT16, rtol=args.tolerance, atol=args.tolerance, model_class=args.model_class, has_position_ids=use_padding, has_attention_mask=use_padding) if args.input_test_file: test_inputs = [] # Each line of test file is a JSON string like: # {"input_ids": [[14698, 257, 1310, 13688, 319, 326]]} with open(args.input_test_file) as read_f: for _, line in enumerate(read_f): line = line.rstrip() data = json.loads(line) input_ids = torch.from_numpy( numpy.asarray(data["input_ids"], dtype=numpy.int64)).to(device) if use_padding: if "attention_mask" in data: numpy_float = numpy.float16 if args.precision == Precision.FLOAT16 else numpy.float32 attention_mask = torch.from_numpy( numpy.asarray(data["attention_mask"], dtype=numpy_float)).to(device) else: padding = -1 attention_mask = ( input_ids != padding).type(torch.float16 if args.precision == Precision.FLOAT16 else torch.float32) input_ids.masked_fill_(input_ids == padding, 0) if "position_ids" in data: position_ids = torch.from_numpy( numpy.asarray(data["position_ids"], dtype=numpy.int64)).to(device) else: position_ids = (attention_mask.long().cumsum(-1) - 1) position_ids.masked_fill_(position_ids < 0, 0) inputs = { "input_ids": input_ids, "position_ids": position_ids, "attention_mask": attention_mask } else: inputs = {"input_ids": input_ids} if model_type == "beam_search_step": beam_select_idx = torch.zeros([1, input_ids.shape[0]]).long() input_log_probs = torch.zeros([input_ids.shape[0], 1]) input_unfinished_sents = torch.ones( [input_ids.shape[0], 1], dtype=torch.bool) inputs.update({ "beam_select_idx": beam_select_idx, "input_log_probs": input_log_probs, "input_unfinished_sents": input_unfinished_sents, }) test_inputs.append(inputs) gpt2tester.test_generation(session, model, device, test_inputs, precision=args.precision, model_class=args.model_class, top_k=20, top_k_no_order=True, max_steps=24, max_inputs=0, verbose=args.verbose, save_test_data=3, save_test_data_dir=Path(output_path).parent) logger.info(f"Done. Output model: {output_path}")
# See the License for the specific language governing permissions and # limitations under the License. import matplotlib import numpy as np import pandas as pd import pymc3 as pm import theano.tensor as tt import pytest import theano from pymc3.theanof import floatX from packaging import version from .helpers import SeededTest if version.parse(matplotlib.__version__) < version.parse('3.3'): matplotlib.use('Agg', warn=False) else: matplotlib.use('Agg') def get_city_data(): """Helper to get city data""" data = pd.read_csv(pm.get_data('srrs2.dat')) cty_data = pd.read_csv(pm.get_data('cty.dat')) data = data[data.state == 'MN'] data['fips'] = data.stfips * 1000 + data.cntyfips cty_data['fips'] = cty_data.stfips * 1000 + cty_data.ctfips data['lradon'] = np.log(np.where(data.activity == 0, .1, data.activity))
def train(self, model_path: Optional[str] = None): """ Main training entry point. Args: model_path: (Optional) Local path to model if model to train has been instantiated from a local path If present, we will try reloading the optimizer/scheduler states from there. """ train_dataloader = self.get_train_dataloader() if self.args.max_steps > 0: t_total = self.args.max_steps num_train_epochs = (self.args.max_steps // (len(train_dataloader) // self.args.gradient_accumulation_steps) + 1) else: t_total = int( len(train_dataloader) // self.args.gradient_accumulation_steps * self.args.num_train_epochs) num_train_epochs = self.args.num_train_epochs optimizer, scheduler = self.get_optimizers(num_training_steps=t_total) # Check if saved optimizer or scheduler states exist if (model_path is not None and os.path.isfile(os.path.join(model_path, "optimizer.pt")) and os.path.isfile(os.path.join(model_path, "scheduler.pt"))): # Load in optimizer and scheduler states optimizer.load_state_dict( torch.load(os.path.join(model_path, "optimizer.pt"), map_location=self.args.device)) scheduler.load_state_dict( torch.load(os.path.join(model_path, "scheduler.pt"))) model = self.model if self.args.fp16: if not is_apex_available(): raise ImportError( "Please install apex from https://www.github.com/nvidia/apex to use fp16 training." ) model, optimizer = amp.initialize( model, optimizer, opt_level=self.args.fp16_opt_level) # multi-gpu training (should be after apex fp16 initialization) if self.args.n_gpu > 1: model = torch.nn.DataParallel(model) # Distributed training (should be after apex fp16 initialization) if self.args.local_rank != -1: model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[self.args.local_rank], output_device=self.args.local_rank, find_unused_parameters=True, ) if self.tb_writer is not None: self.tb_writer.add_text("args", self.args.to_json_string()) self.tb_writer.add_hparams(self.args.to_sanitized_dict(), metric_dict={}) # Train! if is_torch_tpu_available(): total_train_batch_size = self.args.train_batch_size * xm.xrt_world_size( ) else: total_train_batch_size = (self.args.train_batch_size * self.args.gradient_accumulation_steps * (torch.distributed.get_world_size() if self.args.local_rank != -1 else 1)) logger.info("***** Running training *****") logger.info(" Num examples = %d", self.num_examples(train_dataloader)) logger.info(" Num Epochs = %d", num_train_epochs) logger.info(" Instantaneous batch size per device = %d", self.args.per_device_train_batch_size) logger.info( " Total train batch size (w. parallel, distributed & accumulation) = %d", total_train_batch_size) logger.info(" Gradient Accumulation steps = %d", self.args.gradient_accumulation_steps) logger.info(" Total optimization steps = %d", t_total) self.global_step = 0 self.epoch = 0 epochs_trained = 0 steps_trained_in_current_epoch = 0 # Check if continuing training from a checkpoint if model_path is not None: # set global_step to global_step of last saved checkpoint from model path try: self.global_step = int(model_path.split("-")[-1].split("/")[0]) epochs_trained = self.global_step // ( len(train_dataloader) // self.args.gradient_accumulation_steps) steps_trained_in_current_epoch = self.global_step % ( len(train_dataloader) // self.args.gradient_accumulation_steps) logger.info( " Continuing training from checkpoint, will skip to saved global_step" ) logger.info(" Continuing training from epoch %d", epochs_trained) logger.info(" Continuing training from global step %d", self.global_step) logger.info( " Will skip the first %d steps in the first epoch", steps_trained_in_current_epoch) except ValueError: self.global_step = 0 logger.info(" Starting fine-tuning.") tr_loss = 0.0 logging_loss = 0.0 model.zero_grad() train_iterator = trange(epochs_trained, int(num_train_epochs), desc="Epoch", disable=not self.is_local_master()) for epoch in train_iterator: if isinstance(train_dataloader, DataLoader) and isinstance( train_dataloader.sampler, DistributedSampler): train_dataloader.sampler.set_epoch(epoch) if is_torch_tpu_available(): parallel_loader = pl.ParallelLoader( train_dataloader, [self.args.device]).per_device_loader(self.args.device) epoch_iterator = tqdm(parallel_loader, desc="Iteration", disable=not self.is_local_master()) else: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=not self.is_local_master()) for step, inputs in enumerate(epoch_iterator): # Skip past any already trained steps if resuming training if steps_trained_in_current_epoch > 0: steps_trained_in_current_epoch -= 1 continue tr_loss += self._training_step(model, inputs, optimizer) if (step + 1) % self.args.gradient_accumulation_steps == 0 or ( # last step in epoch but step is always smaller than gradient_accumulation_steps len(epoch_iterator) <= self.args.gradient_accumulation_steps and (step + 1) == len(epoch_iterator)): if self.args.fp16: torch.nn.utils.clip_grad_norm_( amp.master_params(optimizer), self.args.max_grad_norm) else: torch.nn.utils.clip_grad_norm_(model.parameters(), self.args.max_grad_norm) if is_torch_tpu_available(): xm.optimizer_step(optimizer) else: optimizer.step() scheduler.step() model.zero_grad() self.global_step += 1 self.epoch = epoch + (step + 1) / len(epoch_iterator) if (self.args.logging_steps > 0 and self.global_step % self.args.logging_steps == 0) or (self.global_step == 1 and self.args.logging_first_step): logs: Dict[str, float] = {} logs["loss"] = (tr_loss - logging_loss) / self.args.logging_steps # backward compatibility for pytorch schedulers logs["learning_rate"] = ( scheduler.get_last_lr()[0] if version.parse(torch.__version__) >= version.parse("1.4") else scheduler.get_lr()[0]) logging_loss = tr_loss self._log(logs) if self.args.evaluate_during_training: self.evaluate() if self.args.save_steps > 0 and self.global_step % self.args.save_steps == 0: # In all cases (even distributed/parallel), self.model is always a reference # to the model we want to save. if hasattr(model, "module"): assert model.module is self.model else: assert model is self.model # Save model checkpoint output_dir = os.path.join( self.args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{self.global_step}") self.save_model(output_dir) if self.is_world_master(): self._rotate_checkpoints() if is_torch_tpu_available(): xm.rendezvous("saving_optimizer_states") xm.save(optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt")) xm.save(scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt")) elif self.is_world_master(): torch.save( optimizer.state_dict(), os.path.join(output_dir, "optimizer.pt")) torch.save( scheduler.state_dict(), os.path.join(output_dir, "scheduler.pt")) if self.args.max_steps > 0 and self.global_step > self.args.max_steps: epoch_iterator.close() break if self.args.max_steps > 0 and self.global_step > self.args.max_steps: train_iterator.close() break if self.args.tpu_metrics_debug: # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.) xm.master_print(met.metrics_report()) if self.tb_writer: self.tb_writer.close() logger.info( "\n\nTraining completed. Do not forget to share your model on huggingface.co/models =)\n\n" ) return TrainOutput(self.global_step, tr_loss / self.global_step)
def main(argv=None, experiment_name="", run_id=0, csv_filename="gpt2_parity_results.csv"): result = {} from transformers import __version__ as transformers_version if version.parse(transformers_version) < version.parse( "3.1.0"): # past_key_values name does not exist in 3.0.2 or older raise RuntimeError("This tool requires transformers 3.1.0 or later.") args = parse_arguments(argv) setup_logger(args.verbose) if not experiment_name: import sys experiment_name = " ".join(argv if argv else sys.argv[1:]) if args.tolerance == 0: args.tolerance = DEFAULT_TOLERANCE[args.precision] logger.info(f"Arguments:{args}") cache_dir = args.cache_dir output_dir = args.output if not args.output.endswith( ".onnx") else os.path.dirname(args.output) prepare_environment(cache_dir, output_dir, args.use_gpu) if args.precision != Precision.FLOAT32: assert args.optimize_onnx, "fp16/int8 requires --optimize_onnx" if args.precision == Precision.FLOAT16: assert args.use_gpu, "fp16 requires --use_gpu" if args.precision == Precision.INT8: assert not args.use_gpu, "quantization only supports CPU" if args.use_external_data_format: assert not args.output.endswith( '.onnx' ), "output shall be a directory for --use_external_data_format" model_class = MODEL_CLASSES[args.model_class][0] use_padding = MODEL_CLASSES[args.model_class][2] if args.model_class == "GPT2LMHeadModel_BeamSearchStep": model_type = "beam_search_step" elif args.model_class == "GPT2LMHeadModel_ConfigurableOneStepSearch": model_type = "configurable_one_step_search" else: model_type = "default" gpt2helper = Gpt2HelperFactory.create_helper(model_type) gpt2tester = Gpt2TesterFactory.create_tester(model_type) config = AutoConfig.from_pretrained(args.model_name_or_path, cache_dir=cache_dir) if model_type == 'beam_search_step': model = model_class.from_pretrained(args.model_name_or_path, config=config, batch_size=1, beam_size=args.beam_size, cache_dir=cache_dir) elif model_type == 'configurable_one_step_search': model = model_class.from_pretrained( args.model_name_or_path, config=config, batch_size=1, beam_size=args.beam_size, ignore_eos=args.ignore_eos, temperature=args.temperature, repetition_penalty=args.repetition_penalty, excluded_token_ids=args.excluded_token_ids, length_penalty=args.length_penalty, do_sample=args.do_sample, do_sample_top_p=args.do_sample_top_p, do_sample_top_k=args.do_sample_top_k, cache_dir=cache_dir) else: model = model_class.from_pretrained(args.model_name_or_path, config=config, cache_dir=cache_dir) device = torch.device("cuda:0" if args.use_gpu else "cpu") model.eval().to(device) if (not args.use_external_data_format) and (config.n_layer > 24): logger.info(f"Try --use_external_data_format when model size > 2GB") onnx_model_paths = gpt2helper.get_onnx_paths( output_dir, args.model_name_or_path, args.model_class, new_folder=args.use_external_data_format, remove_existing=[ "fp32", "fp16", "int8" ]) # Do not remove raw model to save time in parity test raw_onnx_model = onnx_model_paths["raw"] if os.path.exists(raw_onnx_model): logger.warning( f"Skip exporting ONNX model since it existed: {raw_onnx_model}") else: logger.info(f"Exporting ONNX model to {raw_onnx_model}") gpt2helper.export_onnx(model, device, raw_onnx_model, args.verbose, args.use_external_data_format, has_position_ids=use_padding, has_attention_mask=use_padding) fp16_params = {"keep_io_types": args.keep_io_types} if args.io_block_list: fp16_params["keep_io_types"] = args.io_block_list if args.node_block_list: fp16_params["node_block_list"] = args.node_block_list if args.op_block_list: fp16_params["op_block_list"] = args.op_block_list if args.force_fp16_initializers: fp16_params["force_fp16_initializers"] = args.force_fp16_initializers is_io_float16 = (args.precision == Precision.FLOAT16 and not args.keep_io_types) if args.optimize_onnx or args.precision != Precision.FLOAT32: output_path = onnx_model_paths[str(args.precision) if args. precision != Precision.INT8 else 'fp32'] logger.info(f"Optimizing model to {output_path}") gpt2helper.optimize_onnx(raw_onnx_model, output_path, args.precision == Precision.FLOAT16, model.config.num_attention_heads, model.config.hidden_size, args.use_external_data_format, **fp16_params) else: output_path = raw_onnx_model if args.precision == Precision.INT8: logger.info("quantizing model...") QuantizeHelper.quantize_onnx_model(output_path, onnx_model_paths['int8'], args.use_external_data_format) model = QuantizeHelper.quantize_torch_model(model) logger.info("finished quantizing model") output_path = onnx_model_paths['int8'] if args.output.endswith( '.onnx' ) and output_path != args.output and not args.use_external_data_format: import shutil shutil.move(output_path, args.output) output_path = args.output logger.info(f"Output path: {output_path}") model_size_in_MB = int( get_onnx_model_size(output_path, args.use_external_data_format) / 1024 / 1024) session = create_onnxruntime_session(output_path, args.use_gpu, enable_all_optimization=True, verbose=args.verbose) if args.model_class == "GPT2LMHeadModel" and session is not None: parity_result = gpt2helper.test_parity( session, model, device, is_io_float16, rtol=args.tolerance, atol=args.tolerance, model_class=args.model_class, has_position_ids=use_padding, has_attention_mask=use_padding, test_cases_per_run=args.test_cases, total_runs=args.test_runs, verbose=args.verbose) latency = gpt2helper.test_performance(session, model, device, is_io_float16, total_runs=100, use_io_binding=True, model_class=args.model_class, has_position_ids=use_padding, has_attention_mask=use_padding, batch_size=8, sequence_length=1, past_sequence_length=32) if args.precision == Precision.FLOAT16: logger.info(f"fp16 conversion parameters:{fp16_params}") # Write results to file import csv from onnxruntime import __version__ as ort_version latency_name = get_latency_name() csv_file_existed = os.path.exists(csv_filename) with open(csv_filename, mode="a", newline='') as csv_file: column_names = [ "experiment", "run_id", "model_name", "model_class", "gpu", "precision", "optimizer", "test_cases", "runs", "keep_io_types", "io_block_list", "op_block_list", "node_block_list", "force_fp16_initializers", "ORT_TRANSFORMER_OPTIONS", "ORT_CUDA_GEMM_OPTIONS", "onnxruntime", latency_name, "top1_match_rate", "onnx_size_in_MB", "diff_50_percentile", "diff_90_percentile", "diff_95_percentile", "diff_99_percentile", "diff_pass_rate", "nan_rate", "top1_match_rate_per_run" ] csv_writer = csv.DictWriter(csv_file, fieldnames=column_names) if not csv_file_existed: csv_writer.writeheader() row = { "experiment": experiment_name, "run_id": run_id, "model_name": args.model_name_or_path, "model_class": args.model_class, "gpu": args.use_gpu, "precision": args.precision, "optimizer": args.optimize_onnx, "test_cases": args.test_cases, "runs": args.test_runs, "keep_io_types": args.keep_io_types, "io_block_list": args.io_block_list, "op_block_list": args.op_block_list, "node_block_list": args.node_block_list, "force_fp16_initializers": args.force_fp16_initializers, "ORT_TRANSFORMER_OPTIONS": os.getenv('ORT_TRANSFORMER_OPTIONS'), "ORT_CUDA_GEMM_OPTIONS": os.getenv('ORT_CUDA_GEMM_OPTIONS'), "onnxruntime": ort_version, latency_name: f"{latency:.2f}", "diff_50_percentile": parity_result["max_diff_percentile_50"], "diff_90_percentile": parity_result["max_diff_percentile_90"], "diff_95_percentile": parity_result["max_diff_percentile_95"], "diff_99_percentile": parity_result["max_diff_percentile_99"], "diff_pass_rate": parity_result["diff_pass_rate"], "nan_rate": parity_result["nan_rate"], "top1_match_rate": parity_result["top1_match_rate"], "top1_match_rate_per_run": parity_result["top1_match_rate_per_run"], "onnx_size_in_MB": "{}".format(model_size_in_MB), } logger.info(f"result: {row}") result.update(row) csv_writer.writerow(row) if args.input_test_file: test_inputs = [] # Each line of test file is a JSON string like: # {"input_ids": [[14698, 257, 1310, 13688, 319, 326]]} with open(args.input_test_file) as read_f: for _, line in enumerate(read_f): line = line.rstrip() data = json.loads(line) input_ids = torch.from_numpy( numpy.asarray(data["input_ids"], dtype=numpy.int64)).to(device) if use_padding: if "attention_mask" in data: numpy_float = numpy.float16 if is_io_float16 else numpy.float32 attention_mask = torch.from_numpy( numpy.asarray(data["attention_mask"], dtype=numpy_float)).to(device) else: padding = -1 attention_mask = (input_ids != padding).type( torch.float16 if is_io_float16 else torch.float32) input_ids.masked_fill_(input_ids == padding, 0) if "position_ids" in data: position_ids = torch.from_numpy( numpy.asarray(data["position_ids"], dtype=numpy.int64)).to(device) else: position_ids = (attention_mask.long().cumsum(-1) - 1) position_ids.masked_fill_(position_ids < 0, 0) inputs = { "input_ids": input_ids, "position_ids": position_ids, "attention_mask": attention_mask } else: inputs = {"input_ids": input_ids} if model_type == "beam_search_step" or model_type == "configurable_one_step_search": beam_select_idx = torch.zeros([1, input_ids.shape[0]]).long() input_log_probs = torch.zeros([input_ids.shape[0], 1]) input_unfinished_sents = torch.ones( [input_ids.shape[0], 1], dtype=torch.bool) inputs.update({ "beam_select_idx": beam_select_idx, "input_log_probs": input_log_probs, "input_unfinished_sents": input_unfinished_sents, }) test_inputs.append(inputs) gpt2tester.test_generation(session, model, device, test_inputs, precision=args.precision, model_class=args.model_class, top_k=20, top_k_no_order=True, max_steps=24, max_inputs=0, verbose=args.verbose, save_test_data=3, save_test_data_dir=Path(output_path).parent) logger.info(f"Done. Output model: {output_path}") return result
def torch_version_check(): from packaging import version return version.parse(torch.__version__) > version.parse("1.4.0")
def fit(cls, train_data, label, tuning_data=None, time_limits=None, output_directory='./ag_text', feature_columns=None, holdout_frac=None, eval_metric=None, stopping_metric=None, nthreads_per_trial=None, ngpus_per_trial=None, dist_ip_addrs=None, num_trials=None, search_strategy=None, search_options=None, scheduler_options=None, hyperparameters=None, plot_results=None, seed=None, verbosity=2): """Fit models to make predictions based on text inputs. Parameters ---------- train_data : :class:`autogluon.task.tabular_prediction.TabularDataset` or `pandas.DataFrame` Training dataset where rows = individual training examples, columns = features. label : str Name of the label column. It can be a stringBy default, we will search for a column named tuning_data : :class:`autogluon.task.tabular_prediction.TabularDataset` or `pandas.DataFrame`, default = None Another dataset containing validation data reserved for hyperparameter tuning (in same format as training data). If `tuning_data = None`, `fit()` will automatically hold out random examples from `train_data` for validation. time_limits : int or str, default = None Approximately how long `fit()` should run for (wallclock time in seconds if int). String values may instead be used to specify time in different units such as: '1min' or '1hour'. Longer `time_limits` will usually improve predictive accuracy. If not specified, `fit()` will run until all models to try by default have completed training. output_directory : str, default = './ag_text' Path to directory where models and intermediate outputs should be saved. feature_columns : List[str], default = None Which columns of table to consider as predictive features (other columns will be ignored, except for label-column). If None (by default), all columns of table are considered predictive features. holdout_frac : float, default = None Fraction of train_data to holdout as tuning data for optimizing hyperparameters (ignored unless `tuning_data = None`). If None, default value is selected based on the number of training examples. eval_metric : str, default = None The evaluation metric that will be used to evaluate the model's predictive performance. If None, an appropriate default metric will be selected (accuracy for classification, mean-squared-error for regression). Options for classification include: 'acc' (accuracy), 'nll' (negative log-likelihood). Additional options for binary classification include: 'f1' (F1 score), 'mcc' (Matthews coefficient), 'auc' (area under ROC curve). Options for regression include: 'mse' (mean squared error), 'rmse' (root mean squared error), 'mae' (mean absolute error). stopping_metric, default = None Metric which iteratively-trained models use to early stop to avoid overfitting. Defaults to `eval_metric` value (if None). Options are identical to options for `eval_metric`. nthreads_per_trial, default = None The number of threads per individual model training run. By default, all available CPUs are used. ngpus_per_trial, default = None The number of GPUs to use per individual model training run. If unspecified, a default value is chosen based on total number of GPUs available. dist_ip_addrs, default = None List of IP addresses corresponding to remote workers, in order to leverage distributed computation. num_trials : , default = None The number of trials in the HPO search search_strategy : str, default = None Which hyperparameter search algorithm to use. Options include: 'random' (random search), 'bayesopt' (Gaussian process Bayesian optimization), 'skopt' (SKopt Bayesian optimization), 'grid' (grid search), 'hyperband' (Hyperband scheduling with random search), 'bayesopt-hyperband' (Hyperband scheduling with GP-BO search). If unspecified, the default is 'random'. search_options : dict, default = None Options passed to searcher. scheduler_options : dict, default = None Additional kwargs passed to scheduler __init__. hyperparameters : dict, default = None Determines the hyperparameters used by the models. Each hyperparameter may be either fixed value or search space of many values. For example of default hyperparameters, see: `autogluon.task.text_prediction.text_prediction.default()` plot_results : bool, default = None Whether or not to plot intermediate training results during `fit()`. seed : int, default = None Seed value for random state used inside `fit()`. verbosity : int, default = 2 Verbosity levels range from 0 to 4 and control how much information is printed during fit(). Higher levels correspond to more detailed print statements (you can set verbosity = 0 to suppress warnings). If using logging, you can alternatively control amount of information printed via `logger.setLevel(L)`, where `L` ranges from 0 to 50 (Note: higher values of `L` correspond to fewer print statements, opposite of verbosity levels) Returns ------- model A `BertForTextPredictionBasic` object that can be used for making predictions on new data. """ assert dist_ip_addrs is None, 'Training on remote machine is currently not supported.' # Version check of MXNet if version.parse(mxnet.__version__) < version.parse('1.7.0') \ or version.parse(mxnet.__version__) >= version.parse('2.0.0'): raise ImportError( 'You will need to ensure that you have mxnet>=1.7.0, <2.0.0. ' 'For more information about how to install mxnet, you can refer to ' 'https://sxjscience.github.io/KDD2020/ .') if verbosity < 0: verbosity = 0 elif verbosity > 4: verbosity = 4 console_log = verbosity >= 2 logging_config(folder=output_directory, name='ag_text_prediction', logger=logger, level=verbosity2loglevel(verbosity), console=console_log) # Parse the hyper-parameters if hyperparameters is None: hyperparameters = ag_text_prediction_params.create('default') elif isinstance(hyperparameters, str): hyperparameters = ag_text_prediction_params.create(hyperparameters) else: base_params = ag_text_prediction_params.create('default') hyperparameters = merge_params(base_params, hyperparameters) if seed is not None: hyperparameters['seed'] = seed np.random.seed(hyperparameters['seed']) if not isinstance(train_data, pd.DataFrame): train_data = load_pd.load(train_data) # Inference the label if not isinstance(label, list): label = [label] label_columns = [] for ele in label: if isinstance(ele, int): label_columns.append(train_data.columns[ele]) else: label_columns.append(ele) if feature_columns is None: all_columns = list(train_data.columns) feature_columns = [ ele for ele in all_columns if ele not in label_columns ] else: if isinstance(feature_columns, str): feature_columns = [feature_columns] for col in feature_columns: assert col not in label_columns, 'Feature columns and label columns cannot overlap.' assert col in train_data.columns,\ 'Feature columns must be in the pandas dataframe! Received col = "{}", ' \ 'all columns = "{}"'.format(col, train_data.columns) all_columns = feature_columns + label_columns all_columns = [ ele for ele in train_data.columns if ele in all_columns ] if tuning_data is None: if holdout_frac is None: holdout_frac = default_holdout_frac(len(train_data), True) train_data, tuning_data = random_split_train_val( train_data, valid_ratio=holdout_frac) else: if not isinstance(tuning_data, pd.DataFrame): tuning_data = load_pd.load(tuning_data) train_data = train_data[all_columns] tuning_data = tuning_data[all_columns] column_properties = get_column_properties( pd.concat([train_data, tuning_data]), metadata=None, label_columns=label_columns, provided_column_properties=None, categorical_default_handle_missing_value=True) train_data = TabularDataset(train_data, column_properties=column_properties, label_columns=label_columns) tuning_data = TabularDataset( tuning_data, column_properties=train_data.column_properties, label_columns=label_columns) logger.info('Train Dataset:') logger.info(train_data) logger.info('Tuning Dataset:') logger.info(tuning_data) logger.debug('Hyperparameters:') logger.debug(hyperparameters) has_text_column = False for k, v in column_properties.items(): if v.type == _C.TEXT: has_text_column = True break if not has_text_column: raise NotImplementedError('No Text Column is found! This is currently not supported by ' 'the TextPrediction task. You may try to use ' 'TabularPrediction.fit().\n' \ 'The inferred column properties of the training data is {}' .format(train_data)) problem_types = [] label_shapes = [] for label_col_name in label_columns: problem_type, label_shape = infer_problem_type( column_properties=column_properties, label_col_name=label_col_name) problem_types.append(problem_type) label_shapes.append(label_shape) logging.info( 'Label columns={}, Feature columns={}, Problem types={}, Label shapes={}' .format(label_columns, feature_columns, problem_types, label_shapes)) eval_metric, stopping_metric, log_metrics =\ infer_eval_stop_log_metrics(problem_types[0], label_shapes[0], eval_metric=eval_metric, stopping_metric=stopping_metric) logging.info('Eval Metric={}, Stop Metric={}, Log Metrics={}'.format( eval_metric, stopping_metric, log_metrics)) model_candidates = [] for model_type, kwargs in hyperparameters['models'].items(): search_space = kwargs['search_space'] if model_type == 'BertForTextPredictionBasic': model = BertForTextPredictionBasic( column_properties=column_properties, label_columns=label_columns, feature_columns=feature_columns, label_shapes=label_shapes, problem_types=problem_types, stopping_metric=stopping_metric, log_metrics=log_metrics, base_config=None, search_space=search_space, output_directory=output_directory, logger=logger) model_candidates.append(model) else: raise ValueError( 'model_type = "{}" is not supported. You can try to use ' 'model_type = "BertForTextPredictionBasic"'.format( model_type)) assert len( model_candidates) == 1, 'Only one model is supported currently' recommended_resource = get_recommended_resource( nthreads_per_trial=nthreads_per_trial, ngpus_per_trial=ngpus_per_trial) if search_strategy is None: search_strategy = hyperparameters['hpo_params']['search_strategy'] if time_limits is None: time_limits = hyperparameters['hpo_params']['time_limits'] else: if isinstance(time_limits, str): if time_limits.endswith('min'): time_limits = int(float(time_limits[:-3]) * 60) elif time_limits.endswith('hour'): time_limits = int(float(time_limits[:-4]) * 60 * 60) else: raise ValueError( 'The given time_limits="{}" cannot be parsed!'.format( time_limits)) if num_trials is None: num_trials = hyperparameters['hpo_params']['num_trials'] if scheduler_options is None: scheduler_options = hyperparameters['hpo_params'][ 'scheduler_options'] if scheduler_options is None: scheduler_options = dict() if search_strategy.endswith('hyperband'): # Specific defaults for hyperband scheduling scheduler_options['reduction_factor'] = scheduler_options.get( 'reduction_factor', 4) scheduler_options['grace_period'] = scheduler_options.get( 'grace_period', 10) scheduler_options['max_t'] = scheduler_options.get('max_t', 50) if recommended_resource['num_gpus'] == 0: warnings.warn( 'Recommend to use GPU to run the TextPrediction task!') model = model_candidates[0] if plot_results is None: if in_ipynb(): plot_results = True else: plot_results = False model.train(train_data=train_data, tuning_data=tuning_data, resource=recommended_resource, time_limits=time_limits, search_strategy=search_strategy, search_options=search_options, scheduler_options=scheduler_options, num_trials=num_trials, plot_results=plot_results, console_log=verbosity >= 2, ignore_warning=verbosity <= 2) return model