def globals(Globals, **tagmap): '''Apply the tags in `Globals` back into the database.''' global apply cls, tagmap_output = apply.__class__, u", {:s}".format(u', '.join(u"{:s}={:s}".format(internal.utils.string.escape(oldtag), internal.utils.string.escape(newtag)) for oldtag, newtag in six.iteritems(tagmap))) if tagmap else '' count = 0 for ea, res in Globals: ns = func if func.within(ea) else db # grab the current (old) tag state state = ns.tag(ea) # transform the new tag state using the tagmap new = { tagmap.get(name, name) : value for name, value in six.viewitems(res) } # check if the tag mapping resulted in the deletion of a tag if len(new) != len(res): for name in six.viewkeys(res) - six.viewkeys(new): logging.warn(u"{:s}.globals(...{:s}) : Refusing requested tag mapping as it results in the tag \"{:s}\" overwriting the tag \"{:s}\" in the global {:#x}. The value {!s} would be replaced with {!s}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.escape(name, '"'), internal.utils.string.escape(tagmap[name], '"'), ea, internal.utils.string.repr(res[name]), internal.utils.string.repr(res[tagmap[name]]))) pass # check what's going to be overwritten with different values prior to doing it for name in six.viewkeys(state) & six.viewkeys(new): if state[name] == new[name]: continue logging.warn(u"{:s}.globals(...{:s}) : Overwriting tag \"{:s}\" for global at {:#x} with new value {!s}. Old value was {!s}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.escape(name, '"'), ea, internal.utils.string.repr(new[name]), internal.utils.string.repr(state[name]))) # now we can apply the tags to the global address try: [ ns.tag(ea, name, value) for name, value in six.iteritems(new) if state.get(name, dummy) != value ] except: logging.warn(u"{:s}.globals(...{:s}) : Unable to apply tags ({!s}) to global {:#x}.".format('.'.join((__name__, cls.__name__)), tagmap_output, internal.utils.string.repr(new), ea), exc_info=True) # increase our counter count += 1 return count
def once_a_day(midnight_dt, current_data=self.current_data, data_portal=self.data_portal): # process any capital changes that came overnight for capital_change in algo.calculate_capital_changes( midnight_dt, emission_rate=emission_rate, is_interday=True): yield capital_change # set all the timestamps self.simulation_dt = midnight_dt algo.on_dt_changed(midnight_dt) metrics_tracker.handle_market_open( midnight_dt, algo.data_portal, ) # handle any splits that impact any positions or any open orders. assets_we_care_about = ( viewkeys(metrics_tracker.positions) | viewkeys(algo.blotter.open_orders) ) if assets_we_care_about: splits = data_portal.get_splits(assets_we_care_about, midnight_dt) if splits: algo.blotter.process_splits(splits) metrics_tracker.handle_splits(splits)
def setup_params(self, data): params = self.params.copy() valid_scale = ('area', 'count', 'width') if params['scale'] not in valid_scale: msg = "Parameter scale should be one of {}" raise PlotnineError(msg.format(valid_scale)) lookup = { 'biweight': 'biw', 'cosine': 'cos', 'cosine2': 'cos2', 'epanechnikov': 'epa', 'gaussian': 'gau', 'triangular': 'tri', 'triweight': 'triw', 'uniform': 'uni'} with suppress(KeyError): params['kernel'] = lookup[params['kernel'].lower()] if params['kernel'] not in six.viewvalues(lookup): msg = ("kernel should be one of {}. " "You may use the abbreviations {}") raise PlotnineError(msg.format(six.viewkeys(lookup), six.viewvalues())) missing_params = (six.viewkeys(stat_density.DEFAULT_PARAMS) - six.viewkeys(params)) for key in missing_params: params[key] = stat_density.DEFAULT_PARAMS[key] return params
def sync_bay_status(self, ctx): try: LOG.debug('Starting to sync up bay status') osc = clients.OpenStackClients(ctx) status = [bay_status.CREATE_IN_PROGRESS, bay_status.UPDATE_IN_PROGRESS, bay_status.DELETE_IN_PROGRESS] filters = {'status': status} bays = objects.Bay.list(ctx, filters=filters) if not bays: return sid_to_bay_mapping = {bay.stack_id: bay for bay in bays} bay_stack_ids = sid_to_bay_mapping.keys() stacks = osc.heat().stacks.list(global_tenant=True, filters={'id': bay_stack_ids}) sid_to_stack_mapping = {s.id: s for s in stacks} # intersection of bays magnum has and heat has for sid in (six.viewkeys(sid_to_bay_mapping) & six.viewkeys(sid_to_stack_mapping)): stack = sid_to_stack_mapping[sid] bay = sid_to_bay_mapping[sid] self._sync_existing_bay(bay, stack) # the stacks that magnum has but heat doesn't have for sid in (six.viewkeys(sid_to_bay_mapping) - six.viewkeys(sid_to_stack_mapping)): bay = sid_to_bay_mapping[sid] self._sync_missing_heat_stack(bay) except Exception as e: LOG.warn(_LW("Ignore error [%s] when syncing up bay status."), e, exc_info=True)
def _dictionary_merge(dictionary_left, dictionary_right): """Merge two dictionaries preserving values for the same key. :param dictionary_left: A valid dictionary with keys and values. Example: dictionary_left = {1: 'A', 2: ['B', 'C'], 3: []} :param dictionary_right: A valid dictionary with keys and values. Example: dictionary_right = {1: 'A', 2: 'C', 4: 'E'} :return: A merged dictionary, which preserves both values in the situation of a key conflict. Example: {1: ['A', 'A'], 2: [['B', 'C'], 'C'], 3: [[]], 4: ['E']} """ merged_dictionary = {} for key in (viewkeys(dictionary_left) | viewkeys(dictionary_right)): if key in dictionary_left: merged_dictionary.setdefault(key, []).append(dictionary_left[key]) if key in dictionary_right: merged_dictionary.setdefault(key, []).append( dictionary_right[key]) return merged_dictionary
def once_a_day(midnight_dt, current_data=self.current_data, data_portal=self.data_portal): perf_tracker = algo.perf_tracker # Get the positions before updating the date so that prices are # fetched for trading close instead of midnight positions = algo.perf_tracker.position_tracker.positions position_assets = algo.asset_finder.retrieve_all(positions) # set all the timestamps self.simulation_dt = midnight_dt algo.on_dt_changed(midnight_dt) # process any capital changes that came overnight for capital_change in algo.calculate_capital_changes( midnight_dt, emission_rate=emission_rate, is_interday=True): yield capital_change # we want to wait until the clock rolls over to the next day # before cleaning up expired assets. self._cleanup_expired_assets(midnight_dt, position_assets) # handle any splits that impact any positions or any open orders. assets_we_care_about = \ viewkeys(perf_tracker.position_tracker.positions) | \ viewkeys(algo.blotter.open_orders) if assets_we_care_about: splits = data_portal.get_splits(assets_we_care_about, midnight_dt) if splits: algo.blotter.process_splits(splits) perf_tracker.position_tracker.handle_splits(splits)
def use_defaults(self, data): """ Combine data with defaults and set aesthetics from parameters stats should not override this method. Parameters ---------- data : dataframe Data used for drawing the geom. Returns ------- out : dataframe Data used for drawing the geom. """ missing = (self.aesthetics() - six.viewkeys(self.aes_params) - set(data.columns)) for ae in missing-self.REQUIRED_AES: if self.DEFAULT_AES[ae] is not None: data[ae] = self.DEFAULT_AES[ae] missing = (six.viewkeys(self.aes_params) - set(data.columns)) for ae in self.aes_params: data[ae] = self.aes_params[ae] return data
def legend_aesthetics(self, layer, plot): """ Return the aesthetics that contribute to the legend Parameters ---------- layer : Layer Layer whose legend is to be drawn plot : ggplot Plot object Returns ------- matched : list List of the names of the aethetics that contribute to the legend. """ l = layer legend_ae = set(self.key.columns) - {'label'} all_ae = (six.viewkeys(l.mapping) | (plot.mapping if l.inherit_aes else set()) | six.viewkeys(l.stat.DEFAULT_AES)) geom_ae = l.geom.REQUIRED_AES | six.viewkeys(l.geom.DEFAULT_AES) matched = all_ae & geom_ae & legend_ae matched = list(matched - set(l.geom.aes_params)) return matched
def once_a_day(midnight_dt): # Get the positions before updating the date so that prices are # fetched for trading close instead of midnight positions = algo.perf_tracker.position_tracker.positions position_assets = algo.asset_finder.retrieve_all(positions) # set all the timestamps self.simulation_dt = midnight_dt algo.on_dt_changed(midnight_dt) # we want to wait until the clock rolls over to the next day # before cleaning up expired assets. self._cleanup_expired_assets(midnight_dt, position_assets) perf_tracker = algo.perf_tracker # handle any splits that impact any positions or any open orders. assets_we_care_about = \ viewkeys(perf_tracker.position_tracker.positions) | \ viewkeys(algo.blotter.open_orders) if assets_we_care_about: splits = data_portal.get_splits(assets_we_care_about, midnight_dt) if splits: algo.blotter.process_splits(splits) perf_tracker.position_tracker.handle_splits(splits) # call before trading start algo.before_trading_start(current_data)
def validate(self, doc): return doc try: assert set(six.viewkeys(doc)) == { '_id', 'imageId', 'skill', 'creatorId', 'lesionBoundary', 'created'} assert isinstance(doc['imageId'], ObjectId) assert self.model('image', 'isic_archive').find( {'_id': doc['imageId']}).count() # TODO: better use of Enum assert doc['skill'] in {'novice', 'expert'} assert isinstance(doc['creatorId'], ObjectId) assert self.model('user').find( {'_id': doc['creatorId']}).count() assert isinstance(doc['lesionBoundary'], dict) assert set(six.viewkeys(doc['lesionBoundary'])) == { 'type', 'properties', 'geometry'} assert doc['lesionBoundary']['type'] == 'Feature' assert isinstance(doc['lesionBoundary']['properties'], dict) assert set(six.viewkeys(doc['lesionBoundary']['properties'])) <= { 'source', 'startTime', 'stopTime', 'seedPoint', 'tolerance'} assert set(six.viewkeys(doc['lesionBoundary']['properties'])) >= { 'source', 'startTime', 'stopTime'} assert doc['lesionBoundary']['properties']['source'] in { 'autofill', 'manual pointlist'} assert isinstance(doc['lesionBoundary']['properties']['startTime'], datetime.datetime) assert isinstance(doc['lesionBoundary']['properties']['stopTime'], datetime.datetime) assert isinstance(doc['lesionBoundary']['geometry'], dict) assert set(six.viewkeys(doc['lesionBoundary']['geometry'])) == { 'type', 'coordinates'} assert doc['lesionBoundary']['geometry']['type'] == 'Polygon' assert isinstance(doc['lesionBoundary']['geometry']['coordinates'], list) assert len(doc['lesionBoundary']['geometry']['coordinates']) == 1 assert isinstance( doc['lesionBoundary']['geometry']['coordinates'][0], list) assert len(doc['lesionBoundary']['geometry']['coordinates'][0]) > 2 assert doc['lesionBoundary']['geometry']['coordinates'][0][0] == \ doc['lesionBoundary']['geometry']['coordinates'][0][-1] for coord in doc['lesionBoundary']['geometry']['coordinates'][0]: assert isinstance(coord, list) assert len(coord) == 2 assert isinstance(coord[0], (int, float)) assert isinstance(coord[1], (int, float)) assert isinstance(doc['created'], datetime.datetime) except AssertionError: # TODO: message raise ValidationException('') return doc
def sync_bay_status(self, ctx): try: LOG.debug('Starting to sync up bay status') osc = clients.OpenStackClients(ctx) filters = [bay_status.CREATE_IN_PROGRESS, bay_status.UPDATE_IN_PROGRESS, bay_status.DELETE_IN_PROGRESS] bays = objects.Bay.list_all(ctx, filters=filters) if not bays: return sid_to_bay_mapping = {bay.stack_id: bay for bay in bays} bay_stack_ids = sid_to_bay_mapping.keys() stacks = osc.heat().stacks.list(global_tenant=True, filters={'id': bay_stack_ids}) sid_to_stack_mapping = {s.id: s for s in stacks} for sid in (six.viewkeys(sid_to_bay_mapping) & six.viewkeys(sid_to_stack_mapping)): stack = sid_to_stack_mapping[sid] bay = sid_to_bay_mapping[sid] if bay.status != stack.stack_status: old_status = bay.status bay.status = stack.stack_status bay.save() LOG.info(_LI("Sync up bay with id %(id)s from " "%(old_status)s to %(status)s."), {'id': bay.id, 'old_status': old_status, 'status': bay.status}) for sid in (six.viewkeys(sid_to_bay_mapping) - six.viewkeys(sid_to_stack_mapping)): bay = sid_to_bay_mapping[sid] if bay.status == bay_status.DELETE_IN_PROGRESS: bay.destroy() LOG.info(_LI("Bay with id %(id)s has been deleted due " "to stack with id %(sid)s not found in " "Heat."), {'id': bay.id, 'sid': sid}) elif bay.status == bay_status.CREATE_IN_PROGRESS: bay.status = bay_status.CREATE_FAILED bay.save() LOG.info(_LI("Bay with id %(id)s has been set to " "%(status)s due to stack with id %(sid)s " "not found in Heat."), {'id': bay.id, 'status': bay.status, 'sid': sid}) elif bay.status == bay_status.UPDATE_IN_PROGRESS: bay.status = bay_status.UPDATE_FAILED bay.save() LOG.info(_LI("Bay with id %(id)s has been set to " "%(status)s due to stack with id %(sid)s " "not found in Heat."), {'id': bay.id, 'status': bay.status, 'sid': sid}) except Exception as e: LOG.warn(_LW("Ignore error [%s] when syncing up bay status."), e, exc_info=True)
def _clean_running_config_from_removed_nets(self): # Cleanup running config from networks that have been actually # removed but not yet removed from running config. running_config = RunningConfig() nets2remove = (six.viewkeys(running_config.networks) - six.viewkeys(self.runningConfig.networks)) for net in nets2remove: running_config.removeNetwork(net) running_config.save()
def _adjust_az_filters(self, filters): # The intersect of sets gets us applicable filter keys (others ignored) common_keys = six.viewkeys(filters) & six.viewkeys(AZ_ATTRIBUTE_MAP) for key in common_keys: filter_key = AZ_ATTRIBUTE_MAP[key]['agent_key'] filter_vals = filters.pop(key) if filter_vals: filter_vals = [AZ_ATTRIBUTE_MAP[key]['convert_to'](v) for v in filter_vals] filters.setdefault(filter_key, []) filters[filter_key] += filter_vals return filters
def assert_dict_equal(result, expected, path=(), msg="", **kwargs): _check_sets(viewkeys(result), viewkeys(expected), msg, path + (".%s()" % ("viewkeys" if PY2 else "keys"),), "key") failures = [] for k, (resultv, expectedv) in iteritems(dzip_exact(result, expected)): try: assert_equal(resultv, expectedv, path=path + ("[%r]" % k,), msg=msg, **kwargs) except AssertionError as e: failures.append(str(e)) if failures: raise AssertionError("\n".join(failures))
def namespace(cls): # turn all instances of things into read-only attributes attrs,properties,subclass = {},{},{} for k,v in cls.__dict__.items(): if hasattr(v, '__name__'): v.__name__ = '{}.{}'.format(cls.__name__,k) if k.startswith('_') or type(v) is property: attrs[k] = v elif not six.callable(v) or isinstance(v,type): properties[k] = v elif not hasattr(v, '__class__'): subclass[k] = namespace(v) else: attrs[k] = v continue def getprops(obj): result = [] col1,col2 = 0,0 for k,v in obj.items(): col1 = max((col1,len(k))) if isinstance(v, type): val = '<>' elif hasattr(v, '__class__'): val = '{!r}'.format(v) else: raise ValueError(k) doc = v.__doc__.split('\n')[0] if v.__doc__ else None col2 = max((col2,len(val))) result.append((k, val, doc)) return [('{name:{}} : {val:{}} # {doc}' if d else '{name:{}} : {val:{}}').format(col1,col2,name=k,val=v,doc=d) for k,v,d in result] def __repr__(self): props = getprops(properties) descr = ('{{{!s}}} # {}\n' if cls.__doc__ else '{{{!s}}}\n') subs = ['{{{}.{}}}\n...'.format(cls.__name__,k) for k in subclass.keys()] res = descr.format(cls.__name__,cls.__doc__) + '\n'.join(props) if subs: return res + '\n' + '\n'.join(subs) + '\n' return res + '\n' def __setattr__(self, name, value): if name in six.viewkeys(attrs): object.__setattr__(self, name, value) return raise AttributeError('Configuration \'{:s}\' does not have field named \'{:s}\''.format(cls.__name__,name)) attrs['__repr__'] = __repr__ attrs['__setattr__'] = __setattr__ attrs.update((k,property(fget=lambda s,k=k:properties[k])) for k in six.viewkeys(properties)) attrs.update((k,property(fget=lambda s,k=k:subclass[k])) for k in six.viewkeys(subclass)) result = type(cls.__name__, cls.__bases__, attrs) return result()
def distance(left, right, weight={'author': 3, 'year': 3, 'title': 3, 'ENTRYTYPE': 2}): """Simple measure of the difference between two bibtex-field dicts.""" if not (left or right): return 0.0 keys = viewkeys(left) & viewkeys(right) if not keys: return 1.0 weights = {k: weight.get(k, 1) for k in keys} ratios = ( w * difflib.SequenceMatcher(None, left[k], right[k]).ratio() for k, w in weights.items()) return 1 - (sum(ratios) / sum(weights.values()))
def __init__(self, *args, **kwargs): kwargs = data_mapping_as_kwargs(args, kwargs) self._kwargs = kwargs # Will be used to create the geom self.params = copy_keys(kwargs, deepcopy(self.DEFAULT_PARAMS)) self.aes_params = {ae: kwargs[ae] for ae in (self.aesthetics() & six.viewkeys(kwargs))}
def _exp4p_score(self, context): """The main part of Exp4.P. """ advisor_ids = list(six.viewkeys(context)) w = self._modelstorage.get_model()['w'] if len(w) == 0: for i in advisor_ids: w[i] = 1 w_sum = sum(six.viewvalues(w)) action_probs_list = [] for action_id in self.action_ids: weighted_exp = [w[advisor_id] * context[advisor_id][action_id] for advisor_id in advisor_ids] prob_vector = np.sum(weighted_exp) / w_sum action_probs_list.append((1 - self.n_actions * self.p_min) * prob_vector + self.p_min) action_probs_list = np.asarray(action_probs_list) action_probs_list /= action_probs_list.sum() estimated_reward = {} uncertainty = {} score = {} for action_id, action_prob in zip(self.action_ids, action_probs_list): estimated_reward[action_id] = action_prob uncertainty[action_id] = 0 score[action_id] = action_prob self._modelstorage.save_model( {'action_probs': estimated_reward, 'w': w}) return estimated_reward, uncertainty, score
def getProviderResp(): resp = self.request('/oauth/provider', params={ 'redirect': 'http://localhost/#foo/bar', 'list': True }) self.assertStatusOk(resp) self.assertIsInstance(resp.json, list) self.assertEqual(len(resp.json), 1) providerResp = resp.json[0] self.assertSetEqual( set(six.viewkeys(providerResp)), {'id', 'name', 'url'}) self.assertEqual(providerResp['id'], providerInfo['id']) self.assertEqual(providerResp['name'], providerInfo['name']) self.assertRegexpMatches( providerResp['url'], providerInfo['url_re']) redirectParams = urllib.parse.parse_qs( urllib.parse.urlparse(providerResp['url']).query) csrfTokenParts = redirectParams['state'][0].partition('.') token = self.model('token').load( csrfTokenParts[0], force=True, objectId=False) self.assertLess( token['expires'], datetime.datetime.utcnow() + datetime.timedelta(days=0.30)) self.assertEqual( csrfTokenParts[2], 'http://localhost/#foo/bar') return providerResp
def _linthompsamp_score(self, context): """Thompson Sampling""" action_ids = list(six.viewkeys(context)) context_array = np.asarray([context[action_id] for action_id in action_ids]) model = self._model_storage.get_model() B = model['B'] # pylint: disable=invalid-name mu_hat = model['mu_hat'] v = self.R * np.sqrt(24 / self.epsilon * self.context_dimension * np.log(1 / self.delta)) mu_tilde = self.random_state.multivariate_normal( mu_hat.flat, v**2 * np.linalg.inv(B))[..., np.newaxis] estimated_reward_array = context_array.dot(mu_hat) score_array = context_array.dot(mu_tilde) estimated_reward_dict = {} uncertainty_dict = {} score_dict = {} for action_id, estimated_reward, score in zip( action_ids, estimated_reward_array, score_array): estimated_reward_dict[action_id] = float(estimated_reward) score_dict[action_id] = float(score) uncertainty_dict[action_id] = float(score - estimated_reward) return estimated_reward_dict, uncertainty_dict, score_dict
def listResources(self, params): return { 'apiVersion': API_VERSION, 'swaggerVersion': SWAGGER_VERSION, 'apis': [{'path': '/{}'.format(resource)} for resource in sorted(six.viewkeys(docs.routes))] }
def write_csvs(self, asset_map, show_progress=False, invalid_data_behavior='warn'): """Read CSVs as DataFrames from our asset map. Parameters ---------- asset_map : dict[int -> str] A mapping from asset id to file path with the CSV data for that asset show_progress : bool Whether or not to show a progress bar while writing. invalid_data_behavior : {'warn', 'raise', 'ignore'} What to do when data is encountered that is outside the range of a uint32. """ read = partial( read_csv, parse_dates=['day'], index_col='day', dtype=self._csv_dtypes, ) return self.write( ((asset, read(path)) for asset, path in iteritems(asset_map)), assets=viewkeys(asset_map), show_progress=show_progress, invalid_data_behavior=invalid_data_behavior, )
def rebalance(context, data): # Pipeline data will be a dataframe with boolean columns named 'longs' and # 'shorts'. pipeline_data = context.pipeline_data all_assets = pipeline_data.index longs = all_assets[pipeline_data.longs] shorts = all_assets[pipeline_data.shorts] record(universe_size=len(all_assets)) # Build a 2x-leveraged, equal-weight, long-short portfolio. one_third = 1.0 / 3.0 for asset in longs: order_target_percent(asset, one_third) for asset in shorts: order_target_percent(asset, -one_third) # Remove any assets that should no longer be in our portfolio. portfolio_assets = longs | shorts positions = context.portfolio.positions for asset in viewkeys(positions) - set(portfolio_assets): # This will fail if the asset was removed from our portfolio because it # was delisted. if data.can_trade(asset): order_target_percent(asset, 0)
def _decorator(f): args, varargs, varkw, defaults = argspec = getargspec(f) if defaults is None: defaults = () no_defaults = (NO_DEFAULT,) * (len(args) - len(defaults)) args_defaults = list(zip(args, no_defaults + defaults)) if varargs: args_defaults.append((varargs, NO_DEFAULT)) if varkw: args_defaults.append((varkw, NO_DEFAULT)) argset = set(args) | {varargs, varkw} - {None} # Arguments can be declared as tuples in Python 2. if not all(isinstance(arg, str) for arg in args): raise TypeError( "Can't validate functions using tuple unpacking: %s" % (argspec,) ) # Ensure that all processors map to valid names. bad_names = viewkeys(processors) - argset if bad_names: raise TypeError( "Got processors for unknown arguments: %s." % bad_names ) return _build_preprocessed_function( f, processors, args_defaults, varargs, varkw, )
def _convert_asset_timestamp_fields(dict): """ Takes in a dict of Asset init args and converts dates to pd.Timestamps """ for key in (_asset_timestamp_fields & viewkeys(dict)): value = pd.Timestamp(dict[key], tz='UTC') dict[key] = None if pd.isnull(value) else value
def _decorator(f): args, varargs, varkw, defaults = argspec = getargspec(f) if defaults is None: defaults = () no_defaults = (NO_DEFAULT,) * (len(args) - len(defaults)) args_defaults = zip(args, no_defaults + defaults) argset = set(args) # These assumptions simplify the implementation significantly. If you # really want to validate a *args/**kwargs function, you'll have to # implement this here or do it yourself. if varargs: raise TypeError( "Can't validate functions that take *args: %s" % argspec ) if varkw: raise TypeError( "Can't validate functions that take **kwargs: %s" % argspec ) # Arguments can be declared as tuples in Python 2. if not all(isinstance(arg, str) for arg in args): raise TypeError( "Can't validate functions using tuple unpacking: %s" % argspec ) # Ensure that all processors map to valid names. bad_names = viewkeys(processors) - argset if bad_names: raise TypeError( "Got processors for unknown arguments: %s." % bad_names ) return _build_preprocessed_function(f, processors, args_defaults)
def extract(cls, data): """ Extract the value for this parser's field. Field keys in data are matched case insensitively. A MetadataFieldNotFoundException is raised if none of the allowed fields are found. A MultipleFieldException is raised if more than one of the allowed fields are found. """ availableFields = six.viewkeys(data) allowedFields = set(field.lower() for field in cls.allowedFields) foundFields = [field for field in availableFields if field.lower() in allowedFields] if not foundFields: raise MetadataFieldNotFoundException(fields=cls.allowedFields) if len(foundFields) > 1: raise MultipleFieldException(name=cls.name, fields=sorted(foundFields)) field = foundFields.pop() value = data.pop(field) assert(value is None or isinstance(value, six.string_types)) return value
def fill_from_dict (self, bybl): self.nsamps = len (bybl) seenants = set () for a1, a2 in six.viewkeys (bybl): seenants.add (a1) seenants.add (a2) self.ants = np.array (sorted (seenants)) self.nants = self.ants.size self.ant_to_antidx = dict ((num, idx) for idx, num in enumerate (self.ants)) self.ncontrib = np.empty ((self.nsamps,), dtype=np.int) self.vis = np.empty ((self.nsamps,), dtype=np.complex) self.blidxs = np.empty ((self.nsamps, 2), dtype=np.int) self.nperant = np.zeros ((self.nants,), dtype=np.int) for i, (bl, (data, flags)) in enumerate (six.viewitems (bybl)): ok = ~flags self.ncontrib[i] = ok.sum () self.vis[i] = data[ok].mean () i1 = self.ant_to_antidx[bl[0]] i2 = self.ant_to_antidx[bl[1]] self.blidxs[i] = i1, i2 self.nperant[i1] += 1 self.nperant[i2] += 1
def _normalize(d): """ The above parse function generates output of list in dict form i.e. {'abc' : {0: 'xyz', 1: 'pqr'}}. This function normalize it and turn them into proper data type, i.e. {'abc': ['xyz', 'pqr']} Note: if dict has element starts with 10, 11 etc.. this function won't fill blanks. for eg: {'abc': {10: 'xyz', 12: 'pqr'}} will convert to {'abc': ['xyz', 'pqr']} """ newd = {} if isinstance(d, dict) == False: return d # if dictionary. iterate over each element and append to newd for k, v in six.iteritems(d): if isinstance(v, dict): first_key = next(iter(six.viewkeys(v))) if isinstance(first_key, int): temp_new = [] for k1, v1 in v.items(): temp_new.append(_normalize(v1)) newd[k] = temp_new elif first_key == "": newd[k] = v.values()[0] else: newd[k] = _normalize(v) else: newd[k] = v return newd
def computeAverageScores(score): """ Compute the average score for each metric and add it to the score list under the name "Average". Datasets with a score of None are omitted from the average calculation. :param score: The score object to compute the average of. The result of the computation is placed at the head of the list. :type score: list """ sums = defaultdict(float) counts = defaultdict(int) for dataset in score: for metric in dataset['metrics']: if metric['value'] is not None: sums[metric['name']] += float(metric['value']) counts[metric['name']] += 1 metrics = [ { 'name': metricName, 'value': sums[metricName] / float(counts[metricName]) } for metricName in sorted(six.viewkeys(sums))] score.insert(0, { 'dataset': 'Average', 'metrics': metrics })
def generate_timing_breakdown_plot(timing_stats, scaling_var, title, description, plot_file): """ Description Args: timing_stats: a dictionary of the form {proc_count : {model||bench : { var : { stat : val }}}} scaling_var: the variable that accounts for the total runtime title: the title of the plot description: the description of the plot plot_file: the file to write the plot out to Returns: an image element containing the plot file and metadata """ cmap_data = colormaps._viridis_data n_subplots = len(six.viewkeys(timing_stats)) fig, ax = plt.subplots(1, n_subplots+1, figsize=(3*(n_subplots+2), 5)) for plot_num, p_count in enumerate( sorted(six.iterkeys(timing_stats), key=functions.sort_processor_counts)): case_data = timing_stats[p_count] all_timers = set(six.iterkeys(case_data['model'])) | set(six.iterkeys(case_data['bench'])) all_timers = sorted(list(all_timers), reverse=True) cmap_stride = int(len(cmap_data)/(len(all_timers)+1)) colors = {all_timers[i]: cmap_data[i*cmap_stride] for i in range(len(all_timers))} sub_ax = plt.subplot(1, n_subplots+1, plot_num+1) sub_ax.set_title(p_count) sub_ax.set_ylabel('Runtime (s)') for case, var_data in case_data.items(): if case == 'bench': bar_num = 2 else: bar_num = 1 offset = 0 if var_data != {}: for var in sorted(six.iterkeys(var_data), reverse=True): if var != scaling_var: plt.bar(bar_num, var_data[var]['mean'], 0.8, bottom=offset, color=colors[var], label=(var if bar_num == 1 else '_none')) offset += var_data[var]['mean'] plt.bar(bar_num, var_data[scaling_var]['mean']-offset, 0.8, bottom=offset, color=colors[scaling_var], label=(scaling_var if bar_num == 1 else '_none')) sub_ax.set_xticks([1.4, 2.4]) sub_ax.set_xticklabels(('test', 'bench')) plt.legend(loc=6, bbox_to_anchor=(1.05,0.5)) plt.tight_layout() sub_ax = plt.subplot(1, n_subplots+1, n_subplots+1) hid_bar = plt.bar(1, 100) for group in hid_bar: group.set_visible(False) sub_ax.set_visible(False) if livvkit.publish: plt.savefig(os.path.splitext(plot_file)[0]+'.eps', dpi=600) plt.savefig(plot_file) plt.close() return elements.image(title, description, os.path.basename(plot_file))
def variables(self): """Return immutable view of variables in expression.""" return viewkeys(self._variables)
def to_html(sample, stats_object): """ Generate a HTML report from summary statistics and a given sample :param sample: DataFrame containing the sample you want to print :param stats_object: Dictionary containing summary statistics. Should be generated with an appropriate describe() function :return: profile report in HTML format :type: string """ n_obs = stats_object['table']['n'] row_formatters = formatters.row_formatters if not isinstance(sample, pd.DataFrame): raise TypeError('sample must be of type pandas.DataFrame') if not isinstance(stats_object, dict): raise TypeError( 'stats_object must be of type dict. Did you generate this using the spark_df_profiling.describe() function?' ) if set(stats_object.keys()) != {'table', 'variables', 'freq'}: raise TypeError( 'stats_object badly formatted. Did you generate this using the spark_df_profiling-eda.describe() function?' ) # Variables rows_html = '' messages = [] for idx, row in stats_object['variables'].iterrows(): formatted_values = {'varname': idx, 'varid': hash(idx)} for col, value in six.iteritems(row): formatted_values[col] = value_format(value, col) row_classes = {} for col in set(row.index) & six.viewkeys(row_formatters): row_classes[col] = row_formatters[col](row[col]) if row_classes[col] == 'alert' and col in templates.messages: messages.append(templates.messages[col].format( formatted_values, varname=formatters.fmt_varname(idx))) if row['type'] == 'CAT': formatted_values['minifreqtable'] = format_freq_table( idx, stats_object['freq'][idx], n_obs, stats_object['variables'].ix[idx], templates.template('mini_freq_table'), templates.template('mini_freq_table_row'), 3) formatted_values['freqtable'] = format_freq_table( idx, stats_object['freq'][idx], n_obs, stats_object['variables'].ix[idx], templates.template('freq_table'), templates.template('freq_table_row'), 20) if row['distinct_count'] > 50: messages.append(templates.messages['HIGH_CARDINALITY'].format( formatted_values, varname=formatters.fmt_varname(idx))) row_classes['distinct_count'] = 'alert' else: row_classes['distinct_count'] = '' if row['type'] == 'UNIQUE': obs = stats_object['freq'][idx].index formatted_values['firstn'] = pd.DataFrame( obs[0:3], columns=['First 3 values']).to_html(classes='example_values', index=False) formatted_values['lastn'] = pd.DataFrame( obs[-3:], columns=['Last 3 values']).to_html(classes='example_values', index=False) if n_obs > 40: formatted_values['firstn_expanded'] = pd.DataFrame( obs[0:20], index=range(1, 21)).to_html( classes='sample table table-hover', header=False) formatted_values['lastn_expanded'] = pd.DataFrame( obs[-20:], index=range(n_obs - 20 + 1, n_obs + 1)).to_html( classes='sample table table-hover', header=False) else: formatted_values['firstn_expanded'] = pd.DataFrame( obs, index=range(1, n_obs + 1)).to_html( classes='sample table table-hover', header=False) formatted_values['lastn_expanded'] = '' rows_html += templates.row_templates_dict[row['type']].render( values=formatted_values, row_classes=row_classes) if row['type'] in ['CORR', 'CONST']: formatted_values['varname'] = formatters.fmt_varname(idx) messages.append( templates.messages[row['type']].format(formatted_values)) # Overview formatted_values = { k: value_format(v, k) for k, v in six.iteritems(stats_object['table']) } row_classes = {} for col in six.viewkeys( stats_object['table']) & six.viewkeys(row_formatters): row_classes[col] = row_formatters[col](stats_object['table'][col]) if row_classes[col] == 'alert' and col in templates.messages: messages.append(templates.messages[col].format( formatted_values, varname=formatters.fmt_varname(idx))) messages_html = '' for msg in messages: messages_html += templates.message_row.format(message=msg) overview_html = templates.template('overview').render( values=formatted_values, row_classes=row_classes, messages=messages_html) # Add Sample sample_html = templates.template('sample').render( sample_table_html=sample.to_html(classes='sample', index=False)) # TODO: should be done in the template return templates.template('base').render({ 'overview_html': overview_html, 'rows_html': rows_html, 'sample_html': sample_html })
def visualize_boxes_and_labels_on_image_array( image, boxes, classes, scores, category_index, instance_masks=None, instance_boundaries=None, keypoints=None, track_ids=None, use_normalized_coordinates=False, max_boxes_to_draw=20, min_score_thresh=0.5, agnostic_mode=False, line_thickness=4, groundtruth_box_visualization_color="black", skip_scores=False, skip_labels=False, skip_track_ids=False, detection_boundary_mask=None, detection_box_ignored=None, ): """Overlay labeled boxes on an image with formatted scores and label names. This function groups boxes that correspond to the same location and creates a display string for each detection and overlays these on the image. Note that this function modifies the image in place, and returns that same image. Args: image: uint8 numpy array with shape (img_height, img_width, 3) boxes: a numpy array of shape [N, 4] classes: a numpy array of shape [N]. Note that class indices are 1-based, and match the keys in the label map. scores: a numpy array of shape [N] or None. If scores=None, then this function assumes that the boxes to be plotted are groundtruth boxes and plot all boxes as black with no classes or scores. category_index: a dict containing category dictionaries (each holding category index `id` and category name `name`) keyed by category indices. instance_masks: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. instance_boundaries: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. keypoints: a numpy array of shape [N, num_keypoints, 2], can be None track_ids: a numpy array of shape [N] with unique track ids. If provided, color-coding of boxes will be determined by these ids, and not the class indices. use_normalized_coordinates: whether boxes is to be interpreted as normalized coordinates or not. max_boxes_to_draw: maximum number of boxes to visualize. If None, draw all boxes. min_score_thresh: minimum score threshold for a box to be visualized agnostic_mode: boolean (default: False) controlling whether to evaluate in class-agnostic mode or not. This mode will display scores but ignore classes. line_thickness: integer (default: 4) controlling line width of the boxes. groundtruth_box_visualization_color: box color for visualizing groundtruth boxes skip_scores: whether to skip score when drawing a single detection skip_labels: whether to skip label when drawing a single detection skip_track_ids: whether to skip track id when drawing a single detection calibration: dict of {x0,y0,x1,y1} Returns: uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. """ # Create a display string (and color) for every box location, group any boxes # that correspond to the same location. box_to_display_str_map = collections.defaultdict(list) box_to_color_map = collections.defaultdict(str) box_to_instance_masks_map = {} box_to_instance_boundaries_map = {} box_to_keypoints_map = collections.defaultdict(list) box_to_track_ids_map = {} box_to_detection_box_ignored_map = {} if not max_boxes_to_draw: max_boxes_to_draw = boxes.shape[0] for i in range(min(max_boxes_to_draw, boxes.shape[0])): if scores is None or scores[i] > min_score_thresh: box = tuple(boxes[i].tolist()) if instance_masks is not None: box_to_instance_masks_map[box] = instance_masks[i] if instance_boundaries is not None: box_to_instance_boundaries_map[box] = instance_boundaries[i] if keypoints is not None: box_to_keypoints_map[box].extend(keypoints[i]) if track_ids is not None: box_to_track_ids_map[box] = track_ids[i] if detection_box_ignored is not None: box_to_detection_box_ignored_map[box] = detection_box_ignored[ i] if scores is None: box_to_color_map[box] = groundtruth_box_visualization_color else: display_str = "" if not skip_labels: if not agnostic_mode: if classes[i] in six.viewkeys(category_index): class_name = category_index[classes[i]]["name"] else: class_name = "N/A" display_str = str(class_name) if not skip_scores: if not display_str: display_str = "{}%".format(int(100 * scores[i])) else: display_str = "{}: {}%".format(display_str, int(100 * scores[i])) if not skip_track_ids and track_ids is not None: if not display_str: display_str = "ID {}".format(track_ids[i]) else: display_str = "{}: ID {}".format( display_str, track_ids[i]) box_to_display_str_map[box].append(display_str) if agnostic_mode: box_to_color_map[box] = "DarkOrange" elif track_ids is not None: prime_multipler = _get_multiplier_for_color_randomness() box_to_color_map[box] = STANDARD_COLORS[ (prime_multipler * track_ids[i]) % len(STANDARD_COLORS)] else: box_to_color_map[box] = STANDARD_COLORS[ classes[i] % len(STANDARD_COLORS)] if detection_boundary_mask is not None: detection_boundary_mask = np.logical_not( detection_boundary_mask).astype(np.uint8) draw_mask_on_image_array(image, detection_boundary_mask, color="Black", alpha=0.6) # Draw all boxes onto image. for box, color in box_to_color_map.items(): ymin, xmin, ymax, xmax = box draw_bounding_box_on_image_array( image, ymin, xmin, ymax, xmax, color=color, thickness=line_thickness, display_str_list=box_to_display_str_map[box], use_normalized_coordinates=use_normalized_coordinates, detection_box_ignored=box_to_detection_box_ignored_map.get(box), ) return image
def get_boxes(image, boxes, classes, scores, category_index, instance_masks=None, instance_boundaries=None, keypoints=None, track_ids=None, use_normalized_coordinates=False, max_boxes_to_draw=20, min_score_thresh=.5, agnostic_mode=False, line_thickness=4, groundtruth_box_visualization_color='black', skip_scores=False, skip_labels=False, skip_track_ids=False): box_to_display_str_map = collections.defaultdict(list) box_to_color_map = collections.defaultdict(str) box_to_instance_masks_map = {} box_to_instance_boundaries_map = {} box_to_keypoints_map = collections.defaultdict(list) box_to_track_ids_map = {} if not max_boxes_to_draw: max_boxes_to_draw = boxes.shape[0] for i in range(min(max_boxes_to_draw, boxes.shape[0])): if scores is None or scores[i] > min_score_thresh: box = tuple(boxes[i].tolist()) if instance_masks is not None: box_to_instance_masks_map[box] = instance_masks[i] if instance_boundaries is not None: box_to_instance_boundaries_map[box] = instance_boundaries[i] if keypoints is not None: box_to_keypoints_map[box].extend(keypoints[i]) if track_ids is not None: box_to_track_ids_map[box] = track_ids[i] if scores is None: box_to_color_map[box] = groundtruth_box_visualization_color else: display_str = '' if not skip_labels: if not agnostic_mode: if classes[i] in six.viewkeys(category_index): class_name = category_index[classes[i]]['name'] else: class_name = 'N/A' display_str = str(class_name) if not skip_scores: if not display_str: display_str = '{}%'.format(int(100 * scores[i])) else: display_str = '{}: {}%'.format(display_str, int(100 * scores[i])) if not skip_track_ids and track_ids is not None: if not display_str: display_str = 'ID {}'.format(track_ids[i]) else: display_str = '{}: ID {}'.format( display_str, track_ids[i]) box_to_display_str_map[box].append(display_str) if agnostic_mode: box_to_color_map[box] = 'DarkOrange' elif track_ids is not None: prime_multipler = vis_util._get_multiplier_for_color_randomness( ) box_to_color_map[box] = vis_util.STANDARD_COLORS[ (prime_multipler * track_ids[i]) % len(vis_util.STANDARD_COLORS)] else: box_to_color_map[box] = vis_util.STANDARD_COLORS[ classes[i] % len(vis_util.STANDARD_COLORS)] return box_to_display_str_map
def testSettings(self): users = self.users # Only admins should be able to get or set settings for method in ('GET', 'PUT', 'DELETE'): resp = self.request(path='/system/setting', method=method, params={ 'key': 'foo', 'value': 'bar' }, user=users[1]) self.assertStatus(resp, 403) # Only valid setting keys should be allowed resp = self.request(path='/system/setting', method='PUT', params={ 'key': 'foo', 'value': 'bar' }, user=users[0]) self.assertStatus(resp, 400) self.assertEqual(resp.json['field'], 'key') # Only a valid JSON list is permitted resp = self.request(path='/system/setting', method='GET', params={'list': json.dumps('not_a_list')}, user=users[0]) self.assertStatus(resp, 400) resp = self.request(path='/system/setting', method='PUT', params={'list': json.dumps('not_a_list')}, user=users[0]) self.assertStatus(resp, 400) # Set an invalid setting value, should fail resp = self.request(path='/system/setting', method='PUT', params={ 'key': SettingKey.BANNER_COLOR, 'value': 'bar' }, user=users[0]) self.assertStatus(resp, 400) self.assertEqual(resp.json['message'], 'The banner color must be a hex color triplet') # Set a valid value resp = self.request(path='/system/setting', method='PUT', params={ 'key': SettingKey.BANNER_COLOR, 'value': '#121212' }, user=users[0]) self.assertStatusOk(resp) # We should now be able to retrieve it resp = self.request(path='/system/setting', method='GET', params={'key': SettingKey.BANNER_COLOR}, user=users[0]) self.assertStatusOk(resp) self.assertEqual(resp.json, '#121212') # We should now clear the setting resp = self.request(path='/system/setting', method='DELETE', params={'key': SettingKey.BANNER_COLOR}, user=users[0]) self.assertStatusOk(resp) # Setting should now be default setting = Setting().get(SettingKey.BANNER_COLOR) self.assertEqual(setting, SettingDefault.defaults[SettingKey.BANNER_COLOR]) # We should also be able to put several setting using a JSON list resp = self.request(path='/system/setting', method='PUT', params={ 'list': json.dumps([ { 'key': SettingKey.BANNER_COLOR, 'value': '#121212' }, { 'key': SettingKey.COOKIE_LIFETIME, 'value': None }, ]) }, user=users[0]) self.assertStatusOk(resp) # We can get a list as well resp = self.request(path='/system/setting', method='GET', params={ 'list': json.dumps([ SettingKey.BANNER_COLOR, SettingKey.COOKIE_LIFETIME, ]) }, user=users[0]) self.assertStatusOk(resp) self.assertEqual(resp.json[SettingKey.BANNER_COLOR], '#121212') # Try to set each key in turn to test the validation. First test with # am invalid value, then test with the default value. If the value # 'bad' won't trigger a validation error, the key should be present in # the badValues table. badValues = { SettingKey.BRAND_NAME: '', SettingKey.BANNER_COLOR: '', SettingKey.EMAIL_FROM_ADDRESS: '', SettingKey.PRIVACY_NOTICE: '', SettingKey.CONTACT_EMAIL_ADDRESS: '', SettingKey.EMAIL_HOST: {}, SettingKey.SMTP_HOST: '', SettingKey.SMTP_PASSWORD: {}, SettingKey.SMTP_USERNAME: {}, SettingKey.CORS_ALLOW_ORIGIN: {}, SettingKey.CORS_ALLOW_METHODS: {}, SettingKey.CORS_ALLOW_HEADERS: {}, SettingKey.CORS_EXPOSE_HEADERS: {}, } allKeys = dict.fromkeys(six.viewkeys(SettingDefault.defaults)) allKeys.update(badValues) for key in allKeys: resp = self.request(path='/system/setting', method='PUT', params={ 'key': key, 'value': badValues.get(key, 'bad') }, user=users[0]) self.assertStatus(resp, 400) resp = self.request(path='/system/setting', method='PUT', params={ 'key': key, 'value': json.dumps( SettingDefault.defaults.get(key, '')) }, user=users[0]) self.assertStatusOk(resp) resp = self.request( path='/system/setting', method='PUT', params={'list': json.dumps([{ 'key': key, 'value': None }])}, user=users[0]) self.assertStatusOk(resp)
def testRegisterAndLoginBcrypt(self): """ Test user registration and logging in. """ cherrypy.config['auth']['hash_alg'] = 'bcrypt' # Set this to minimum so test runs faster. cherrypy.config['auth']['bcrypt_rounds'] = 4 params = { 'email': 'bad_email', 'login': '******', 'firstName': 'First', 'lastName': 'Last', 'password': '******' } # First test all of the required parameters. self.ensureRequiredParams(path='/user', method='POST', required=six.viewkeys(params)) # Now test parameter validation resp = self.request(path='/user', method='POST', params=params) self.assertValidationError(resp, 'password') self.assertEqual(cherrypy.config['users']['password_description'], resp.json['message']) params['password'] = '******' resp = self.request(path='/user', method='POST', params=params) self.assertValidationError(resp, 'login') # Make login something that violates the regex but doesn't contain @ params['login'] = '******' resp = self.request(path='/user', method='POST', params=params) self.assertValidationError(resp, 'login') self.assertEqual(cherrypy.config['users']['login_description'], resp.json['message']) params['login'] = '******' resp = self.request(path='/user', method='POST', params=params) self.assertValidationError(resp, 'email') # Now successfully create the user params['email'] = '*****@*****.**' resp = self.request(path='/user', method='POST', params=params) self.assertStatusOk(resp) self._verifyUserDocument(resp.json) user = User().load(resp.json['_id'], force=True) self.assertEqual(user['hashAlg'], 'bcrypt') # Try logging in without basic auth, should get 401 resp = self.request(path='/user/authentication', method='GET') self.assertStatus(resp, 401) # Bad authentication header resp = self.request(path='/user/authentication', method='GET', additionalHeaders=[('Girder-Authorization', 'Basic Not-Valid-64')]) self.assertStatus(resp, 401) self.assertEqual('Invalid HTTP Authorization header', resp.json['message']) resp = self.request(path='/user/authentication', method='GET', additionalHeaders=[('Girder-Authorization', 'Basic NotValid')]) self.assertStatus(resp, 401) self.assertEqual('Invalid HTTP Authorization header', resp.json['message']) # Login with unregistered email resp = self.request(path='/user/authentication', method='GET', basicAuth='[email protected]:badpassword') self.assertStatus(resp, 401) self.assertEqual('Login failed.', resp.json['message']) # Correct email, but wrong password resp = self.request(path='/user/authentication', method='GET', basicAuth='[email protected]:badpassword') self.assertStatus(resp, 401) self.assertEqual('Login failed.', resp.json['message']) # Login successfully with email resp = self.request(path='/user/authentication', method='GET', basicAuth='[email protected]:good:password') self.assertStatusOk(resp) self.assertHasKeys(resp.json, ['authToken']) self.assertHasKeys(resp.json['authToken'], ['token', 'expires']) self._verifyAuthCookie(resp) # Invalid login resp = self.request(path='/user/authentication', method='GET', basicAuth='badlogin:good:password') self.assertStatus(resp, 401) self.assertEqual('Login failed.', resp.json['message']) # Login successfully with fallback Authorization header resp = self.request(path='/user/authentication', method='GET', basicAuth='goodlogin:good:password', authHeader='Authorization') self.assertStatusOk(resp) # Test secure cookie validation with self.assertRaises(ValidationException): Setting().set(SettingKey.SECURE_COOKIE, 'bad value') # Set secure cookie value Setting().set(SettingKey.SECURE_COOKIE, True) # Login successfully with login resp = self.request(path='/user/authentication', method='GET', basicAuth='goodlogin:good:password') self.assertStatusOk(resp) # Make sure we got a nice (secure) cookie self._verifyAuthCookie(resp, secure=True) # Test user/me resp = self.request(path='/user/me', method='GET', user=user) self.assertStatusOk(resp) self.assertEqual(resp.json['login'], user['login'])
def handle_param_renaming_with_kwargs(self, new_name, old_names, context, new_name_value, new_name_is_provided, user_kwargs, transform_old=lambda x: x): """Look for errors with a particular parameter in an invocation Exemple ------- >>> def fn(newname='default', **kwargs): ... newname, kwargs = deprecation_pool.handle_param_renaming_with_kwargs( ... new_name='newname', old_names={'oldname': '0.2.3'}, ... new_name_value=newname, context='the fn function', ... new_name_is_provided=newname != 'default', ... user_kwargs=kwargs, ... ) ... return newname >>> fn() # Nothing happens 'default' >>> fn(newname='default') # Nothing happens 'default' >>> fn(oldname='aha') # A warning is issued the first time WARNING:root:`oldname` is deprecated since v0.2.3, use `newname` 'aha' >>> fn(newname='default', oldname='default') # A warning is issued the first time WARNING:root:`oldname` is deprecated since v0.2.3, use `newname` 'default' >>> fn(newname='default', oldname='aha') # A warning is issued the first time WARNING:root:`oldname` is deprecated since v0.2.3, use `newname` 'aha' >>> fn(newname='aha', oldname='default') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): NameError: Using both `newname` and `oldname`, `oldname` is deprecated >>> fn(newname='aha', oldname='aha') # doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): NameError: Using both `newname` and `oldname`, `oldname` is deprecated """ deprecated_names_used = six.viewkeys(old_names) & six.viewkeys( user_kwargs) if len(deprecated_names_used) == 0: return new_name_value, user_kwargs n = deprecated_names_used.pop() if new_name_is_provided: raise NameError( 'Using both `{}` and `{}` in `{}`, `{}` is deprecated'.format( new_name, n, context, n, )) key = (context, new_name, n) if key not in self._seen: self._seen.add(key) logging.warning( '`{}` parameter in `{}` is deprecated since v{}, use `{}` instead' .format( n, context, old_names[n], new_name, )) v = transform_old(user_kwargs[n]) del user_kwargs[n] return v, user_kwargs
def to_html(sample, stats_object): """Generate a HTML report from summary statistics and a given sample. Parameters ---------- sample : DataFrame the sample you want to print stats_object : dict Summary statistics. Should be generated with an appropriate describe() function Returns ------- str containing profile report in HTML format Notes ----- * This function as to be refactored since it's huge and it contains inner functions """ n_obs = stats_object['table']['n'] value_formatters = formatters.value_formatters row_formatters = formatters.row_formatters if not isinstance(sample, pd.DataFrame): raise TypeError("sample must be of type pandas.DataFrame") if not isinstance(stats_object, dict): raise TypeError( "stats_object must be of type dict. Did you generate this using the pandas_profiling.describe() function?" ) if not set({'table', 'variables', 'freq', 'correlations'}).issubset( set(stats_object.keys())): raise TypeError( "stats_object badly formatted. Did you generate this using the pandas_profiling.describe() function?" ) def fmt(value, name): if pd.isnull(value): return "" if name in value_formatters: return value_formatters[name](value) elif isinstance(value, float): return value_formatters[formatters.DEFAULT_FLOAT_FORMATTER](value) else: try: return unicode(value) # Python 2 except NameError: return str(value) # Python 3 def _format_row(freq, label, max_freq, row_template, n, extra_class=''): if max_freq != 0: width = int(freq / max_freq * 99) + 1 else: width = 1 if width > 20: label_in_bar = freq label_after_bar = "" else: label_in_bar = " " label_after_bar = freq return row_template.render(label=label, width=width, count=freq, percentage='{:2.1f}'.format(freq / n * 100), extra_class=extra_class, label_in_bar=label_in_bar, label_after_bar=label_after_bar) def freq_table(freqtable, n, table_template, row_template, max_number_to_print, nb_col=6): freq_rows_html = u'' if max_number_to_print > n: max_number_to_print = n if max_number_to_print < len(freqtable): freq_other = sum(freqtable.iloc[max_number_to_print:]) min_freq = freqtable.values[max_number_to_print] else: freq_other = 0 min_freq = 0 freq_missing = n - sum(freqtable) max_freq = max(freqtable.values[0], freq_other, freq_missing) # TODO: Correctly sort missing and other for label, freq in six.iteritems( freqtable.iloc[0:max_number_to_print]): freq_rows_html += _format_row(freq, label, max_freq, row_template, n) if freq_other > min_freq: freq_rows_html += _format_row( freq_other, "Other values (%s)" % (freqtable.count() - max_number_to_print), max_freq, row_template, n, extra_class='other') if freq_missing > min_freq: freq_rows_html += _format_row(freq_missing, "(Missing)", max_freq, row_template, n, extra_class='missing') return table_template.render(rows=freq_rows_html, varid=hash(idx), nb_col=nb_col) def extreme_obs_table(freqtable, table_template, row_template, number_to_print, n, ascending=True): # If it's mixed between base types (str, int) convert to str. Pure "mixed" types are filtered during type discovery if "mixed" in freqtable.index.inferred_type: freqtable.index = freqtable.index.astype(str) sorted_freqTable = freqtable.sort_index() if ascending: obs_to_print = sorted_freqTable.iloc[:number_to_print] else: obs_to_print = sorted_freqTable.iloc[-number_to_print:] freq_rows_html = '' max_freq = max(obs_to_print.values) for label, freq in six.iteritems(obs_to_print): freq_rows_html += _format_row(freq, label, max_freq, row_template, n) return table_template.render(rows=freq_rows_html) # Variables rows_html = u"" messages = [] for idx, row in stats_object['variables'].iterrows(): formatted_values = {'varname': idx, 'varid': hash(idx)} row_classes = {} for col, value in six.iteritems(row): formatted_values[col] = fmt(value, col) for col in set(row.index) & six.viewkeys(row_formatters): row_classes[col] = row_formatters[col](row[col]) if row_classes[col] == "alert" and col in templates.messages: messages.append(templates.messages[col].format( formatted_values, varname=idx)) if row['type'] in {'CAT', 'BOOL'}: formatted_values['minifreqtable'] = freq_table( stats_object['freq'][idx], n_obs, templates.template('mini_freq_table'), templates.template('mini_freq_table_row'), 3, templates.mini_freq_table_nb_col[row['type']]) if row['distinct_count'] > 50: messages.append(templates.messages['HIGH_CARDINALITY'].format( formatted_values, varname=idx)) row_classes['distinct_count'] = "alert" else: row_classes['distinct_count'] = "" if row['type'] == 'UNIQUE': obs = stats_object['freq'][idx].index formatted_values['firstn'] = pd.DataFrame( obs[0:3], columns=["First 3 values"]).to_html(classes="example_values", index=False) formatted_values['lastn'] = pd.DataFrame( obs[-3:], columns=["Last 3 values"]).to_html(classes="example_values", index=False) if row['type'] == 'UNSUPPORTED': formatted_values['varname'] = idx messages.append( templates.messages[row['type']].format(formatted_values)) elif row['type'] in {'CORR', 'CONST', 'RECODED'}: formatted_values['varname'] = idx messages.append( templates.messages[row['type']].format(formatted_values)) else: formatted_values['freqtable'] = freq_table( stats_object['freq'][idx], n_obs, templates.template('freq_table'), templates.template('freq_table_row'), 10) formatted_values['firstn_expanded'] = extreme_obs_table( stats_object['freq'][idx], templates.template('freq_table'), templates.template('freq_table_row'), 5, n_obs, ascending=True) formatted_values['lastn_expanded'] = extreme_obs_table( stats_object['freq'][idx], templates.template('freq_table'), templates.template('freq_table_row'), 5, n_obs, ascending=False) rows_html += templates.row_templates_dict[row['type']].render( values=formatted_values, row_classes=row_classes) # Overview formatted_values = { k: fmt(v, k) for k, v in six.iteritems(stats_object['table']) } row_classes = {} for col in six.viewkeys( stats_object['table']) & six.viewkeys(row_formatters): row_classes[col] = row_formatters[col](stats_object['table'][col]) if row_classes[col] == "alert" and col in templates.messages: messages.append(templates.messages[col].format(formatted_values, varname=idx)) messages_html = u'' for msg in messages: messages_html += templates.message_row.format(message=msg) overview_html = templates.template('overview').render( values=formatted_values, row_classes=row_classes, messages=messages_html) # Add plot of matrix correlation pearson_matrix = plot.correlation_matrix( stats_object['correlations']['pearson'], 'Pearson') spearman_matrix = plot.correlation_matrix( stats_object['correlations']['spearman'], 'Spearman') correlations_html = templates.template('correlations').render( values={ 'pearson_matrix': pearson_matrix, 'spearman_matrix': spearman_matrix }) # Add sample sample_html = templates.template('sample').render( sample_table_html=sample.to_html(classes="sample")) # TODO: should be done in the template return templates.template('base').render({ 'overview_html': overview_html, 'rows_html': rows_html, 'sample_html': sample_html, 'correlation_html': correlations_html, 'dataframe_name': dataframe_name, 'statement': statement, })
def to_html(sample, stats_object): """ Generate a HTML report from summary statistics and a given sample Parameters ---------- sample: DataFrame containing the sample you want to print stats_object: Dictionary containing summary statistics. Should be generated with an appropriate describe() function Returns ------- str, containing profile report in HTML format """ n_obs = stats_object['table']['n'] value_formatters = formatters.value_formatters row_formatters = formatters.row_formatters if not isinstance(sample, pd.DataFrame): raise TypeError("sample must be of type pandas.DataFrame") if not isinstance(stats_object, dict): raise TypeError( "stats_object must be of type dict. Did you generate this using the spark_df_profiling.describe() function?" ) if set(stats_object.keys()) != {'table', 'variables', 'freq'}: raise TypeError( "stats_object badly formatted. Did you generate this using the spark_df_profiling-eda.describe() function?" ) def fmt(value, name): if pd.isnull(value): return "" if name in value_formatters: return value_formatters[name](value) elif isinstance(value, float): return value_formatters[formatters.DEFAULT_FLOAT_FORMATTER](value) else: if sys.version_info.major == 3: return str(value) else: return unicode(value) def freq_table(freqtable, n, var_table, table_template, row_template, max_number_of_items_in_table): local_var_table = var_table.copy() freq_other_prefiltered = freqtable["***Other Values***"] freq_other_prefiltered_num = freqtable[ "***Other Values Distinct Count***"] freqtable = freqtable.drop( ["***Other Values***", "***Other Values Distinct Count***"]) freq_rows_html = u'' freq_other = sum( freqtable[max_number_of_items_in_table:]) + freq_other_prefiltered freq_missing = var_table["n_missing"] max_freq = max(freqtable.values[0], freq_other, freq_missing) try: min_freq = freqtable.values[max_number_of_items_in_table] except IndexError: min_freq = 0 # TODO: Correctly sort missing and other def format_row(freq, label, extra_class=''): width = int(freq / float(max_freq) * 99) + 1 if width > 20: label_in_bar = freq label_after_bar = "" else: label_in_bar = " " label_after_bar = freq return row_template.render(label=label, width=width, count=freq, percentage='{:2.1f}'.format( freq / float(n) * 100), extra_class=extra_class, label_in_bar=label_in_bar, label_after_bar=label_after_bar) for label, freq in six.iteritems( freqtable[0:max_number_of_items_in_table]): freq_rows_html += format_row(freq, label) if freq_other > min_freq: freq_rows_html += format_row( freq_other, "Other values (%s)" % (freqtable.count() + freq_other_prefiltered_num - max_number_of_items_in_table), extra_class='other') if freq_missing > min_freq: freq_rows_html += format_row(freq_missing, "(Missing)", extra_class='missing') return table_template.render(rows=freq_rows_html, varid=hash(idx)) # Variables rows_html = u"" messages = [] for idx, row in stats_object['variables'].iterrows(): formatted_values = {'varname': idx, 'varid': hash(idx)} row_classes = {} for col, value in six.iteritems(row): formatted_values[col] = fmt(value, col) for col in set(row.index) & six.viewkeys(row_formatters): row_classes[col] = row_formatters[col](row[col]) if row_classes[col] == "alert" and col in templates.messages: messages.append(templates.messages[col].format( formatted_values, varname=formatters.fmt_varname(idx))) if row['type'] == 'CAT': formatted_values['minifreqtable'] = freq_table( stats_object['freq'][idx], n_obs, stats_object['variables'].ix[idx], templates.template('mini_freq_table'), templates.template('mini_freq_table_row'), 3) formatted_values['freqtable'] = freq_table( stats_object['freq'][idx], n_obs, stats_object['variables'].ix[idx], templates.template('freq_table'), templates.template('freq_table_row'), 20) if row['distinct_count'] > 50: messages.append(templates.messages['HIGH_CARDINALITY'].format( formatted_values, varname=formatters.fmt_varname(idx))) row_classes['distinct_count'] = "alert" else: row_classes['distinct_count'] = "" if row['type'] == 'UNIQUE': obs = stats_object['freq'][idx].index formatted_values['firstn'] = pd.DataFrame( obs[0:3], columns=["First 3 values"]).to_html(classes="example_values", index=False) formatted_values['lastn'] = pd.DataFrame( obs[-3:], columns=["Last 3 values"]).to_html(classes="example_values", index=False) if n_obs > 40: formatted_values['firstn_expanded'] = pd.DataFrame( obs[0:20], index=range(1, 21)).to_html( classes="sample table table-hover", header=False) formatted_values['lastn_expanded'] = pd.DataFrame( obs[-20:], index=range(n_obs - 20 + 1, n_obs + 1)).to_html( classes="sample table table-hover", header=False) else: formatted_values['firstn_expanded'] = pd.DataFrame( obs, index=range(1, n_obs + 1)).to_html( classes="sample table table-hover", header=False) formatted_values['lastn_expanded'] = '' rows_html += templates.row_templates_dict[row['type']].render( values=formatted_values, row_classes=row_classes) if row['type'] in {'CORR', 'CONST'}: formatted_values['varname'] = formatters.fmt_varname(idx) messages.append( templates.messages[row['type']].format(formatted_values)) # Overview formatted_values = { k: fmt(v, k) for k, v in six.iteritems(stats_object['table']) } row_classes = {} for col in six.viewkeys( stats_object['table']) & six.viewkeys(row_formatters): row_classes[col] = row_formatters[col](stats_object['table'][col]) if row_classes[col] == "alert" and col in templates.messages: messages.append(templates.messages[col].format( formatted_values, varname=formatters.fmt_varname(idx))) messages_html = u'' for msg in messages: messages_html += templates.message_row.format(message=msg) overview_html = templates.template('overview').render( values=formatted_values, row_classes=row_classes, messages=messages_html) # Sample sample_html = templates.template('sample').render( sample_table_html=sample.to_html(classes="sample")) # TODO: should be done in the template return templates.template('base').render({ 'overview_html': overview_html, 'rows_html': rows_html, 'sample_html': sample_html })
def visualize_boxes_and_labels_on_image_array( # 수정한 부분 1-2 item, w, h, # 수정한 부분 1-2 image, boxes, classes, scores, category_index, instance_masks=None, instance_boundaries=None, keypoints=None, track_ids=None, use_normalized_coordinates=False, max_boxes_to_draw=20, min_score_thresh=.5, agnostic_mode=False, line_thickness=4, groundtruth_box_visualization_color='black', skip_scores=False, skip_labels=False, skip_track_ids=False): """Overlay labeled boxes on an image with formatted scores and label names. This function groups boxes that correspond to the same location and creates a display string for each detection and overlays these on the image. Note that this function modifies the image in place, and returns that same image. Args: image: uint8 numpy array with shape (img_height, img_width, 3) boxes: a numpy array of shape [N, 4] classes: a numpy array of shape [N]. Note that class indices are 1-based, and match the keys in the label map. scores: a numpy array of shape [N] or None. If scores=None, then this function assumes that the boxes to be plotted are groundtruth boxes and plot all boxes as black with no classes or scores. category_index: a dict containing category dictionaries (each holding category index `id` and category name `name`) keyed by category indices. instance_masks: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. instance_boundaries: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. keypoints: a numpy array of shape [N, num_keypoints, 2], can be None track_ids: a numpy array of shape [N] with unique track ids. If provided, color-coding of boxes will be determined by these ids, and not the class indices. use_normalized_coordinates: whether boxes is to be interpreted as normalized coordinates or not. max_boxes_to_draw: maximum number of boxes to visualize. If None, draw all boxes. min_score_thresh: minimum score threshold for a box to be visualized agnostic_mode: boolean (default: False) controlling whether to evaluate in class-agnostic mode or not. This mode will display scores but ignore classes. line_thickness: integer (default: 4) controlling line width of the boxes. groundtruth_box_visualization_color: box color for visualizing groundtruth boxes skip_scores: whether to skip score when drawing a single detection skip_labels: whether to skip label when drawing a single detection skip_track_ids: whether to skip track id when drawing a single detection Returns: uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. """ # Create a display string (and color) for every box location, group any boxes # that correspond to the same location. box_to_display_str_map = collections.defaultdict(list) box_to_color_map = collections.defaultdict(str) box_to_instance_masks_map = {} box_to_instance_boundaries_map = {} box_to_keypoints_map = collections.defaultdict(list) box_to_track_ids_map = {} # print("실행 후 w = {} , h = {}".format(w, h)) facecut_w9 = int(round(w / 9)) facecut_w4 = int(round(w / 4)) facecut_w10 = int(round(w / 10)) facecut_w3 = int(round(w / 3)) label_str = "" if not max_boxes_to_draw: max_boxes_to_draw = boxes.shape[0] for i in range(min(max_boxes_to_draw, boxes.shape[0])): # min_score_thresh = 0.1 if scores is None or scores[i] > min_score_thresh: box = tuple(boxes[i].tolist()) if instance_masks is not None: box_to_instance_masks_map[box] = instance_masks[i] if instance_boundaries is not None: box_to_instance_boundaries_map[box] = instance_boundaries[i] if keypoints is not None: box_to_keypoints_map[box].extend(keypoints[i]) if track_ids is not None: box_to_track_ids_map[box] = track_ids[i] if scores is None: box_to_color_map[box] = groundtruth_box_visualization_color else: display_str = '' if not skip_labels: if not agnostic_mode: if classes[i] in six.viewkeys(category_index): class_name = category_index[classes[i]]['name'] # 클래스가 N/A라고 나오는 경우 else: class_name = 'N/A' display_str = str(class_name) if not skip_scores: if not display_str: display_str = '{}%'.format(int(100 * scores[i])) else: display_str = '{}: {}%'.format(display_str, int(100 * scores[i])) # 결과 퍼센트 출력 #print(display_str+", ") label_str += '{}, {}%'.format( class_name, int(100 * scores[i])) + ", " if not skip_track_ids and track_ids is not None: if not display_str: display_str = 'ID {}'.format(track_ids[i]) else: display_str = '{}: ID {}'.format( display_str, track_ids[i]) box_to_display_str_map[box].append(display_str) if agnostic_mode: box_to_color_map[box] = STANDARD_COLORS[i] print(str(i), str(STANDARD_COLORS[i])) elif track_ids is not None: prime_multipler = _get_multiplier_for_color_randomness() box_to_color_map[box] = STANDARD_COLORS[ (prime_multipler * track_ids[i]) % len(STANDARD_COLORS)] else: box_to_color_map[box] = STANDARD_COLORS[ classes[i] % len(STANDARD_COLORS)] # Draw all boxes onto image. for box, color in box_to_color_map.items(): ymin, xmin, ymax, xmax = box if instance_masks is not None: draw_mask_on_image_array(image, box_to_instance_masks_map[box], color=color) if instance_boundaries is not None: draw_mask_on_image_array(image, box_to_instance_boundaries_map[box], color='red', alpha=1.0) draw_bounding_box_on_image_array( image, ymin, xmin, ymax, xmax, color=color, thickness=line_thickness, display_str_list=box_to_display_str_map[box], use_normalized_coordinates=use_normalized_coordinates) # 수정한 부분 1-3 if int(facecut_w9 * 2) <= int(xmin * w) and int(xmax * w) <= int( facecut_w9 * 7): image1 = image[0:h, facecut_w9 * 2:facecut_w9 * 7] output_path = "D:\\DFDC\\nose\\test_output\\nose\\v0.3\\total_real_test\\side_output1\\" cv2.imwrite(output_path + item, image1) # 정면이면 if문 통과 if int(facecut_w4 * 1) <= int(xmin * w) and int(xmax * w) <= int( facecut_w4 * 3): image2 = image[0:h, facecut_w4 * 1:facecut_w4 * 3] output_path = "D:\\DFDC\\nose\\test_output\\nose\\v0.3\\total_real_test\\side_output2\\" cv2.imwrite(output_path + item, image2) if int(facecut_w10 * 3) <= int(xmin * w) and int(xmax * w) <= int( facecut_w10 * 7): image3 = image[0:h, facecut_w10 * 3:facecut_w10 * 7] output_path = "D:\\DFDC\\nose\\test_output\\nose\\v0.3\\total_real_test\\side_output3\\" cv2.imwrite(output_path + item, image3) if int(facecut_w3 * 1) <= int(xmin * w) and int(xmax * w) <= int( facecut_w3 * 2): image4 = image[0:h, facecut_w3 * 1:facecut_w3 * 2] output_path = "D:\\DFDC\\nose\\test_output\\nose\\v0.3\\total_real_test\\side_output4\\" cv2.imwrite(output_path + item, image4) # 수정한 부분 1-3 끝 if keypoints is not None: draw_keypoints_on_image_array( image, box_to_keypoints_map[box], color=color, radius=line_thickness / 2, use_normalized_coordinates=use_normalized_coordinates) return image, label_str
def keys(self): return [key_validator(k) for k in six.viewkeys(self._json_data)]
def keys(self): return six.viewkeys(self._json_data)
def get_commit_history(self, revisions): """Return the commit history specified by the revisions. Args: revisions (dict): A dictionary of revisions to generate history for, as returned by :py:meth:`parse_revision_spec`. Returns: list of dict: The list of history entries, in order. The dictionaries have the following keys: ``commit_id``: The unique identifier of the commit. ``parent_id``: The unique identifier of the parent commit. ``author_name``: The name of the commit's author. ``author_email``: The e-mail address of the commit's author. ``author_date``: The date the commit was authored. ``committer_name``: The committer's name. ``committer_email``: The e-mail address of the committer. ``committer_date``: The date the commit was committed. ``commit_message``: The commit's message. Raises: rbtools.clients.errors.SCMError: The history is non-linear or there is a commit with no parents. """ log_fields = { 'commit_id': b'%H', 'parent_id': b'%P', 'author_name': b'%an', 'author_email': b'%ae', 'author_date': b'%ad', 'committer_name': b'%cn', 'committer_email': b'%ce', 'committer_date': b'%cd', 'commit_message': b'%B', } # 0x1f is the ASCII field separator. It is a non-printable character # that should not appear in any field in `git log`. log_format = b'%x1f'.join(six.itervalues(log_fields)) log_entries = execute([ self.git, b'log', b'-z', b'--reverse', b'--pretty=format:%s' % log_format, b'--date=iso8601-strict', b'%s..%s' % (bytes(revisions['base']), bytes(revisions['tip'])), ], ignore_errors=True, none_on_ignored_error=True, results_unicode=True) if not log_entries: return None history = [] field_names = six.viewkeys(log_fields) for log_entry in log_entries.split(self._NUL): fields = log_entry.split(self._FIELD_SEP) entry = dict(zip(field_names, fields)) parents = entry['parent_id'].split() if len(parents) > 1: raise SCMError( 'The Git SCMClient only supports posting commit histories ' 'that are entirely linear.') elif len(parents) == 0: raise SCMError( 'The Git SCMClient only supports posting commits that ' 'have exactly one parent.') history.append(entry) return history
def qplot(x=None, y=None, data=None, facets=None, margins=False, geom='auto', xlim=None, ylim=None, log='', main=None, xlab=None, ylab=None, asp=None, **kwargs): """ Quick plot Parameters ---------- x: str | array_like x aesthetic y: str | array_like y aesthetic data: pandas.DataFrame Data frame to use (optional). If not specified, will create one, extracting arrays from the current environment. geom: str | list *geom(s)* to do the drawing. If ``auto``, defaults to 'point' if ``x`` and ``y`` are specified or 'histogram' if only ``x`` is specified. xlim: tuple x-axis limits ylim: tuple y-axis limits log: 'x' | 'y' | 'xy' Which variables to log transform. main: str Plot title xlab: str x-axis label ylab: str y-axis label asp: str | float The y/x aspect ratio. kwargs: dict Arguments passed on to the geom. Returns ------- p: ggplot ggplot object """ # Extract all recognizable aesthetic mappings from the parameters # String values e.g "I('red')", "I(4)" are not treated as mappings environment = EvalEnvironment.capture(1) aesthetics = {} if x is None else {'x': x} if y is not None: aesthetics['y'] = y def is_mapping(value): """ Return True if value is not enclosed in I() function """ with suppress(AttributeError): return not (value.startswith('I(') and value.endswith(')')) return True def I(value): return value I_env = EvalEnvironment([{'I': I}]) for ae in six.viewkeys(kwargs) & all_aesthetics: value = kwargs[ae] if is_mapping(value): aesthetics[ae] = value else: kwargs[ae] = I_env.eval(value) # List of geoms if is_string(geom): geom = [geom] elif isinstance(geom, tuple): geom = list(geom) if data is None: data = pd.DataFrame() # Work out plot data, and modify aesthetics, if necessary def replace_auto(lst, str2): """ Replace all occurences of 'auto' in with str2 """ for i, value in enumerate(lst): if value == 'auto': lst[i] = str2 return lst if 'auto' in geom: if 'sample' in aesthetics: replace_auto(geom, 'qq') elif y is None: # If x is discrete we choose geom_bar & # geom_histogram otherwise. But we need to # evaluate the mapping to find out the dtype env = environment.with_outer_namespace( {'factor': pd.Categorical}) if isinstance(aesthetics['x'], six.string_types): try: x = env.eval(aesthetics['x'], inner_namespace=data) except Exception: msg = "Could not evaluate aesthetic 'x={}'" raise PlotnineError(msg.format(aesthetics['x'])) elif not hasattr(aesthetics['x'], 'dtype'): x = np.asarray(aesthetics['x']) if x.dtype.kind in DISCRETE_KINDS: replace_auto(geom, 'bar') else: replace_auto(geom, 'histogram') else: if x is None: if pdtypes.is_list_like(aesthetics['y']): aesthetics['x'] = range(len(aesthetics['y'])) xlab = 'range(len(y))' ylab = 'y' else: # We could solve the issue in layer.compute_asthetics # but it is not worth the extra complexity raise PlotnineError( "Cannot infer how long x should be.") replace_auto(geom, 'point') p = ggplot(aes(**aesthetics), data=data, environment=environment) def get_facet_type(facets): with suppress(PlotnineError): parse_grid_facets(facets) return 'grid' with suppress(PlotnineError): parse_wrap_facets(facets) return 'wrap' warn("Could not determine the type of faceting, " "therefore no faceting.") return 'null' if facets: facet_type = get_facet_type(facets) if facet_type == 'grid': p += facet_grid(facets, margins=margins) elif facet_type == 'wrap': p += facet_wrap(facets) else: p += facet_null() # Add geoms for g in geom: geom_name = 'geom_{}'.format(g) geom_klass = Registry[geom_name] stat_name = 'stat_{}'.format(geom_klass.DEFAULT_PARAMS['stat']) stat_klass = Registry[stat_name] # find params recognized = (six.viewkeys(kwargs) & (six.viewkeys(geom_klass.DEFAULT_PARAMS) | geom_klass.aesthetics() | six.viewkeys(stat_klass.DEFAULT_PARAMS) | stat_klass.aesthetics())) recognized = recognized - six.viewkeys(aesthetics) params = {ae: kwargs[ae] for ae in recognized} p += geom_klass(**params) if 'x' in log: p += scale_x_log10() if 'y' in log: p += scale_y_log10() if xlab: p += xlabel(xlab) if ylab: p += ylabel(ylab) if main: p += ggtitle(main) if asp: p += theme(aspect_ratio=asp) return p
def update_deleted(self, job): """ High-level method that replicates a single partition that doesn't belong on this node. :param job: a dict containing info about the partition to be replicated """ def tpool_get_suffixes(path): return [ suff for suff in os.listdir(path) if len(suff) == 3 and isdir(join(path, suff)) ] self.replication_count += 1 self.logger.increment('partition.delete.count.%s' % (job['device'], )) headers = dict(self.default_headers) headers['X-Backend-Storage-Policy-Index'] = int(job['policy']) failure_devs_info = set() begin = time.time() handoff_partition_deleted = False try: responses = [] suffixes = tpool.execute(tpool_get_suffixes, job['path']) synced_remote_regions = {} delete_objs = None if suffixes: for node in job['nodes']: self.stats['rsync'] += 1 kwargs = {} if node['region'] in synced_remote_regions and \ self.conf.get('sync_method', 'rsync') == 'ssync': kwargs['remote_check_objs'] = \ synced_remote_regions[node['region']] # candidates is a dict(hash=>timestamp) of objects # for deletion success, candidates = self.sync(node, job, suffixes, **kwargs) if success: with Timeout(self.http_timeout): conn = http_connect(node['replication_ip'], node['replication_port'], node['device'], job['partition'], 'REPLICATE', '/' + '-'.join(suffixes), headers=headers) conn.getresponse().read() if node['region'] != job['region']: synced_remote_regions[node['region']] = viewkeys( candidates) else: failure_devs_info.add( (node['replication_ip'], node['device'])) responses.append(success) for cand_objs in synced_remote_regions.values(): if delete_objs is None: delete_objs = cand_objs else: delete_objs = delete_objs & cand_objs if self.handoff_delete: # delete handoff if we have had handoff_delete successes delete_handoff = len([resp for resp in responses if resp]) >= \ self.handoff_delete else: # delete handoff if all syncs were successful delete_handoff = len(responses) == len(job['nodes']) and \ all(responses) if delete_handoff: self.stats['remove'] += 1 if (self.conf.get('sync_method', 'rsync') == 'ssync' and delete_objs is not None): self.logger.info(_("Removing %s objects"), len(delete_objs)) _junk, error_paths = self.delete_handoff_objs( job, delete_objs) # if replication works for a hand-off device and it failed, # the remote devices which are target of the replication # from the hand-off device will be marked. Because cleanup # after replication failed means replicator needs to # replicate again with the same info. if error_paths: failure_devs_info.update([ (failure_dev['replication_ip'], failure_dev['device']) for failure_dev in job['nodes'] ]) else: self.delete_partition(job['path']) handoff_partition_deleted = True elif not suffixes: self.delete_partition(job['path']) handoff_partition_deleted = True except (Exception, Timeout): self.logger.exception(_("Error syncing handoff partition")) self._add_failure_stats(failure_devs_info) finally: target_devs_info = set([(target_dev['replication_ip'], target_dev['device']) for target_dev in job['nodes']]) self.stats['success'] += len(target_devs_info - failure_devs_info) if not handoff_partition_deleted: self.handoffs_remaining += 1 self.partition_times.append(time.time() - begin) self.logger.timing_since('partition.delete.timing', begin)
def _scan_directory(directory, output_format, ssd): """Scan a directory for log files and write the final model to stdout.""" if output_format == _OUTPUT_FORMAT_LINES: print('directory =', directory) model_spec_filename = os.path.join(directory, 'model_spec.json') if not tf.io.gfile.exists(model_spec_filename): print('file {} not found; skipping'.format(model_spec_filename)) if output_format == _OUTPUT_FORMAT_LINES: print() return with tf.io.gfile.GFile(model_spec_filename, 'r') as handle: model_spec = schema_io.deserialize(handle.read()) paths = [] oneofs = dict() def populate_oneofs(path, oneof): paths.append(path) oneofs[path] = oneof schema.map_oneofs_with_paths(populate_oneofs, model_spec) all_path_logits = analyze_mobile_search_lib.read_path_logits(directory) if not all_path_logits: print( 'event data missing from directory {}; skipping'.format(directory)) if output_format == _OUTPUT_FORMAT_LINES: print() return global_step = max(all_path_logits) if output_format == _OUTPUT_FORMAT_LINES: print('global_step = {:d}'.format(global_step)) all_path_logit_keys = six.viewkeys(all_path_logits[global_step]) oneof_keys = six.viewkeys(oneofs) if all_path_logit_keys != oneof_keys: raise ValueError( 'OneOf key mismatch. Present in event files but not in model_spec: {}. ' 'Present in model_spec but not in event files: {}'.format( all_path_logit_keys - oneof_keys, oneof_keys - all_path_logit_keys)) indices = [] for path in paths: index = np.argmax(all_path_logits[global_step][path]) indices.append(index) indices_str = ':'.join(map(str, indices)) if output_format == _OUTPUT_FORMAT_LINES: print('indices = {:s}'.format(indices_str)) cost_model_time = mobile_cost_model.estimate_cost(indices, ssd) if output_format == _OUTPUT_FORMAT_LINES: print('cost_model = {:f}'.format(cost_model_time)) if output_format == _OUTPUT_FORMAT_LINES: print() elif output_format == _OUTPUT_FORMAT_CSV: fields = [indices_str, global_step, directory, cost_model_time] print(','.join(map(str, fields)))
def visualize_boxes_and_labels_on_image_array( image, boxes, classes, scores, category_index, instance_masks=None, instance_boundaries=None, keypoints=None, keypoint_scores=None, keypoint_edges=None, track_ids=None, use_normalized_coordinates=False, max_boxes_to_draw=20, min_score_thresh=.5, agnostic_mode=False, line_thickness=4, mask_alpha=.4, groundtruth_box_visualization_color='black', skip_boxes=False, skip_scores=False, skip_labels=False, skip_track_ids=False): """Overlay labeled boxes on an image with formatted scores and label names. This function groups boxes that correspond to the same location and creates a display string for each detection and overlays these on the image. Note that this function modifies the image in place, and returns that same image. Args: image: uint8 numpy array with shape (img_height, img_width, 3) boxes: a numpy array of shape [N, 4] classes: a numpy array of shape [N]. Note that class indices are 1-based, and match the keys in the label map. scores: a numpy array of shape [N] or None. If scores=None, then this function assumes that the boxes to be plotted are groundtruth boxes and plot all boxes as black with no classes or scores. category_index: a dict containing category dictionaries (each holding category index `id` and category name `name`) keyed by category indices. instance_masks: a uint8 numpy array of shape [N, image_height, image_width], can be None. instance_boundaries: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. keypoints: a numpy array of shape [N, num_keypoints, 2], can be None. keypoint_scores: a numpy array of shape [N, num_keypoints], can be None. keypoint_edges: A list of tuples with keypoint indices that specify which keypoints should be connected by an edge, e.g. [(0, 1), (2, 4)] draws edges from keypoint 0 to 1 and from keypoint 2 to 4. track_ids: a numpy array of shape [N] with unique track ids. If provided, color-coding of boxes will be determined by these ids, and not the class indices. use_normalized_coordinates: whether boxes is to be interpreted as normalized coordinates or not. max_boxes_to_draw: maximum number of boxes to visualize. If None, draw all boxes. min_score_thresh: minimum score threshold for a box or keypoint to be visualized. agnostic_mode: boolean (default: False) controlling whether to evaluate in class-agnostic mode or not. This mode will display scores but ignore classes. line_thickness: integer (default: 4) controlling line width of the boxes. mask_alpha: transparency value between 0 and 1 (default: 0.4). groundtruth_box_visualization_color: box color for visualizing groundtruth boxes skip_boxes: whether to skip the drawing of bounding boxes. skip_scores: whether to skip score when drawing a single detection skip_labels: whether to skip label when drawing a single detection skip_track_ids: whether to skip track id when drawing a single detection Returns: uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. """ # Create a display string (and color) for every box location, group any boxes # that correspond to the same location. box_to_display_str_map = collections.defaultdict(list) box_to_color_map = collections.defaultdict(str) box_to_instance_masks_map = {} box_to_instance_boundaries_map = {} box_to_keypoints_map = collections.defaultdict(list) box_to_keypoint_scores_map = collections.defaultdict(list) box_to_track_ids_map = {} if not max_boxes_to_draw: max_boxes_to_draw = boxes.shape[0] for i in range(boxes.shape[0]): if max_boxes_to_draw == len(box_to_color_map): break if scores is None or scores[i] > min_score_thresh: box = tuple(boxes[i].tolist()) if instance_masks is not None: box_to_instance_masks_map[box] = instance_masks[i] if instance_boundaries is not None: box_to_instance_boundaries_map[box] = instance_boundaries[i] if keypoints is not None: box_to_keypoints_map[box].extend(keypoints[i]) if keypoint_scores is not None: box_to_keypoint_scores_map[box].extend(keypoint_scores[i]) if track_ids is not None: box_to_track_ids_map[box] = track_ids[i] if scores is None: box_to_color_map[box] = groundtruth_box_visualization_color else: display_str = '' if not skip_labels: if not agnostic_mode: if classes[i] in six.viewkeys(category_index): class_name = category_index[classes[i]]['name'] else: class_name = 'N/A' display_str = str(class_name) if not skip_scores: if not display_str: display_str = '{}%'.format(round(100 * scores[i])) else: display_str = '{}: {}%'.format(display_str, round(100 * scores[i])) if not skip_track_ids and track_ids is not None: if not display_str: display_str = 'ID {}'.format(track_ids[i]) else: display_str = '{}: ID {}'.format( display_str, track_ids[i]) box_to_display_str_map[box].append(display_str) if agnostic_mode: box_to_color_map[box] = 'DarkOrange' elif track_ids is not None: prime_multipler = _get_multiplier_for_color_randomness() box_to_color_map[box] = STANDARD_COLORS[ (prime_multipler * track_ids[i]) % len(STANDARD_COLORS)] else: box_to_color_map[box] = STANDARD_COLORS[ classes[i] % len(STANDARD_COLORS)] # Draw all boxes onto image. for box, color in box_to_color_map.items(): ymin, xmin, ymax, xmax = box if instance_masks is not None: draw_mask_on_image_array(image, box_to_instance_masks_map[box], color=color, alpha=mask_alpha) if instance_boundaries is not None: draw_mask_on_image_array(image, box_to_instance_boundaries_map[box], color='red', alpha=1.0) draw_bounding_box_on_image_array( image, ymin, xmin, ymax, xmax, color=color, thickness=0 if skip_boxes else line_thickness, display_str_list=box_to_display_str_map[box], use_normalized_coordinates=use_normalized_coordinates) if keypoints is not None: keypoint_scores_for_box = None if box_to_keypoint_scores_map: keypoint_scores_for_box = box_to_keypoint_scores_map[box] draw_keypoints_on_image_array( image, box_to_keypoints_map[box], keypoint_scores_for_box, min_score_thresh=min_score_thresh, color=color, radius=line_thickness / 2, use_normalized_coordinates=use_normalized_coordinates, keypoint_edges=keypoint_edges, keypoint_edge_color=color, keypoint_edge_width=line_thickness // 2) return image
def visualize_boxes_and_labels_on_image_array( image, boxes, classes, scores, category_index, instance_boundaries=None, track_ids=None, use_normalized_coordinates=False, max_boxes_to_draw=20, min_score_thresh=.5, agnostic_mode=False, line_thickness=4, groundtruth_box_visualization_color='black', skip_scores=False, skip_labels=False, skip_track_ids=False): """Overlay labeled boxes on an image with formatted scores and label names. This function groups boxes that correspond to the same location and creates a display string for each detection and overlays these on the image. Note that this function modifies the image in place, and returns that same image. Args: image: uint8 numpy array with shape (img_height, img_width, 3) boxes: a numpy array of shape [N, 4] classes: a numpy array of shape [N]. Note that class indices are 1-based, and match the keys in the label map. scores: a numpy array of shape [N] or None. If scores=None, then this function assumes that the boxes to be plotted are groundtruth boxes and plot all boxes as black with no classes or scores. category_index: a dict containing category dictionaries (each holding category index `id` and category name `name`) keyed by category indices. instance_masks: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. instance_boundaries: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. keypoints: a numpy array of shape [N, num_keypoints, 2], can be None track_ids: a numpy array of shape [N] with unique track ids. If provided, color-coding of boxes will be determined by these ids, and not the class indices. use_normalized_coordinates: whether boxes is to be interpreted as normalized coordinates or not. max_boxes_to_draw: maximum number of boxes to visualize. If None, draw all boxes. min_score_thresh: minimum score threshold for a box to be visualized agnostic_mode: boolean (default: False) controlling whether to evaluate in class-agnostic mode or not. This mode will display scores but ignore classes. line_thickness: integer (default: 4) controlling line width of the boxes. groundtruth_box_visualization_color: box color for visualizing groundtruth boxes skip_scores: whether to skip score when drawing a single detection skip_labels: whether to skip label when drawing a single detection skip_track_ids: whether to skip track id when drawing a single detection Returns: uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. """ # Create a display string (and color) for every box location, group any boxes # that correspond to the same location. if len(boxes) < 1: print("No boxes") return image box_to_display_str_map = collections.defaultdict(list) box_to_color_map = collections.defaultdict(str) box_to_instance_boundaries_map = {} box_to_track_ids_map = {} if not max_boxes_to_draw: max_boxes_to_draw = boxes.shape[0] for i in range(min(max_boxes_to_draw, boxes.shape[0])): if scores is None or scores[i] > min_score_thresh: box = tuple(boxes[i].tolist()) if instance_boundaries is not None: box_to_instance_boundaries_map[box] = instance_boundaries[i] if track_ids is not None: box_to_track_ids_map[box] = track_ids[i] if scores is None: box_to_color_map[box] = groundtruth_box_visualization_color else: display_str = '' if not skip_labels: if not agnostic_mode: if classes[i] in six.viewkeys(category_index): class_name = category_index[classes[i]]['name'] else: class_name = 'N/A' display_str = str(class_name) if not skip_scores: if not display_str: display_str = '{}%'.format(int(100 * scores[i])) else: display_str = '{}: {}%'.format(display_str, int(100 * scores[i])) if not skip_track_ids and track_ids is not None: if not display_str: display_str = 'ID {}'.format(track_ids[i]) else: display_str = '{}: ID {}'.format( display_str, track_ids[i]) box_to_display_str_map[box].append(display_str) if agnostic_mode: box_to_color_map[box] = 'DarkOrange' elif track_ids is not None: prime_multipler = _get_multiplier_for_color_randomness() box_to_color_map[box] = STANDARD_COLORS[ (prime_multipler * track_ids[i]) % len(STANDARD_COLORS)] else: box_to_color_map[box] = STANDARD_COLORS[ classes[i] % len(STANDARD_COLORS)] # Draw all boxes onto image. for box, color in box_to_color_map.items(): #ymin, xmin, ymax, xmax = box #the boxes returned by the TF OD API's pretrained models returns boxes # in the format (ymin, xmin, ymax, xmax), normalized to [0, 1]. Thus, this # function needs to convert them to the format expected by WOD, namely # (center_x, center_y, width, height) center_x, center_y, width, height = box xmin = center_x - width / 2 xmax = center_x + width / 2 ymin = center_y - height / 2 ymax = center_y + height / 2 draw_bounding_box_on_image_array( image, ymin, xmin, ymax, xmax, color=color, thickness=line_thickness, display_str_list=box_to_display_str_map[box], use_normalized_coordinates=use_normalized_coordinates) return image
def _promote_kwargs(cls, kwargs, optional=None, read_only=None): if optional is None: optional = set() if not isinstance(optional, set): raise TypeError( "Optional kwargs must be given as a set, not {}".format(optional) ) if read_only is None: read_only = set() if not isinstance(read_only, set): raise TypeError( "Read only kwargs must be given as a set, not {}".format(read_only) ) class_name = cls.__name__ try: type_params = cls._type_params[0] except TypeError: raise TypeError( "Cannot instantiate a generic {}; the item types must be specified".format( class_name ) ) missing_args = ( six.viewkeys(type_params) - six.viewkeys(kwargs) - optional - read_only ) if len(missing_args) > 0: raise ProxyTypeError( "Missing required keyword arguments to {}: {}".format( class_name, ", ".join(six.moves.map(repr, missing_args)) ) ) provided_read_only_args = six.viewkeys(kwargs) & read_only if len(provided_read_only_args) > 0: raise ProxyTypeError( "Read only keyword argument to {}: {}".format( class_name, ", ".join(six.moves.map(repr, provided_read_only_args)) ) ) promoted_kwargs = {} for field_name, val in six.iteritems(kwargs): try: field_cls = type_params[field_name] except KeyError: raise ProxyTypeError( "{} has no field {!r}".format(class_name, field_name) ) if val is None or isinstance(val, NoneType) and field_name in optional: continue try: promoted_val = field_cls._promote(val) except ProxyTypeError as e: raise ProxyTypeError( "In field {!r} of {}, expected {}, but got {}: {}".format( field_name, class_name, field_cls, type(val), val ), e, ) promoted_kwargs[field_name] = promoted_val return promoted_kwargs
def update(self, tree): new_data = defaultdict(dict) new_hashes = {} reftest_nodes = [] old_files = defaultdict( set, {k: set(viewkeys(v)) for k, v in iteritems(self._data)}) changed = False reftest_changes = False for source_file in tree: rel_path = source_file.rel_path file_hash = source_file.hash is_new = rel_path not in self._path_hash hash_changed = False if not is_new: old_hash, old_type = self._path_hash[rel_path] old_files[old_type].remove(rel_path) if old_hash != file_hash: new_type, manifest_items = source_file.manifest_items() hash_changed = True else: new_type, manifest_items = old_type, self._data[old_type][ rel_path] if old_type == "reftest" and new_type != old_type: reftest_changes = True else: new_type, manifest_items = source_file.manifest_items() if new_type in ("reftest", "reftest_node"): reftest_nodes.extend(manifest_items) if is_new or hash_changed: reftest_changes = True elif new_type: new_data[new_type][rel_path] = set(manifest_items) new_hashes[rel_path] = (file_hash, new_type) if is_new or hash_changed: changed = True if reftest_changes or old_files["reftest"] or old_files["reftest_node"]: reftests, reftest_nodes, changed_hashes = self._compute_reftests( reftest_nodes) new_data["reftest"] = reftests new_data["reftest_node"] = reftest_nodes new_hashes.update(changed_hashes) else: new_data["reftest"] = self._data["reftest"] new_data["reftest_node"] = self._data["reftest_node"] if any(itervalues(old_files)): changed = True self._data = new_data self._path_hash = new_hashes return changed
def has_param_annotations(self): return bool(six.viewkeys(self.annotations) - {"return"})
def testSettings(self): users = self.users # Only admins should be able to get or set settings for method in ('GET', 'PUT', 'DELETE'): resp = self.request(path='/system/setting', method=method, params={ 'key': 'foo', 'value': 'bar' }, user=users[1]) self.assertStatus(resp, 403) # Only valid setting keys should be allowed obj = ['oauth', 'geospatial', '_invalid_'] resp = self.request(path='/system/setting', method='PUT', params={ 'key': 'foo', 'value': json.dumps(obj) }, user=users[0]) self.assertStatus(resp, 400) self.assertEqual(resp.json['field'], 'key') # Only a valid JSON list is permitted resp = self.request(path='/system/setting', method='GET', params={'list': json.dumps('not_a_list')}, user=users[0]) self.assertStatus(resp, 400) resp = self.request(path='/system/setting', method='PUT', params={'list': json.dumps('not_a_list')}, user=users[0]) self.assertStatus(resp, 400) # Set an invalid setting value, should fail resp = self.request(path='/system/setting', method='PUT', params={ 'key': SettingKey.PLUGINS_ENABLED, 'value': json.dumps(obj) }, user=users[0]) self.assertStatus(resp, 400) self.assertEqual(resp.json['message'], 'Required plugin _invalid_ does not exist.') # Set a valid value resp = self.request(path='/system/setting', method='PUT', params={ 'key': SettingKey.PLUGINS_ENABLED, 'value': json.dumps(['geospatial', 'oauth']) }, user=users[0]) self.assertStatusOk(resp) # We should now be able to retrieve it resp = self.request(path='/system/setting', method='GET', params={'key': SettingKey.PLUGINS_ENABLED}, user=users[0]) self.assertStatusOk(resp) self.assertEqual(set(resp.json), set(['geospatial', 'oauth'])) # We should now clear the setting resp = self.request(path='/system/setting', method='DELETE', params={'key': SettingKey.PLUGINS_ENABLED}, user=users[0]) self.assertStatusOk(resp) # Setting should now be () setting = Setting().get(SettingKey.PLUGINS_ENABLED) self.assertEqual(setting, []) # We should be able to ask for a different default setting = Setting().get(SettingKey.PLUGINS_ENABLED, default=None) self.assertEqual(setting, None) # We should also be able to put several setting using a JSON list resp = self.request(path='/system/setting', method='PUT', params={ 'list': json.dumps([ { 'key': SettingKey.PLUGINS_ENABLED, 'value': json.dumps(()) }, { 'key': SettingKey.COOKIE_LIFETIME, 'value': None }, ]) }, user=users[0]) self.assertStatusOk(resp) # We can get a list as well resp = self.request(path='/system/setting', method='GET', params={ 'list': json.dumps([ SettingKey.PLUGINS_ENABLED, SettingKey.COOKIE_LIFETIME, ]) }, user=users[0]) self.assertStatusOk(resp) self.assertEqual(resp.json[SettingKey.PLUGINS_ENABLED], []) # We can get the default values, or ask for no value if the current # value is taken from the default resp = self.request(path='/system/setting', method='GET', params={ 'key': SettingKey.PLUGINS_ENABLED, 'default': 'default' }, user=users[0]) self.assertStatusOk(resp) self.assertEqual(resp.json, []) resp = self.request(path='/system/setting', method='GET', params={ 'key': SettingKey.COOKIE_LIFETIME, 'default': 'none' }, user=users[0]) self.assertStatusOk(resp) self.assertEqual(resp.json, None) # But we have to ask for a sensible value in the default parameter resp = self.request(path='/system/setting', method='GET', params={ 'key': SettingKey.COOKIE_LIFETIME, 'default': 'bad_value' }, user=users[0]) self.assertStatus(resp, 400) # Try to set each key in turn to test the validation. First test with # am invalid value, then test with the default value. If the value # 'bad' won't trigger a validation error, the key should be present in # the badValues table. badValues = { SettingKey.BRAND_NAME: '', SettingKey.BANNER_COLOR: '', SettingKey.EMAIL_FROM_ADDRESS: '', SettingKey.CONTACT_EMAIL_ADDRESS: '', SettingKey.EMAIL_HOST: {}, SettingKey.SMTP_HOST: '', SettingKey.CORS_ALLOW_ORIGIN: {}, SettingKey.CORS_ALLOW_METHODS: {}, SettingKey.CORS_ALLOW_HEADERS: {}, } allKeys = dict.fromkeys(six.viewkeys(SettingDefault.defaults)) allKeys.update(badValues) for key in allKeys: resp = self.request(path='/system/setting', method='PUT', params={ 'key': key, 'value': badValues.get(key, 'bad') }, user=users[0]) self.assertStatus(resp, 400) resp = self.request(path='/system/setting', method='PUT', params={ 'key': key, 'value': json.dumps( SettingDefault.defaults.get(key, '')) }, user=users[0]) self.assertStatusOk(resp) resp = self.request( path='/system/setting', method='PUT', params={'list': json.dumps([{ 'key': key, 'value': None }])}, user=users[0]) self.assertStatusOk(resp) resp = self.request(path='/system/setting', method='GET', params={ 'key': key, 'default': 'default' }, user=users[0]) self.assertStatusOk(resp)
def validate_schema_and_query_ast(schema, query_ast): """Validate the supplied GraphQL schema and query_ast. This method wraps around graphql-core's validation to enforce a stricter requirement of the schema -- all directives supported by the compiler must be declared by the schema, regardless of whether each directive is used in the query or not. Args: schema: GraphQL schema object, created using the GraphQL library query_ast: abstract syntax tree representation of a GraphQL query Returns: list containing schema and/or query validation errors """ core_graphql_errors = validate(schema, query_ast) # The following directives appear in the core-graphql library, but are not supported by the # GraphQL compiler. unsupported_default_directives = frozenset([ frozenset([ "include", frozenset(["FIELD", "FRAGMENT_SPREAD", "INLINE_FRAGMENT"]), frozenset(["if"]), ]), frozenset([ "skip", frozenset(["FIELD", "FRAGMENT_SPREAD", "INLINE_FRAGMENT"]), frozenset(["if"]), ]), ]) # This directive is supported and ignored by the compiler, since it is meant as an indication # to the user that a field should not be used. supported_default_directive = frozenset([ frozenset([ "deprecated", frozenset(["FIELD_DEFINITION", "ENUM_VALUE"]), frozenset(["reason"]) ]) ]) # Directives expected by the graphql compiler. expected_directives = { frozenset([ directive.name, frozenset(directive.locations), frozenset(six.viewkeys(directive.args)), ]) for directive in DIRECTIVES } # Directives provided in the parsed graphql schema. actual_directives = { frozenset([ directive.name, frozenset(directive.locations), frozenset(six.viewkeys(directive.args)), ]) for directive in schema.get_directives() } # Directives missing from the actual directives provided. missing_directives = expected_directives - actual_directives if missing_directives: missing_message = (u"The following directives were missing from the " u"provided schema: {}".format(missing_directives)) core_graphql_errors.append(missing_message) # Directives that are not specified by the core graphql library. Note that Graphql-core # automatically injects default directives into the schema, regardless of whether # the schema supports said directives. Hence, while the directives contained in # unsupported_default_directives are incompatible with the graphql-compiler, we allow them to # be present in the parsed schema string. extra_directives = (actual_directives - expected_directives - unsupported_default_directives - supported_default_directive) if extra_directives: extra_message = ( u"The following directives were supplied in the given schema, but are not " u"not supported by the GraphQL compiler: {}".format( extra_directives)) core_graphql_errors.append(extra_message) return core_graphql_errors
def independent_sample(structure, increase_ops_probability=None, increase_filters_probability=None, hierarchical=True, name=None, temperature=1.0): """Generate a search space specification for an RL controller model. Each OneOf value is sampled independently of every other; hence the name. Args: structure: Nested data structure containing OneOf objects to search over. increase_ops_probability: Scalar float Tensor or None. If not None, we will randomly enable all possible operations instead of just the selected operations with this probability. increase_filters_probability: Scalar float Tensor or None. If not None, we will randomly use the largest possible filter sizes with this probability. hierarchical: Boolean. If true, the values of the outputs `sample_log_prob` and `entropy` will only take into account subgraphs that are enabled at the current training step. name: Optional name for the newly created TensorFlow scope. temperature: Positive scalar controlling the temperature to use when sampling from the RL controller. Returns: A tuple (new_structure, dist_info) where `new_structure` is a copy of `structure` annotated with mask tensors, and `dist_info` is a dictionary containing information about the sampling distribution which contains the following keys: - entropy: Scalar float Tensor, entropy of the current probability distribution. - logits_by_path: OrderedDict of rank-1 Tensors, sample-independent logits for each OneOf in `structure`. Names are derived from OneOf paths. - logits_by_tag: OrderedDict of rank-1 Tensors, sample-independent logits for each OneOf in `structure`. Names are derived from OneOf tags. - sample_log_prob: Scalar float Tensor, log-probability of the current sample associated with `new_structure`. """ with tf.variable_scope(name, 'independent_sample'): temperature = tf.convert_to_tensor(temperature, tf.float32) dist_info = { 'entropy': tf.constant(0, tf.float32), 'logits_by_path': collections.OrderedDict(), 'logits_by_tag': collections.OrderedDict(), 'sample_log_prob': tf.constant(0, tf.float32), } tag_counters = collections.Counter() entropies = dict() log_probs = dict() is_active = dict() def visit(tuple_path, oneof): """Visit a OneOf node in `structure`.""" string_path = '/'.join(map(str, tuple_path)) num_choices = len(oneof.choices) logits = tf.get_variable(name='logits/' + string_path, initializer=tf.initializers.zeros(), shape=[num_choices], dtype=tf.float32) logits = logits / temperature tag_name = '{:s}_{:d}'.format(oneof.tag, tag_counters[oneof.tag]) tag_counters[oneof.tag] += 1 dist_info['logits_by_path'][string_path] = logits dist_info['logits_by_tag'][tag_name] = logits dist = tfp.distributions.OneHotCategorical(logits=logits, dtype=tf.float32) entropies[tuple_path] = dist.entropy() sample_mask = dist.sample() sample_log_prob = dist.log_prob(sample_mask) if oneof.tag == basic_specs.OP_TAG: sample_mask, sample_log_prob = _replace_sample_with_probability( sample_mask, sample_log_prob, increase_ops_probability, tf.constant([1.0 / num_choices] * num_choices, tf.float32)) elif oneof.tag == basic_specs.FILTERS_TAG: # NOTE: While np.argmax() was originally designed to work with integer # filter sizes, it will also work with any object type that supports # "less than" and "greater than" operations. sample_mask, sample_log_prob = _replace_sample_with_probability( sample_mask, sample_log_prob, increase_filters_probability, tf.one_hot(np.argmax(oneof.choices), len(oneof.choices))) log_probs[tuple_path] = sample_log_prob for i in range(len(oneof.choices)): tuple_subpath = tuple_path + ('choices', i) is_active[tuple_subpath] = tf.greater(tf.abs(sample_mask[i]), 1e-6) return schema.OneOf(choices=oneof.choices, tag=oneof.tag, mask=sample_mask) new_structure = schema.map_oneofs_with_tuple_paths(visit, structure) assert six.viewkeys(entropies) == six.viewkeys(log_probs) for path in entropies: path_is_active = tf.constant(True) if hierarchical: for i in range(len(path) + 1): if path[:i] in is_active: path_is_active = tf.logical_and( path_is_active, is_active[path[:i]]) path_is_active = tf.cast(path_is_active, tf.float32) dist_info['entropy'] += entropies[path] * path_is_active dist_info['sample_log_prob'] += log_probs[path] * path_is_active return (new_structure, dist_info)
def countries(self): """A set-like object of the country codes supplied by this reader. """ return viewkeys(self._readers)
def _filter_kwds(kwds, function_or_method): argspec = getfullargspec(function_or_method) keys_ = argspec.args[len(argspec.args) - ( 0 if argspec.defaults is None else len(argspec.defaults)):] filtered_kwds = {k: kwds[k] for k in six.viewkeys(kwds) & keys_} return filtered_kwds
def visualize_boxes_and_labels_on_image_array( file_name, image, boxes, classes, scores, category_index, instance_masks=None, instance_boundaries=None, keypoints=None, track_ids=None, use_normalized_coordinates=False, max_boxes_to_draw=20, min_score_thresh=.5, agnostic_mode=False, line_thickness=4, groundtruth_box_visualization_color='black', skip_scores=False, skip_labels=False, skip_track_ids=False): """Overlay labeled boxes on an image with formatted scores and label names. This function groups boxes that correspond to the same location and creates a display string for each detection and overlays these on the image. Note that this function modifies the image in place, and returns that same image. Args: image: uint8 numpy array with shape (img_height, img_width, 3) boxes: a numpy array of shape [N, 4] classes: a numpy array of shape [N]. Note that class indices are 1-based, and match the keys in the label map. scores: a numpy array of shape [N] or None. If scores=None, then this function assumes that the boxes to be plotted are groundtruth boxes and plot all boxes as black with no classes or scores. category_index: a dict containing category dictionaries (each holding category index `id` and category name `name`) keyed by category indices. instance_masks: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. instance_boundaries: a numpy array of shape [N, image_height, image_width] with values ranging between 0 and 1, can be None. keypoints: a numpy array of shape [N, num_keypoints, 2], can be None track_ids: a numpy array of shape [N] with unique track ids. If provided, color-coding of boxes will be determined by these ids, and not the class indices. use_normalized_coordinates: whether boxes is to be interpreted as normalized coordinates or not. max_boxes_to_draw: maximum number of boxes to visualize. If None, draw all boxes. min_score_thresh: minimum score threshold for a box to be visualized agnostic_mode: boolean (default: False) controlling whether to evaluate in class-agnostic mode or not. This mode will display scores but ignore classes. line_thickness: integer (default: 4) controlling line width of the boxes. groundtruth_box_visualization_color: box color for visualizing groundtruth boxes skip_scores: whether to skip score when drawing a single detection skip_labels: whether to skip label when drawing a single detection skip_track_ids: whether to skip track id when drawing a single detection Returns: uint8 numpy array with shape (img_height, img_width, 3) with overlaid boxes. """ # Create a display string (and color) for every box location, group any boxes # that correspond to the same location. new_xml = False classes_names = {} box_to_display_str_map = collections.defaultdict(list) box_to_color_map = collections.defaultdict(str) box_to_instance_masks_map = {} box_to_instance_boundaries_map = {} box_to_keypoints_map = collections.defaultdict(list) box_to_track_ids_map = {} if not max_boxes_to_draw: max_boxes_to_draw = boxes.shape[0] for i in range(min(max_boxes_to_draw, boxes.shape[0])): if scores is None or scores[i] > min_score_thresh: box = tuple(boxes[i].tolist()) if instance_masks is not None: box_to_instance_masks_map[box] = instance_masks[i] if instance_boundaries is not None: box_to_instance_boundaries_map[box] = instance_boundaries[i] if keypoints is not None: box_to_keypoints_map[box].extend(keypoints[i]) if track_ids is not None: box_to_track_ids_map[box] = track_ids[i] if scores is None: box_to_color_map[box] = groundtruth_box_visualization_color else: display_str = '' if not skip_labels: if not agnostic_mode: if classes[i] in six.viewkeys(category_index): class_name = category_index[classes[i]]['name'] classes_names[i] = str(class_name) new_xml = True else: class_name = 'N/A' display_str = str(class_name) if not skip_scores: if not display_str: display_str = '{}%'.format(int(100 * scores[i])) else: display_str = '{}: {}%'.format(display_str, int(100 * scores[i])) if not skip_track_ids and track_ids is not None: if not display_str: display_str = 'ID {}'.format(track_ids[i]) else: display_str = '{}: ID {}'.format( display_str, track_ids[i]) box_to_display_str_map[box].append(display_str) if agnostic_mode: box_to_color_map[box] = 'DarkOrange' elif track_ids is not None: prime_multipler = _get_multiplier_for_color_randomness() box_to_color_map[box] = STANDARD_COLORS[ (prime_multipler * track_ids[i]) % len(STANDARD_COLORS)] else: box_to_color_map[box] = STANDARD_COLORS[ classes[i] % len(STANDARD_COLORS)] array_position = [] im_height, im_width, shape = image.shape # Draw all boxes onto image. for box, color in box_to_color_map.items(): ymin, xmin, ymax, xmax = box dict_position = {'xmin': 0, 'xmax': 0, 'ymin': 0, 'ymax': 0} dict_position['ymin'] = ymin * im_height dict_position['xmin'] = xmin * im_width dict_position['ymax'] = ymax * im_height dict_position['xmax'] = xmax * im_width array_position.append(dict_position) if instance_masks is not None: draw_mask_on_image_array(image, box_to_instance_masks_map[box], color=color) if instance_boundaries is not None: draw_mask_on_image_array(image, box_to_instance_boundaries_map[box], color='red', alpha=1.0) draw_bounding_box_on_image_array( image, ymin, xmin, ymax, xmax, color=color, thickness=line_thickness, display_str_list=box_to_display_str_map[box], use_normalized_coordinates=use_normalized_coordinates) if keypoints is not None: draw_keypoints_on_image_array( image, box_to_keypoints_map[box], color=color, radius=line_thickness / 2, use_normalized_coordinates=use_normalized_coordinates) if new_xml != False: xml = generate_xml.GenerateXml(array_position, im_width, im_height, classes_names, file_name) xml.gerenate_basic_structure() return image
def _update_config(self): self._set_config(six.viewkeys(self._action_handlers))
def create_evaluation_lists(self): # return reference status_OUT = None # declare variables citation_map = None baseline_list = None derived_binary_list = None derived_raw_list = None cutoff_value = None publication_id_list = None publication_id = None publication_dict = None data_set_map = None data_set_id_list = None data_set_id = None data_set_found_map = None baseline_score = -1 derived_score = -1 # get citation_map citation_map = self.get_citation_map() # init lists baseline_list = self.set_baseline_list([]) derived_binary_list = self.set_derived_binary_list([]) derived_raw_list = self.set_derived_raw_list([]) # cutoffs cutoff_value = self.get_cutoff() # so we can get publication ID list publication_id_list = list(six.viewkeys(citation_map)) publication_id_list.sort() # loop over publications, and then data sets within. for publication_id in publication_id_list: # DEBUG if (self.debug_flag == True): print("Publication ID: {}".format(publication_id)) #-- END DEBUG --# # get publication map publication_dict = citation_map.get(publication_id, None) # get the data set map and ID list. data_set_map = publication_dict.get(self.JSON_NAME_DATA_SET_MAP, None) data_set_id_list = list(six.viewkeys(data_set_map)) data_set_id_list.sort() # loop over data set ID list. for data_set_id in data_set_id_list: # DEBUG if (self.debug_flag == True): print("==> Data Set ID: {}".format(data_set_id)) #-- END DEBUG --# # get the data_set_found_map data_set_found_map = data_set_map.get(data_set_id, None) # get the scores. baseline_score = data_set_found_map.get( self.RESULT_TYPE_BASELINE, 0.0) derived_score = data_set_found_map.get( self.RESULT_TYPE_DERIVED, 0.0) # DEBUG if (self.debug_flag == True): print(" baseline: {}".format(baseline_score)) print(" derived.: {}".format(derived_score)) #-- END DEBUG --# # add them to the lists baseline_list.append(baseline_score) derived_raw_list.append(derived_score) if derived_score > cutoff_value: derived_binary_list.append(1.0) else: derived_binary_list.append(0.0) #-- END binary value assignment --# #-- END loop over data set IDs. --# #-- END loop over publication IDs. --# return status_OUT