def factory(cls, web3, class_name=None, **kwargs): kwargs['web3'] = web3 normalizers = { 'abi': normalize_abi, 'address': partial(normalize_address, kwargs['web3'].ens), 'bytecode': normalize_bytecode, 'bytecode_runtime': normalize_bytecode, } contract = PropertyCheckingFactory(class_name or cls.__name__, (cls, ), kwargs, normalizers=normalizers) setattr(contract, 'functions', ContractFunctions(contract.abi, contract.web3)) setattr(contract, 'events', ContractEvents(contract.abi, contract.web3)) setattr(contract, 'fallback', Contract.get_fallback_function(contract.abi, contract.web3)) return contract
def parse_stratagem_results_to_influx(measurement, fs_name, stratagem_results_json): parse_fns = { "size_distribution": partial(parse_size_distribution, measurement, fs_name, labels), "user_distribution": partial(parse_user_distribution, measurement, fs_name), } group_counters = stratagem_results_json.get("group_counters") return pipe( [], partial( reduce, lambda out, cur: out + [(cur.get("name"), cur.get("counters"))], group_counters), partial(filter, lambda xs: xs[0] not in ["warn_fids", "purge_fids"]), partial(map, lambda xs, parse_fns=parse_fns: parse_fns[xs[0]](xs[1])), partial(flatten), )
def printable_train_data(malform_data_dir, okform_data_dir, ids, extractor, feature_names, start, end=None, title_transform_func=make_capitalized_title, exclude_labels=None, exclude_word_positions=set([0])): """ Adapted to PULS requirement: - auxil file is read to get the additional prepreocessed features Parameters ------------ malform_data_dir: string the directory where the malformed data reside okform_data_dir: string the directory where the correctly formed data reside ids: list of string document ids extractor: FeatureExtractor the feature extractor feature_names: list of string the feature names start, end: int how many titles to extract title_transform_func: function funtion that accepts the title and transforms it into some badly capitalized version exclude_labels: iterable of str labels that we don't consider Returns ------------ Generator of str: each str is one sentence, each line in the str is one token in the sent """ feature_names += ['y'] # add the label feature name malform_data_dir = Path(malform_data_dir) # take care of this ["tickerSymbol",["NYSE","SKT"]] # /cs/taatto/home/hxiao/capitalization-recovery/corpus/puls-format-capitalized/3987E0BD03749C996A04B881079AD753.auxil clean_tag = (lambda t: t[0] if isinstance(t, list) else t) get_tokens = partial(map, partial(get_in, ['token'])) get_tags = partial(map, compose(clean_tag, partial(get_in, ['pos']))) get_lemmas = partial(map, partial(get_in, ['lemma'])) n_collected = 0 for i, id_ in enumerate(ids): if i < start: continue if i % 1000 == 0: logger.info("Collected %d" % n_collected) logger.info("Finished %d" % i) if end is not None and i >= end: logger.info("Reached %d. Terminate." % end) break try: malform_auxil_path = (malform_data_dir / Path(id_)).with_suffix('.auxil') with malform_auxil_path.open(encoding='utf8') as f: logger.debug('processing: {}'.format(id_)) # to get the last line lines = f.readlines() if len(lines) == 0: raise EmptyFileError( 'auxil file empty: {}'.format(malform_auxil_path)) l = lines[-1] data = json.loads(l.strip()) okform_auxil_path = str( (okform_data_dir / Path(id_)).with_suffix('.auxil')) okform_paf_path = str( (okform_data_dir / Path(id_)).with_suffix('.paf')) good_title_sents, body_sents = separate_title_from_body( okform_auxil_path, okform_paf_path) # extract the tokens doc = [[t['token'] for t in sent['features']] for sent in body_sents] good_title_sents = list(good_title_sents) bad_title_sents = data['sents'] if not isinstance(bad_title_sents, list): raise InvalidTitleError( 'bad_title_sents not a list: {}'.format( bad_title_sents)) # we only consider headline that contains only ONE sentence if (len(good_title_sents) == 1 and len(bad_title_sents) == 1): good_sent = good_title_sents[0] bad_sent = bad_title_sents[0] good_title_tokens = get_tokens(good_sent['features']) bad_title_tokens = get_tokens(bad_sent['features']) # some validity checking if len(good_title_tokens) != len(bad_title_tokens): raise TitleInconsistencyError('{}\n{}'.format( good_title_tokens, bad_title_tokens)) good_title_tokens_lower = map(lambda s: s.lower(), good_title_tokens) bad_title_tokens_lower = map(lambda s: s.lower(), bad_title_tokens) if (good_title_tokens_lower != bad_title_tokens_lower): raise TitleInconsistencyError('{}\n{}'.format( good_title_tokens_lower, bad_title_tokens_lower)) tags = get_tags(bad_sent['features']) lemmas = get_lemmas(bad_sent['features']) # tag validity checking for tag in tags: if not (tag is None or isinstance(tag, basestring)): raise InvalidTitleError( '{}: tag {} not string'.format(id_, tag)) # get malformed title tokens words = convert_to_trainable_format(good_title_tokens, title_transform_func, extractor, doc=doc, pos=tags, lemma=lemmas) # format the features in the required form res = unicode() for i, word in enumerate(words): if (i not in exclude_word_positions and exclude_labels and word['y'] not in exclude_labels): word_feature_str = u'\t'.join([ unicode(word[feature_name]) for feature_name in feature_names ]) res += word_feature_str + '\n' n_collected += 1 yield id_, res else: raise TitleInconsistencyError( '# of title sentences more than 1: {}'.format(id_)) except (IOError, TitleInconsistencyError, InvalidTitleError, EmptyFileError): logger.debug(traceback.format_exc()) continue except: logger.error(traceback.format_exc()) continue
size_distribution_name_table = { "size < 1m": "less_than_1m", "1m <= size < 1g": "greater_than_equal_1m_less_than_1g", "size >= 1g": "greater_than_equal_1g", "size >= 1t": "greater_than_equal_1t", } labels = { "less_than_1m": "<\\\ 1\\\ Mib", "greater_than_equal_1m_less_than_1g": ">\\\=\\\ 1\\\ Mib\\\,\\\ <\\\ 1\\\ GiB", "greater_than_equal_1g": ">\\\=\\\ 1\\\ GiB", "greater_than_equal_1t": ">\\\=\\\ 1\\\ TiB", } filter_out_other_counter = partial( filter, lambda counter: counter.get("name").lower() != "other") def flatten(xs): return [item for l in xs for item in l] def tuple_to_equals(xs): return "{}={}".format(*xs) def create_stratagem_influx_point(measurement, tags, fields): return "{},{} {} {}".format(measurement, ",".join(map(tuple_to_equals, tags)), ",".join(map(tuple_to_equals, fields)), "").rstrip()
from chroma_api.authentication import AnonymousAuthentication, PatchedDjangoAuthorization from chroma_core.services.job_scheduler.job_scheduler_client import JobSchedulerClient from tastypie.validation import Validation from chroma_api.validation_utils import validate from chroma_api.utils import custom_response, dehydrate_command, StatefulModelResource from chroma_core.models import ( StratagemConfiguration, ManagedHost, ManagedMdt, ManagedTargetMount, ManagedFilesystem, Command, get_fs_id_from_identifier, ) get_bundle_int_val = compose(partial(flip, int, 10), str) # Postgres can store numbers up to 8 bytes (+9,223,372,036,854,775,807). These values will ultimately be passed back to the web # interface, where they will be used by javascript. Therefore, the maximum size of an integer is limited to the # maximum size allowed by javascript, which is Number.MAX_SAFE_INTEGER (9,007,199,254,740,991). MAX_SAFE_INTEGER = 9007199254740991 def get_duration_type(duration_key): return duration_key.split("_")[0].capitalize() def get_duration(duration_key, bundle): try: duration = bundle.data.get(duration_key) and get_bundle_int_val( bundle.data.get(duration_key))
from collections import namedtuple from typing import Iterable from toolz.functoolz import partial import premailer import pystache from wedding.model import Party Invitation = namedtuple('Invitation', ['recipient', 'subject', 'message']) def _url_sub(field: str, value: str, url: str) -> str: return url.replace(field, urllib.parse.quote(value)) _substitute_party = partial(_url_sub, '{partyId}') _substitute_guest = partial(_url_sub, '{guestId}') def build_invitations(invitation_template: str, body_template: str, envelope_template: str, party: Party) -> Iterable[Invitation]: invitation_url = _substitute_party(party.id, invitation_template) envelope_url = _substitute_party(party.id, envelope_template) return (Invitation( guest.email, 'Jenny and Jesse are Getting Married!', premailer.transform( pystache.render( body_template, { 'partyName': party.title,
def get_latest_testrun(testrun_id): tr = get_latest_test_run(testrun_id) valid = partial(only_pretty_polarion, tr) fields = filter(valid, dir(tr)) for attr in fields: print_kv(tr, attr)
def __payload(self, event): body = json.loads(event['body']) return option.cata(partial(compose(list, map), self.__codec.decode), lambda: self.__codec.decode(body))( body.get('items'))