def _validate_positive_integer(val, argname): try: return int(val) except ValueError: raise exception.Error("%s should be an integer" % argname) if val < 0: raise exception.Error("%s should be a positive integer" % argname)
def read_all_lines(config, sentences): source_to_num, _, _, _ = load_dictionaries(config) lines = [] for sent in sentences: line = [] for w in sent.strip().split(): if config.factors == 1: w = [source_to_num[0][w] if w in source_to_num[0] else 1] else: w = [ source_to_num[i][f] if f in source_to_num[i] else 1 for (i, f) in enumerate(w.split('|')) ] if len(w) != config.factors: raise exception.Error( 'Expected {0} factors, but input word has {1}\n'. format(config.factors, len(w))) line.append(w) lines.append(line) lines = numpy.array(lines) lengths = numpy.array(map(lambda l: len(l), lines)) lengths = numpy.array(lengths) idxs = lengths.argsort() lines = lines[idxs] #merge into batches batches = [] for i in range(0, len(lines), config.valid_batch_size): batch = lines[i:i + config.valid_batch_size] batches.append(batch) return batches, idxs
def api_get_long(self, req, target, last_c='', interval=1): req['format'] = 'json' self.status = '' last_cont = {'continue': last_c} while True: c_req = req.copy() c_req.update(last_cont) try: result = self.s.get(lang_api, params=c_req, headers=headers).json() except: print('api_get_long: Try again after %d sec...' % interval) print(req, target, last_c) time.sleep(interval) for t in self.api_get_long(req, target, last_cont['continue'], interval * 2): yield t break if 'error' in result: print(req, target, last_c) raise exception.Error(result['error']) if 'warnings' in result: print('api_get_long: Warning: %s' % result['warnings']) print(req, target, last_c) if target in result: yield result[target] if 'continue' not in result: break last_cont = result['continue']
def delete_stack_user(self, user_id): user = self.client.users.get(user_id) # FIXME (shardy) : need to test, do we still need this retry logic? # Copied from user.py, but seems like something we really shouldn't # need to do, no bug reference in the original comment (below)... # tempory hack to work around an openstack bug. # seems you can't delete a user first time - you have to try # a couple of times - go figure! tmo = eventlet.Timeout(10) status = 'WAITING' reason = 'Timed out trying to delete user' try: while status == 'WAITING': try: user.delete() status = 'DELETED' except Exception as ce: reason = str(ce) logger.warning("Problem deleting user %s: %s" % (user_id, reason)) eventlet.sleep(1) except eventlet.Timeout as t: if t is not tmo: # not my timeout raise else: status = 'TIMEDOUT' finally: tmo.cancel() if status != 'DELETED': raise exception.Error(reason)
def validate_pid(pid): '''Check if we've received a valid PID''' p = int(pid) if p >= PID_MIN and p <= PID_MAX: return True else: raise exception.Error('PID out of range %d-%d: %d' % (PID_MIN, PID_MAX, p))
def __init__(self, models, configs, beam_size, beta): """Sets some things up then calls _beam_search() to do the real work. Args: models: a sequence of RNN or Transformer objects. configs: a sequence of model configs (argparse.Namespace objects). beam_size: an integer specifying the beam width. beta: a float between 0.0 and 1.0 specifying the value of bias """ self._models = models self._configs = configs self._beam_size = beam_size self.beta = beta with tf.name_scope('beam_search'): # Define placeholders. self.inputs = sampler_inputs.SamplerInputs() # Create model adapters to get a consistent interface to # Transformer and RNN models. model_adapters = [] for i, (model, config) in enumerate(zip(models, configs)): with tf.name_scope('model_adapter_{}'.format(i)) as scope: if config.model_type == 'transformer': adapter = transformer_inference.ModelAdapter( model, config, scope) else: assert config.model_type == 'rnn' adapter = rnn_inference.ModelAdapter( model, config, scope) model_adapters.append(adapter) # Check that individual models are compatible with each other. vocab_sizes = [a.target_vocab_size for a in model_adapters] if len(set(vocab_sizes)) > 1: raise exception.Error('Cannot ensemble models with different ' 'target vocabulary sizes') target_vocab_size = vocab_sizes[0] # Build the graph to do the actual work. sequences, scores = _beam_search( model_adapters=model_adapters, beam_size=beam_size, batch_size_x=self.inputs.batch_size_x, max_translation_len=self.inputs.max_translation_len, normalization_alpha=self.inputs.normalization_alpha, vocab_size=target_vocab_size, eos_id=0, last_translation=self.inputs.last_translation, last_translation_len=self.inputs.last_translation_len, beta=self.beta) # print(sequences, scores) self._outputs = sequences, scores
def client_login(self, pwd=None): data = {'logintoken': self.query_tokens('login')['logintoken']} self.tokens['login'] = data['logintoken'] data['username'] = bot_name data['password'] = pwd if pwd is not None else self.pwd data['action'] = 'clientlogin' data['loginreturnurl'] = 'https://zh.wikipedia.org/' self.pwd = pwd result = self.api_post(data)['clientlogin'] if 'error' in result: raise exception.Error(result['error']) if 'warnings' in result: print('Warning: %s' % result['warnings']) if result.get('status') == 'PASS': self.tokens['csrf'] = self.query_tokens('csrf').get('csrftoken') return None else: raise exception.Error(result.get('message', 'Login Failed'))
def api_get(self, req, target, interval=1): req['format'] = 'json' self.status = '' try: result = self.s.get(lang_api, params=req, headers=headers).json() except: # json.decoder.JSONDecodeError: print('api_get: Try again after %d sec...' % interval) print(req, target) time.sleep(interval) return self.api_get(req, target, interval * 2) # print(json.dumps(result, indent=4, sort_keys=True)) if 'error' in result: print(req, target) raise exception.Error(result['error']) if 'warnings' in result: print('api_get: Warning: %s' % result['warnings']) print(req, target) return result.get(target)
def read_all_lines(config, sentences, batch_size): source_to_num, _, _, _ = load_dictionaries(config) if config.source_vocab_sizes != None: assert len(config.source_vocab_sizes) == len(source_to_num) for d, vocab_size in zip(source_to_num, config.source_vocab_sizes): if vocab_size != None and vocab_size > 0: for key, idx in d.items(): if idx >= vocab_size: del d[key] lines = [] for sent in sentences: line = [] for w in sent.strip().split(): if config.factors == 1: w = [source_to_num[0][w] if w in source_to_num[0] else 1] else: w = [ source_to_num[i][f] if f in source_to_num[i] else 1 for (i, f) in enumerate(w.split('|')) ] if len(w) != config.factors: raise exception.Error( 'Expected {0} factors, but input word has {1}\n'. format(config.factors, len(w))) line.append(w) lines.append(line) lines = numpy.array(lines) lengths = numpy.array(map(lambda l: len(l), lines)) lengths = numpy.array(lengths) idxs = lengths.argsort() lines = lines[idxs] #merge into batches batches = [] for i in range(0, len(lines), batch_size): batch = lines[i:i + batch_size] batches.append(batch) return batches, idxs
def pid(self): if self._pid is not None and validate_pid(self._pid): return int(self._pid) else: raise exception.Error('Invalid PID: %d' % self._pid)