def plot_experiment(experiment_spec, experiment_df): ''' Plot the variable specs vs fitness vector of an experiment, where each point is a trial. ref colors: https://plot.ly/python/heatmaps-contours-and-2dhistograms-tutorial/#plotlys-predefined-color-scales ''' y_cols = ['fitness'] + FITNESS_COLS x_cols = ps.difference(experiment_df.columns.tolist(), y_cols) fig = viz.tools.make_subplots(rows=len(y_cols), cols=len(x_cols), shared_xaxes=True, shared_yaxes=True) fitness_sr = experiment_df['fitness'] min_fitness = fitness_sr.values.min() max_fitness = fitness_sr.values.max() for row_idx, y in enumerate(y_cols): for col_idx, x in enumerate(x_cols): x_sr = experiment_df[x] guard_cat_x = x_sr.astype(str) if x_sr.dtype == 'object' else x_sr trace = viz.go.Scatter( y=experiment_df[y], yaxis=f'y{row_idx+1}', x=guard_cat_x, xaxis=f'x{col_idx+1}', showlegend=False, mode='markers', marker={ 'symbol': 'circle-open-dot', 'color': experiment_df['fitness'], 'opacity': 0.5, # dump first quarter of colorscale that is too bright 'cmin': min_fitness - 0.25 * (max_fitness - min_fitness), 'cmax': max_fitness, 'colorscale': 'YIGnBu', 'reversescale': True }, ) fig.append_trace(trace, row_idx + 1, col_idx + 1) fig.layout[f'xaxis{col_idx+1}'].update(title='<br>'.join(ps.chunk(x, 20)), zerolinewidth=1, categoryarray=sorted(guard_cat_x.unique())) fig.layout[f'yaxis{row_idx+1}'].update(title=y, rangemode='tozero') fig.layout.update(title=f'experiment graph: {experiment_spec["name"]}', width=max(600, len(x_cols) * 300), height=700) viz.plot(fig) return fig
def add_identical_offers(collection_name, offer_limit, n_highest, provenance=None): collection = get_collection("mpnoffers") model = load_model_from_s3(collection_name) now = datetime.now() mongo_filter = { "validThrough": { "$gt": now }, "siteCollection": collection_name } if provenance: mongo_filter["provenance"] = provenance offers = collection.find( mongo_filter, projection=MONGO_PROJECTION, limit=offer_limit, ) offers_list = list(offers) result = list( add_identical_offers_to_batch(batch, model, collection_name, n_highest) for batch in pydash.chunk(offers_list, CHUNK_SIZE)) return result
def plot_experiment(experiment_spec, experiment_df): ''' Plot the variable specs vs fitness vector of an experiment, where each point is a trial. ref colors: https://plot.ly/python/heatmaps-contours-and-2dhistograms-tutorial/#plotlys-predefined-color-scales ''' y_cols = ['fitness'] + FITNESS_COLS x_cols = ps.difference(experiment_df.columns.tolist(), y_cols) fig = viz.tools.make_subplots(rows=len(y_cols), cols=len(x_cols), shared_xaxes=True, shared_yaxes=True) fitness_sr = experiment_df['fitness'] min_fitness = fitness_sr.values.min() max_fitness = fitness_sr.values.max() for row_idx, y in enumerate(y_cols): for col_idx, x in enumerate(x_cols): x_sr = experiment_df[x] guard_cat_x = x_sr.astype(str) if x_sr.dtype == 'object' else x_sr trace = viz.go.Scatter( y=experiment_df[y], yaxis=f'y{row_idx+1}', x=guard_cat_x, xaxis=f'x{col_idx+1}', showlegend=False, mode='markers', marker={ 'symbol': 'circle-open-dot', 'color': experiment_df['fitness'], 'opacity': 0.5, # dump first quarter of colorscale that is too bright 'cmin': min_fitness - 0.50 * (max_fitness - min_fitness), 'cmax': max_fitness, 'colorscale': 'YIGnBu', 'reversescale': True }, ) fig.append_trace(trace, row_idx + 1, col_idx + 1) fig.layout[f'xaxis{col_idx+1}'].update(title='<br>'.join(ps.chunk(x, 20)), zerolinewidth=1, categoryarray=sorted(guard_cat_x.unique())) fig.layout[f'yaxis{row_idx+1}'].update(title=y, rangemode='tozero') fig.layout.update(title=f'experiment graph: {experiment_spec["name"]}', width=max(600, len(x_cols) * 300), height=700) viz.plot(fig) return fig
def read_spec_and_run(spec_file, spec_name, lab_mode): '''Read a spec and run it in lab mode''' logger.info( f'Running lab spec_file:{spec_file} spec_name:{spec_name} in mode:{lab_mode}' ) if lab_mode in TRAIN_MODES: spec = spec_util.get(spec_file, spec_name) else: # eval mode if '@' in lab_mode: lab_mode, prename = lab_mode.split('@') spec = spec_util.get_eval_spec(spec_file, spec_name, prename) else: spec = spec_util.get(spec_file, spec_name) if 'spec_params' not in spec: run_spec(spec, lab_mode) else: # spec is parametrized; run them in parallel param_specs = spec_util.get_param_specs(spec) num_pro = spec['meta']['param_spec_process'] # can't use Pool since it cannot spawn nested Process, which is needed for VecEnv and parallel sessions. So these will run and wait by chunks workers = [ mp.Process(target=run_spec, args=(spec, lab_mode)) for spec in param_specs ] for chunk_w in ps.chunk(workers, num_pro): for w in chunk_w: w.start() for w in chunk_w: w.join()
def process_in_chunks(keys, process, chunk_size=10000): docs = [] chunks = py_.chunk(keys, chunk_size) pbar = tqdm(total=len(keys)) for chunk in chunks: docs.extend(process(chunk)) pbar.update(len(chunk)) pbar.close() return docs
def plot_experiment(experiment_spec, experiment_df, metrics_cols): ''' Plot the metrics vs. specs parameters of an experiment, where each point is a trial. ref colors: https://plot.ly/python/heatmaps-contours-and-2dhistograms-tutorial/#plotlys-predefined-color-scales ''' y_cols = metrics_cols x_cols = ps.difference(experiment_df.columns.tolist(), y_cols + ['trial']) fig = subplots.make_subplots(rows=len(y_cols), cols=len(x_cols), shared_xaxes=True, shared_yaxes=True, print_grid=False) strength_sr = experiment_df['strength'] min_strength, max_strength = strength_sr.min(), strength_sr.max() for row_idx, y in enumerate(y_cols): for col_idx, x in enumerate(x_cols): x_sr = experiment_df[x] guard_cat_x = x_sr.astype(str) if x_sr.dtype == 'object' else x_sr trace = go.Scatter( y=experiment_df[y], yaxis=f'y{row_idx+1}', x=guard_cat_x, xaxis=f'x{col_idx+1}', showlegend=False, mode='markers', marker={ 'symbol': 'circle-open-dot', 'color': strength_sr, 'opacity': 0.5, # dump first portion of colorscale that is too bright 'cmin': min_strength - 0.5 * (max_strength - min_strength), 'cmax': max_strength, 'colorscale': 'YlGnBu', 'reversescale': False }, ) fig.add_trace(trace, row_idx + 1, col_idx + 1) fig.update_xaxes(title_text='<br>'.join(ps.chunk(x, 20)), zerolinewidth=1, categoryarray=sorted(guard_cat_x.unique()), row=len(y_cols), col=col_idx + 1) fig.update_yaxes(title_text=y, rangemode='tozero', row=row_idx + 1, col=1) fig.layout.update(title=f'experiment graph: {experiment_spec["name"]}', width=100 + 300 * len(x_cols), height=200 + 300 * len(y_cols)) plot(fig) graph_prepath = experiment_spec['meta']['graph_prepath'] save_image(fig, f'{graph_prepath}_experiment_graph.png') # save important graphs in prepath directly prepath = experiment_spec['meta']['prepath'] save_image(fig, f'{prepath}_experiment_graph.png') return fig
def get_latex_im_body(envs): latex_ims = [] for env in envs: env = guard_env_name(env) latex_im = f'\subfloat{{\includegraphics[width=1.22in]{{images/{env}_multi_trial_graph_mean_returns_ma_vs_frames.png}}}}' latex_ims.append(latex_im) im_matrix = ps.chunk(latex_ims, 4) latex_im_body = '\\\\\n'.join([' & \n'.join(row) for row in im_matrix]) return latex_im_body
def average_implied_volatility(self, tenor: str, strike_reference: EdrDataReference, relative_strike: Real, *, real_time: bool = False, request_id: Optional[str] = None, source: Optional[str] = None) -> pd.Series: """ Weighted average implied volatility :param tenor: relative date representation of expiration date e.g. 1m :param strike_reference: reference for strike level :param relative_strike: strike relative to reference :param real_time: whether to retrieve intraday data instead of EOD :param request_id: service request id, if any :param source: name of function caller :return: time series of the average implied volatility """ if real_time: raise NotImplementedError('real-time basket implied vol not implemented') ref_string, relative_strike = preprocess_implied_vol_strikes_eq(VolReference(strike_reference.value), relative_strike) log_debug(request_id, _logger, 'where tenor=%s, strikeReference=%s, relativeStrike=%s', tenor, ref_string, relative_strike) where = dict(tenor=tenor, strikeReference=ref_string, relativeStrike=relative_strike) tasks = [] for i, chunked_assets in enumerate(chunk(self.get_marquee_ids(), 3)): query = GsDataApi.build_market_data_query( chunked_assets, QueryType.IMPLIED_VOLATILITY, where=where, source=source, real_time=real_time) tasks.append(partial(GsDataApi.get_market_data, query, request_id)) results = ThreadPoolManager.run_async(tasks) vol_data = pd.concat(results) actual_weights = self.get_actual_weights(request_id) # Add in today's data if not real_time and DataContext.current.end_date >= datetime.date.today(): vol_data = ts.append_last_for_measure(vol_data, self.get_marquee_ids(), QueryType.IMPLIED_VOLATILITY, where, source=source, request_id=request_id) vol_data.index.rename('date', inplace=True) vols = vol_data.pivot_table('impliedVolatility', ['date'], 'assetId') vols.reindex(self.get_marquee_ids(), axis=1) vols.index.name = None # Necessary when current values appended - set weights index to match vols index actual_weights = actual_weights.reindex(vols.index).fillna(method='pad') return actual_weights.mul(vols).sum(axis=1, skipna=False)
def send_message(self) -> List[SlackResponse]: parent_message = self.client.chat_postMessage(channel=self.channel, text="*Snow Peak 入荷情報*") # blocks are no more than 50 items allowed. blocks_list = chunk(self.blocks, 50) parent_message_ts = parent_message['ts'] for blocks in blocks_list: message = Message(channel=self.channel, blocks=blocks) self.client.chat_postMessage(**message, thread_ts=parent_message_ts) return parent_message
def bulk_compliant_status(cls, imeis): """Method to get IMEIs status from CORE in bulk.""" non_compliant_list, compliant_list = [], [] imei_list = pydash.flatten_deep(imeis) imei_chunks = pydash.chunk(imei_list, 1000) url = cls.core_api_v2 + '/imei-batch' for chunk in imei_chunks: response = requests.post(url, json={'imeis': chunk}) compliant, non_compliant = Utilities.get_bulk_compliant_imeis(response) non_compliant_list = pydash.interleave(non_compliant_list, non_compliant) compliant_list = pydash.interleave(compliant_list, compliant) return compliant_list, non_compliant_list
def generate_interpolation_frames(trainer, latents_low, latents_high, num_frames, batch_size=50, device="cuda:0"): trainer.GAN.eval() num_rows = 1 latent_dim = trainer.GAN.latent_dim image_size = trainer.GAN.image_size # latents and noise #latents_low = torch.randn(num_rows ** 2, latent_dim).cuda(self.rank) #latents_high = torch.randn(num_rows ** 2, latent_dim).cuda(self.rank) ratios = torch.linspace(0., 1., num_frames) chunks = list(py_.chunk(ratios, size=batch_size)) latents_low = latents_low.unsqueeze(0) latents_high = latents_high.unsqueeze(0) ret = None for i, chunk_ratios in enumerate(chunks): batch_latents = [] for j, ratio in enumerate(chunk_ratios): if (i + j) == 0: interp_latents = latents_low elif (i + j) == len(ratios) - 1: interp_latents = latents_high else: interp_latents = slerp(ratio, latents_low, latents_high) batch_latents.append(interp_latents) stacked_latents = torch.vstack(batch_latents).to(device) generated_images = trainer.generate_truncated( trainer.GAN.GE, stacked_latents).cpu() if ret is None: ret = generated_images else: ret = ret.stack(generated_images, axis=0) return ret
def test_chunk(case, expected): assert _.chunk(*case) == expected
def list_case_books(): casebooks = chunk(CaseBook.select(), 4) return render_template('list_case_books.html', casebooks=casebooks)
async def command_clear(message, client, *args, **kwargs): if message.content.startswith('!clear'): logger.info('Run command !clear') # If no arguments were found for the command, clear the entire channel command_args = kwargs.pop('command_args', []) if not command_args: # Get all the messages in the channel messages = client.logs_from(message.channel) messages = [m async for m in messages] # Try bulk deleting the messages in chunks of 100 for chunk in pydash.chunk(messages, 100): await client.delete_messages(chunk) messages = pydash.difference(messages, chunk) # Individually delete each message which could not be deleted # via bulk for message_ in messages: await client.delete_message(message_) return True # If arguments were found for the command, do different things else: command_args = list(command_args) # If there is more than one argument, do nothing if len(command_args) > 1: return True # Save the first argument to a variable to readability clear_amount = command_args[0] # If the first argument is an integer, only clear so many messages if type(clear_amount) is int: # TODO: Support retrieving more than 100 messages # If clear_amount is greater than 100, send an apology if clear_amount > 100: await client.send_message( message.channel, 'I can only clear 100 messages at a time.' ) return True # Get the messages to be deleted messages = client.logs_from( message.channel, limit=clear_amount) messages = [m async for m in messages] # Try bulk deleting the messages in chunks of 100 for chunk in pydash.chunk(messages, 100): await client.delete_messages(chunk) messages = pydash.difference(messages, chunk) # Individually delete each message which could not be deleted # via bulk for message_ in messages: await client.delete_message(message_) return True return False