def _urlretrieve(url, filename): for _ in range(3): try: return urlretrieve(url, filename=filename) except Exception as e: logger.info('Download xls error: type <>, msg <>, url <>, filename <>'.format( e.__class__, e, url, filename))
def parse_ext_args(): """Parse external arguments. Returns ------- parsed_args : :class:`argparse.Namespace` Parsed program arguments. """ parser = ArgumentParser("luminosity-function-fitting") parser.add_argument( '--task', type=str.lower, choices=['sample', 'retrieve', 'resume'], help="task to perform: sample; retrieve samples; or resume sampling") parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus']) parser.add_argument('--prescription', type=str.lower, choices=['native', 'poisson', 'symlg']) parser.add_argument( '--jump', action='store_true', help="change zeus sampler jump condition; used for multimodal cases") parser.add_argument('--use-prior', action='store_true') parser.add_argument('--use-constraint', action='store_true') parser.add_argument('--nonautostop', action='store_true', help="do not stop sampling after convergence detected") parser.add_argument('--quiet', action='store_false') parser.add_argument('--model-name', type=str, default='quasar_PLE') parser.add_argument('--data-files', type=str, nargs=2, default=None) parser.add_argument('--prior-file', type=str, default=None) parser.add_argument('--fixed-file', type=str, default=None) parser.add_argument('--chain-file', type=str, default=None) parser.add_argument('--nwalkers', type=int, default=100) parser.add_argument('--nsteps', type=int, default=10000) parser.add_argument('--thinby', type=int, default=1) parser.add_argument('--skip-chains', type=int, default=None, help="only plot chain traces every interval") parser.add_argument('--burnin', type=int, default=None) parser.add_argument('--reduce', type=int, default=None) parsed_args = parser.parse_args() parsed_args.chain_file = parsed_args.chain_file.rstrip('.h5') parsed_args.chain_file += "_{}_{}_{}_by{}.h5".format( parsed_args.prescription, parsed_args.nwalkers, sci_notation(parsed_args.nsteps), parsed_args.thinby) logger.info("\n---Program configuration---\n%s\n", pformat(vars(parsed_args)).lstrip("{").rstrip("}")) return parsed_args
def publish(self): """Method which will publish the deposition linked with the id. .. warning: After publishing it is not possible to delete anything. Exception --------- ZenodoException raise if token not define (token = None) or if connection return status >= 400 """ # Test if token was defined self._verify_token() publish_url = (self.depositions_url + '/{}/actions/publish'.format(self.deposition_id)) request = requests.post(publish_url, params={'access_token': self.token}) self.status_code = request.status_code self._check_status_code(self.status_code) # Test that everything went as expected if self.status_code == 202: logger.info('Deposition id: {}'.format(self.deposition_id)) logger.debug('Status code: {}'.format(self.status_code)) return
def set_metadata(self, metadata): """Method to set metadata from a file or a dictionary. Parameter --------- metadata: str, dict - if it is a string it is the name of file which contains zenodo metadata (yaml format). It will read the metadata and set the attribute _metadata to it - if dictionary, it set the attribute _metadata to it Attribute --------- _metadata: dict dictionary which contains Zenodo metadata """ if type(metadata) is str: logger.info('Metadata provided by file: {}'.format(metadata)) self._metadata = self._read_metadata(metadata) elif type(metadata) is dict: logger.info('Metadata provided by dictionary.') if self._metadata is None: self._metadata = metadata else: self._metadata.update(metadata) elif metadata is None: logger.warn('Metadata are empty') self._metadata = metadata self._check_minimal()
def validate(self): """Method which is verifying that the metadata does have the correct type and if the dependencies are respected. The dependencies have to be check because the value of a metadata can implied the presence of another one. For example, if *upload_type* (which is a necessary metadata) has the value *publication* that implied the presence of the metadata *publication_type*. """ # Check if the minimal set of information are provided self._check_minimal() # Check validity of the license (if open or embargoed) self._check_license_availability() try: jsonschema.validate(self._metadata, self._schema) except jsonschema.exceptions.ValidationError as err: error = 'ValidationError: {}'.format(err.message) logger.error(error) raise ZenodoMetadataException(error) logger.info('Metadata should be ok to use for upload')
def extract_biases(lumfunc_param_chain, pool=None): """Extract relativistic biases from a luminosity function parameter chain. Parameters ---------- lumfunc_model_chain : :class:`numpy.ndarray` Luminosity function parameter chain. pool : :class:`multiprocessing.Pool` *or None, optional* Multiprocessing pool (default is `None`). Returns ------- bias_samples : :class:`numpy.ndarray` Relativistic bias samples. """ mapping = pool.imap if pool else map num_cpus = cpu_count() if pool else 1 logger.info("Resampling relativistic biases with %i CPUs...\n", num_cpus) bias_samples = list(tqdm( mapping(compute_biases_from_lumfunc, lumfunc_param_chain), total=len(lumfunc_param_chain), mininterval=15, file=sys.stdout )) logger.info("... finished.\n") bias_samples = np.asarray(bias_samples) return bias_samples
def initialise(): """Initialise program. Returns ------- program_configuration : :class:`argparse.Namespace` Parsed program configuration parameters. """ parser = ArgumentParser("shift-chain-samples") parser.add_argument('--original-param-file', type=str) parser.add_argument('--new-param-file', type=str) parser.add_argument('--model', type=str, default='quasar_PLE') parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus'], default='zeus') parser.add_argument('--chain-file', type=str, default=None) parser.add_argument('--burnin', type=int, default=None) parser.add_argument('--reduction', type=int, default=1) program_configuration = parser.parse_args() logger.info("\n---Program configuration---\n%s\n", pformat(vars(program_configuration)).lstrip("{").rstrip("}")) return program_configuration
def view_extracts(chain): """View the extracted chain of relativistic biases. Parameters ---------- chain : :class:`numpy.ndarray` Chain. Returns ------- chain_fig, contour_fig : :class:`matplotlib.figure.Figure` Chain and contour figures. """ LEVELS = [0.39346934, 0.86466472] QUANTILES = [0.1587, 0.5, 0.8413] COLOUR = '#A3C1AD' CORNER_OPTIONS = dict( color=COLOUR, quantiles=QUANTILES, levels=LEVELS, labels=[lab.format(progrc.redshift) for lab in LABELS], label_kwargs={'visible': False}, plot_datapoints=False, plot_contours=True, fill_contours=True, range=(0.999,)*NDIM, show_titles=True, title_fmt='.3f', quiet=True, rasterized=True, ) plt.close('all') chain_fig, axes = plt.subplots(NDIM, figsize=(12, NDIM), sharex=True) for param_idx in range(NDIM): ax = axes[param_idx] ax.plot( chain[:, param_idx], color=COLOUR, alpha=0.66, rasterized=True ) ax.set_xlim(0, len(chain)) ax.set_ylabel(LABELS[param_idx]) axes[-1].set_xlabel("steps") fig_file = str(output_path).replace('.h5', '.chains.pdf') if SAVEFIG: chain_fig.savefig(fig_file) logger.info("Saved chain plot of relativistic bias samples.\n") contour_fig = corner.corner( chain, bins=160, smooth=.75, smooth1d=.95, **CORNER_OPTIONS ) fig_file = str(output_path).replace('.h5', '.contours.pdf') if SAVEFIG: contour_fig.savefig(fig_file) logger.info("Saved contour plot of relativistic bias samples.\n") return contour_fig
def save_extracts(): """Save extracted relativistic bias chains. Returns ------- :class:`pathlib.Path` Chain output file path. """ infile = PATHOUT/progrc.chain_file redshift_tag = "_z{:.2f}".format(progrc.redshift) threshold_tag = "_m{:.1f}".format(brightness_threshold) prefix = "relbias" + redshift_tag + threshold_tag + "_" outfile = PATHOUT/(prefix + progrc.chain_file) with hp.File(infile, 'r') as indata, hp.File(outfile, 'w') as outdata: outdata.create_group('extract') outdata.create_dataset('extract/chain', data=extracted_chain) if progrc.sampler == 'emcee': outdata.create_dataset( 'extract/log_prob', data=np.ravel(indata['mcmc/log_prob'][burnin::reduction, :]) ) logger.info("Extracted chain saved to %s.\n", outfile) return outfile
def initialise(): """Initialise program. Returns ------- program_configuration : :class:`argparse.Namespace` Parsed program configuration parameters. """ parser = ArgumentParser("tracer-number-density") parser.add_argument('--task', type=str, choices=['extract', 'load']) parser.add_argument('--model-name', type=str) parser.add_argument('--redshift', type=float) parser.add_argument('--threshold', type=float) parser.add_argument('--apparent_to_absolute', action='store_true') parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus']) parser.add_argument('--chain-file', type=str, default=None) parser.add_argument('--burnin', type=int, default=None) parser.add_argument('--reduction', type=int, default=1) program_configuration = parser.parse_args() logger.info("\n---Program configuration---\n%s\n", pformat(vars(program_configuration)).lstrip("{").rstrip("}")) return program_configuration
def read_chains(): """Load and process chains from file. Returns ------- flat_chain : :class:`numpy.ndarray` Flattened chains. """ # Read chains into memory. chain_file = PATHOUT / progrc.chain_file if progrc.sampler == 'emcee': reader = mc.backends.HDFBackend(chain_file, read_only=True) elif progrc.sampler == 'zeus': chain_data = hp.File(chain_file, 'r') reader = chain_data['mcmc'] logger.info("Loaded chain file: %s.\n", chain_file) # Process chains by burn-in and thinning. if progrc.burnin is None or progrc.reduction == 0: try: autocorr_time = reader.get_autocorr_time() except AttributeError: autocorr_time = reader['autocorr_time'][()] except mc.autocorr.AutocorrError as warning: logger.warning(warning) autocorr_time = None if progrc.burnin is None: try: _burnin = 4 * int(np.max(autocorr_time)) # can change 4 to 2 except (TypeError, ValueError): _burnin = 0 else: _burnin = progrc.burnin if progrc.reduction == 0: try: _reduction = int(np.min(autocorr_time)) // 5 # can change 5 to 2 except (TypeError, ValueError): _reduction = 1 else: _reduction = progrc.reduction # Flatten chains. if progrc.sampler == 'emcee': flat_chain = reader.get_chain(flat=True, discard=_burnin, thin=_reduction) elif progrc.sampler == 'zeus': flat_chain = reader['chain'][_burnin::_reduction, :, :]\ .reshape((-1, len(parameters))) chain_data.close() logger.info("Chain flattened with %i burn-in and %i thinning.\n", _burnin, _reduction) return flat_chain, _burnin, _reduction
def save_chains(shifted_chains): """Save shifted parameter sample chains. Parameters ---------- shifted_chains : :class:`numpy.ndarray` Shifted chains. """ infile = PATHOUT / progrc.chain_file outfile = PATHOUT / progrc.chain_file.replace(".h5", "shifted.h5") with hp.File(infile, 'r') as indata, hp.File(outfile, 'w') as outdata: outdata.create_group('mcmc') outdata.create_dataset('mcmc/chain', data=shifted_chains) outdata.create_dataset('mcmc/autocorr_time', data=indata['mcmc/autocorr_time'][()]) logger.info("Verify medians: {}.\n".format( np.squeeze([ np.squeeze( corner.quantile(shifted_param_chain.reshape( -1, len(PARAMETERS[progrc.model])), q=[0.5])) for shifted_param_chain in np.transpose(shifted_chains) ]))) logger.info("Shifted chain saved to %s.\n", outfile)
def initialise(): """Initialise program. Returns ------- program_configuration : :class:`argparse.Namespace` Parsed program configuration parameters. """ parser = ArgumentParser("relativistic-bias-constraint") parser.add_argument('--model-name', type=str) parser.add_argument('--redshift', type=float) parser.add_argument('--threshold', type=float) parser.add_argument('--convert-to-source', action='store_true') parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus']) parser.add_argument('--chain-file', type=str, default=None) parser.add_argument('--burnin', type=int, default=None) parser.add_argument('--reduction', type=int, default=1) program_configuration = parser.parse_args() logger.info( "\n---Program configuration---\n%s\n", pformat(vars(program_configuration)).lstrip("{").rstrip("}") ) return program_configuration
def upload_metadata(self, _id=None): """Upload metadata to Zenodo repository. After creating the request and upload the file(s) we need to update the metadata needed by Zenodo related to the record. Parameters ---------- metadata: dict dictionary which contains zenodo metadata. It should be a dictionary with one key 'metadata' associated to another dictionary which contains the Zenodo metadata. _id: int deposition id of the record where the metadata will be updated """ self._verify_token() metadata_to_upload = self.set_metadata() if _id is not None: self.deposition_id = _id # Create the url to upload with the deposition_id url = self.depositions_url + '/{}'.format(self.deposition_id) logger.info('url: {}'.format(url)) headers = {"Content-Type": "application/json"} request = requests.put(url, params={'access_token': self.token}, data=json.dumps(metadata_to_upload), headers=headers) self.status_code = request.status_code self._check_status_code(self.status_code)
def save_distilled(): """Save the distilled relativistic factor quantiles Returns ------- :class:`pathlib.Path` Chain output file path. """ prefix = "relpole_" redshift_tag = "z{:.2f}".format(progrc.redshift) chain_suffix = progrc.chain_file.replace("relcrct_", "") if redshift_tag not in progrc.chain_file: prefix += redshift_tag + "_" outfile = PATHOUT / progrc.chain_subdir / (prefix + chain_suffix) with hp.File(outfile, 'w') as outdata: for ell in [0, 2]: for q in [-2, -1, 0, 1, 2]: outdata.create_dataset('{}/{}'.format(ell, q), data=np.asarray( distilled_quantiles[ell][q])) logger.info("Distilled chains saved to %s.\n", outfile)
def _urlretrieve(url, filename): for _ in range(3): try: return urlretrieve(url, filename=filename) except Exception as e: logger.info( 'Download xls error: type <>, msg <>, url <>, filename <>'. format(e.__class__, e, url, filename))
def close(spider, reason): closed = getattr(spider, 'closed', None) if callable(closed): return closed(reason) mongo = StorageMongo('csindex') logger.info(' Total_data count: <{}>'.format(len(total_data))) mongo.insert2mongo(total_data) mongo.eliminate() mongo.close() del total_data[:]
def distill_factor_quantiles(correction_chain, pool=None): """Distill relativistic correction factor quantiles from a relativistic correction chain. Parameters ---------- correction_chain : :class:`numpy.ndarray` Relativistic correction_chain chain. pool : :class:`multiprocessing.Pool` or None, optional Multiprocessing pool (default is `None`). Returns ------- factor_quantiles : dict of dict of :class:`numpy.ndarray` Relativistic correction factor quantiles. """ mapping = pool.imap if pool else map num_cpus = cpu_count() if pool else 1 quantile_levels = [0.022750, 0.158655, 0.5, 0.841345, 0.977250] factor_quantiles = {0: {}, 2: {}} logger.info( "Distilling relativistic correction factors at redshift %.2f " + "with %i CPUs...\n", progrc.redshift, num_cpus) factor_chain = np.asarray( list( tqdm(mapping(compute_factors_from_corrections, correction_chain), total=len(correction_chain), mininterval=1, file=sys.stdout))) factor_0_q = np.asarray([ corner.quantile(factor_chain[:, 0, k_idx], q=quantile_levels) for k_idx, k in enumerate(wavenumbers) ]) factor_2_q = np.asarray([ corner.quantile(factor_chain[:, 1, k_idx], q=quantile_levels) for k_idx, k in enumerate(wavenumbers) ]) for q_idx, q in enumerate([-2, -1, 0, 1, 2]): factor_quantiles[0][q] = factor_0_q[:, q_idx] factor_quantiles[2][q] = factor_2_q[:, q_idx] logger.info("... finished.\n") return factor_quantiles
def spider_indexes(): today = datetime.now().strftime('%Y-%m-%d') if today not in is_workday(): return sse_start = time.time() crawl_sse_index() logger.info('SSE site crawl Done!\n') szse_start = time.time() crawl_szse_index() logger.info('SZSE site crawl Done!\n') cn_start = time.time() crawl_cn_index() logger.info('CNINDES site crawl Done!\n') cs_start = time.time() crawl_cs_index() logger.info('CSINDEX site crawl Done!\n') end = time.time() subject = u'指数成分股抓取完成' text = u'上交所网站抓取时间: %s\n' % (szse_start - sse_start) + \ u'深交所网站抓取时间: %s\n' % (cn_start - szse_start) + \ u'CNINDEX 网站抓取时间: %s\n' % (cs_start - cn_start) + \ u'CSINDEX 网站抓取时间: %s\n' % (end - cs_start) Sender(receivers=receiver).send_email(subject, text)
def view_extracts(chain): """View the extracted chain of tracer number density. Parameters ---------- chain : :class:`numpy.ndarray` Chain. Returns ------- chain_fig, contour_fig : :class:`matplotlib.figure.Figure` Chain and contour figures. """ _labels = [lab.format(progrc.redshift) for lab in LABELS] LEVELS = [0.6826895, 0.9544997] QUANTILES = [0.1587, 0.5, 0.8413] COLOUR = '#A3C1AD' CORNER_OPTIONS = dict( color=COLOUR, quantiles=QUANTILES, levels=LEVELS, labels=_labels, label_kwargs={'visible': False}, title_fmt='.5f', plot_datapoints=False, plot_contours=True, fill_contours=True, quiet=True, range=(0.999, ) * len(_labels), rasterized=True, show_titles=True, ) plt.close('all') distribution_fig = corner.corner(chain, bins=160, smooth=.75, smooth1d=.95, **CORNER_OPTIONS) fig_file = str(output_path).replace('.h5', '.pdf') if SAVEFIG: distribution_fig.savefig(fig_file) logger.info("Saved distribution plot of tracer number density samples.\n") return distribution_fig
def set_schema(self, schema): if type(schema) is str: logger.info('Schema file use: {}'.format(schema)) self._schema = self._read_schema(schema) elif type(schema) is dict: logger.info('Schema provided through dictionary object') if self._schema is None: self._schema = schema else: self._schema.update(schema) else: message = 'Something is wrong with the schema: {}.'.format(schema) logger.error(message) raise ZenodoMetadataException(message)
def crawl_cs_index(): from scrapy.crawler import CrawlerProcess try: # 中证指数网站 # cp = CrawlerProcess() # cp.crawl(CsindexSpider()) # cp.start() if run_path.endswith('.py'): os.system('python %s' % run_path) elif run_path.endswith('.pyc'): os.system('python %s' % run_path[:-1]) except Exception as e: logger.info('CSIndex crawl error: type <{typ}>, msg <{msg}>\n'.format( typ=e.__class__, msg=e))
def load_samples(): """Load samples of relativistic biases from file. Returns ------- bias_samples : :class:`numpy.ndarray` Relativistic bias samples. """ chain_file = PATHOUT / progrc.chain_subdir / progrc.chain_file with hp.File(chain_file, 'r') as chain_data: bias_samples = chain_data['extract/chain'][()] logger.info("Loaded bias samples from file: %s.\n", chain_file) return bias_samples
def load_samples(): """Load samples of relativistic corrections from file. Returns ------- correction_samples : :class:`numpy.ndarray` Relativistic correction samples. """ chain_file = PATHOUT / progrc.chain_subdir / progrc.chain_file with hp.File(chain_file, 'r') as chain_data: correction_samples = chain_data['distill/chain'][()] logger.info("Loaded correction samples from file: %s.\n", chain_file) return correction_samples
def _read_metadata(fmetadata): """Method to read Zenodo metadata file """ logger.info('Read metadata from: {}'.format(fmetadata)) try: with open(fmetadata) as f: _metadata = yaml.load(f) except FileNotFoundError as err: message = 'Metadata file not founded.'.format(fmetadata) logger.error(message) raise ZenodoMetadataException(message) # change communities identifier in lower case (only format accepted by zenodo) if 'communities' in _metadata: for _com in _metadata['communities']: _com['identifier'] = _com['identifier'].lower() return _metadata
def zipdata(files, zipname='data.zip'): """Method to zip files which will be uploaded to the data repository. Parameters ---------- files: list a list of string which contains the path of the files which will be part of the zip archive. Path will be conserved. zipname: str, optional Name of the zip file to create. Default: data.zip """ logger.info('Zip the files to create archive: {}'.format(zipname)) # writing files to a zipfile with ZipFile(zipname, 'w') as zip: # writing each file one by one for file in files: zip.write(file)
def distill_corrections(bias_chain, pool=None): """Distill relativistic corrections and correction factors from a relativistic bias chain. Parameters ---------- bias_chain : :class:`numpy.ndarray` Relativistic bias parameter chain. pool : :class:`multiprocessing.Pool` or None, optional Multiprocessing pool (default is `None`). Returns ------- correction_chain : :class:`numpy.ndarray` Relativistic correction samples. factor_chain : :class:`numpy.ndarray` Relativistic correction factor samples. """ mapping = pool.imap if pool else map num_cpus = cpu_count() if pool else 1 logger.info( "Distilling relativistic corrections/correction factors " + "with %i CPUs...\n", num_cpus) correction_chain = np.asarray( list( tqdm(mapping(compute_corrections_from_biases, bias_chain), total=len(bias_chain), mininterval=1, file=sys.stdout))) factor_chain = list( tqdm(mapping(compute_factors_from_corrections, correction_chain), total=len(correction_chain), mininterval=1, file=sys.stdout)) logger.info("... finished.\n") return correction_chain, factor_chain
def eliminate(self): must_remove_indexes = [] need_indexes = self.need_index mongo_indexes = self.get_data_from_mongo(unset=False, including_sign=False, query=False) latest_docs = getattr(self, 'latest_indexes') diff_set = mongo_indexes - need_indexes # If have `diff_set`, group by `p_code`, then send email normal_indexes, no_normal_indexes = self.think_indexes_issue(diff_set) print 'set(mong0) - set(web)=', len(diff_set) print 'Normal:', len(normal_indexes), 'No Normal:', len(no_normal_indexes) for ps_key in normal_indexes: if ps_key in latest_docs: try: _id = latest_docs[ps_key][0]['_id'] out_dt = latest_docs[ps_key][0]['out_dt'] if not out_dt: query = {'_id': _id} setdata = { '$set': {'sign': '1', 'stat': 2, 'upt': datetime.now(), 'out_dt': date.today().strftime("%Y%m%d")} } self.collection.update(query, setdata) must_remove_indexes.append(ps_key) except (AttributeError, IndexError, KeyError) as e: logger.info('`Eliminate` crawl error: type <{typ}>, msg <{msg}>'.format(typ=e.__class__, msg=e)) if self.insert_mongo_data: subject = '%s 网站: 网页新增, 插入数据库的记录(新增),望检查' % self.using_category.upper() self.send_email(subject, self.insert_mongo_data, prefix='web') if no_normal_indexes: subject = '%s 网站: 可能有问题的指数成分股(未剔除), 请检查' % self.using_category.upper() self.send_email(subject, no_normal_indexes) if must_remove_indexes: subject = '%s 网站: 没有问题的指数成分股(已剔除), 望检查' % self.using_category.upper() self.send_email(subject, must_remove_indexes)
def delete(self, _id=None): """Method to delete deposition. Parameters ---------- _id: int deposition id of the record to delete .. note:: it worked only if it is not publish. Exception --------- ZenodoException raise if token not define (token = None) or if connection return status >= 400 """ # Test if token was defined self._verify_token() # Use provided if if not None. If not provided use self.deposition_id if _id is not None: self.deposition_id = _id # Create the request url request_url = (self.depositions_url + '/{}'.format(self.deposition_id)) logger.info('Delete url: {}'.format(request_url)) try: request = requests.delete(request_url, params={'access_token': self.token}) self.status_code = request.status_code logger.debug('Status code: {}'.format(self.status_code)) if self.status_code >= 400: raise ZenodoException except ZenodoException: message = 'Request_url does not exist or bad token. ' \ 'Error: {}'.format(self.status_code) logger.error(message) raise ZenodoException(message)
def save_distilled(): """Save the distilled relativistic correction and correction factor chains. Returns ------- :class:`pathlib.Path` Chain output file path. """ infile = PATHOUT / progrc.chain_subdir / progrc.chain_file chain_suffix = progrc.chain_file.replace("relbias_", "") redshift_tag = "z{:.2f}".format(progrc.redshift) prefix_0, prefix_1 = "relcrct_", "relfact_" if redshift_tag not in progrc.chain_file: prefix_0 += redshift_tag + "_" prefix_1 += redshift_tag + "_k{}_".format(progrc.wavenumber) outfile_0 = PATHOUT / progrc.chain_subdir / (prefix_0 + chain_suffix) outfile_1 = PATHOUT / progrc.chain_subdir / (prefix_1 + chain_suffix) with hp.File(infile, 'r') as indata, hp.File(outfile_0, 'w') as outdata: outdata.create_group('distill') try: indata.copy('extract/log_prob', outdata['distill']) except KeyError: pass outdata.create_dataset('distill/chain', data=distilled_chains[0]) with hp.File(infile, 'r') as indata, hp.File(outfile_1, 'w') as outdata: outdata.create_group('distill') try: indata.copy('extract/log_prob', outdata['distill']) except KeyError: pass outdata.create_dataset('distill/chain', data=distilled_chains[1]) logger.info("Distilled chains saved to %s and %s.\n", outfile_0, outfile_1)
def read_chains(): """Load and process chains from file. Returns ------- chains : :class:`numpy.ndarray` Flattened chains. """ chain_file = PATHOUT / progrc.chain_file if progrc.sampler == 'emcee': reader = mc.backends.HDFBackend(chain_file, read_only=True) chains = reader.get_chain() elif progrc.sampler == 'zeus': with hp.File(chain_file, 'r') as chain_data: chains = chain_data['mcmc']['chain'][()] logger.info("Loaded chain file: %s.\n", chain_file) return chains