def _urlretrieve(url, filename):
for _ in range(3):
try:
return urlretrieve(url, filename=filename)
except Exception as e:
logger.info('Download xls error: type <>, msg <>, url <>, filename <>'.format(
e.__class__, e, url, filename))
def parse_ext_args():
"""Parse external arguments.
Returns
-------
parsed_args : :class:`argparse.Namespace`
Parsed program arguments.
"""
parser = ArgumentParser("luminosity-function-fitting")
parser.add_argument(
'--task',
type=str.lower,
choices=['sample', 'retrieve', 'resume'],
help="task to perform: sample; retrieve samples; or resume sampling")
parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus'])
parser.add_argument('--prescription',
type=str.lower,
choices=['native', 'poisson', 'symlg'])
parser.add_argument(
'--jump',
action='store_true',
help="change zeus sampler jump condition; used for multimodal cases")
parser.add_argument('--use-prior', action='store_true')
parser.add_argument('--use-constraint', action='store_true')
parser.add_argument('--nonautostop',
action='store_true',
help="do not stop sampling after convergence detected")
parser.add_argument('--quiet', action='store_false')
parser.add_argument('--model-name', type=str, default='quasar_PLE')
parser.add_argument('--data-files', type=str, nargs=2, default=None)
parser.add_argument('--prior-file', type=str, default=None)
parser.add_argument('--fixed-file', type=str, default=None)
parser.add_argument('--chain-file', type=str, default=None)
parser.add_argument('--nwalkers', type=int, default=100)
parser.add_argument('--nsteps', type=int, default=10000)
parser.add_argument('--thinby', type=int, default=1)
parser.add_argument('--skip-chains',
type=int,
default=None,
help="only plot chain traces every interval")
parser.add_argument('--burnin', type=int, default=None)
parser.add_argument('--reduce', type=int, default=None)
parsed_args = parser.parse_args()
parsed_args.chain_file = parsed_args.chain_file.rstrip('.h5')
parsed_args.chain_file += "_{}_{}_{}_by{}.h5".format(
parsed_args.prescription, parsed_args.nwalkers,
sci_notation(parsed_args.nsteps), parsed_args.thinby)
logger.info("\n---Program configuration---\n%s\n",
pformat(vars(parsed_args)).lstrip("{").rstrip("}"))
return parsed_args
def publish(self):
"""Method which will publish the deposition linked with the id.
.. warning:
After publishing it is not possible to delete anything.
Exception
---------
ZenodoException
raise if token not define (token = None) or
if connection return status >= 400
"""
# Test if token was defined
self._verify_token()
publish_url = (self.depositions_url +
'/{}/actions/publish'.format(self.deposition_id))
request = requests.post(publish_url,
params={'access_token': self.token})
self.status_code = request.status_code
self._check_status_code(self.status_code)
# Test that everything went as expected
if self.status_code == 202:
logger.info('Deposition id: {}'.format(self.deposition_id))
logger.debug('Status code: {}'.format(self.status_code))
return
def set_metadata(self, metadata):
"""Method to set metadata from a file or a dictionary.
Parameter
---------
metadata: str, dict
- if it is a string it is the name of file which contains
zenodo metadata (yaml format). It will read the metadata
and set the attribute _metadata to it
- if dictionary, it set the attribute _metadata to it
Attribute
---------
_metadata: dict
dictionary which contains Zenodo metadata
"""
if type(metadata) is str:
logger.info('Metadata provided by file: {}'.format(metadata))
self._metadata = self._read_metadata(metadata)
elif type(metadata) is dict:
logger.info('Metadata provided by dictionary.')
if self._metadata is None:
self._metadata = metadata
else:
self._metadata.update(metadata)
elif metadata is None:
logger.warn('Metadata are empty')
self._metadata = metadata
self._check_minimal()
def validate(self):
"""Method which is verifying that the metadata does have the correct type
and if the dependencies are respected.
The dependencies have to be check because the value of a
metadata can implied the presence of another one. For example,
if *upload_type* (which is a necessary metadata) has the value
*publication* that implied the presence of the metadata
*publication_type*.
"""
# Check if the minimal set of information are provided
self._check_minimal()
# Check validity of the license (if open or embargoed)
self._check_license_availability()
try:
jsonschema.validate(self._metadata, self._schema)
except jsonschema.exceptions.ValidationError as err:
error = 'ValidationError: {}'.format(err.message)
logger.error(error)
raise ZenodoMetadataException(error)
logger.info('Metadata should be ok to use for upload')
def extract_biases(lumfunc_param_chain, pool=None):
"""Extract relativistic biases from a luminosity function parameter
chain.
Parameters
----------
lumfunc_model_chain : :class:`numpy.ndarray`
Luminosity function parameter chain.
pool : :class:`multiprocessing.Pool` *or None, optional*
Multiprocessing pool (default is `None`).
Returns
-------
bias_samples : :class:`numpy.ndarray`
Relativistic bias samples.
"""
mapping = pool.imap if pool else map
num_cpus = cpu_count() if pool else 1
logger.info("Resampling relativistic biases with %i CPUs...\n", num_cpus)
bias_samples = list(tqdm(
mapping(compute_biases_from_lumfunc, lumfunc_param_chain),
total=len(lumfunc_param_chain), mininterval=15, file=sys.stdout
))
logger.info("... finished.\n")
bias_samples = np.asarray(bias_samples)
return bias_samples
def initialise():
"""Initialise program.
Returns
-------
program_configuration : :class:`argparse.Namespace`
Parsed program configuration parameters.
"""
parser = ArgumentParser("shift-chain-samples")
parser.add_argument('--original-param-file', type=str)
parser.add_argument('--new-param-file', type=str)
parser.add_argument('--model', type=str, default='quasar_PLE')
parser.add_argument('--sampler',
type=str.lower,
choices=['emcee', 'zeus'],
default='zeus')
parser.add_argument('--chain-file', type=str, default=None)
parser.add_argument('--burnin', type=int, default=None)
parser.add_argument('--reduction', type=int, default=1)
program_configuration = parser.parse_args()
logger.info("\n---Program configuration---\n%s\n",
pformat(vars(program_configuration)).lstrip("{").rstrip("}"))
return program_configuration
def view_extracts(chain):
"""View the extracted chain of relativistic biases.
Parameters
----------
chain : :class:`numpy.ndarray`
Chain.
Returns
-------
chain_fig, contour_fig : :class:`matplotlib.figure.Figure`
Chain and contour figures.
"""
LEVELS = [0.39346934, 0.86466472]
QUANTILES = [0.1587, 0.5, 0.8413]
COLOUR = '#A3C1AD'
CORNER_OPTIONS = dict(
color=COLOUR,
quantiles=QUANTILES,
levels=LEVELS,
labels=[lab.format(progrc.redshift) for lab in LABELS],
label_kwargs={'visible': False},
plot_datapoints=False,
plot_contours=True,
fill_contours=True,
range=(0.999,)*NDIM,
show_titles=True,
title_fmt='.3f',
quiet=True,
rasterized=True,
)
plt.close('all')
chain_fig, axes = plt.subplots(NDIM, figsize=(12, NDIM), sharex=True)
for param_idx in range(NDIM):
ax = axes[param_idx]
ax.plot(
chain[:, param_idx], color=COLOUR, alpha=0.66, rasterized=True
)
ax.set_xlim(0, len(chain))
ax.set_ylabel(LABELS[param_idx])
axes[-1].set_xlabel("steps")
fig_file = str(output_path).replace('.h5', '.chains.pdf')
if SAVEFIG:
chain_fig.savefig(fig_file)
logger.info("Saved chain plot of relativistic bias samples.\n")
contour_fig = corner.corner(
chain, bins=160, smooth=.75, smooth1d=.95, **CORNER_OPTIONS
)
fig_file = str(output_path).replace('.h5', '.contours.pdf')
if SAVEFIG:
contour_fig.savefig(fig_file)
logger.info("Saved contour plot of relativistic bias samples.\n")
return contour_fig
def save_extracts():
"""Save extracted relativistic bias chains.
Returns
-------
:class:`pathlib.Path`
Chain output file path.
"""
infile = PATHOUT/progrc.chain_file
redshift_tag = "_z{:.2f}".format(progrc.redshift)
threshold_tag = "_m{:.1f}".format(brightness_threshold)
prefix = "relbias" + redshift_tag + threshold_tag + "_"
outfile = PATHOUT/(prefix + progrc.chain_file)
with hp.File(infile, 'r') as indata, hp.File(outfile, 'w') as outdata:
outdata.create_group('extract')
outdata.create_dataset('extract/chain', data=extracted_chain)
if progrc.sampler == 'emcee':
outdata.create_dataset(
'extract/log_prob',
data=np.ravel(indata['mcmc/log_prob'][burnin::reduction, :])
)
logger.info("Extracted chain saved to %s.\n", outfile)
return outfile
def initialise():
"""Initialise program.
Returns
-------
program_configuration : :class:`argparse.Namespace`
Parsed program configuration parameters.
"""
parser = ArgumentParser("tracer-number-density")
parser.add_argument('--task', type=str, choices=['extract', 'load'])
parser.add_argument('--model-name', type=str)
parser.add_argument('--redshift', type=float)
parser.add_argument('--threshold', type=float)
parser.add_argument('--apparent_to_absolute', action='store_true')
parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus'])
parser.add_argument('--chain-file', type=str, default=None)
parser.add_argument('--burnin', type=int, default=None)
parser.add_argument('--reduction', type=int, default=1)
program_configuration = parser.parse_args()
logger.info("\n---Program configuration---\n%s\n",
pformat(vars(program_configuration)).lstrip("{").rstrip("}"))
return program_configuration
def read_chains():
"""Load and process chains from file.
Returns
-------
flat_chain : :class:`numpy.ndarray`
Flattened chains.
"""
# Read chains into memory.
chain_file = PATHOUT / progrc.chain_file
if progrc.sampler == 'emcee':
reader = mc.backends.HDFBackend(chain_file, read_only=True)
elif progrc.sampler == 'zeus':
chain_data = hp.File(chain_file, 'r')
reader = chain_data['mcmc']
logger.info("Loaded chain file: %s.\n", chain_file)
# Process chains by burn-in and thinning.
if progrc.burnin is None or progrc.reduction == 0:
try:
autocorr_time = reader.get_autocorr_time()
except AttributeError:
autocorr_time = reader['autocorr_time'][()]
except mc.autocorr.AutocorrError as warning:
logger.warning(warning)
autocorr_time = None
if progrc.burnin is None:
try:
_burnin = 4 * int(np.max(autocorr_time)) # can change 4 to 2
except (TypeError, ValueError):
_burnin = 0
else:
_burnin = progrc.burnin
if progrc.reduction == 0:
try:
_reduction = int(np.min(autocorr_time)) // 5 # can change 5 to 2
except (TypeError, ValueError):
_reduction = 1
else:
_reduction = progrc.reduction
# Flatten chains.
if progrc.sampler == 'emcee':
flat_chain = reader.get_chain(flat=True,
discard=_burnin,
thin=_reduction)
elif progrc.sampler == 'zeus':
flat_chain = reader['chain'][_burnin::_reduction, :, :]\
.reshape((-1, len(parameters)))
chain_data.close()
logger.info("Chain flattened with %i burn-in and %i thinning.\n", _burnin,
_reduction)
return flat_chain, _burnin, _reduction
def save_chains(shifted_chains):
"""Save shifted parameter sample chains.
Parameters
----------
shifted_chains : :class:`numpy.ndarray`
Shifted chains.
"""
infile = PATHOUT / progrc.chain_file
outfile = PATHOUT / progrc.chain_file.replace(".h5", "shifted.h5")
with hp.File(infile, 'r') as indata, hp.File(outfile, 'w') as outdata:
outdata.create_group('mcmc')
outdata.create_dataset('mcmc/chain', data=shifted_chains)
outdata.create_dataset('mcmc/autocorr_time',
data=indata['mcmc/autocorr_time'][()])
logger.info("Verify medians: {}.\n".format(
np.squeeze([
np.squeeze(
corner.quantile(shifted_param_chain.reshape(
-1, len(PARAMETERS[progrc.model])),
q=[0.5]))
for shifted_param_chain in np.transpose(shifted_chains)
])))
logger.info("Shifted chain saved to %s.\n", outfile)
def initialise():
"""Initialise program.
Returns
-------
program_configuration : :class:`argparse.Namespace`
Parsed program configuration parameters.
"""
parser = ArgumentParser("relativistic-bias-constraint")
parser.add_argument('--model-name', type=str)
parser.add_argument('--redshift', type=float)
parser.add_argument('--threshold', type=float)
parser.add_argument('--convert-to-source', action='store_true')
parser.add_argument('--sampler', type=str.lower, choices=['emcee', 'zeus'])
parser.add_argument('--chain-file', type=str, default=None)
parser.add_argument('--burnin', type=int, default=None)
parser.add_argument('--reduction', type=int, default=1)
program_configuration = parser.parse_args()
logger.info(
"\n---Program configuration---\n%s\n",
pformat(vars(program_configuration)).lstrip("{").rstrip("}")
)
return program_configuration
def upload_metadata(self, _id=None):
"""Upload metadata to Zenodo repository.
After creating the request and upload the file(s) we need to update
the metadata needed by Zenodo related to the record.
Parameters
----------
metadata: dict
dictionary which contains zenodo metadata. It should be
a dictionary with one key 'metadata' associated to another
dictionary which contains the Zenodo metadata.
_id: int
deposition id of the record where the metadata will be updated
"""
self._verify_token()
metadata_to_upload = self.set_metadata()
if _id is not None:
self.deposition_id = _id
# Create the url to upload with the deposition_id
url = self.depositions_url + '/{}'.format(self.deposition_id)
logger.info('url: {}'.format(url))
headers = {"Content-Type": "application/json"}
request = requests.put(url,
params={'access_token': self.token},
data=json.dumps(metadata_to_upload),
headers=headers)
self.status_code = request.status_code
self._check_status_code(self.status_code)
def save_distilled():
"""Save the distilled relativistic factor quantiles
Returns
-------
:class:`pathlib.Path`
Chain output file path.
"""
prefix = "relpole_"
redshift_tag = "z{:.2f}".format(progrc.redshift)
chain_suffix = progrc.chain_file.replace("relcrct_", "")
if redshift_tag not in progrc.chain_file:
prefix += redshift_tag + "_"
outfile = PATHOUT / progrc.chain_subdir / (prefix + chain_suffix)
with hp.File(outfile, 'w') as outdata:
for ell in [0, 2]:
for q in [-2, -1, 0, 1, 2]:
outdata.create_dataset('{}/{}'.format(ell, q),
data=np.asarray(
distilled_quantiles[ell][q]))
logger.info("Distilled chains saved to %s.\n", outfile)
def _urlretrieve(url, filename):
for _ in range(3):
try:
return urlretrieve(url, filename=filename)
except Exception as e:
logger.info(
'Download xls error: type <>, msg <>, url <>, filename <>'.
format(e.__class__, e, url, filename))
def close(spider, reason):
closed = getattr(spider, 'closed', None)
if callable(closed):
return closed(reason)
mongo = StorageMongo('csindex')
logger.info(' Total_data count: <{}>'.format(len(total_data)))
mongo.insert2mongo(total_data)
mongo.eliminate()
mongo.close()
del total_data[:]
def close(spider, reason):
closed = getattr(spider, 'closed', None)
if callable(closed):
return closed(reason)
mongo = StorageMongo('csindex')
logger.info(' Total_data count: <{}>'.format(len(total_data)))
mongo.insert2mongo(total_data)
mongo.eliminate()
mongo.close()
del total_data[:]
def distill_factor_quantiles(correction_chain, pool=None):
"""Distill relativistic correction factor quantiles from a
relativistic correction chain.
Parameters
----------
correction_chain : :class:`numpy.ndarray`
Relativistic correction_chain chain.
pool : :class:`multiprocessing.Pool` or None, optional
Multiprocessing pool (default is `None`).
Returns
-------
factor_quantiles : dict of dict of :class:`numpy.ndarray`
Relativistic correction factor quantiles.
"""
mapping = pool.imap if pool else map
num_cpus = cpu_count() if pool else 1
quantile_levels = [0.022750, 0.158655, 0.5, 0.841345, 0.977250]
factor_quantiles = {0: {}, 2: {}}
logger.info(
"Distilling relativistic correction factors at redshift %.2f " +
"with %i CPUs...\n", progrc.redshift, num_cpus)
factor_chain = np.asarray(
list(
tqdm(mapping(compute_factors_from_corrections, correction_chain),
total=len(correction_chain),
mininterval=1,
file=sys.stdout)))
factor_0_q = np.asarray([
corner.quantile(factor_chain[:, 0, k_idx], q=quantile_levels)
for k_idx, k in enumerate(wavenumbers)
])
factor_2_q = np.asarray([
corner.quantile(factor_chain[:, 1, k_idx], q=quantile_levels)
for k_idx, k in enumerate(wavenumbers)
])
for q_idx, q in enumerate([-2, -1, 0, 1, 2]):
factor_quantiles[0][q] = factor_0_q[:, q_idx]
factor_quantiles[2][q] = factor_2_q[:, q_idx]
logger.info("... finished.\n")
return factor_quantiles
def spider_indexes():
today = datetime.now().strftime('%Y-%m-%d')
if today not in is_workday():
return
sse_start = time.time()
crawl_sse_index()
logger.info('SSE site crawl Done!\n')
szse_start = time.time()
crawl_szse_index()
logger.info('SZSE site crawl Done!\n')
cn_start = time.time()
crawl_cn_index()
logger.info('CNINDES site crawl Done!\n')
cs_start = time.time()
crawl_cs_index()
logger.info('CSINDEX site crawl Done!\n')
end = time.time()
subject = u'指数成分股抓取完成'
text = u'上交所网站抓取时间: %s\n' % (szse_start - sse_start) + \
u'深交所网站抓取时间: %s\n' % (cn_start - szse_start) + \
u'CNINDEX 网站抓取时间: %s\n' % (cs_start - cn_start) + \
u'CSINDEX 网站抓取时间: %s\n' % (end - cs_start)
Sender(receivers=receiver).send_email(subject, text)
def view_extracts(chain):
"""View the extracted chain of tracer number density.
Parameters
----------
chain : :class:`numpy.ndarray`
Chain.
Returns
-------
chain_fig, contour_fig : :class:`matplotlib.figure.Figure`
Chain and contour figures.
"""
_labels = [lab.format(progrc.redshift) for lab in LABELS]
LEVELS = [0.6826895, 0.9544997]
QUANTILES = [0.1587, 0.5, 0.8413]
COLOUR = '#A3C1AD'
CORNER_OPTIONS = dict(
color=COLOUR,
quantiles=QUANTILES,
levels=LEVELS,
labels=_labels,
label_kwargs={'visible': False},
title_fmt='.5f',
plot_datapoints=False,
plot_contours=True,
fill_contours=True,
quiet=True,
range=(0.999, ) * len(_labels),
rasterized=True,
show_titles=True,
)
plt.close('all')
distribution_fig = corner.corner(chain,
bins=160,
smooth=.75,
smooth1d=.95,
**CORNER_OPTIONS)
fig_file = str(output_path).replace('.h5', '.pdf')
if SAVEFIG:
distribution_fig.savefig(fig_file)
logger.info("Saved distribution plot of tracer number density samples.\n")
return distribution_fig
def set_schema(self, schema):
if type(schema) is str:
logger.info('Schema file use: {}'.format(schema))
self._schema = self._read_schema(schema)
elif type(schema) is dict:
logger.info('Schema provided through dictionary object')
if self._schema is None:
self._schema = schema
else:
self._schema.update(schema)
else:
message = 'Something is wrong with the schema: {}.'.format(schema)
logger.error(message)
raise ZenodoMetadataException(message)
def crawl_cs_index():
from scrapy.crawler import CrawlerProcess
try:
# 中证指数网站
# cp = CrawlerProcess()
# cp.crawl(CsindexSpider())
# cp.start()
if run_path.endswith('.py'):
os.system('python %s' % run_path)
elif run_path.endswith('.pyc'):
os.system('python %s' % run_path[:-1])
except Exception as e:
logger.info('CSIndex crawl error: type <{typ}>, msg <{msg}>\n'.format(
typ=e.__class__, msg=e))
def load_samples():
"""Load samples of relativistic biases from file.
Returns
-------
bias_samples : :class:`numpy.ndarray`
Relativistic bias samples.
"""
chain_file = PATHOUT / progrc.chain_subdir / progrc.chain_file
with hp.File(chain_file, 'r') as chain_data:
bias_samples = chain_data['extract/chain'][()]
logger.info("Loaded bias samples from file: %s.\n", chain_file)
return bias_samples
def load_samples():
"""Load samples of relativistic corrections from file.
Returns
-------
correction_samples : :class:`numpy.ndarray`
Relativistic correction samples.
"""
chain_file = PATHOUT / progrc.chain_subdir / progrc.chain_file
with hp.File(chain_file, 'r') as chain_data:
correction_samples = chain_data['distill/chain'][()]
logger.info("Loaded correction samples from file: %s.\n", chain_file)
return correction_samples
def _read_metadata(fmetadata):
"""Method to read Zenodo metadata file
"""
logger.info('Read metadata from: {}'.format(fmetadata))
try:
with open(fmetadata) as f:
_metadata = yaml.load(f)
except FileNotFoundError as err:
message = 'Metadata file not founded.'.format(fmetadata)
logger.error(message)
raise ZenodoMetadataException(message)
# change communities identifier in lower case (only format accepted by zenodo)
if 'communities' in _metadata:
for _com in _metadata['communities']:
_com['identifier'] = _com['identifier'].lower()
return _metadata
def zipdata(files, zipname='data.zip'):
"""Method to zip files which will be uploaded to the data repository.
Parameters
----------
files: list
a list of string which contains the path of the files which will be
part of the zip archive. Path will be conserved.
zipname: str, optional
Name of the zip file to create. Default: data.zip
"""
logger.info('Zip the files to create archive: {}'.format(zipname))
# writing files to a zipfile
with ZipFile(zipname, 'w') as zip:
# writing each file one by one
for file in files:
zip.write(file)
def distill_corrections(bias_chain, pool=None):
"""Distill relativistic corrections and correction factors from a
relativistic bias chain.
Parameters
----------
bias_chain : :class:`numpy.ndarray`
Relativistic bias parameter chain.
pool : :class:`multiprocessing.Pool` or None, optional
Multiprocessing pool (default is `None`).
Returns
-------
correction_chain : :class:`numpy.ndarray`
Relativistic correction samples.
factor_chain : :class:`numpy.ndarray`
Relativistic correction factor samples.
"""
mapping = pool.imap if pool else map
num_cpus = cpu_count() if pool else 1
logger.info(
"Distilling relativistic corrections/correction factors " +
"with %i CPUs...\n", num_cpus)
correction_chain = np.asarray(
list(
tqdm(mapping(compute_corrections_from_biases, bias_chain),
total=len(bias_chain),
mininterval=1,
file=sys.stdout)))
factor_chain = list(
tqdm(mapping(compute_factors_from_corrections, correction_chain),
total=len(correction_chain),
mininterval=1,
file=sys.stdout))
logger.info("... finished.\n")
return correction_chain, factor_chain
def eliminate(self):
must_remove_indexes = []
need_indexes = self.need_index
mongo_indexes = self.get_data_from_mongo(unset=False, including_sign=False, query=False)
latest_docs = getattr(self, 'latest_indexes')
diff_set = mongo_indexes - need_indexes
# If have `diff_set`, group by `p_code`, then send email
normal_indexes, no_normal_indexes = self.think_indexes_issue(diff_set)
print 'set(mong0) - set(web)=', len(diff_set)
print 'Normal:', len(normal_indexes), 'No Normal:', len(no_normal_indexes)
for ps_key in normal_indexes:
if ps_key in latest_docs:
try:
_id = latest_docs[ps_key][0]['_id']
out_dt = latest_docs[ps_key][0]['out_dt']
if not out_dt:
query = {'_id': _id}
setdata = {
'$set': {'sign': '1', 'stat': 2, 'upt': datetime.now(),
'out_dt': date.today().strftime("%Y%m%d")}
}
self.collection.update(query, setdata)
must_remove_indexes.append(ps_key)
except (AttributeError, IndexError, KeyError) as e:
logger.info('`Eliminate` crawl error: type <{typ}>, msg <{msg}>'.format(typ=e.__class__, msg=e))
if self.insert_mongo_data:
subject = '%s 网站: 网页新增， 插入数据库的记录(新增)，望检查' % self.using_category.upper()
self.send_email(subject, self.insert_mongo_data, prefix='web')
if no_normal_indexes:
subject = '%s 网站: 可能有问题的指数成分股(未剔除)， 请检查' % self.using_category.upper()
self.send_email(subject, no_normal_indexes)
if must_remove_indexes:
subject = '%s 网站: 没有问题的指数成分股(已剔除)， 望检查' % self.using_category.upper()
self.send_email(subject, must_remove_indexes)
def delete(self, _id=None):
"""Method to delete deposition.
Parameters
----------
_id: int
deposition id of the record to delete
.. note::
it worked only if it is not publish.
Exception
---------
ZenodoException
raise if token not define (token = None) or if connection
return status >= 400
"""
# Test if token was defined
self._verify_token()
# Use provided if if not None. If not provided use self.deposition_id
if _id is not None:
self.deposition_id = _id
# Create the request url
request_url = (self.depositions_url + '/{}'.format(self.deposition_id))
logger.info('Delete url: {}'.format(request_url))
try:
request = requests.delete(request_url,
params={'access_token': self.token})
self.status_code = request.status_code
logger.debug('Status code: {}'.format(self.status_code))
if self.status_code >= 400:
raise ZenodoException
except ZenodoException:
message = 'Request_url does not exist or bad token. ' \
'Error: {}'.format(self.status_code)
logger.error(message)
raise ZenodoException(message)
def save_distilled():
"""Save the distilled relativistic correction and correction factor
chains.
Returns
-------
:class:`pathlib.Path`
Chain output file path.
"""
infile = PATHOUT / progrc.chain_subdir / progrc.chain_file
chain_suffix = progrc.chain_file.replace("relbias_", "")
redshift_tag = "z{:.2f}".format(progrc.redshift)
prefix_0, prefix_1 = "relcrct_", "relfact_"
if redshift_tag not in progrc.chain_file:
prefix_0 += redshift_tag + "_"
prefix_1 += redshift_tag + "_k{}_".format(progrc.wavenumber)
outfile_0 = PATHOUT / progrc.chain_subdir / (prefix_0 + chain_suffix)
outfile_1 = PATHOUT / progrc.chain_subdir / (prefix_1 + chain_suffix)
with hp.File(infile, 'r') as indata, hp.File(outfile_0, 'w') as outdata:
outdata.create_group('distill')
try:
indata.copy('extract/log_prob', outdata['distill'])
except KeyError:
pass
outdata.create_dataset('distill/chain', data=distilled_chains[0])
with hp.File(infile, 'r') as indata, hp.File(outfile_1, 'w') as outdata:
outdata.create_group('distill')
try:
indata.copy('extract/log_prob', outdata['distill'])
except KeyError:
pass
outdata.create_dataset('distill/chain', data=distilled_chains[1])
logger.info("Distilled chains saved to %s and %s.\n", outfile_0, outfile_1)
def read_chains():
"""Load and process chains from file.
Returns
-------
chains : :class:`numpy.ndarray`
Flattened chains.
"""
chain_file = PATHOUT / progrc.chain_file
if progrc.sampler == 'emcee':
reader = mc.backends.HDFBackend(chain_file, read_only=True)
chains = reader.get_chain()
elif progrc.sampler == 'zeus':
with hp.File(chain_file, 'r') as chain_data:
chains = chain_data['mcmc']['chain'][()]
logger.info("Loaded chain file: %s.\n", chain_file)
return chains
