def get_apis(config):
"""
Get list of APIs to test, based on command line options and their availability.
"""
# if we import it in the header, it messes up with coverage results
import reikna.cluda as cluda
conf_api_id = config.option.api
if conf_api_id == "supported":
api_ids = cluda.supported_api_ids()
else:
if not cluda.supports_api(conf_api_id):
raise Exception("Requested API " + conf_api_id + " is not supported.")
api_ids = [conf_api_id]
apis = [cluda.get_api(api_id) for api_id in api_ids]
return apis, api_ids
def thread(request):
api_id, pnum, dnum = request.param
api = get_api(api_id)
platform = api.get_platforms()[pnum]
device = platform.get_devices()[dnum]
thread = api.Thread(device)
yield thread
# Computations may retain references to the Thread objects,
# so we need to clear the cache first so that the thread could be destroyed
# as it goes out of scope.
# CUDA is sensitive to the exact timing of the destruction
# because of the stateful nature of its API.
clear_computation_cache()
def get_apis(config):
"""
Get list of APIs to test, based on command line options and their availability.
"""
# if we import it in the header, it messes up with coverage results
import reikna.cluda as cluda
conf_api_id = config.option.api
if conf_api_id == "supported":
api_ids = cluda.supported_api_ids()
else:
if not cluda.supports_api(conf_api_id):
raise Exception("Requested API " + conf_api_id +
" is not supported.")
api_ids = [conf_api_id]
apis = [cluda.get_api(api_id) for api_id in api_ids]
return apis, api_ids
def worker_gpu(system,
samples,
measurements,
seed,
repetitions=1,
api_id='ocl',
device_num=0):
result_set = ResultSet()
rng = numpy.random.RandomState(seed)
api = get_api(api_id)
device = api.get_platforms()[0].get_devices()[device_num]
thread = api.Thread(device)
for representation in used_representations(measurements):
generate_input_state = generate_gpu.prepare_generate_input_state(
thread, system, representation, samples)
apply_matrix = generate_gpu.prepare_apply_matrix(
thread, system, representation, samples)
prepared_meters = {
label: measurement.meter.prepare_gpu(thread, system,
representation, samples)
for label, measurement in measurements.items()
if representation in measurement.representations
}
for repetition in range(repetitions):
in_state = generate_input_state(make_seed(rng))
out_state = apply_matrix(in_state, make_seed(rng))
for label, measurement in measurements.items():
if 'in' in measurement.stages and representation in measurement.representations:
result = prepared_meters[label](in_state)
result_set.add_result(result, label, 'in', representation)
if 'out' in measurement.stages and representation in measurement.representations:
result = prepared_meters[label](out_state)
result_set.add_result(result, label, 'out', representation)
return result_set
def pytest_generate_tests(metafunc):
config = metafunc.config
if 'thread' in metafunc.fixturenames:
api_opt = config.option.api
api_ids = supported_api_ids() if api_opt == 'supported' else [api_opt]
vals = []
ids = []
for api_id in api_ids:
api = get_api(api_id)
devices = find_devices(
api,
include_devices=config.option.device_include_mask,
include_duplicate_devices=config.option.
include_duplicate_devices)
for pnum in sorted(devices.keys()):
dnums = sorted(devices[pnum])
for dnum in dnums:
vals.append((api_id, pnum, dnum))
ids.append("{api_id}:{pnum}:{dnum}".format(api_id=api_id,
pnum=pnum,
dnum=dnum))
if len(vals) == 0:
raise RuntimeError(
"Neither PyCUDA nor PyOpenCL could find any suitable devices. "
"Check your system configuration.")
metafunc.parametrize("thread", vals, ids=ids, indirect=True)
if 'transform_type' in metafunc.fixturenames:
if config.option.transform == 'all':
vals = ['NTT', 'FFT']
else:
vals = [config.option.transform]
metafunc.parametrize("transform_type", vals)
def pytest_generate_tests(metafunc):
config = metafunc.config
if 'thread' in metafunc.fixturenames:
api_opt = config.option.api
api_ids = supported_api_ids() if api_opt == 'supported' else [api_opt]
vals = []
ids = []
for api_id in api_ids:
api = get_api(api_id)
devices = find_devices(
api,
include_devices=config.option.device_include_mask,
include_duplicate_devices=config.option.
include_duplicate_devices)
for pnum in sorted(devices.keys()):
dnums = sorted(devices[pnum])
for dnum in dnums:
vals.append((api_id, pnum, dnum))
ids.append("{api_id}:{pnum}:{dnum}".format(api_id=api_id,
pnum=pnum,
dnum=dnum))
metafunc.parametrize("thread", vals, ids=ids, indirect=True)
if 'transform_type' in metafunc.fixturenames:
if config.option.transform == 'all':
vals = ['NTT', 'FFT']
else:
vals = [config.option.transform]
metafunc.parametrize("transform_type", vals)
def _get_objects(self):
api_obj = cluda.get_api(self.api_id)
platform = api_obj.get_platforms()[self.platform_id]
return api_obj, platform, platform.get_devices()[self.device_id]
def make_thread(api_id, device_num):
api = get_api(api_id)
device = api.get_platforms()[0].get_devices()[device_num]
thread = api.Thread(device)
return thread
def simulate_mp(dirname,
system,
ensembles,
samples_per_ensemble,
measurements,
seed,
api_id='ocl',
gpu_name_filters=[]):
fname = f"{dirname}/result_sets modes={system.modes} seed={seed}.pickle"
if Path(fname).exists():
with open(fname, 'rb') as f:
result_sets = pickle.load(f)
else:
result_sets = {}
ensembles = ensembles - len(result_sets)
print(fname, "- remaining ensembles:", ensembles)
if ensembles <= 0:
return ResultSet.merge(list(result_sets.values())[:ensembles])
api = get_api(api_id)
device_nums = [
device_num for device_num, device in enumerate(api.get_platforms()
[0].get_devices())
if len(gpu_name_filters) == 0 or any(name in device.name
for name in gpu_name_filters)
]
# Just in case, to make the code below airtight.
device_nums = device_nums[:ensembles]
result_queue = Queue()
command_queues = [Queue() for worker_id in range(len(device_nums))]
processes = [
Thread(target=worker_gpu,
args=(worker_id, result_queue, command_queue, system,
samples_per_ensemble, measurements, api_id, device_num))
for worker_id, (
command_queue,
device_num) in enumerate(zip(command_queues, device_nums))
]
for process in processes:
process.start()
rng = numpy.random.RandomState(seed)
# Initial tasks
for worker_id in range(len(device_nums)):
seed = make_seed(rng)
while seed in result_sets:
seed = make_seed(rng)
command_queues[worker_id].put(seed)
ensembles_started = len(device_nums)
ensembles_finished = 0
while True:
worker_id, worker_seed, worker_result_set = result_queue.get()
result_sets[worker_seed] = worker_result_set
ensembles_finished += 1
if ensembles_finished == ensembles:
break
if ensembles_started < ensembles:
seed = make_seed(rng)
while seed in result_sets:
seed = make_seed(rng)
command_queues[worker_id].put(seed)
for command_queue in command_queues:
command_queue.put('stop')
for process in processes:
process.join()
with open(fname, 'wb') as f:
pickle.dump(result_sets, f)
return ResultSet.merge(result_sets.values())
def worker_gpu(worker_id, out_queue, in_queue, system, samples, measurements,
api_id, device_num):
api = get_api(api_id)
device = api.get_platforms()[0].get_devices()[device_num]
thread = api.Thread(device)
prepared_calls = {}
for representation in used_representations(measurements):
generate_input_state = generate_gpu.prepare_generate_input_state(
thread, system, representation, samples)
apply_matrix = generate_gpu.prepare_apply_matrix(
thread, system, representation, samples)
prepared_meters = {
label: measurement.meter.prepare_gpu(thread, system,
representation, samples)
for label, measurement in measurements.items()
if representation in measurement.representations
}
prepared_calls[representation] = dict(
generate_input_state=generate_input_state,
apply_matrix=apply_matrix,
prepared_meters=prepared_meters)
while True:
command = in_queue.get()
if command == 'stop':
print(f"Worker {worker_id}: stopping")
break
seed = command
result_set = ResultSet()
print(f"{worker_id}: processing seed {seed}")
for representation in used_representations(measurements):
#print(f"{worker_id}: processing representation {representation}")
rng = numpy.random.RandomState(seed)
in_state = prepared_calls[representation]['generate_input_state'](
make_seed(rng))
out_state = prepared_calls[representation]['apply_matrix'](
in_state, make_seed(rng))
for label, measurement in measurements.items():
if 'in' in measurement.stages and representation in measurement.representations:
result = prepared_calls[representation]['prepared_meters'][
label](in_state)
result_set.add_result(result, label, 'in', representation)
if 'out' in measurement.stages and representation in measurement.representations:
result = prepared_calls[representation]['prepared_meters'][
label](out_state)
result_set.add_result(result, label, 'out', representation)
out_queue.put((worker_id, seed, result_set))
def find_periods(algorithm, lightcurves, freqs, batch_size=1,
doGPU=False, doCPU=False, doSaveMemory=False,
doRemoveTerrestrial=False,
doRemoveWindow=False,
doUsePDot=False, doSingleTimeSegment=False,
freqs_to_remove=None,
phase_bins=20, mag_bins=10,
doParallel=False,
Ncore=4):
if doRemoveTerrestrial and (freqs_to_remove is not None) and not (algorithm=="LS" or algorithm=="GCE_LS_AOV" or algorithm=="GCE_LS" or algorithm=="GCE_LS_AOV_x3"):
for pair in freqs_to_remove:
idx = np.where((freqs < pair[0]) | (freqs > pair[1]))[0]
freqs = freqs[idx]
periods_best, significances = [], []
pdots = np.zeros((len(lightcurves),))
print('Period finding lightcurves...')
if doGPU:
if algorithm == "CE":
from cuvarbase.ce import ConditionalEntropyAsyncProcess
proc = ConditionalEntropyAsyncProcess(use_double=True, use_fast=True, phase_bins=phase_bins, mag_bins=mag_bins, phase_overlap=1, mag_overlap=1, only_keep_best_freq=True)
if doSaveMemory:
periods_best, significances = proc.batched_run_const_nfreq(lightcurves, batch_size=batch_size, freqs = freqs, only_keep_best_freq=True,show_progress=True,returnBestFreq=True)
else:
results = proc.batched_run_const_nfreq(lightcurves, batch_size=batch_size, freqs = freqs, only_keep_best_freq=True,show_progress=True,returnBestFreq=False)
cnt = 0
for lightcurve, out in zip(lightcurves,results):
periods = 1./out[0]
entropies = out[1]
significance = np.abs(np.mean(entropies)-np.min(entropies))/np.std(entropies)
period = periods[np.argmin(entropies)]
periods_best.append(period)
significances.append(significance)
elif algorithm == "BLS":
from cuvarbase.bls import eebls_gpu_fast
for ii,data in enumerate(lightcurves):
if np.mod(ii,10) == 0:
print("%d/%d"%(ii,len(lightcurves)))
copy = np.ma.copy(data).T
powers = eebls_gpu_fast(copy[:,0],copy[:,1], copy[:,2],
freq_batch_size=batch_size,
freqs = freqs)
significance = np.abs(np.mean(powers)-np.max(powers))/np.std(powers)
freq = freqs[np.argmax(powers)]
period = 1.0/freq
periods_best.append(period)
significances.append(significance)
elif algorithm == "LS":
from cuvarbase.lombscargle import LombScargleAsyncProcess, fap_baluev
nfft_sigma, spp = 10, 10
ls_proc = LombScargleAsyncProcess(use_double=True,
sigma=nfft_sigma)
if doSaveMemory:
periods_best, significances = ls_proc.batched_run_const_nfreq(lightcurves, batch_size=batch_size, use_fft=True, samples_per_peak=spp, returnBestFreq=True, freqs = freqs, doRemoveTerrestrial=doRemoveTerrestrial, freqs_to_remove=freqs_to_remove)
else:
results = ls_proc.batched_run_const_nfreq(lightcurves,
batch_size=batch_size,
use_fft=True,
samples_per_peak=spp,
returnBestFreq=False,
freqs = freqs)
for data, out in zip(lightcurves,results):
freqs, powers = out
if doRemoveTerrestrial and (freqs_to_remove is not None):
for pair in freqs_to_remove:
idx = np.where((freqs < pair[0]) | (freqs > pair[1]))[0]
freqs = freqs[idx]
powers = powers[idx]
copy = np.ma.copy(data).T
fap = fap_baluev(copy[:,0], copy[:,2], powers, np.max(freqs))
idx = np.argmin(fap)
period = 1./freqs[idx]
significance = 1./fap[idx]
periods_best.append(period)
significances.append(significance)
ls_proc.finish()
elif algorithm == "PDM":
from cuvarbase.pdm import PDMAsyncProcess
kind, nbins = 'binned_linterp', 10
pdm_proc = PDMAsyncProcess()
for lightcurve in lightcurves:
results = pdm_proc.run([lightcurve], kind=kind, nbins=nbins)
pdm_proc.finish()
powers = results[0]
significance = np.abs(np.mean(powers)-np.max(powers))/np.std(powers)
freq = freqs[np.argmax(powers)]
period = 1.0/freq
periods_best.append(period)
significances.append(significance)
elif algorithm == "GCE":
from gcex.gce import ConditionalEntropy
ce = ConditionalEntropy(phase_bins=phase_bins, mag_bins=mag_bins)
if doUsePDot:
num_pdots = 10
max_pdot = 1e-10
min_pdot = 1e-12
pdots_to_test = -np.logspace(np.log10(min_pdot), np.log10(max_pdot), num_pdots)
pdots_to_test = np.append(0,pdots_to_test)
#pdots_to_test = np.array([-2.365e-11])
else:
pdots_to_test = np.array([0.0])
if doSingleTimeSegment:
tt = np.empty((0,1))
for lightcurve in lightcurves:
tt = np.unique(np.append(tt, lightcurve[0]))
maxn = -np.inf
lightcurves_stack = []
for lightcurve in lightcurves:
if doSingleTimeSegment:
xy, x_ind, y_ind = np.intersect1d(tt, lightcurve[0],
return_indices=True)
mag_array = 999*np.ones(tt.shape)
magerr_array = 999*np.ones(tt.shape)
mag_array[x_ind] = lightcurve[1][y_ind]
magerr_array[x_ind] = lightcurve[2][y_ind]
lightcurve = (tt, mag_array, magerr_array)
else:
idx = np.argsort(lightcurve[0])
tmin = np.min(lightcurve[0])
lightcurve = (lightcurve[0][idx]-tmin,
lightcurve[1][idx],
lightcurve[2][idx])
lightcurve_stack = np.vstack((lightcurve[0],
lightcurve[1],
lightcurve[2])).T
lightcurves_stack.append(lightcurve_stack)
if len(idx) > maxn:
maxn = len(idx)
periods_best = np.zeros((len(lightcurves),1))
significances = np.zeros((len(lightcurves),1))
pdots = np.zeros((len(lightcurves),1))
pdots_split = np.array_split(pdots_to_test,len(pdots_to_test))
for ii, pdot in enumerate(pdots_split):
print("Running pdot %d / %d" % (ii+1, len(pdots_split)))
print("Number of lightcurves: %d" % len(lightcurves_stack))
print("Max length of lightcurves: %d" % maxn)
print("Batch size: %d" % batch_size)
print("Number of frequency bins: %d" % len(freqs))
print("Number of phase bins: %d" % phase_bins)
print("Number of magnitude bins: %d" % mag_bins)
results = ce.batched_run_const_nfreq(lightcurves_stack, batch_size, freqs, pdot, show_progress=False)
periods = 1./freqs
for jj, (lightcurve, entropies2) in enumerate(zip(lightcurves,results)):
for kk, entropies in enumerate(entropies2):
significance = np.abs(np.mean(entropies)-np.min(entropies))/np.std(entropies)
period = periods[np.argmin(entropies)]
if significance > significances[jj]:
periods_best[jj] = period
significances[jj] = significance
pdots[jj] = pdot[kk]*1.0
pdots, periods_best, significances = pdots.flatten(), periods_best.flatten(), significances.flatten()
elif (algorithm == "ECE") or (algorithm == "EAOV") or (algorithm == "ELS"):
if algorithm == "ECE":
from periodfind.ce import ConditionalEntropy
#ce = ConditionalEntropy(phase_bins=phase_bins, mag_bins=mag_bins)
ce = ConditionalEntropy(phase_bins, mag_bins)
elif algorithm == "EAOV":
from periodfind.aov import AOV
aov = AOV(phase_bins)
elif algorithm == "ELS":
from periodfind.ls import LombScargle
ls = LombScargle()
if doUsePDot:
num_pdots = 10
max_pdot = 1e-10
min_pdot = 1e-12
pdots_to_test = -np.logspace(np.log10(min_pdot), np.log10(max_pdot), num_pdots)
pdots_to_test = np.append(0,pdots_to_test)
#pdots_to_test = np.array([-2.365e-11])
else:
pdots_to_test = np.array([0.0])
if doSingleTimeSegment:
tt = np.empty((0,1))
for lightcurve in lightcurves:
tt = np.unique(np.append(tt, lightcurve[0]))
maxn = -np.inf
time_stack, mag_stack = [], []
for lightcurve in lightcurves:
if doSingleTimeSegment:
xy, x_ind, y_ind = np.intersect1d(tt, lightcurve[0],
return_indices=True)
mag_array = 999*np.ones(tt.shape)
magerr_array = 999*np.ones(tt.shape)
mag_array[x_ind] = lightcurve[1][y_ind]
magerr_array[x_ind] = lightcurve[2][y_ind]
lightcurve = (tt, mag_array, magerr_array)
else:
idx = np.argsort(lightcurve[0])
tmin = np.min(lightcurve[0])
lightcurve = (lightcurve[0][idx]-tmin,
lightcurve[1][idx],
lightcurve[2][idx])
lightcurve_stack = np.vstack((lightcurve[0],
lightcurve[1],
lightcurve[2])).T
time_stack.append(lightcurve[0].astype(np.float32))
lc = lightcurve[1]
lc = (lc - np.min(lc))/(np.max(lc)-np.min(lc))
mag_stack.append(lc.astype(np.float32))
if len(idx) > maxn:
maxn = len(idx)
print("Number of lightcurves: %d" % len(time_stack))
print("Max length of lightcurves: %d" % maxn)
print("Batch size: %d" % batch_size)
print("Number of frequency bins: %d" % len(freqs))
print("Number of phase bins: %d" % phase_bins)
print("Number of magnitude bins: %d" % mag_bins)
periods = (1.0/freqs).astype(np.float32)
pdots_to_test = pdots_to_test.astype(np.float32)
periods_best = np.zeros((len(lightcurves),1))
significances = np.zeros((len(lightcurves),1))
pdots = np.zeros((len(lightcurves),1))
if algorithm == "ECE":
data_out = ce.calc(time_stack, mag_stack, periods, pdots_to_test)
elif algorithm == "EAOV":
data_out = aov.calc(time_stack, mag_stack, periods, pdots_to_test)
elif algorithm == "ELS":
data_out = ls.calc(time_stack, mag_stack, periods, pdots_to_test)
for ii, stat in enumerate(data_out):
if np.isnan(stat.significance):
raise ValueError("Oops... significance is nan... something went wrong")
periods_best[ii] = stat.params[0]
pdots[ii] = stat.params[1]
significances[ii] = stat.significance
pdots, periods_best, significances = pdots.flatten(), periods_best.flatten(), significances.flatten()
elif (algorithm == "ECE_periodogram") or (algorithm == "EAOV_periodogram") or (algorithm == "ELS_periodogram"):
if algorithm.split("_")[0] == "ECE":
from periodfind.ce import ConditionalEntropy
#ce = ConditionalEntropy(phase_bins=phase_bins, mag_bins=mag_bins)
ce = ConditionalEntropy(phase_bins, mag_bins)
elif algorithm.split("_")[0] == "EAOV":
from periodfind.aov import AOV
aov = AOV(phase_bins)
elif algorithm.split("_")[0] == "ELS":
from periodfind.ls import LombScargle
ls = LombScargle()
if doUsePDot:
num_pdots = 10
max_pdot = 1e-10
min_pdot = 1e-12
pdots_to_test = -np.logspace(np.log10(min_pdot), np.log10(max_pdot), num_pdots)
pdots_to_test = np.append(0,pdots_to_test)
#pdots_to_test = np.array([-2.365e-11])
else:
pdots_to_test = np.array([0.0])
if doSingleTimeSegment:
tt = np.empty((0,1))
for lightcurve in lightcurves:
tt = np.unique(np.append(tt, lightcurve[0]))
maxn = -np.inf
time_stack, mag_stack = [], []
for lightcurve in lightcurves:
if doSingleTimeSegment:
xy, x_ind, y_ind = np.intersect1d(tt, lightcurve[0],
return_indices=True)
mag_array = 999*np.ones(tt.shape)
magerr_array = 999*np.ones(tt.shape)
mag_array[x_ind] = lightcurve[1][y_ind]
magerr_array[x_ind] = lightcurve[2][y_ind]
lightcurve = (tt, mag_array, magerr_array)
else:
idx = np.argsort(lightcurve[0])
tmin = np.min(lightcurve[0])
lightcurve = (lightcurve[0][idx]-tmin,
lightcurve[1][idx],
lightcurve[2][idx])
lightcurve_stack = np.vstack((lightcurve[0],
lightcurve[1],
lightcurve[2])).T
time_stack.append(lightcurve[0].astype(np.float32))
lc = lightcurve[1]
lc = (lc - np.min(lc))/(np.max(lc)-np.min(lc))
mag_stack.append(lc.astype(np.float32))
if len(idx) > maxn:
maxn = len(idx)
print("Number of lightcurves: %d" % len(time_stack))
print("Max length of lightcurves: %d" % maxn)
print("Batch size: %d" % batch_size)
print("Number of frequency bins: %d" % len(freqs))
print("Number of phase bins: %d" % phase_bins)
print("Number of magnitude bins: %d" % mag_bins)
periods = (1.0/freqs).astype(np.float32)
pdots_to_test = pdots_to_test.astype(np.float32)
periods_best = []
significances = np.zeros((len(lightcurves),1))
pdots = np.zeros((len(lightcurves),1))
if algorithm.split("_")[0] == "ECE":
data_out = ce.calc(time_stack, mag_stack, periods, pdots_to_test, output='periodogram')
elif algorithm.split("_")[0] == "EAOV":
data_out = aov.calc(time_stack, mag_stack, periods, pdots_to_test, output='periodogram')
elif algorithm.split("_")[0] == "ELS":
data_out = ls.calc(time_stack, mag_stack, periods, pdots_to_test, output='periodogram')
for ii, stat in enumerate(data_out):
if algorithm.split("_")[0] == "ECE":
significance = np.abs(np.mean(stat.data)-np.min(stat.data))/np.std(stat.data)
period = periods[np.argmin(stat.data)]
elif algorithm.split("_")[0] == "EAOV":
significance = np.abs(np.mean(stat.data)-np.min(stat.data))/np.std(stat.data)
period = periods[np.argmin(stat.data)]
elif algorithm.split("_")[0] == "ELS":
significance = np.abs(np.mean(stat.data)-np.max(stat.data))/np.std(stat.data)
period = periods[np.argmax(stat.data)]
if np.isnan(significance):
raise ValueError("Oops... significance is nan... something went wrong")
periods_best.append({'period': period, 'data': stat.data})
pdots[ii] = pdots_to_test[0]
significances[ii] = significance
pdots, significances = pdots.flatten(), significances.flatten()
elif algorithm == "GCE_LS_AOV":
nfreqs_to_keep = 100
nfreqs_to_keep = 50
df = freqs[1]-freqs[0]
from cuvarbase.lombscargle import LombScargleAsyncProcess, fap_baluev
from gcex.gce import ConditionalEntropy
from ztfperiodic.pyaov.pyaov import aovw, amhw
ce = ConditionalEntropy(phase_bins=phase_bins, mag_bins=mag_bins)
freqs_to_keep = {}
for jj in range(len(lightcurves)):
freqs_to_keep[jj] = np.empty((0,1))
pdot = np.array([0.0])
if doSingleTimeSegment:
tt = np.empty((0,1))
for lightcurve in lightcurves:
tt = np.unique(np.append(tt, lightcurve[0]))
maxn = -np.inf
lightcurves_stack = []
for lightcurve in lightcurves:
if doSingleTimeSegment:
xy, x_ind, y_ind = np.intersect1d(tt, lightcurve[0],
return_indices=True)
mag_array = 999*np.ones(tt.shape)
magerr_array = 999*np.ones(tt.shape)
mag_array[x_ind] = lightcurve[1][y_ind]
magerr_array[x_ind] = lightcurve[2][y_ind]
lightcurve = (tt, mag_array, magerr_array)
else:
tmin = np.min(lightcurve[0])
idx = np.argsort(lightcurve[0])
lightcurve = (lightcurve[0][idx]-tmin,
lightcurve[1][idx],
lightcurve[2][idx])
lightcurve_stack = np.vstack((lightcurve[0],
lightcurve[1],
lightcurve[2])).T
lightcurves_stack.append(lightcurve_stack)
if len(idx) > maxn:
maxn = len(idx)
print("Number of lightcurves: %d" % len(lightcurves_stack))
print("Max length of lightcurves: %d" % maxn)
print("Batch size: %d" % batch_size)
print("Number of frequency bins: %d" % len(freqs))
print("Number of phase bins: %d" % phase_bins)
print("Number of magnitude bins: %d" % mag_bins)
results = ce.batched_run_const_nfreq(lightcurves_stack, batch_size, freqs, pdot, show_progress=False)
for jj, (lightcurve, entropies2) in enumerate(zip(lightcurves,results)):
for kk, entropies in enumerate(entropies2):
freqs_tmp = np.copy(freqs)
if doRemoveTerrestrial and (freqs_to_remove is not None):
for pair in freqs_to_remove:
idx = np.where((freqs_tmp < pair[0]) | (freqs_tmp > pair[1]))[0]
freqs_tmp = freqs_tmp[idx]
entropies = entropies[idx]
significance = np.abs(np.mean(entropies)-entropies)/np.std(entropies)
idx = np.argsort(significance)[::-1]
freqs_to_keep[jj] = np.append(freqs_to_keep[jj],
freqs_tmp[idx[:nfreqs_to_keep]])
nfft_sigma, spp = 10, 10
ls_proc = LombScargleAsyncProcess(use_double=True,
sigma=nfft_sigma)
results = ls_proc.batched_run_const_nfreq(lightcurves,
batch_size=batch_size,
use_fft=True,
samples_per_peak=spp,
returnBestFreq=False,
freqs = freqs)
for jj, (data, out) in enumerate(zip(lightcurves,results)):
freqs, powers = out
if doRemoveTerrestrial and (freqs_to_remove is not None):
for pair in freqs_to_remove:
idx = np.where((freqs < pair[0]) | (freqs > pair[1]))[0]
freqs = freqs[idx]
powers = powers[idx]
copy = np.ma.copy(data).T
fap = fap_baluev(copy[:,0], copy[:,2], powers, np.max(freqs))
idx = np.argsort(fap)
freqs_to_keep[jj] = np.append(freqs_to_keep[jj],
freqs[idx[:nfreqs_to_keep]])
ls_proc.finish()
if doParallel:
from joblib import Parallel, delayed
res = Parallel(n_jobs=Ncore)(delayed(calc_AOV)(data, freqs_to_keep[jj], df) for jj, data in enumerate(lightcurves))
periods_best = [x[0] for x in res]
significances = [x[1] for x in res]
else:
for jj, data in enumerate(lightcurves):
if np.mod(jj,10) == 0:
print("%d/%d"%(jj,len(lightcurves)))
period, significance = calc_AOV(data, freqs_to_keep[jj], df)
periods_best.append(period)
significances.append(significance)
elif algorithm == "GCE_LS_AOV_x3":
nfreqs_to_keep = 100
nfreqs_to_keep = 50
df = freqs[1]-freqs[0]
niter = 3
from cuvarbase.lombscargle import LombScargleAsyncProcess, fap_baluev
from gcex.gce import ConditionalEntropy
from ztfperiodic.pyaov.pyaov import aovw, amhw
ce = ConditionalEntropy(phase_bins=phase_bins, mag_bins=mag_bins)
freqs_to_keep = {}
for jj in range(len(lightcurves)):
freqs_to_keep[jj] = np.empty((0,1))
pdot = np.array([0.0])
if doSingleTimeSegment:
tt = np.empty((0,1))
for lightcurve in lightcurves:
tt = np.unique(np.append(tt, lightcurve[0]))
maxn = -np.inf
lightcurves_stack = []
for lightcurve in lightcurves:
if doSingleTimeSegment:
xy, x_ind, y_ind = np.intersect1d(tt, lightcurve[0],
return_indices=True)
mag_array = 999*np.ones(tt.shape)
magerr_array = 999*np.ones(tt.shape)
mag_array[x_ind] = lightcurve[1][y_ind]
magerr_array[x_ind] = lightcurve[2][y_ind]
lightcurve = (tt, mag_array, magerr_array)
else:
tmin = np.min(lightcurve[0])
idx = np.argsort(lightcurve[0])
lightcurve = (lightcurve[0][idx]-tmin,
lightcurve[1][idx],
lightcurve[2][idx])
lightcurve_stack = np.vstack((lightcurve[0],
lightcurve[1],
lightcurve[2])).T
lightcurves_stack.append(lightcurve_stack)
if len(idx) > maxn:
maxn = len(idx)
print("Number of lightcurves: %d" % len(lightcurves_stack))
print("Max length of lightcurves: %d" % maxn)
print("Batch size: %d" % batch_size)
print("Number of frequency bins: %d" % len(freqs))
print("Number of phase bins: %d" % phase_bins)
print("Number of magnitude bins: %d" % mag_bins)
entropies_all = {}
for nn in range(niter):
results = ce.batched_run_const_nfreq(lightcurves_stack, batch_size, freqs, pdot, show_progress=False)
for jj, (lightcurve, entropies2) in enumerate(zip(lightcurves,results)):
if nn == 0:
entropies_all[jj] = np.empty((0,len(freqs)))
for kk, entropies in enumerate(entropies2):
entropies_all[jj] = np.append(entropies_all[jj],
np.atleast_2d(entropies),
axis=0)
for jj in entropies_all.keys():
entropies = np.median(entropies_all[jj], axis=0)
freqs_tmp = np.copy(freqs)
if doRemoveTerrestrial and (freqs_to_remove is not None):
for pair in freqs_to_remove:
idx = np.where((freqs_tmp < pair[0]) | (freqs_tmp > pair[1]))[0]
freqs_tmp = freqs_tmp[idx]
entropies = entropies[idx]
significance = np.abs(np.mean(entropies)-entropies)/np.std(entropies)
idx = np.argsort(significance)[::-1]
freqs_to_keep[jj] = np.append(freqs_to_keep[jj],
freqs_tmp[idx[:nfreqs_to_keep]])
nfft_sigma, spp = 10, 10
freqs_all = {}
powers_all = {}
for nn in range(niter):
ls_proc = LombScargleAsyncProcess(use_double=True,
sigma=nfft_sigma)
results = ls_proc.batched_run_const_nfreq(lightcurves,
batch_size=batch_size,
use_fft=True,
samples_per_peak=spp,
returnBestFreq=False,
freqs = freqs)
for jj, (data, out) in enumerate(zip(lightcurves,results)):
freqs, powers = out
if nn == 0:
freqs_all[jj] = freqs
powers_all[jj] = np.empty((0,len(freqs)))
powers_all[jj] = np.append(powers_all[jj],
np.atleast_2d(powers),
axis=0)
ls_proc.finish()
for jj in powers_all.keys():
data = lightcurves[jj]
freqs = freqs_all[jj]
powers = np.median(powers_all[jj], axis=0)
if doRemoveTerrestrial and (freqs_to_remove is not None):
for pair in freqs_to_remove:
idx = np.where((freqs < pair[0]) | (freqs > pair[1]))[0]
freqs = freqs[idx]
powers = powers[idx]
copy = np.ma.copy(data).T
fap = fap_baluev(copy[:,0], copy[:,2], powers, np.max(freqs))
idx = np.argsort(fap)
freqs_to_keep[jj] = np.append(freqs_to_keep[jj],
freqs[idx[:nfreqs_to_keep]])
for jj, data in enumerate(lightcurves):
if np.mod(jj,10) == 0:
print("%d/%d"%(jj,len(lightcurves)))
copy = np.ma.copy(data).T
copy[:,1] = (copy[:,1] - np.min(copy[:,1])) \
/ (np.max(copy[:,1]) - np.min(copy[:,1]))
freqs, aovs = np.empty((0,1)), np.empty((0,1))
for ii, fr0 in enumerate(freqs_to_keep[jj]):
err = copy[:,2]
aov, frtmp, _ = amhw(copy[:,0], copy[:,1], err,
fr0=fr0-50*df,
fstop=fr0+50*df,
fstep=df/2.0,
nh2=4)
idx = np.where(frtmp > 0)[0]
aovs = np.append(aovs,aov[idx])
freqs = np.append(freqs,frtmp[idx])
significance = np.abs(np.mean(aovs)-np.max(aovs))/np.std(aovs)
periods = 1./freqs
significance = np.max(aovs)
period = periods[np.argmax(aovs)]
periods_best.append(period)
significances.append(significance)
elif algorithm == "GCE_LS":
from cuvarbase.lombscargle import LombScargleAsyncProcess, fap_baluev
from gcex.gce import ConditionalEntropy
from ztfperiodic.pyaov.pyaov import aovw, amhw
ce = ConditionalEntropy(phase_bins=phase_bins, mag_bins=mag_bins)
pdot = np.array([0.0])
if doSingleTimeSegment:
tt = np.empty((0,1))
for lightcurve in lightcurves:
tt = np.unique(np.append(tt, lightcurve[0]))
maxn = -np.inf
lightcurves_stack = []
for lightcurve in lightcurves:
if doSingleTimeSegment:
xy, x_ind, y_ind = np.intersect1d(tt, lightcurve[0],
return_indices=True)
mag_array = 999*np.ones(tt.shape)
magerr_array = 999*np.ones(tt.shape)
mag_array[x_ind] = lightcurve[1][y_ind]
magerr_array[x_ind] = lightcurve[2][y_ind]
lightcurve = (tt, mag_array, magerr_array)
else:
tmin = np.min(lightcurve[0])
idx = np.argsort(lightcurve[0])
lightcurve = (lightcurve[0][idx]-tmin,
lightcurve[1][idx],
lightcurve[2][idx])
lightcurve_stack = np.vstack((lightcurve[0],
lightcurve[1])).T
lightcurves_stack.append(lightcurve_stack)
if len(idx) > maxn:
maxn = len(idx)
print("Number of lightcurves: %d" % len(lightcurves_stack))
print("Max length of lightcurves: %d" % maxn)
print("Batch size: %d" % batch_size)
print("Number of frequency bins: %d" % len(freqs))
print("Number of phase bins: %d" % phase_bins)
print("Number of magnitude bins: %d" % mag_bins)
results2 = ce.batched_run_const_nfreq(lightcurves_stack, batch_size, freqs, pdot, show_progress=False)
nfft_sigma, spp = 10, 10
ls_proc = LombScargleAsyncProcess(use_double=True,
sigma=nfft_sigma)
results1 = ls_proc.batched_run_const_nfreq(lightcurves,
batch_size=batch_size,
use_fft=True,
samples_per_peak=spp,
returnBestFreq=False,
freqs = freqs)
for jj, (data, out, entropies2) in enumerate(zip(lightcurves, results1, results2)):
entropies = entropies2[0]
freqs1, powers = out
if doRemoveTerrestrial and (freqs_to_remove is not None):
for pair in freqs_to_remove:
idx = np.where((freqs1 < pair[0]) | (freqs1 > pair[1]))[0]
freqs1 = freqs1[idx]
powers = powers[idx]
entropies = entropies[idx]
#copy = np.ma.copy(data).T
#fap = fap_baluev(copy[:,0], copy[:,2], powers, np.max(freqs))
#significance1 = 1./fap
significance1 = np.abs(powers-np.mean(powers))/np.std(powers)
significance2 = np.abs(np.mean(entropies)-entropies)/np.std(entropies)
significance = significance1*significance2
#significance = significance2
idx = np.argmax(significance)
period = 1./freqs[idx]
significance = significance[idx]
periods_best.append(period)
significances.append(significance)
ls_proc.finish()
elif algorithm == "FFT":
from cuvarbase.lombscargle import fap_baluev
from reikna import cluda
from reikna.fft.fft import FFT
T = 30.0/86400.0
fs = 1.0/T
api = cluda.get_api('cuda')
dev = api.get_platforms()[0].get_devices()[0]
thr = api.Thread(dev)
x = np.arange(0.0, 12.0/24.0, T).astype(np.complex128)
fft = FFT(x, axes=(0,))
fftc = fft.compile(thr, fast_math=True)
lightcurves_stack = []
period_min, period_max = 60.0/86400.0, 12.0*3600.0/86400.0
freq_min, freq_max = 1/period_max, 1/period_min
for ii, lightcurve in enumerate(lightcurves):
bins_tmp = np.arange(np.min(lightcurve[0]), np.max(lightcurve[1]), 1/24.0)
bins = np.vstack((bins_tmp[:-12],bins_tmp[12:]))
n, bins_tmp = ztfperiodic.utils.overlapping_histogram(lightcurve[0], bins)
idx = np.argmax(n)
bins_max = bins[:,idx]
x = np.arange(bins_max[0], bins_max[0] + 12.0/24.0, T)
y = np.interp(x, lightcurve[0], lightcurve[1]).astype(np.complex128)
yerr = np.interp(x, lightcurve[0], lightcurve[2])
if len(y) == 0:
periods_best.append(-1)
significances.append(-1)
continue
y = y - np.median(y)
y = y * np.hanning(len(y))
dev = thr.to_device(y)
fftc(dev, dev)
Y = dev.get()
powers = np.abs(Y)
N = len(y)
freqs = np.linspace(0.0, 1.0/(2.0*T), N/2)
powers = powers[:int(N/2)]
idx = np.where((freqs >= freq_min) & (freqs <= freq_max))[0]
freqs, powers = freqs[idx], powers[idx]
powers = powers * freqs**2
significance = np.abs(np.median(powers)-np.max(powers))/np.std(powers)
freq = freqs[np.argmax(powers)]
period = 1.0/freq
periods_best.append(period)
significances.append(significance)
elif doCPU:
periods = 1/freqs
period_jobs=1
if algorithm == "LS":
from astropy.stats import LombScargle
for ii,data in enumerate(lightcurves):
if np.mod(ii,1) == 0:
print("%d/%d"%(ii,len(lightcurves)))
copy = np.ma.copy(data).T
nrows, ncols = copy.shape
if nrows == 1:
periods_best.append(-1)
significances.append(-1)
continue
ls = LombScargle(copy[:,0], copy[:,1], copy[:,2])
power = ls.power(freqs)
fap = ls.false_alarm_probability(power,maximum_frequency=np.max(freqs))
idx = np.argmin(fap)
significance = 1./fap[idx]
period = 1./freqs[idx]
periods_best.append(period)
significances.append(significance)
elif algorithm == "CE":
from ztfperiodic.period import CE
for ii,data in enumerate(lightcurves):
if np.mod(ii,1) == 0:
print("%d/%d"%(ii,len(lightcurves)))
copy = np.ma.copy(data).T
copy[:,1] = (copy[:,1] - np.min(copy[:,1])) \
/ (np.max(copy[:,1]) - np.min(copy[:,1]))
entropies = []
for period in periods:
entropy = CE(period, data=copy, xbins=phase_bins, ybins=mag_bins)
entropies.append(entropy)
significance = np.abs(np.mean(entropies)-np.min(entropies))/np.std(entropies)
period = periods[np.argmin(entropies)]
periods_best.append(period)
significances.append(significance)
elif algorithm == "AOV":
from ztfperiodic.pyaov.pyaov import aovw, amhw
for ii,data in enumerate(lightcurves):
if np.mod(ii,10) == 0:
print("%d/%d"%(ii,len(lightcurves)))
copy = np.ma.copy(data).T
copy[:,1] = (copy[:,1] - np.min(copy[:,1])) \
/ (np.max(copy[:,1]) - np.min(copy[:,1]))
aov, fr, _ = amhw(copy[:,0], copy[:,1], copy[:,2],
fstop=np.max(1.0/periods),
fstep=1/periods[0])
significance = np.abs(np.mean(aov)-np.max(aov))/np.std(aov)
period = periods[np.argmax(aov)]
periods_best.append(period)
significances.append(significance)
elif algorithm == "AOV_cython":
from AOV_cython import aov as pyaov
for ii,data in enumerate(lightcurves):
if np.mod(ii,10) == 0:
print("%d/%d"%(ii,len(lightcurves)))
copy = np.ma.copy(data).T
copy[:,1] = (copy[:,1] - np.min(copy[:,1])) \
/ (np.max(copy[:,1]) - np.min(copy[:,1]))
aov = pyaov(freqs, copy[:,0], copy[:,1],
np.mean(copy[:,1]),
len(copy[:,0]), 10, len(freqs))
significance = np.abs(np.mean(aov)-np.max(aov))/np.std(aov)
freq = freqs[np.argmax(aov)]
period = 1.0/freq
periods_best.append(period)
significances.append(significance)
return np.array(periods_best), np.array(significances), np.array(pdots)