def do_create(lsst, nobjects=1000):
try:
lsst.drop_collection('y')
print('y dropped')
except:
pass
ra = 0
decl = 0
window = 180.
stepper = st.Stepper()
requests = []
for i in range(nobjects):
obj = {
'loc': [(random.random() * 2 * window - window),
(random.random() * 2 * window - window)]
}
# lsst.y.insert( obj )
requests.append(pymongo.InsertOne(obj))
try:
lsst.y.bulk_write(requests)
except BulkWriteError as bwe:
print('error in bulk write', bwe.details)
exit()
stepper.show_step('y created')
# lsst.Object.aggregate( [ {'$match' : {'chunkId': 516} }, { '$project': { 'loc': [ '$ra', '$decl' ] } }, {'$limit': 1000}, {'$out': 'y'} ] )
print(lsst.y.count())
result = lsst.y.find()
for i, o in enumerate(result):
print(o)
if i > 10:
break
stepper = st.Stepper()
try:
lsst.y.create_index([('loc.0', pymongo.ASCENDING)])
except pymongo.errors.PyMongoError as e:
print('error create index on ra', e)
stepper.show_step('index loc.0 creation')
stepper = st.Stepper()
try:
lsst.y.create_index([('loc.1', pymongo.ASCENDING)])
except pymongo.errors.PyMongoError as e:
print('error create index on decl', e)
stepper.show_step('index loc.1 creation')
stepper = st.Stepper()
try:
lsst.y.create_index([('loc', pymongo.GEO2D)])
except pymongo.errors.PyMongoError as e:
print('error create_geo_index', e)
stepper.show_step('index loc creation')
test9(lsst.y)
def try_geo(lsst, objects=10):
print('============================================ try_geo')
recreate = True
if recreate:
try:
lsst.drop_collection('geo')
print('geo created')
except:
pass
geo = lsst.geo
creation(geo, objects)
aggregate(geo, 'geo')
geo.update_many({}, {'$inc': {'loc.0': -180, 'loc.1': -180}})
stepper = st.Stepper()
geo.create_index('deepSourceId', unique=True)
stepper.show_step('index created' + str(geo.index_information()))
# create_geo_index(geo)
else:
geo = lsst.geo
# stepper = Stepper()
result = geo.find({}, VIEW)
for i, o in enumerate(result):
print('object read', o)
if i > 5:
break
ra = 45.
decl = 0.
ext = 10.
bottomleft = [ra - ext, decl - ext]
topright = [ra + ext, decl + ext]
stepper = st.Stepper()
result = geo.find(
{'loc': {
'$geoWithin': {
'$box': [bottomleft, topright]
}
}}, VIEW)
stepper.show_step('geo find')
for i, o in enumerate(result):
print(o)
if i > 5:
break
def test5(dataset):
stepper = st.Stepper()
ra_left = -4.0
decl_bottom = -3.2
ra_right = -3.99
decl_top = -3.16
try:
query = {
'loc': {
'$geoWithin': {
'$box': [[ra_left, decl_bottom], [ra_right, decl_top]]
}
}
}
result = dataset.find(query, VIEW)
for o in result:
print(o)
except pymongo.errors.PyMongoError as e:
print('error', e)
stepper.show_step(
'SELECT ra, decl FROM Object WHERE qserv_areaspec_box({}, {}, {}, {});'
.format(ra_left + 180., decl_bottom, ra_right + 180., decl_top))
def do_select(lsst, limit, ra, decl, window):
try:
lsst.drop_collection('z')
print('z dropped')
except:
pass
ext = window
bottomleft = [ ra - ext, decl - ext ]
topright = [ ra + ext, decl + ext ]
p = [
{'$geoNear':
{
'near': [ra, decl],
'query': { 'loc': { '$geoWithin': {'$box': [bottomleft, topright] } } },
'limit': limit,
'distanceField': 'dist',
}
},
{'$project': {'_id':1, 'ra':1, 'decl':1} },
{'$out': 'z'},
]
stepper = st.Stepper()
result = lsst.Object.aggregate(p, allowDiskUse=True)
t = stepper.show_step('aggregate create z')
c = lsst.z.count()
print('Objects in the selection:', c, 'window:', bottomleft, topright)
return bottomleft, topright, t, c
def test9(dataset):
stepper = st.Stepper()
try:
min_ra = dataset.find({}, {
'_id': 0,
'loc': 1
}).sort('loc.0', 1).limit(1)[0]['loc'][0]
max_ra = dataset.find({}, {
'_id': 0,
'loc': 1
}).sort('loc.0', -1).limit(1)[0]['loc'][0]
min_decl = dataset.find({}, {
'_id': 0,
'loc': 1
}).sort('loc.1', 1).limit(1)[0]['loc'][1]
max_decl = dataset.find({}, {
'_id': 0,
'loc': 1
}).sort('loc.1', -1).limit(1)[0]['loc'][1]
except pymongo.errors.PyMongoError as e:
print('error min, max', e)
print('ra= [', min_ra, ',', max_ra, ']')
print('decl= [', min_decl, ',', max_decl, ']')
stepper.show_step(
'select min(ra), max(ra), min(decl), max(decl) from Object;')
def on_code_execute(self,gui=None,step=None):
"""The controller calls this when the user hits the 'execute' button.
When step is True the timer isn't started.
This is the point from where the whole show starts."""
self.logger.debug("on_code_execute called")
worldlines = self.controller.get_worldwin_text()
if _DEBUG > 1: print self.__class__,"worldlines",worldlines
codelines = self.controller.get_codewin_text()
if len(worldlines) == 0 or len(codelines) == 0:
self.controller.give_warning(_("Please load a world and program before executing."))
return None
codestring = "\n".join(codelines)
if _DEBUG > 1: print self.__class__,"codestring",codestring
logicworld = self.on_world_reload(worldlines)
if not logicworld:# on_world_reload gives signals the controller in case of error
return None
# logicworld is made a class member because stepper doesn't pass the
# logicworld object when it calls updateWorldBitmapAfterMove.
# Because we want all the logic to use the same logicworld object we
# made it a class member so that updateWorldBitmapAfterMove uses the
# self.logicworld to pass on to the controller.
# see also updateWorldBitmapAfterMove and GvrController.word_state_changed
self.logicworld = logicworld
# GuiWorld gets a GvrModel reference because it expects a object with a
# updateWorldBitmapAfterMove(oldcoords) method which it calls when the
# world state is changed.
myGuiWorld = guiWorld.GuiWorld(self,logicworld)
try:
# stepper can be passed a object to highlight the current line of code
# This object is 'debugger' which just provides the method stepper
# calls: setLine
# See stepper.Stepper.step
self.myStepper = stepper.Stepper(codestring, myGuiWorld,self.controller)
except Exception,info:
self.controller.give_error(str(info))
def test6(dataset):
stepper = st.Stepper()
try:
result = dataset.create_index([('y_instFlux', pymongo.ASCENDING)])
except pymongo.errors.PyMongoError as e:
print('error create index on y_instFlux', e)
stepper.show_step('create index on y_instFlux')
def test3(dataset):
stepper = st.Stepper()
try:
result = dataset.count()
print(result)
except pymongo.errors.PyMongoError as e:
print('error', e)
stepper.show_step('select count(*) from ForcedSource')
def test7(dataset):
stepper = st.Stepper()
try:
filter = {'y_instFlux': {'$gt': 5.0}}
result = dataset.count(filter)
print(result)
except pymongo.errors.PyMongoError as e:
print('error', e)
stepper.show_step('select count(*) from Object where y_instFlux > 5;')
def test4(dataset):
stepper = st.Stepper()
id = 2322374716295173
try:
result = dataset.find({'deepSourceId': id}, VIEW)
for o in result:
print(o)
except pymongo.errors.PyMongoError as e:
print('error', e)
stepper.show_step(
'SELECT ra, decl FROM Object WHERE deepSourceId = {};'.format(id))
def create_geo_index(dataset):
stepper = st.Stepper()
try:
dataset.create_index([('loc', pymongo.GEO2D)])
# dataset.create_index( {'loc': 1}, '2d' )
except pymongo.errors.PyMongoError as e:
print('error create_geo_index', e)
stepper.show_step('create_geo_index')
infos = dataset.index_information()
for i in infos:
print(' > ', i, ': ', infos[i])
def test8(dataset):
stepper = st.Stepper()
try:
result = dataset.create_index([('ra', pymongo.ASCENDING)])
except pymongo.errors.PyMongoError as e:
print('error create index on ra', e)
try:
result = dataset.create_index([('decl', pymongo.ASCENDING)])
except pymongo.errors.PyMongoError as e:
print('error create index on decl', e)
stepper.show_step('create indexes on ra, decl')
def aggregate(dataset, col):
stepper = st.Stepper()
try:
dataset.aggregate([{
'$addFields': {
'loc': ['$ra', '$decl']
}
}, {
'$out': col
}])
except:
print('error aggregate')
stepper.show_step('aggregate')
def test9(dataset):
stepper = st.Stepper()
try:
min_ra = dataset.find().sort('ra', 1).limit(1)[0]['ra']
max_ra = dataset.find().sort('ra', -1).limit(1)[0]['ra']
min_decl = dataset.find().sort('decl', 1).limit(1)[0]['decl']
max_decl = dataset.find().sort('decl', -1).limit(1)[0]['decl']
except pymongo.errors.PyMongoError as e:
print('error min, max', e)
print('ra in [', min_ra, ',', max_ra, ']')
print('decl in [', min_decl, ',', max_decl, ']')
stepper.show_step(
'select min(ra), max(ra), min(decl), max(decl) from Object;')
def creation(geo, objects=100):
stepper = st.Stepper()
for i in range(objects):
obj = dict()
obj['ra'] = random.random() * 360.0
obj['decl'] = random.random() * 360.0
obj['deepSourceId'] = int(random.random() * 100000000)
try:
geo.insert_one(obj)
if i < 5:
print('object created', obj)
except:
print('error')
stepper.show_step('objects created')
def test14(dataset):
stepper = st.Stepper()
try:
filter = {
'$and': [{
'psfFlux': {
'$gt': 0.1
}
}, {
'psfFlux': {
'$lt': 0.2
}
}]
}
result = dataset.count(filter)
print(result)
except pymongo.errors.PyMongoError as e:
print('error', e)
stepper.show_step(
'select count(*) from ForcedSource where psfFlux between 0.1 and 0.2;')
def test12(dataset):
stepper = st.Stepper()
try:
filter = {
'$and': [{
'flux_sinc': {
'$gt': 2
}
}, {
'flux_sinc': {
'$lt': 3
}
}]
}
result = dataset.count(filter)
print(result)
except pymongo.errors.PyMongoError as e:
print('error', e)
stepper.show_step(
'select count(*) from Source where flux_sinc between 2 and 3;')
def __init__(self, *args, **kwargs):
# Create camera and start stream
self.cam = cam.CameraStream().start()
# Create user interface
self.userinterface = gui.UserInterface(self)
# Add RUN function to button 1 to start Scan
self.start_page = self.userinterface.frames[
self.userinterface.StartPage]
self.start_page.add_function_to_button1(self.run)
self.shot_frame = [[] for x in range(200)]
# Create Stepper and add function to button 4
self.stepper = step.Stepper("/dev/ttyACM0")
self.start_page.add_function_to_button4(self.stepper.moveBack)
self.stepper_set = 0
self.start_page.add_function_to_button2(self.stepper.moveStep)
# Create image processor
self.img_proc = ImageProcessor()
def one_image(image_id, dataframe=None):
print('starting one_image', image_id)
stepper = step.Stepper()
if dataframe is None:
data = dataset.Dataset(image_id)
image_id = data.image_id
ra = data.ra
dec = data.dec
image = data.image
r = data.r
c = data.c
else:
image_id = dataframe[0]
ra = dataframe[1]
dec = dataframe[2]
r = dataframe[3]
c = dataframe[4]
image = np.array(dataframe[5])
background, dispersion, x, y = compute_background(image)
max_y = np.max(y)
print('max y', max_y, 'background', background, 'dispersion', dispersion)
#========================================================================================
stepper.show_step('background computed')
clustering = Clustering(ra, dec)
clusters = clustering(image, background, dispersion)
#========================================================================================
stepper.show_step('== clusters computed')
stars = job.setup_db()
radius = 0.0004
cluster_found = 0
all_matches = []
for cid, cluster in enumerate(clusters):
found = False
matches = 0
for oid, o in enumerate(
stars.find(
{
'center': {
'$geoWithin': {
'$centerSphere':
[[cluster.ra(), cluster.dec()], radius]
}
}
}, {
'_id': 0,
'where': 1,
'center': 1
})):
if not found:
cluster_found += 1
found = True
matches += 1
# print('image [', r, c, '] star found (clusterid:', cid, ') object id:', oid, ') (object:', o)
all_matches.append(matches)
print(' -> ', cluster_found, 'found vs. ', len(clusters),
'clusters in image. Match efficiency',
sum(all_matches) / len(clusters))
"""
image = add_crosses(image, clusters)
_ = main_ax[r, c].imshow(image, interpolation='none')
"""
#========================================================================================
stepper.show_step('full image')
return image, cluster_found, len(clusters), sum(all_matches)
def __call__(self,
image,
background,
dispersion,
factor=conf.THRESHOLD_FACTOR):
"""
principle:
- we then start a scan of the convolution image (CI):
- at every position we detect if there is a peak:
- we extract a 3x3 region of the CI centered at the current position
- a peak is detected when ALL pixels around the center of this little region are below the center.
- when a peak is detected, we get the cluster (the group of pixels around a peak):
- accumulate pixels circularly around the peak until the sum of pixels at a given distance
is lower than the threshold
- we compute the integral of pixel values of the cluster
- this list of clusters is returned.
"""
def extend_image(image, margin):
ext_shape = np.array(image.shape) + 2 * margin
ext_image = np.zeros(ext_shape)
ext_image[margin:-margin, margin:-margin] = image
return ext_image
# make a copy with a border of half
half = self.pattern_size // 2
ext_image = extend_image(image, half)
# build the convolution product image
cp_image = self.convolution_image(ext_image)
# make a copy with a border of 1
ext_cp_image = extend_image(cp_image, 1)
stepper = step.Stepper()
# ========================================================================================
stepper.show_step(' image prepared for clustering')
peaks = []
# scan the convolution image to detect peaks and build clusters
threshold = background + factor * dispersion
for rnum, row in enumerate(image):
for cnum, col in enumerate(row):
if cp_image[rnum, cnum] <= threshold:
continue
if not self.has_peak(ext_cp_image, rnum + 1, cnum + 1):
continue
peaks.append((rnum, cnum))
# ========================================================================================
stepper.show_step(' peaks detected')
clusters = []
for peak in peaks:
rnum = peak[0]
cnum = peak[1]
integral, radius = self.spread_peak(image, threshold, rnum, cnum)
if radius == 0:
continue
clusters.append(
Cluster(self.ra0, self.dec0, rnum, cnum, image[rnum, cnum],
integral))
# ========================================================================================
stepper.show_step(' clusters built')
# sort by integrals then by top
if len(clusters) > 0:
max_top = max(clusters, key=lambda cl: cl.top).top
clusters.sort(key=lambda cl: cl.integral + cl.top / max_top,
reverse=True)
# ========================================================================================
stepper.show_step(' clusters sorted')
# results
return clusters
def analyze(x):
image, cluster_found, clusters, all_matches = one_image(x[0], x)
text = ' -> %d found vs. %d clusters in image. Match efficiency %f' % (
cluster_found, clusters, all_matches / clusters)
# return x[5]
return 'image %d %s' % (x[0], text)
if __name__ == '__main__':
a = args.get_args()
print('rows=%d columns=%d pixels=%d graphic=%s' %
(conf.IMAGES_IN_RA, conf.IMAGES_IN_DEC, conf.PIXELS_PER_DEGREE,
conf.HAS_GRAPHIC))
stepper = step.Stepper()
if job.HAS_JOBLIB:
num_cores = multiprocessing.cpu_count()
print('core number:', num_cores)
if job.HAS_FUTURES:
exe = concurrent.futures.ProcessPoolExecutor()
submissions = []
if job.HAS_SPARK:
spark = SparkSession \
.builder \
.appName("LSSTSim") \
os.path.join(os.getcwd(), os.path.dirname(__file__)))
sys.path.append(__location__)
import stepper
import camera
from multiprocessing import Process
import time
def CoupledCapture(n, directory, stp, cam):
kwargs = {'n': n, 'directory': __location__ + '/' + directory}
#sp = Process(target=stp.discreteRotation, kwargs=kwargs)
cp = Process(target=cam.collectSeries, kwargs=kwargs)
# sp.start()
cp.start()
# delay = 0.5
# time.sleep(delay)
# stp.discreteRotation(**kwargs)
timeshift = -0.05
fps = 6
t = n * (1 / fps) + timeshift
stp.continuousRotation(t=t, nTurn=2)
cp.join()
if __name__ == '__main__':
cam = camera.Camera()
stp = stepper.Stepper(autoEnable=True)
CoupledCapture(16, 'capture', stp, cam)
stp.disable()
def create_reference_catalog():
"""
We simulate a sky region extended by one image width
besides the basic conf.IMAGES_IN_RA x conf.IMAGES_IN_DEC
"""
stepper = step.Stepper()
if job.HAS_JOBLIB:
num_cores = multiprocessing.cpu_count()
print('core number:', num_cores)
if job.HAS_FUTURES:
exe = concurrent.futures.ProcessPoolExecutor()
region_ra0 = conf.RA0 - conf.IMAGE_RA_SIZE
region_ra1 = region_ra0 + conf.IMAGE_RA_SIZE * (conf.IMAGES_IN_RA + 2)
region_dec0 = conf.DEC0 - conf.IMAGE_DEC_SIZE
region_dec1 = region_dec0 + conf.IMAGE_DEC_SIZE * (conf.IMAGES_IN_DEC + 2)
print('simulation region [', region_ra0, ':', region_ra1, ',', region_dec0,
':', region_dec1, ']')
submissions = []
if job.HAS_JOBLIB:
n_jobs = num_cores
# n_jobs = 1
submissions = Parallel(n_jobs=n_jobs)\
(delayed(simul_one_object)(region_ra0, region_ra1, region_dec0, region_dec1) for n in range(conf.NOBJECTS))
else:
for n in range(conf.NOBJECTS):
if job.HAS_FUTURES:
o = exe.submit(simul_one_object, region_ra0, region_ra1,
region_dec0, region_dec1)
else:
o = simul_one_object(region_ra0, region_ra1, region_dec0,
region_dec1)
submissions.append(o)
#========================================================================================
stepper.show_step('all submissions done')
stars = None
if job.HAS_MONGODB:
client = pymongo.MongoClient(job.MONGO_URL)
lsst = client.lsst
recreate = True
if recreate:
try:
stars = lsst.stars
lsst.drop_collection('stars')
except:
pass
stars = lsst.stars
#========================================================================================
stepper.show_step('Db initialized')
objects = dict()
for s in submissions:
if job.HAS_FUTURES:
o = s.result()
else:
o = s
objects[o.id] = o
# ========================================================================================
stepper.show_step('all objects created {}'.format(conf.NOBJECTS))
if job.HAS_MONGODB:
for o_id in objects:
o = objects[o_id]
object = o.to_db()
object['center'] = {'type': 'Point', 'coordinates': [o.ra, o.dec]}
try:
id = stars.insert_one(object)
# print('object inserted', o.ra, o.dec)
except Exception as e:
print('oops')
print(e.message)
sys.exit()
#========================================================================================
stepper.show_step('all objects inserted')
stars.create_index([('center', '2dsphere')])
#========================================================================================
stepper.show_step('2D index created')
fields = dict()
for i in range(header['TFIELDS']):
j = i + 1
fields[header['TTYPE{}'.format(j)]] = (header['TFORM{}'.format(j)],
header['TUNIT{}'.format(j)])
print("Fields = ", fields)
for row in range(10):
print(hdu[1].data[row])
data = hdu[1].data
s1 = stp.Stepper()
s = stp.Stepper()
"""
Transpose the data table columns => rows
Work upon a "subset" = % of the full data table
And apply the transposition by blocks ("steps" = # blocks)
"""
def build(sc, data, subset, steps):
# will create a list of transposed arrays
if subset <= 0:
return None
if subset > 100:
subset = 100
def __init__(self):
logging.info('Starting application')
self.root = ttk.Tk()
self.delay = 1000
self.timeout = 1000
self.rep_rate = 0.5
self.loc= 0.00
self._update_count = 0
self.measure_plasma_params = False
self.save_plasma_params = False
self.save_raw = False
self.plasma_params_filename = ttk.StringVar()
self.plasma_params_filename.set('test.txt')
self.raw_data_folder = ttk.StringVar()
self.raw_data_folder.set('data/')
self.scan_interval = ttk.StringVar()
self.scan_interval.set('20.0')
self.scan_number = ttk.StringVar()
self.scan_number.set('4')
self.scan_samples = ttk.StringVar()
self.scan_samples.set('10')
self.manual_disp = ttk.StringVar()
self.manual_disp.set('2.0')
self.curr_location = ttk.StringVar()
self.curr_location.set('Current Location: {:.2f}mm'.format(0.00))
self.plasma_density = ttk.StringVar()
self.plasma_density.set('0.00+/-0.00')
self.plasma_temp = ttk.StringVar()
self.plasma_temp.set('0.00+/-0.00')
self.status = ttk.StringVar()
self.status.set('Starting')
self.frame = ttk.Frame(self.root)
self.frame.pack()
status_label = ttk.Label(self.frame,textvariable = self.status)
status_label.pack()
file_entrylabel = ttk.Label(self.frame,text = 'Save to file: ')
file_entrylabel.pack()
file_entry = CopyPasteBox(self.frame,textvariable = self.plasma_params_filename)
file_entry.pack()
raw_folder_entrylabel = ttk.Label(self.frame,text = 'Raw data folder: ')
raw_folder_entrylabel.pack()
raw_folder_entry = CopyPasteBox(self.frame,textvariable = self.raw_data_folder)
raw_folder_entry.pack()
scan_lengthlabel = ttk.Label(self.frame,text = 'Distance to scan over (mm): ')
scan_lengthlabel.pack()
scan_length = CopyPasteBox(self.frame,textvariable = self.scan_interval)
scan_length.pack()
scan_pointslabel = ttk.Label(self.frame,text = 'Number of points to scan: ')
scan_pointslabel.pack()
scan_points = CopyPasteBox(self.frame,textvariable = self.scan_number)
scan_points.pack()
scan_samplelabel = ttk.Label(self.frame,text = 'Number of samples: ')
scan_samplelabel.pack()
scan_sample = CopyPasteBox(self.frame,textvariable = self.scan_samples)
scan_sample.pack()
manual_entrylabel = ttk.Label(self.frame,text = 'Manual displacement (+/- mm): ')
manual_entrylabel.pack()
manual_entry = CopyPasteBox(self.frame,textvariable = self.manual_disp)
manual_entry.pack()
location_label = ttk.Label(self.frame,textvariable = self.curr_location)
location_label.pack()
#plasma_density_label = ttk.Label(self.frame,textvariable = self.plasma_density)
#plasma_density_label.pack()
#plasma_temp_label = ttk.Label(self.frame,textvariable = self.plasma_temp)
#plasma_temp_label.pack()
#saveparamsbutton = ttk.Button(self.frame,text = 'Measure Plasma Params',command = self.flip_measure_plasma_params)
#saveparamsbutton.pack()
#saveparamsbutton = ttk.Button(self.frame,text = 'Save Plasma Params',command = self.flip_save_plasma_params)
#saveparamsbutton.pack()
self.saveshotsbutton = ttk.Button(self.frame,text = 'Save Shots',command = self.save_shots)
self.saveshotsbutton.pack()
self.scanbutton = ttk.Button(self.frame,text = 'Scan Plasma Chamber',command = self.scan)
self.scanbutton.pack()
displacebutton = ttk.Button(self.frame,text = 'Manually Displace',command = self.manual_displacement)
displacebutton.pack()
zerobutton = ttk.Button(self.frame,text = 'Zero',command = self.zero_stepper)
zerobutton.pack()
self.init_scope()
self.stepper = stepper.Stepper('COM4')
self.continuous_update()
def do_join(lsst, nobjects, bottomleft, topright, max_dist):
dra = {
'$abs': {
'$subtract': [{
'$arrayElemAt': ['$ns.loc', 0]
}, {
'$arrayElemAt': ['$loc', 0]
}]
}
}
dra2 = {'$multiply': [dra, dra]}
ddecl = {
'$abs': {
'$subtract': [{
'$arrayElemAt': ['$ns.loc', 1]
}, {
'$arrayElemAt': ['$loc', 1]
}]
}
}
ddecl2 = {'$multiply': [ddecl, ddecl]}
dist = {'$sqrt': {'$add': [dra2, ddecl2]}}
"""
quand on rencontre [$_id1, $ns._id2] est-ce qu'il existe le couple [$_id2, $ns._id1] ?
"""
p2 = [
{
'$geoNear': {
'near': [0, 0],
'query': {
'loc': {
'$geoWithin': {
'$box': [bottomleft, topright]
}
}
},
'limit': nobjects,
'distanceField': 'dist',
}
},
{
'$lookup': {
'from': 'z',
'localField': 'y.loc',
'foreignField': 'z.loc',
'as': 'ns'
}
},
{
'$unwind': '$ns'
},
{
'$redact': {
'$cond': [{
'$eq': ["$_id", "$ns._id"]
}, "$$PRUNE", "$$KEEP"]
}
},
# {'$redact': { '$cond': [{ '$eq': ["$_id", "$ns._id"] }, "$$PRUNE", "$$KEEP" ] } },
{
'$addFields': {
'dist': dist
}
},
# {'$match': { '$and': [ { 'dist': { '$gt': 0 } }, { 'dist': { '$lt': max_dist } } ] } },
# {'$project': {'_id': 1, 'loc':1, 'ns.loc':1, 'dist': 1}},
{
'$project': {
'_id': 1,
'ns._id': 1,
'dist': 1
}
},
# {'$project': {'_id': 1}},
# {'$sort': {'dist': 1 } }
# {'$limit': 5},
]
stepper = st.Stepper()
result = lsst.y.aggregate(p2, allowDiskUse=True)
t = stepper.show_step('aggregate join')
i = 0
for i, o in enumerate(result):
print(i, o)
print(i, 'neighbours')
return t, i
runs = 10
records = args.records
block = args.block
steps = args.steps
schema = StructType([
StructField("run", IntegerType(), True),
StructField("image", ArrayType(DoubleType(), True))
])
partitions = 10000 # to have 256Mb files
if create:
print('creating data with', records, 'records made of blocks of', block,
'doubles')
stepper = st.Stepper()
rdd = sc.parallelize(range(runs * records), partitions).map(lambda x: (int(
random.random() * runs), np.random.rand(block).tolist()))
stepper.show_step('create data')
df = spark.createDataFrame(rdd, schema)
stepper.show_step('create dataframe')
# input("ok")
df.write.mode("overwrite").save("./images")
stepper.show_step('write data')
else:
stepper = st.Stepper()
print('reading data and applying', steps, 'steps to them')
df = spark.read.load("./images")
stepper.show_step('read data')
df = df.filter(df.run == 3)
async def run():
s = stepper.Stepper()
async with s:
pts = []
#for v in voltages:
# for sp in speeds:
# current = await measure(s, sp, v, s.CONTROL_MODE_VOLTAGE)
# pts.append((v, sp, current))
print(pts)
v1 = 0.3
v2 = 0.8
sp1 = 75
sp2 = 150
c11 = await measure(s, sp1, v1, s.CONTROL_MODE_VOLTAGE)
c12 = await measure(s, sp1, v2, s.CONTROL_MODE_VOLTAGE)
c21 = await measure(s, sp2, v1, s.CONTROL_MODE_VOLTAGE)
c22 = await measure(s, sp2, v2, s.CONTROL_MODE_VOLTAGE)
r1 = (v2 - v1) / (c12 - c11)
r2 = (v2 - v1) / (c22 - c21)
r = (r1 + r2) / 2
print(r1, r2)
km1 = -r * (c21 - c11) / (sp2 - sp1)
km2 = -r * (c22 - c12) / (sp2 - sp1)
km = (km1 + km2) / 2
print(km1, km2)
by_speed = {}
#by_speed[sp1] = [(v1, c11), (v2, c12)]
#by_speed[sp2] = [(v1, c21), (v2, c22)]
for (v, sp, c) in pts:
by_speed[sp] = by_speed.get(sp, []) + [(v, c)]
for (sp, pts) in by_speed.items():
pts = zip(*pts)
plt.plot(*pts)
for sp in speeds:
v1 = voltages[0]
v2 = voltages[-1]
c1 = (v1 - km * sp) / r
c2 = (v2 - km * sp) / r
plt.plot([v1, v2], [c1, c2])
pts = zip(*pts)
plt.plot(*pts)
plt.show()
await s.program_immediate(MOTOR, s.cmd_tuning(km, r))
pts = []
currents = np.linspace(0.2, 1, 2)
for i in currents:
for sp in speeds:
current = await measure(s,
sp,
i,
s.CONTROL_MODE_CURRENT_OL,
voltage=False)
pts.append((i, sp, current))
by_current = {}
for (i, sp, c) in pts:
by_current[i] = by_current.get(i, []) + [(sp, c)]
for (sp, pts) in by_current.items():
pts = zip(*pts)
plt.plot(*pts)
plt.show()
def get_minmax(allp):
s2 = stp.Stepper()
minmax = allp.agg(F.min('RA'), F.max('RA'), F.min('DEC'),
F.max('DEC')).collect()
s2.show_step("get min-max")
print(minmax)
import stepper
for k in range(3):
p = stepper.Stepper(k + 1)
p.run(8, 500, 1)
