def do_create(lsst, nobjects=1000): try: lsst.drop_collection('y') print('y dropped') except: pass ra = 0 decl = 0 window = 180. stepper = st.Stepper() requests = [] for i in range(nobjects): obj = { 'loc': [(random.random() * 2 * window - window), (random.random() * 2 * window - window)] } # lsst.y.insert( obj ) requests.append(pymongo.InsertOne(obj)) try: lsst.y.bulk_write(requests) except BulkWriteError as bwe: print('error in bulk write', bwe.details) exit() stepper.show_step('y created') # lsst.Object.aggregate( [ {'$match' : {'chunkId': 516} }, { '$project': { 'loc': [ '$ra', '$decl' ] } }, {'$limit': 1000}, {'$out': 'y'} ] ) print(lsst.y.count()) result = lsst.y.find() for i, o in enumerate(result): print(o) if i > 10: break stepper = st.Stepper() try: lsst.y.create_index([('loc.0', pymongo.ASCENDING)]) except pymongo.errors.PyMongoError as e: print('error create index on ra', e) stepper.show_step('index loc.0 creation') stepper = st.Stepper() try: lsst.y.create_index([('loc.1', pymongo.ASCENDING)]) except pymongo.errors.PyMongoError as e: print('error create index on decl', e) stepper.show_step('index loc.1 creation') stepper = st.Stepper() try: lsst.y.create_index([('loc', pymongo.GEO2D)]) except pymongo.errors.PyMongoError as e: print('error create_geo_index', e) stepper.show_step('index loc creation') test9(lsst.y)
def try_geo(lsst, objects=10): print('============================================ try_geo') recreate = True if recreate: try: lsst.drop_collection('geo') print('geo created') except: pass geo = lsst.geo creation(geo, objects) aggregate(geo, 'geo') geo.update_many({}, {'$inc': {'loc.0': -180, 'loc.1': -180}}) stepper = st.Stepper() geo.create_index('deepSourceId', unique=True) stepper.show_step('index created' + str(geo.index_information())) # create_geo_index(geo) else: geo = lsst.geo # stepper = Stepper() result = geo.find({}, VIEW) for i, o in enumerate(result): print('object read', o) if i > 5: break ra = 45. decl = 0. ext = 10. bottomleft = [ra - ext, decl - ext] topright = [ra + ext, decl + ext] stepper = st.Stepper() result = geo.find( {'loc': { '$geoWithin': { '$box': [bottomleft, topright] } }}, VIEW) stepper.show_step('geo find') for i, o in enumerate(result): print(o) if i > 5: break
def test5(dataset): stepper = st.Stepper() ra_left = -4.0 decl_bottom = -3.2 ra_right = -3.99 decl_top = -3.16 try: query = { 'loc': { '$geoWithin': { '$box': [[ra_left, decl_bottom], [ra_right, decl_top]] } } } result = dataset.find(query, VIEW) for o in result: print(o) except pymongo.errors.PyMongoError as e: print('error', e) stepper.show_step( 'SELECT ra, decl FROM Object WHERE qserv_areaspec_box({}, {}, {}, {});' .format(ra_left + 180., decl_bottom, ra_right + 180., decl_top))
def do_select(lsst, limit, ra, decl, window): try: lsst.drop_collection('z') print('z dropped') except: pass ext = window bottomleft = [ ra - ext, decl - ext ] topright = [ ra + ext, decl + ext ] p = [ {'$geoNear': { 'near': [ra, decl], 'query': { 'loc': { '$geoWithin': {'$box': [bottomleft, topright] } } }, 'limit': limit, 'distanceField': 'dist', } }, {'$project': {'_id':1, 'ra':1, 'decl':1} }, {'$out': 'z'}, ] stepper = st.Stepper() result = lsst.Object.aggregate(p, allowDiskUse=True) t = stepper.show_step('aggregate create z') c = lsst.z.count() print('Objects in the selection:', c, 'window:', bottomleft, topright) return bottomleft, topright, t, c
def test9(dataset): stepper = st.Stepper() try: min_ra = dataset.find({}, { '_id': 0, 'loc': 1 }).sort('loc.0', 1).limit(1)[0]['loc'][0] max_ra = dataset.find({}, { '_id': 0, 'loc': 1 }).sort('loc.0', -1).limit(1)[0]['loc'][0] min_decl = dataset.find({}, { '_id': 0, 'loc': 1 }).sort('loc.1', 1).limit(1)[0]['loc'][1] max_decl = dataset.find({}, { '_id': 0, 'loc': 1 }).sort('loc.1', -1).limit(1)[0]['loc'][1] except pymongo.errors.PyMongoError as e: print('error min, max', e) print('ra= [', min_ra, ',', max_ra, ']') print('decl= [', min_decl, ',', max_decl, ']') stepper.show_step( 'select min(ra), max(ra), min(decl), max(decl) from Object;')
def on_code_execute(self,gui=None,step=None): """The controller calls this when the user hits the 'execute' button. When step is True the timer isn't started. This is the point from where the whole show starts.""" self.logger.debug("on_code_execute called") worldlines = self.controller.get_worldwin_text() if _DEBUG > 1: print self.__class__,"worldlines",worldlines codelines = self.controller.get_codewin_text() if len(worldlines) == 0 or len(codelines) == 0: self.controller.give_warning(_("Please load a world and program before executing.")) return None codestring = "\n".join(codelines) if _DEBUG > 1: print self.__class__,"codestring",codestring logicworld = self.on_world_reload(worldlines) if not logicworld:# on_world_reload gives signals the controller in case of error return None # logicworld is made a class member because stepper doesn't pass the # logicworld object when it calls updateWorldBitmapAfterMove. # Because we want all the logic to use the same logicworld object we # made it a class member so that updateWorldBitmapAfterMove uses the # self.logicworld to pass on to the controller. # see also updateWorldBitmapAfterMove and GvrController.word_state_changed self.logicworld = logicworld # GuiWorld gets a GvrModel reference because it expects a object with a # updateWorldBitmapAfterMove(oldcoords) method which it calls when the # world state is changed. myGuiWorld = guiWorld.GuiWorld(self,logicworld) try: # stepper can be passed a object to highlight the current line of code # This object is 'debugger' which just provides the method stepper # calls: setLine # See stepper.Stepper.step self.myStepper = stepper.Stepper(codestring, myGuiWorld,self.controller) except Exception,info: self.controller.give_error(str(info))
def test6(dataset): stepper = st.Stepper() try: result = dataset.create_index([('y_instFlux', pymongo.ASCENDING)]) except pymongo.errors.PyMongoError as e: print('error create index on y_instFlux', e) stepper.show_step('create index on y_instFlux')
def test3(dataset): stepper = st.Stepper() try: result = dataset.count() print(result) except pymongo.errors.PyMongoError as e: print('error', e) stepper.show_step('select count(*) from ForcedSource')
def test7(dataset): stepper = st.Stepper() try: filter = {'y_instFlux': {'$gt': 5.0}} result = dataset.count(filter) print(result) except pymongo.errors.PyMongoError as e: print('error', e) stepper.show_step('select count(*) from Object where y_instFlux > 5;')
def test4(dataset): stepper = st.Stepper() id = 2322374716295173 try: result = dataset.find({'deepSourceId': id}, VIEW) for o in result: print(o) except pymongo.errors.PyMongoError as e: print('error', e) stepper.show_step( 'SELECT ra, decl FROM Object WHERE deepSourceId = {};'.format(id))
def create_geo_index(dataset): stepper = st.Stepper() try: dataset.create_index([('loc', pymongo.GEO2D)]) # dataset.create_index( {'loc': 1}, '2d' ) except pymongo.errors.PyMongoError as e: print('error create_geo_index', e) stepper.show_step('create_geo_index') infos = dataset.index_information() for i in infos: print(' > ', i, ': ', infos[i])
def test8(dataset): stepper = st.Stepper() try: result = dataset.create_index([('ra', pymongo.ASCENDING)]) except pymongo.errors.PyMongoError as e: print('error create index on ra', e) try: result = dataset.create_index([('decl', pymongo.ASCENDING)]) except pymongo.errors.PyMongoError as e: print('error create index on decl', e) stepper.show_step('create indexes on ra, decl')
def aggregate(dataset, col): stepper = st.Stepper() try: dataset.aggregate([{ '$addFields': { 'loc': ['$ra', '$decl'] } }, { '$out': col }]) except: print('error aggregate') stepper.show_step('aggregate')
def test9(dataset): stepper = st.Stepper() try: min_ra = dataset.find().sort('ra', 1).limit(1)[0]['ra'] max_ra = dataset.find().sort('ra', -1).limit(1)[0]['ra'] min_decl = dataset.find().sort('decl', 1).limit(1)[0]['decl'] max_decl = dataset.find().sort('decl', -1).limit(1)[0]['decl'] except pymongo.errors.PyMongoError as e: print('error min, max', e) print('ra in [', min_ra, ',', max_ra, ']') print('decl in [', min_decl, ',', max_decl, ']') stepper.show_step( 'select min(ra), max(ra), min(decl), max(decl) from Object;')
def creation(geo, objects=100): stepper = st.Stepper() for i in range(objects): obj = dict() obj['ra'] = random.random() * 360.0 obj['decl'] = random.random() * 360.0 obj['deepSourceId'] = int(random.random() * 100000000) try: geo.insert_one(obj) if i < 5: print('object created', obj) except: print('error') stepper.show_step('objects created')
def test14(dataset): stepper = st.Stepper() try: filter = { '$and': [{ 'psfFlux': { '$gt': 0.1 } }, { 'psfFlux': { '$lt': 0.2 } }] } result = dataset.count(filter) print(result) except pymongo.errors.PyMongoError as e: print('error', e) stepper.show_step( 'select count(*) from ForcedSource where psfFlux between 0.1 and 0.2;')
def test12(dataset): stepper = st.Stepper() try: filter = { '$and': [{ 'flux_sinc': { '$gt': 2 } }, { 'flux_sinc': { '$lt': 3 } }] } result = dataset.count(filter) print(result) except pymongo.errors.PyMongoError as e: print('error', e) stepper.show_step( 'select count(*) from Source where flux_sinc between 2 and 3;')
def __init__(self, *args, **kwargs): # Create camera and start stream self.cam = cam.CameraStream().start() # Create user interface self.userinterface = gui.UserInterface(self) # Add RUN function to button 1 to start Scan self.start_page = self.userinterface.frames[ self.userinterface.StartPage] self.start_page.add_function_to_button1(self.run) self.shot_frame = [[] for x in range(200)] # Create Stepper and add function to button 4 self.stepper = step.Stepper("/dev/ttyACM0") self.start_page.add_function_to_button4(self.stepper.moveBack) self.stepper_set = 0 self.start_page.add_function_to_button2(self.stepper.moveStep) # Create image processor self.img_proc = ImageProcessor()
def one_image(image_id, dataframe=None): print('starting one_image', image_id) stepper = step.Stepper() if dataframe is None: data = dataset.Dataset(image_id) image_id = data.image_id ra = data.ra dec = data.dec image = data.image r = data.r c = data.c else: image_id = dataframe[0] ra = dataframe[1] dec = dataframe[2] r = dataframe[3] c = dataframe[4] image = np.array(dataframe[5]) background, dispersion, x, y = compute_background(image) max_y = np.max(y) print('max y', max_y, 'background', background, 'dispersion', dispersion) #======================================================================================== stepper.show_step('background computed') clustering = Clustering(ra, dec) clusters = clustering(image, background, dispersion) #======================================================================================== stepper.show_step('== clusters computed') stars = job.setup_db() radius = 0.0004 cluster_found = 0 all_matches = [] for cid, cluster in enumerate(clusters): found = False matches = 0 for oid, o in enumerate( stars.find( { 'center': { '$geoWithin': { '$centerSphere': [[cluster.ra(), cluster.dec()], radius] } } }, { '_id': 0, 'where': 1, 'center': 1 })): if not found: cluster_found += 1 found = True matches += 1 # print('image [', r, c, '] star found (clusterid:', cid, ') object id:', oid, ') (object:', o) all_matches.append(matches) print(' -> ', cluster_found, 'found vs. ', len(clusters), 'clusters in image. Match efficiency', sum(all_matches) / len(clusters)) """ image = add_crosses(image, clusters) _ = main_ax[r, c].imshow(image, interpolation='none') """ #======================================================================================== stepper.show_step('full image') return image, cluster_found, len(clusters), sum(all_matches)
def __call__(self, image, background, dispersion, factor=conf.THRESHOLD_FACTOR): """ principle: - we then start a scan of the convolution image (CI): - at every position we detect if there is a peak: - we extract a 3x3 region of the CI centered at the current position - a peak is detected when ALL pixels around the center of this little region are below the center. - when a peak is detected, we get the cluster (the group of pixels around a peak): - accumulate pixels circularly around the peak until the sum of pixels at a given distance is lower than the threshold - we compute the integral of pixel values of the cluster - this list of clusters is returned. """ def extend_image(image, margin): ext_shape = np.array(image.shape) + 2 * margin ext_image = np.zeros(ext_shape) ext_image[margin:-margin, margin:-margin] = image return ext_image # make a copy with a border of half half = self.pattern_size // 2 ext_image = extend_image(image, half) # build the convolution product image cp_image = self.convolution_image(ext_image) # make a copy with a border of 1 ext_cp_image = extend_image(cp_image, 1) stepper = step.Stepper() # ======================================================================================== stepper.show_step(' image prepared for clustering') peaks = [] # scan the convolution image to detect peaks and build clusters threshold = background + factor * dispersion for rnum, row in enumerate(image): for cnum, col in enumerate(row): if cp_image[rnum, cnum] <= threshold: continue if not self.has_peak(ext_cp_image, rnum + 1, cnum + 1): continue peaks.append((rnum, cnum)) # ======================================================================================== stepper.show_step(' peaks detected') clusters = [] for peak in peaks: rnum = peak[0] cnum = peak[1] integral, radius = self.spread_peak(image, threshold, rnum, cnum) if radius == 0: continue clusters.append( Cluster(self.ra0, self.dec0, rnum, cnum, image[rnum, cnum], integral)) # ======================================================================================== stepper.show_step(' clusters built') # sort by integrals then by top if len(clusters) > 0: max_top = max(clusters, key=lambda cl: cl.top).top clusters.sort(key=lambda cl: cl.integral + cl.top / max_top, reverse=True) # ======================================================================================== stepper.show_step(' clusters sorted') # results return clusters
def analyze(x): image, cluster_found, clusters, all_matches = one_image(x[0], x) text = ' -> %d found vs. %d clusters in image. Match efficiency %f' % ( cluster_found, clusters, all_matches / clusters) # return x[5] return 'image %d %s' % (x[0], text) if __name__ == '__main__': a = args.get_args() print('rows=%d columns=%d pixels=%d graphic=%s' % (conf.IMAGES_IN_RA, conf.IMAGES_IN_DEC, conf.PIXELS_PER_DEGREE, conf.HAS_GRAPHIC)) stepper = step.Stepper() if job.HAS_JOBLIB: num_cores = multiprocessing.cpu_count() print('core number:', num_cores) if job.HAS_FUTURES: exe = concurrent.futures.ProcessPoolExecutor() submissions = [] if job.HAS_SPARK: spark = SparkSession \ .builder \ .appName("LSSTSim") \
os.path.join(os.getcwd(), os.path.dirname(__file__))) sys.path.append(__location__) import stepper import camera from multiprocessing import Process import time def CoupledCapture(n, directory, stp, cam): kwargs = {'n': n, 'directory': __location__ + '/' + directory} #sp = Process(target=stp.discreteRotation, kwargs=kwargs) cp = Process(target=cam.collectSeries, kwargs=kwargs) # sp.start() cp.start() # delay = 0.5 # time.sleep(delay) # stp.discreteRotation(**kwargs) timeshift = -0.05 fps = 6 t = n * (1 / fps) + timeshift stp.continuousRotation(t=t, nTurn=2) cp.join() if __name__ == '__main__': cam = camera.Camera() stp = stepper.Stepper(autoEnable=True) CoupledCapture(16, 'capture', stp, cam) stp.disable()
def create_reference_catalog(): """ We simulate a sky region extended by one image width besides the basic conf.IMAGES_IN_RA x conf.IMAGES_IN_DEC """ stepper = step.Stepper() if job.HAS_JOBLIB: num_cores = multiprocessing.cpu_count() print('core number:', num_cores) if job.HAS_FUTURES: exe = concurrent.futures.ProcessPoolExecutor() region_ra0 = conf.RA0 - conf.IMAGE_RA_SIZE region_ra1 = region_ra0 + conf.IMAGE_RA_SIZE * (conf.IMAGES_IN_RA + 2) region_dec0 = conf.DEC0 - conf.IMAGE_DEC_SIZE region_dec1 = region_dec0 + conf.IMAGE_DEC_SIZE * (conf.IMAGES_IN_DEC + 2) print('simulation region [', region_ra0, ':', region_ra1, ',', region_dec0, ':', region_dec1, ']') submissions = [] if job.HAS_JOBLIB: n_jobs = num_cores # n_jobs = 1 submissions = Parallel(n_jobs=n_jobs)\ (delayed(simul_one_object)(region_ra0, region_ra1, region_dec0, region_dec1) for n in range(conf.NOBJECTS)) else: for n in range(conf.NOBJECTS): if job.HAS_FUTURES: o = exe.submit(simul_one_object, region_ra0, region_ra1, region_dec0, region_dec1) else: o = simul_one_object(region_ra0, region_ra1, region_dec0, region_dec1) submissions.append(o) #======================================================================================== stepper.show_step('all submissions done') stars = None if job.HAS_MONGODB: client = pymongo.MongoClient(job.MONGO_URL) lsst = client.lsst recreate = True if recreate: try: stars = lsst.stars lsst.drop_collection('stars') except: pass stars = lsst.stars #======================================================================================== stepper.show_step('Db initialized') objects = dict() for s in submissions: if job.HAS_FUTURES: o = s.result() else: o = s objects[o.id] = o # ======================================================================================== stepper.show_step('all objects created {}'.format(conf.NOBJECTS)) if job.HAS_MONGODB: for o_id in objects: o = objects[o_id] object = o.to_db() object['center'] = {'type': 'Point', 'coordinates': [o.ra, o.dec]} try: id = stars.insert_one(object) # print('object inserted', o.ra, o.dec) except Exception as e: print('oops') print(e.message) sys.exit() #======================================================================================== stepper.show_step('all objects inserted') stars.create_index([('center', '2dsphere')]) #======================================================================================== stepper.show_step('2D index created')
fields = dict() for i in range(header['TFIELDS']): j = i + 1 fields[header['TTYPE{}'.format(j)]] = (header['TFORM{}'.format(j)], header['TUNIT{}'.format(j)]) print("Fields = ", fields) for row in range(10): print(hdu[1].data[row]) data = hdu[1].data s1 = stp.Stepper() s = stp.Stepper() """ Transpose the data table columns => rows Work upon a "subset" = % of the full data table And apply the transposition by blocks ("steps" = # blocks) """ def build(sc, data, subset, steps): # will create a list of transposed arrays if subset <= 0: return None if subset > 100: subset = 100
def __init__(self): logging.info('Starting application') self.root = ttk.Tk() self.delay = 1000 self.timeout = 1000 self.rep_rate = 0.5 self.loc= 0.00 self._update_count = 0 self.measure_plasma_params = False self.save_plasma_params = False self.save_raw = False self.plasma_params_filename = ttk.StringVar() self.plasma_params_filename.set('test.txt') self.raw_data_folder = ttk.StringVar() self.raw_data_folder.set('data/') self.scan_interval = ttk.StringVar() self.scan_interval.set('20.0') self.scan_number = ttk.StringVar() self.scan_number.set('4') self.scan_samples = ttk.StringVar() self.scan_samples.set('10') self.manual_disp = ttk.StringVar() self.manual_disp.set('2.0') self.curr_location = ttk.StringVar() self.curr_location.set('Current Location: {:.2f}mm'.format(0.00)) self.plasma_density = ttk.StringVar() self.plasma_density.set('0.00+/-0.00') self.plasma_temp = ttk.StringVar() self.plasma_temp.set('0.00+/-0.00') self.status = ttk.StringVar() self.status.set('Starting') self.frame = ttk.Frame(self.root) self.frame.pack() status_label = ttk.Label(self.frame,textvariable = self.status) status_label.pack() file_entrylabel = ttk.Label(self.frame,text = 'Save to file: ') file_entrylabel.pack() file_entry = CopyPasteBox(self.frame,textvariable = self.plasma_params_filename) file_entry.pack() raw_folder_entrylabel = ttk.Label(self.frame,text = 'Raw data folder: ') raw_folder_entrylabel.pack() raw_folder_entry = CopyPasteBox(self.frame,textvariable = self.raw_data_folder) raw_folder_entry.pack() scan_lengthlabel = ttk.Label(self.frame,text = 'Distance to scan over (mm): ') scan_lengthlabel.pack() scan_length = CopyPasteBox(self.frame,textvariable = self.scan_interval) scan_length.pack() scan_pointslabel = ttk.Label(self.frame,text = 'Number of points to scan: ') scan_pointslabel.pack() scan_points = CopyPasteBox(self.frame,textvariable = self.scan_number) scan_points.pack() scan_samplelabel = ttk.Label(self.frame,text = 'Number of samples: ') scan_samplelabel.pack() scan_sample = CopyPasteBox(self.frame,textvariable = self.scan_samples) scan_sample.pack() manual_entrylabel = ttk.Label(self.frame,text = 'Manual displacement (+/- mm): ') manual_entrylabel.pack() manual_entry = CopyPasteBox(self.frame,textvariable = self.manual_disp) manual_entry.pack() location_label = ttk.Label(self.frame,textvariable = self.curr_location) location_label.pack() #plasma_density_label = ttk.Label(self.frame,textvariable = self.plasma_density) #plasma_density_label.pack() #plasma_temp_label = ttk.Label(self.frame,textvariable = self.plasma_temp) #plasma_temp_label.pack() #saveparamsbutton = ttk.Button(self.frame,text = 'Measure Plasma Params',command = self.flip_measure_plasma_params) #saveparamsbutton.pack() #saveparamsbutton = ttk.Button(self.frame,text = 'Save Plasma Params',command = self.flip_save_plasma_params) #saveparamsbutton.pack() self.saveshotsbutton = ttk.Button(self.frame,text = 'Save Shots',command = self.save_shots) self.saveshotsbutton.pack() self.scanbutton = ttk.Button(self.frame,text = 'Scan Plasma Chamber',command = self.scan) self.scanbutton.pack() displacebutton = ttk.Button(self.frame,text = 'Manually Displace',command = self.manual_displacement) displacebutton.pack() zerobutton = ttk.Button(self.frame,text = 'Zero',command = self.zero_stepper) zerobutton.pack() self.init_scope() self.stepper = stepper.Stepper('COM4') self.continuous_update()
def do_join(lsst, nobjects, bottomleft, topright, max_dist): dra = { '$abs': { '$subtract': [{ '$arrayElemAt': ['$ns.loc', 0] }, { '$arrayElemAt': ['$loc', 0] }] } } dra2 = {'$multiply': [dra, dra]} ddecl = { '$abs': { '$subtract': [{ '$arrayElemAt': ['$ns.loc', 1] }, { '$arrayElemAt': ['$loc', 1] }] } } ddecl2 = {'$multiply': [ddecl, ddecl]} dist = {'$sqrt': {'$add': [dra2, ddecl2]}} """ quand on rencontre [$_id1, $ns._id2] est-ce qu'il existe le couple [$_id2, $ns._id1] ? """ p2 = [ { '$geoNear': { 'near': [0, 0], 'query': { 'loc': { '$geoWithin': { '$box': [bottomleft, topright] } } }, 'limit': nobjects, 'distanceField': 'dist', } }, { '$lookup': { 'from': 'z', 'localField': 'y.loc', 'foreignField': 'z.loc', 'as': 'ns' } }, { '$unwind': '$ns' }, { '$redact': { '$cond': [{ '$eq': ["$_id", "$ns._id"] }, "$$PRUNE", "$$KEEP"] } }, # {'$redact': { '$cond': [{ '$eq': ["$_id", "$ns._id"] }, "$$PRUNE", "$$KEEP" ] } }, { '$addFields': { 'dist': dist } }, # {'$match': { '$and': [ { 'dist': { '$gt': 0 } }, { 'dist': { '$lt': max_dist } } ] } }, # {'$project': {'_id': 1, 'loc':1, 'ns.loc':1, 'dist': 1}}, { '$project': { '_id': 1, 'ns._id': 1, 'dist': 1 } }, # {'$project': {'_id': 1}}, # {'$sort': {'dist': 1 } } # {'$limit': 5}, ] stepper = st.Stepper() result = lsst.y.aggregate(p2, allowDiskUse=True) t = stepper.show_step('aggregate join') i = 0 for i, o in enumerate(result): print(i, o) print(i, 'neighbours') return t, i
runs = 10 records = args.records block = args.block steps = args.steps schema = StructType([ StructField("run", IntegerType(), True), StructField("image", ArrayType(DoubleType(), True)) ]) partitions = 10000 # to have 256Mb files if create: print('creating data with', records, 'records made of blocks of', block, 'doubles') stepper = st.Stepper() rdd = sc.parallelize(range(runs * records), partitions).map(lambda x: (int( random.random() * runs), np.random.rand(block).tolist())) stepper.show_step('create data') df = spark.createDataFrame(rdd, schema) stepper.show_step('create dataframe') # input("ok") df.write.mode("overwrite").save("./images") stepper.show_step('write data') else: stepper = st.Stepper() print('reading data and applying', steps, 'steps to them') df = spark.read.load("./images") stepper.show_step('read data') df = df.filter(df.run == 3)
async def run(): s = stepper.Stepper() async with s: pts = [] #for v in voltages: # for sp in speeds: # current = await measure(s, sp, v, s.CONTROL_MODE_VOLTAGE) # pts.append((v, sp, current)) print(pts) v1 = 0.3 v2 = 0.8 sp1 = 75 sp2 = 150 c11 = await measure(s, sp1, v1, s.CONTROL_MODE_VOLTAGE) c12 = await measure(s, sp1, v2, s.CONTROL_MODE_VOLTAGE) c21 = await measure(s, sp2, v1, s.CONTROL_MODE_VOLTAGE) c22 = await measure(s, sp2, v2, s.CONTROL_MODE_VOLTAGE) r1 = (v2 - v1) / (c12 - c11) r2 = (v2 - v1) / (c22 - c21) r = (r1 + r2) / 2 print(r1, r2) km1 = -r * (c21 - c11) / (sp2 - sp1) km2 = -r * (c22 - c12) / (sp2 - sp1) km = (km1 + km2) / 2 print(km1, km2) by_speed = {} #by_speed[sp1] = [(v1, c11), (v2, c12)] #by_speed[sp2] = [(v1, c21), (v2, c22)] for (v, sp, c) in pts: by_speed[sp] = by_speed.get(sp, []) + [(v, c)] for (sp, pts) in by_speed.items(): pts = zip(*pts) plt.plot(*pts) for sp in speeds: v1 = voltages[0] v2 = voltages[-1] c1 = (v1 - km * sp) / r c2 = (v2 - km * sp) / r plt.plot([v1, v2], [c1, c2]) pts = zip(*pts) plt.plot(*pts) plt.show() await s.program_immediate(MOTOR, s.cmd_tuning(km, r)) pts = [] currents = np.linspace(0.2, 1, 2) for i in currents: for sp in speeds: current = await measure(s, sp, i, s.CONTROL_MODE_CURRENT_OL, voltage=False) pts.append((i, sp, current)) by_current = {} for (i, sp, c) in pts: by_current[i] = by_current.get(i, []) + [(sp, c)] for (sp, pts) in by_current.items(): pts = zip(*pts) plt.plot(*pts) plt.show()
def get_minmax(allp): s2 = stp.Stepper() minmax = allp.agg(F.min('RA'), F.max('RA'), F.min('DEC'), F.max('DEC')).collect() s2.show_step("get min-max") print(minmax)
import stepper for k in range(3): p = stepper.Stepper(k + 1) p.run(8, 500, 1)