def calculate_stuff(splits=10, start=0): ''' Runs the spreadsheet, first splitting it into `splits` spreadsheets and then joining them. ''' if type(splits) is not int or type(start) is not int: raise TypeError # We are going to split this into 10 smaller pieces through the magic of mod operations! woo. split_data = [] spreadsheets = [] for i in range(start, splits): data_i = data.where(data.SOURCEID % splits == i) split_data.append(data_i) lookup_i = sp.base_lookup(data_i) # The parameter "-1" is the season that tells data_cut not to make # any cuts on the data. sp_i = sp.spreadsheet_write(data_i, lookup_i, -1, path2 + 'sp%d.fits' % i, flags=256, per=True, graded=False, rob=True, colorslope=True) # EEEEE this is a flag to come and find this section of code try: now = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") except: now = 'sometime' print "finished chunk %d at %s" % (i, now)
def calculate_stuff( splits = 10, start=0 ): ''' Runs the spreadsheet, first splitting it into `splits` spreadsheets and then joining them. ''' if type(splits) is not int or type(start) is not int: raise TypeError # We are going to split this into 10 smaller pieces through the magic of mod operations! woo. split_data = [] spreadsheets = [] for i in range(start, splits): data_i = data.where(data.SOURCEID % splits == i) split_data.append(data_i) lookup_i = sp.base_lookup(data_i) # The parameter "-1" is the season that tells data_cut not to make # any cuts on the data. sp_i = sp.spreadsheet_write(data_i, lookup_i, -1, path2+'sp%d.fits'%i, flags=256, per=True, graded=False, rob=True, colorslope=True) # EEEEE this is a flag to come and find this section of code try: now = datetime.datetime.strftime(datetime.datetime.now(), "%Y-%m-%d %H:%M:%S") except: now = 'sometime' print "finished chunk %d at %s" % (i, now)
def test(): ''' Runs spread_write_test. ''' sp.spread_write_test (data, sp.base_lookup(data, sourceid_offset=44508700000000))
def test(): ''' Runs spread_write_test. ''' sp.spread_write_test (data, sp.base_lookup(data))
def test(): ''' Runs spread_write_test. ''' sp.spread_write_test(data, sp.base_lookup(data, sourceid_offset=44508700000000))
def calculate_color_slope_ratios_versus_time_baseline(delta_t_list=None, date_offset=0, shuffle_dates=False): """ Calculates the color slope ratios for each possible time baseline. """ autovars_photometry = variables_photometry.where( np.in1d(variables_photometry.SOURCEID, autovars_true.SOURCEID) & (variables_photometry.MEANMJDOBS >= np.min(variables_photometry.MEANMJDOBS) + date_offset)) date_list = np.sort(list(set(np.floor(autovars_photometry.MEANMJDOBS)))) # I think this is the only place I need to shuffle? Maybe? if shuffle_dates: random.shuffle(autovars_photometry.MEANMJDOBS) n_positive_slope_list = [] n_negative_slope_list = [] n_undef_slope_list = [] n_variables_list = [] # calculate these properties: # most basic properties = [{'name':'time_baseline', 'function':get_timespan, 'target':'data', 'container':[]}, {'name':'n_obs', 'function':get_n_obs, 'target':'data', 'container':[]}] # how many variables, using different stetson cuts? properties.extend( [{'name':'n_variables_{0}'.format(x), 'function':partial(get_n_variables_stetson, stetson=x), 'target':'spreadsheet', 'container':[]} for x in (0.8, 1.0, 1.2) ] ) # basic properties -- j, h, k amplitudes / deviations properties.extend( [{'name':'median_{0}_{1}'.format(x,y1), 'function':partial(compute_function_on_column,y2,'{0}_ranger'.format(x)), 'target':'spreadsheet', 'container':[] } for x in ('j', 'h', 'k') for y1, y2 in zip(('amplitude', 'deviation'), (np.median, mad))] ) # now broken down into Class properties.extend( [{'name':'{2}_median_{0}_{1}'.format(x,y1, z1), 'function':partial(compute_function_on_column_selected,y2,'{0}_ranger'.format(x),z2), 'target':'spreadsheet', 'container':[] } for x in ('j', 'h', 'k') for y1, y2 in zip(('amplitude', 'deviation'), (np.median, mad)) for z1, z2 in zip(('protostar', 'disk', 'nondisk'), (protostar_IDs, disk_IDs, nondisk_IDs))] ) # For a bunch of different obs_initial's... # (note: when we test this, we may want to only go from the FIRST observation, and then add in the complexity of all the other ones later) # also note: we'll probably want to truncate the MJDs to the nearest integer to avoid any complications from the whole four-exposures-per-band-per-night thing. last_n_obs = 0 for obs_initial in date_list: print "Starting from: MJD %f" % obs_initial longest_time_separation = max(date_list) - obs_initial if delta_t_list == None: delta_t_list = range(int(np.ceil(longest_time_separation))) for delta_t in delta_t_list: relevant_data = autovars_photometry.where( (autovars_photometry.MEANMJDOBS >= obs_initial) & (autovars_photometry.MEANMJDOBS < obs_initial + delta_t) ) n_obs = len(list(set(np.floor(relevant_data.MEANMJDOBS)))) print "%d observations over %d (delta-t) days" % (n_obs, delta_t) if n_obs < 2: print "not enough observations for this to be called 'variability'. Continuing." continue elif n_obs == last_n_obs: print "No new observations have been added. Continuing." continue else: last_n_obs = n_obs # note: # if you don't add any new observations between this delta-t # and the last delta-t, DON'T COMPUTE A SPREADSHEET. just continue. # if timespan_of_relevant_data <= (delta_t - 1): # print timespan_of_relevant_data, delta_t, "timespan is too short to be considered in this delta_t bin. Continuing." # continue relevant_lookup = spread3.base_lookup(relevant_data, 0 ) relevant_spreadsheet = spread3.spreadsheet_write_efficient( 20, relevant_data, relevant_lookup, -1, None, flags=0, colorslope=True, rob=True) # then run color_slope_filtering on it... relevant_khk_spreadsheet = filter_color_slopes( relevant_spreadsheet, 'hk', lower_obs_limit=n_obs/3, upper_obs_limit=n_obs*1.5) relevant_khk_spreadsheet_no_slope_confidence = filter_color_slopes( relevant_spreadsheet, 'hk', slope_confidence=None, lower_obs_limit=n_obs/3, upper_obs_limit=n_obs*1.5) targets = {} targets['spreadsheet'] = relevant_spreadsheet targets['data'] = relevant_data # and extract which guys have colors in the relevant ranges! n_positive_slope = len(relevant_khk_spreadsheet[ (np.degrees(np.arctan(relevant_khk_spreadsheet.khk_slope)) > 25) ]) n_negative_slope = len(relevant_khk_spreadsheet[ (np.degrees(np.arctan(relevant_khk_spreadsheet.khk_slope)) < -25) ]) n_undef_slope = (len(relevant_khk_spreadsheet_no_slope_confidence) - (n_positive_slope + n_negative_slope) ) n_variables = len(relevant_spreadsheet[relevant_spreadsheet.Stetson >= 1.0]) for prop in properties: prop['container'].append(prop['function'](targets[prop['target']]) ) n_positive_slope_list.append( n_positive_slope ) n_negative_slope_list.append( n_negative_slope ) n_undef_slope_list.append( n_undef_slope ) n_variables_list.append( n_variables) break color_slope_ratios_table = atpy.Table() addc = color_slope_ratios_table.add_column addc("n_positive_slope", n_positive_slope_list) addc("n_negative_slope", n_negative_slope_list) addc("n_undef_slope", n_undef_slope_list) addc("n_variables", n_variables_list) for prop in properties: addc(prop['name'], prop['container']) return color_slope_ratios_table