def qc_sd(args): """ %prog sdtest input.matrix output.matrix run quality control on segregation distortions in each SNP. """ p = OptionParser(qc_sd.__doc__) p.add_option("-i", "--input", help=SUPPRESS_HELP) p.add_option("-o", "--output", help=SUPPRESS_HELP) p.add_option('--population', default='RIL', choices=('RIL', 'F2', 'BCFn'), help = "population type") p.add_option('--sig_cutoff', default = 1e-2, type='float', help = "set the chi square test cutoff. 0(less strigent) to 1(more strigent)") q = OptionGroup(p, "format options") p.add_option_group(q) q.add_option('--homo1', default="A", help='character for homozygous genotype') q.add_option("--homo2", default='B', help="character for alternative homozygous genotype") q.add_option('--hete', default='X', help='character for heterozygous genotype') q.add_option('--missing', default='-', help='character for missing value') opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) inmap, outmap = args inputmatrix = opts.input or inmap outputmatrix = opts.output or outmap if opts.sig_cutoff >=1 or opts.sig_cutoff <= 0: eprint('the cutoff chi square test should be smaller than 1 and larger than 0') sys.exit(1) chr_order, chr_nums = getChunk(inputmatrix) map_reader = pd.read_csv(inputmatrix, delim_whitespace=True, index_col=[0, 1], iterator=True) Good_SNPs = [] for chrom in chr_order: print('{}...'.format(chrom)) chunk = chr_nums[chrom] df_chr_tmp = map_reader.get_chunk(chunk) df_chr_tmp_num = df_chr_tmp.replace([opts.homo1, opts.homo2, opts.hete, opts.missing], [0, 2, 1, 9]) ob0, ob2 = (df_chr_tmp_num==0).sum(axis=1), (df_chr_tmp_num==2).sum(axis=1) obsum = ob0 + ob2 exp0, exp2 = (obsum*0.75, obsum*0.25) if opts.population == 'BCFn' else (obsum*0.5, obsum*0.5) df_chi = pd.DataFrame(dict(zip(['ob0', 'ob2', 'exp0', 'exp2'], [ob0, ob2, exp0, exp2]))) min_cond = ((df_chi['ob0']>5) & (df_chi['ob2']>5)).values pval_cond = chisquare([df_chi['ob0'], df_chi['ob2']], [df_chi['exp0'], df_chi['exp2']]).pvalue >= opts.sig_cutoff good_snp = df_chr_tmp.loc[(min_cond & pval_cond), :] Good_SNPs.append(good_snp) df1 = pd.concat(Good_SNPs) before_snp_num = sum(chr_nums.values()) after_snp_num = df1.shape[0] pct = after_snp_num/float(before_snp_num)*100 print('{} SNP markers before quality control.'.format(before_snp_num)) print('{}({:.1f}%) markers left after the quality control.'.format(after_snp_num, pct)) df1.to_csv(outputmatrix, sep='\t', index=True)
def format(args): """ %prog format corrected.matrix convert corrected genotype matrix file to other formats(mstmap, joinmap, r/qtl) for the genetic mapping software. Example: `python -m schnablelab.imputation.GC format test.map --mstmap --mstmap_pop_type RIL2` will generate `test.mstmap` for MSTmap use. """ p = OptionParser(format.__doc__) p.add_option("-i", "--input", help=SUPPRESS_HELP) p.add_option("--mstmap", default=False, action="store_true", help='convert to MSTmap format') p.add_option("--rqtl", default=False, action="store_true", help='convert to R/qtl format') p.add_option("--joinmap", default=False, action="store_true", help='convert to JoinMap format') q = OptionGroup(p, "format options for input matrix file") p.add_option_group(q) q.add_option('--homo1', default="A", choices=('a', 'A'), help='character for homozygous genotype') q.add_option("--homo2", default='B', choices=('b', 'B'), help="character for alternative homozygous genotype") q.add_option('--hete', default='X', choices=('h', 'H', 'X'), help='character for heterozygous genotype') q.add_option('--missing', default='-', choices=('-', 'U'), help='character for missing value') r = OptionGroup(p, "parameters for MSTmap") p.add_option_group(r) r.add_option( '--mstmap_pop_type', help='Possible values are DH and RILd, where d is any natural number. \ For example, RIL6 means a RIL population at generation 6. \ You should use RIL2 for F2. Use DH for BC1, DH and Hap.') r.add_option( "--population_name", default='LinkageGroup', help= "ives a name for the mapping population. It can be any string of letters (a-z, A-Z) or digits (0-9)" ) r.add_option('--distance_function', default='kosambi', choices=('kosambi', 'haldane'), help="choose Kosambi's and Haldane's distance functions") r.add_option( '--cut_off_p_value', default=0.000001, help= 'specifies the threshold to be used for clustering the markers into LGs' ) r.add_option('--no_map_dist', default=15, help='check mstmap manual for details') r.add_option('--no_map_size', default=5, help='check mstmap manual for details') r.add_option( '--missing_threshold', default=0.4, help= 'any marker with more than this value will be removed completely without being mapped' ) r.add_option( '--estimation_before_clustering', default='no', choices=('yes', 'no'), help= 'if yes, MSTmap will try to estimate missing data before clustering the markers into linkage groups' ) r.add_option('--detect_bad_data', default='yes', choices=('yes', 'no'), help='if yes turn on the error detection feature in MSTmap') r.add_option('--objective_function', default='COUNT', choices=('COUNT', 'ML'), help='specifies the objective function') s = OptionGroup(p, "parameters for JoinMap and R/qtl") p.add_option_group(s) s.add_option( '--pop_type', default='RIL', choices=('RIL', 'F2'), help= 'specify mapping population type. Contact me if you need supports for other population types' ) opts, args = p.parse_args(args) if len(args) != 1: sys.exit(not p.print_help()) inmap, = args inputmatrix = opts.input or inmap if (not opts.rqtl) and (not opts.joinmap) and (not opts.mstmap): eprint("ERROR: add at least one output format option.") sys.exit(1) if opts.mstmap: if not opts.mstmap_pop_type: eprint("ERROR: please choose population type for mstmap format.") sys.exit(1) if not (opts.mstmap_pop_type.startswith('RIL') or opts.mstmap_pop_type == 'DH'): eprint('ERROR: only RILd and DH supported in MSTmap') sys.exit(1) opf = inputmatrix.rsplit(".", 1)[0] + '.mstmap' # output file if Path(opf).exists(): eprint( "ERROR: Filename collision. The future output file `{}` exists" .format(opf)) sys.exit(1) df = pd.read_csv(inputmatrix, delim_whitespace=True) cols = list(df.columns[2:]) cols.insert(0, 'locus_name') df['locus_name'] = df.iloc[:, 0].astype( 'str') + '_' + df.iloc[:, 1].astype('str') df = df[cols] print(df.head()) snp_num, sm_num = df.shape[0], df.shape[1] - 1 f1 = open(opf, 'w') f1.write(mst_header.format(opts.mstmap_pop_type, opts.population_name, opts.distance_function, opts.cut_off_p_value, \ opts.no_map_dist, opts.no_map_size, opts.missing_threshold, opts.estimation_before_clustering, opts.detect_bad_data, \ opts.objective_function, snp_num, sm_num)) f1.close() df.to_csv(opf, sep='\t', index=False, mode='a') print('Done, check file {}!'.format(opf)) if opts.joinmap: opf = inputmatrix.rsplit(".", 1)[0] + '.joinmap.xlsx' # output file if Path(opf).exists(): eprint( "ERROR: Filename collision. The future output file `{}` exists" .format(opf)) sys.exit(1) df = pd.read_csv(inputmatrix, delim_whitespace=True) need_reps, reps = [], [] if opts.homo1 != 'a': need_reps.append(opts.homo1) reps.append('a') if opts.homo2 != 'b': need_reps.append(opts.homo2) reps.append('b') if opts.hete != 'h': need_reps.append(opts.hete) reps.append('h') if opts.missing != '-': need_reps.append(opts.missing) reps.append('-') if need_reps: df = df.replace(need_reps, reps) cols = list(df.columns[2:]) cols.insert(0, 'Classification') cols.insert(0, 'locus_name') df['locus_name'] = df.iloc[:, 0].astype( 'str') + '_' + df.iloc[:, 1].astype('str') df['Classification'] = '(a,h,b)' df = df[cols] df.to_excel(opf) print( 'Done! Now you can load the genotype data into the JoinMap project from the MS-Excel spreadsheet {} to a dataset node.' .format(opf)) if opts.rqtl: opf = inputmatrix.rsplit(".", 1)[0] + '.rqtl.csv' # output file if Path(opf).exists(): eprint( "ERROR: Filename collision. The future output file `{}` exists" .format(opf)) sys.exit(1) df = pd.read_csv(inputmatrix, delim_whitespace=True) need_reps, reps = [], [] if opts.homo1 != 'A': need_reps.append(opts.homo1) reps.append('A') if opts.homo2 != 'B': need_reps.append(opts.homo2) reps.append('B') if opts.hete != 'H': need_reps.append(opts.hete) reps.append('H') if opts.missing != '-': need_reps.append(opts.missing) reps.append('-') if need_reps: df = df.replace(need_reps, reps) cols = list(df.columns[2:]) cols.insert(0, 'id') df['id'] = df.iloc[:, 0].astype('str') + '_' + df.iloc[:, 1].astype('str') df = df[cols] df.loc[-1] = 1 df.index = df.index + 1 df = df.sort_index() df.iloc[0, 0] = np.nan df.to_csv(opf, index=False, na_rep='') print('Done, check file {}!'.format(opf))
def qc_hetero(args): """ %prog qc_hetero input.matrix output.matrix run quality control on the continuous same homozygous in heterozygous region. """ p = OptionParser(qc_hetero.__doc__) p.add_option("-i", "--input", help=SUPPRESS_HELP) p.add_option("-o", "--output", help=SUPPRESS_HELP) p.add_option("--read_len", default=150, type='int', help="read length for SNP calling") p.add_option("--logfile", default='GC.qc_hetero.info', help="specify the file saving binning info") q = OptionGroup(p, "format options") p.add_option_group(q) q.add_option('--homo1', default="A", help='character for homozygous genotype') q.add_option("--homo2", default='B', help="character for alternative homozygous genotype") q.add_option('--hete', default='X', help='character for heterozygous genotype') q.add_option('--missing', default='-', help='character for missing value') r = OptionGroup(p, 'advanced options') p.add_option_group(r) r.add_option( '--nonhetero_lens', default=8, type='int', help= 'number of non heterozygous between two heterozygous in a heterozygous region' ) r.add_option( '--min_homo', default=2, type='int', help= 'number of continuous homozygous within the read length in the heterozygous region' ) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) inmap, outmap = args inputmatrix = opts.input or inmap outputmatrix = opts.output or outmap logging.basicConfig(filename=opts.logfile, level=logging.DEBUG, format="%(asctime)s:%(levelname)s:%(message)s") chr_order, chr_nums = getChunk(inputmatrix) map_reader = pd.read_csv(inputmatrix, delim_whitespace=True, index_col=[0, 1], iterator=True) Good_SNPs = [] for chrom in chr_order: print('{}...'.format(chrom)) logging.debug(chrom) chunk = chr_nums[chrom] df_chr_tmp = map_reader.get_chunk(chunk) chr_idx = df_chr_tmp.index df_chr_tmp_num = df_chr_tmp.replace( [opts.homo1, opts.homo2, opts.hete, opts.missing], [0, 2, 1, 9]).loc[chrom] chr_bin_ids = [] for sm in df_chr_tmp_num: geno_grouper = (df_chr_tmp_num[sm].diff(1) != 0).astype('int').cumsum() idx, geno, lens = [], [], [] for __, grp_geno in df_chr_tmp_num[sm].groupby(geno_grouper): idx.append(grp_geno.index) geno.append(grp_geno.unique()[0]) lens.append(grp_geno.shape[0]) df_grp_geno = pd.DataFrame( dict(zip(['idx', 'geno', 'lens'], [idx, geno, lens]))) df_grp_geno['type'] = df_grp_geno['geno'].apply( lambda x: 1 if x == 1 else 0 ) # 1: hetero genotype 0: others(homo1, homo2, missing) type_grouper = (df_grp_geno['type'].diff(1) != 0).astype('int').cumsum() for __, grp_type in df_grp_geno.groupby(type_grouper): if grp_type['type'].unique()[0] == 0: nonhetero_lens = grp_type['lens'].sum() if nonhetero_lens <= opts.nonhetero_lens: for __, row in grp_type.iterrows(): if row.geno == 0 or row.geno == 2: bin_ids = get_blocks(row['idx'].values, dist=opts.read_len, block_size=opts.min_homo) if bin_ids: for bin_index in bin_ids: if bin_index not in chr_bin_ids: chr_bin_ids.append(bin_index) if chr_bin_ids: dropping_ids = [] merged_bin_ids = sort_merge_sort(chr_bin_ids) for idx_block in merged_bin_ids: logging.debug('positions: {}'.format(idx_block)) genos_block = df_chr_tmp_num.loc[idx_block, :] missings = genos_block.apply(lambda x: (x == 9).sum(), axis=1) heteros = genos_block.apply(lambda x: (x == 1).sum(), axis=1) dropping_index = list( pd.concat([missings, heteros], axis=1).sort_values([0, 1]).index[1:]) dropping_ids.extend(dropping_index) df_chr_tmp = df_chr_tmp.drop(dropping_ids, level=1) Good_SNPs.append(df_chr_tmp) df1 = pd.concat(Good_SNPs) before_snp_num = sum(chr_nums.values()) after_snp_num = df1.shape[0] pct = after_snp_num / float(before_snp_num) * 100 print('{} SNP markers before quality control.'.format(before_snp_num)) print('{}({:.1f}%) markers left after the quality control.'.format( after_snp_num, pct)) df1.to_csv(outputmatrix, sep='\t', index=True) print('Done! Check {} for running details.'.format(opts.logfile))
def qc_missing(args): """ %prog filtermissing input.matrix output.matrix run quality control of the missing genotypes in the input.matrix before starting the correction. """ p = OptionParser(qc_missing.__doc__) p.add_option("-i", "--input", help=SUPPRESS_HELP) p.add_option("-o", "--output", help=SUPPRESS_HELP) p.add_option( '--cutoff_snp', default=0.5, type='float', help="SNP with missing rate higher than this value will be removed") p.add_option( '--rm_bad_samples', default=False, action="store_true", help= 'remove bad samples after controlling the SNPs with high missing rate') p.add_option( '--cutoff_sample', type='float', help= "sample missing rate higher than this value will be removed after controlling the SNP missing rate" ) q = OptionGroup(p, "format options") p.add_option_group(q) q.add_option('--homo1', default="A", help='character for homozygous genotype') q.add_option("--homo2", default='B', help="character for alternative homozygous genotype") q.add_option('--hete', default='X', help='character for heterozygous genotype') q.add_option('--missing', default='-', help='character for missing value') opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) if opts.rm_bad_samples and not opts.cutoff_sample: eprint( 'missing value cutoff for --cutoff_sample option must be specified when --rm_bad_samples added.' ) sys.exit(1) inmap, outmap = args inputmatrix = opts.input or inmap outputmatrix = opts.output or outmap chr_order, chr_nums = getChunk(inputmatrix) map_reader = pd.read_csv(inputmatrix, delim_whitespace=True, index_col=[0, 1], iterator=True) Good_SNPs = [] for chrom in chr_order: print('{}...'.format(chrom)) chunk = chr_nums[chrom] df_chr_tmp = map_reader.get_chunk(chunk) df_chr_tmp_num = df_chr_tmp.replace( [opts.homo1, opts.homo2, opts.hete, opts.missing], [0, 2, 1, 9]) sample_num = df_chr_tmp_num.shape[1] good_rates = df_chr_tmp_num.apply(lambda x: (x == 9).sum() / sample_num, axis=1) <= opts.cutoff_snp good_snp = df_chr_tmp.loc[good_rates, :] Good_SNPs.append(good_snp) df1 = pd.concat(Good_SNPs) before_snp_num = sum(chr_nums.values()) after_snp_num, before_sm_num = df1.shape pct = after_snp_num / float(before_snp_num) * 100 print('{} SNP markers before quality control.'.format(before_snp_num)) print('{}({:.1f}%) markers left after the quality control.'.format( after_snp_num, pct)) if opts.rm_bad_samples: print('start quality control on samples') good_samples = df1.apply(lambda x: (x == opts.missing).sum() / after_snp_num, axis=0) <= opts.cutoff_sample df2 = df1.loc[:, good_samples] after_sm_num = df2.shape[1] pct_sm = after_sm_num / float(before_sm_num) * 100 print('{} samples before quality control.'.format(before_sm_num)) print('{}({:.1f}%) markers left after the quality control.'.format( after_sm_num, pct_sm)) df2.to_csv(outputmatrix, sep='\t', index=True) else: df1.to_csv(outputmatrix, sep='\t', index=True)
def correct(args): """ %prog correct config.txt input.matrix Correct wrong genotype calls and impute missing values in biparental populations """ p = OptionParser(correct.__doc__) p.add_option("-c", "--configfile", help=SUPPRESS_HELP) p.add_option("-m", "--matrixfile", help=SUPPRESS_HELP) p.add_option('--itertimes', default=7, type='int', help='maximum correction times to reach the stablized status') q = OptionGroup(p, "output options") p.add_option_group(q) q.add_option('--opp', default="'infer'", help='specify the prefix of the output file names') q.add_option("--logfile", default='GC.correct.log', help="specify the file saving running info") q.add_option( '--debug', default=False, action="store_true", help= 'trun on the debug mode that will generate a tmp file containing both original and corrected genotypes for debug use' ) p.set_cpus(cpus=8) opts, args = p.parse_args(args) if len(args) != 2: sys.exit(not p.print_help()) configfile, mapfile = args inputmatrix = opts.matrixfile or mapfile inputconfig = opts.configfile or configfile opf = inputmatrix.rsplit( ".", 1)[0] + '.corrected.map' if opts.opp == "'infer'" else '{}.map'.format( opts.opp) # output file if Path(opf).exists(): eprint("ERROR: Filename collision. The future output file `{}` exists". format(opf)) sys.exit(1) cpus = opts.cpus if sys.version_info[:2] < (2, 7): logging.debug("Python version: {0}. CPUs set to 1.".\ format(sys.version.splitlines()[0].strip())) cpus = 1 logging.basicConfig(filename=opts.logfile, level=logging.DEBUG, format="%(asctime)s:%(levelname)s:%(message)s") cargs = ParseConfig(inputconfig) if cargs.win_size % 2 == 0: eprint("ERROR: The slding window value cannot be even") sys.exit(1) logging.debug("Parameters in config file: {0}".format(cargs.__dict__)) chr_order, chr_nums = getChunk(inputmatrix) map_reader = pd.read_csv(inputmatrix, delim_whitespace=True, index_col=[0, 1], iterator=True) tmp_chr_list = [] for chrom in chr_order: logging.debug('{}...'.format(chrom)) print('{}...'.format(chrom)) chunk = chr_nums[chrom] df_chr_tmp = map_reader.get_chunk(chunk) marker_num, sample_num = df_chr_tmp.shape logging.debug('{} contains {} markers and {} samples.'.format( chrom, marker_num, sample_num)) tmp_sm_list = [] for sm in df_chr_tmp: logging.debug('Start correcting {}...'.format(sm)) orig_seq = df_chr_tmp[sm] orig_idx = orig_seq.index seq_no_idx = orig_seq.reset_index(drop=True) seq_no_idx_num = seq_no_idx.replace( [cargs.gt_a, cargs.gt_b, cargs.gt_h, cargs.gt_miss], [0, 2, 1, 9]) if seq_no_idx_num.shape[0] <= cargs.win_size: logging.debug( 'number of markers smaller than the window size, omit...') final_seq_no_idx = seq_no_idx else: logging.debug('correction round 1...') correct_obj = CorrectOO(cargs, seq_no_idx_num) corrected_n = get_corrected_num(seq_no_idx_num, correct_obj.corrected) round_n = 2 while round_n <= opts.itertimes: logging.debug('correction round %s...' % round_n) corrected_obj = CorrectOO(cargs, correct_obj.corrected) corrected_n_new = get_corrected_num( seq_no_idx_num, corrected_obj.corrected) round_n += 1 if (corrected_n_new - corrected_n) / float(corrected_n + 0.01) <= 0.01: break else: corrected_n = corrected_n_new final_seq_no_idx = corrected_obj.corrected.replace( [0, 2, 1, 9], [cargs.gt_a, cargs.gt_b, cargs.gt_h, cargs.gt_miss]) final_seq_no_idx.index = orig_idx final_seq = final_seq_no_idx tmp_sm_list.append(final_seq) df_sm_tmp = pd.concat(tmp_sm_list, axis=1) tmp_chr_list.append(df_sm_tmp) df_corrected = pd.concat(tmp_chr_list) df_corrected.to_csv(opf, sep='\t', index=True) if opts.debug: logging.debug('generating the tmp file for debug use...') df_uncorrected = pd.read_csv(inputmatrix, delim_whitespace=True, index_col=[0, 1]) df_debug = df_corrected.where(df_corrected == df_uncorrected, other=df_corrected + '(' + df_uncorrected + ')') df_debug.to_csv(opf + '.debug', sep='\t', index=True) print('Done!')
def quick_search(args): ''' %prog quick_search Perform quick serach and get ids for items from search resutls ''' p = OptionParser(quick_search.__doc__) p.add_option('-o', '--output', default="searches.csv", help='specify output file') p.add_option('--geom', default="lindsay_james.geojson", help='speficy the geojson file containing the geometry info') p.add_option('--cloud', default=True, action='store_false', help='disable cloud filter if add --cloud option') p.add_option('--coverage', default=False, action='store_true', help='add area coverage filter if add --coverage option') p.add_option('--instrument', default=False, action='store_true', help='add instrument filter if add --instrument option') p.add_option('--date_range', default=True, action='store_false', help='disable date filter if add --date_range option') p.add_option('--map_footprint', default=False, action='store_true', help='add mapping footprints if add --map_footprint option') q = OptionGroup(p, "options for date filter") p.add_option_group(q) q.add_option('--start', default="2018-01-01", help='the start date. use yyyy-mm-dd format.') q.add_option('--end', help='the end date. use yyyy-mm-dd format') r = OptionGroup(p, "options for request") p.add_option_group(r) r.add_option( '--item_types', default='PSScene4Band', help='specify the item types. use comma separated if more than one item' ) opts, args = p.parse_args(args) if len(args) != 0: sys.exit(not p.print_help()) # all filters Filters, Filters_names = [], [] geojson_fn = default_geojson if opts.geom == 'lindsay_james.geojson' else opts.geom with open(geojson_fn) as f: data = json.load(f) geometry = data['features'][0]['geometry'] filter_geom = Filter().get_filter_geometry(geometry) Filters.append(filter_geom) Filters_names.append(filter_geom['field_name']) if opts.cloud: filter_cloud = Filter().get_filter_cloud() Filters.append(filter_cloud) Filters_names.append(filter_cloud['field_name']) if opts.coverage: filter_coverage = Filter().get_filter_coverage() Filters.append(filter_coverage) Filters_names.append(filter_coverage['field_name']) if opts.instrument: filter_instrument = Filter().get_filter_instrument() Filters.append(filter_instrument) Filters_names.append(filter_instrument['field_name']) if opts.date_range: st, ed = opts.start, opts.end filter_date = Filter().get_filter_date(st=st, ed=ed) Filters.append(filter_date) Filters_names.append('date_range') Final_Filters = {'type': 'AndFilter', 'config': Filters} print('Applied Filters: %s' % (', '.join(Filters_names))) # request items = opts.item_types.split(',') print('Item Types: ', items) Requests = Request(item_types=items) Final_Requests = Requests.get_request(Final_Filters) #rint(Final_Requests) # post client = Client() res = client.ses.post(client.url_quick_search, json=Final_Requests, params={"_page_size": 10}) geojson = res.json( ) # important keys: _links(current link and link of the next page), features(id) link_first_page = geojson['_links']['_first'] ids, cloud_cover, item_type, assets_url = [], [], [], [] def parse_page(session, search_url, map_footprint): ''' loop pages and extract ids from each page ''' res = session.get(search_url) if map_footprint: url = geojsonio.display(res.text) page = res.json() for feature in page['features']: id = feature['id'] cc = feature['properties']['cloud_cover'] it = feature['properties']['item_type'] au = feature['_links']['assets'] ids.append(id) cloud_cover.append(cc) item_type.append(it) assets_url.append(au) next_url = page["_links"].get("_next") if next_url: parse_page(session, next_url, map_footprint) parse_page(client.ses, link_first_page, opts.map_footprint) df = pd.DataFrame( dict( zip(['id', 'cloud_cover', 'item_type', 'assets_url'], [ids, cloud_cover, item_type, assets_url]))) df.to_csv(opts.output, index=False, sep='\t') print('%s items found, please check search resutls in %s!' % (df.shape[0], opts.output))
def stat(args): ''' %prog stat Check availabe images on Planet ''' p = OptionParser(stat.__doc__) p.add_option('-o', '--output', default="stats.csv", help='specify output file') p.add_option('--geom', default="lindsay_james.geojson", help='speficy the geojson file containing the geometry info') p.add_option('--cloud', default=True, action='store_false', help='disable cloud filter if add --cloud option') p.add_option('--coverage', default=False, action='store_true', help='add area coverage filter if add --coverage option') p.add_option('--instrument', default=False, action='store_true', help='add instrument filter if add --instrument option') p.add_option('--date_range', default=True, action='store_false', help='disable date filter if add --date_range option') q = OptionGroup(p, "options for date filter") p.add_option_group(q) q.add_option('--start', default="2018-01-01", help='the start date. use yyyy-mm-dd format.') q.add_option('--end', help='the end date. use yyyy-mm-dd format') r = OptionGroup(p, "options for request") p.add_option_group(r) r.add_option('--interval', default="year", choices=('year', 'month', 'week', 'day', 'hour'), help='specify the interval in the request') r.add_option( '--item_types', default='PSScene4Band,PSOrthoTile', help='specify the item types. use comma separated if more than one item' ) opts, args = p.parse_args(args) if len(args) != 0: sys.exit(not p.print_help()) # all filters Filters, Filters_names = [], [] geojson_fn = default_geojson if opts.geom == 'lindsay_james.geojson' else opts.geom with open(geojson_fn) as f: data = json.load(f) geometry = data['features'][0]['geometry'] filter_geom = Filter().get_filter_geometry(geometry) Filters.append(filter_geom) Filters_names.append(filter_geom['field_name']) if opts.cloud: filter_cloud = Filter().get_filter_cloud() Filters.append(filter_cloud) Filters_names.append(filter_cloud['field_name']) if opts.coverage: filter_coverage = Filter().get_filter_coverage() Filters.append(filter_coverage) Filters_names.append(filter_coverage['field_name']) if opts.instrument: filter_instrument = Filter().get_filter_instrument() Filters.append(filter_instrument) Filters_names.append(filter_instrument['field_name']) if opts.date_range: st, ed = opts.start, opts.end filter_date = Filter().get_filter_date(st=st, ed=ed) Filters.append(filter_date) Filters_names.append(filter_date['field_name']) Final_Filters = {'type': 'AndFilter', 'config': Filters} print('Applied Filters: %s' % (', '.join(Filters_names))) # request Requests = Request(item_types=opts.item_types.split(','), interval=opts.interval) Final_Requests = Requests.get_request(Final_Filters) # post client = Client() res = client.ses.post(client.url_stat, json=Final_Requests) df_stat = pd.DataFrame.from_dict( res.json()['buckets'])[['start_time', 'count']] print(df_stat) df_stat.to_csv(opts.output, index=False, sep='\t') print('also saved to %s!' % opts.output)