def compute_output(api, args): """ Creates a sample based on a `train_set`, source or dataset. """ samples = None # variables from command-line options resume = args.resume_ sample_ids = args.sample_ids_ output = args.predictions # there's only one sample to be generated at present args.max_parallel_clusters = 1 # sample cannot be published yet. args.public_sample = False # It is compulsory to have a description to publish either datasets or # clusters if (not args.description_ and (args.public_sample or args.public_dataset)): sys.exit("You should provide a description to publish.") # When using --new-fields, it is compulsory to specify also a dataset # id if args.new_fields and not args.dataset: sys.exit("To use --new-fields you must also provide a dataset id" " to generate the new dataset from it.") path = u.check_dir(output) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) csv_properties = {} # If logging is required set the file for logging log = None if args.log_file: u.check_dir(args.log_file) log = args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # basic pre-sample step: creating or retrieving the source related info source, resume, csv_properties, fields = pms.get_source_info( api, args, resume, csv_properties, session_file, path, log) # basic pre-sample step: creating or retrieving the dataset related info dataset_properties = pms.get_dataset_info(api, args, resume, source, csv_properties, fields, session_file, path, log) (_, datasets, _, resume, csv_properties, fields) = dataset_properties if args.sample_file: # sample is retrieved from the contents of the given local JSON file sample, csv_properties, fields = u.read_local_resource( args.sample_file, csv_properties=csv_properties) samples = [sample] sample_ids = [sample['resource']] else: # sample is retrieved from the remote object samples, sample_ids, resume = psa.samples_processing( datasets, samples, sample_ids, api, args, resume, session_file=session_file, path=path, log=log) if samples: sample = samples[0] # We update the sample's public state if needed if sample: if isinstance(sample, basestring): # build the query string from the sample options sample = u.check_resource(sample, api.get_sample) samples[0] = sample if (args.public_sample or (args.shared_flag and r.shared_changed(args.shared, sample))): sample_args = {} if args.shared_flag and r.shared_changed(args.shared, sample): sample_args.update(shared=args.shared) if args.public_sample: sample_args.update(r.set_publish_sample_args(args)) if sample_args: sample = r.update_sample(sample, sample_args, args, api=api, path=path, session_file=session_file) samples[0] = sample # We get the fields of the sample if we haven't got # them yet and need them if sample and psa.needs_sample_fields(args): fields = psa.get_sample_fields(sample, csv_properties, args) sample_file(samples[0], fields, args, api, path=path, session_file=session_file) u.print_generated_files(path, log_file=session_file, verbosity=args.verbosity) if args.reports: clear_reports(path) if args.upload: upload_reports(args.reports, path)
def compute_output(api, args): """ Creates a sample based on a `train_set`, source or dataset. """ samples = None # variables from command-line options resume = args.resume_ sample_ids = args.sample_ids_ output = args.predictions # there's only one sample to be generated at present args.max_parallel_clusters = 1 # sample cannot be published yet. args.public_sample = False # It is compulsory to have a description to publish either datasets or # clusters if (not args.description_ and (args.public_sample or args.public_dataset)): sys.exit("You should provide a description to publish.") # When using --new-fields, it is compulsory to specify also a dataset # id if args.new_fields and not args.dataset: sys.exit("To use --new-fields you must also provide a dataset id" " to generate the new dataset from it.") path = u.check_dir(output) session_file = u"%s%s%s" % (path, os.sep, SESSIONS_LOG) csv_properties = {} # If logging is required set the file for logging log = None if args.log_file: u.check_dir(args.log_file) log = args.log_file # If --clear_logs the log files are cleared clear_log_files([log]) # basic pre-sample step: creating or retrieving the source related info source, resume, csv_properties, fields = pms.get_source_info( api, args, resume, csv_properties, session_file, path, log) # basic pre-sample step: creating or retrieving the dataset related info dataset_properties = pms.get_dataset_info( api, args, resume, source, csv_properties, fields, session_file, path, log) (_, datasets, _, resume, csv_properties, fields) = dataset_properties if args.sample_file: # sample is retrieved from the contents of the given local JSON file sample, csv_properties, fields = u.read_local_resource( args.sample_file, csv_properties=csv_properties) samples = [sample] sample_ids = [sample['resource']] else: # sample is retrieved from the remote object samples, sample_ids, resume = psa.samples_processing( datasets, samples, sample_ids, api, args, resume, session_file=session_file, path=path, log=log) if samples: sample = samples[0] # We update the sample's public state if needed if sample: if isinstance(sample, basestring): # build the query string from the sample options sample = u.check_resource(sample, api.get_sample) samples[0] = sample if (args.public_sample or (args.shared_flag and r.shared_changed(args.shared, sample))): sample_args = {} if args.shared_flag and r.shared_changed(args.shared, sample): sample_args.update(shared=args.shared) if args.public_sample: sample_args.update(r.set_publish_sample_args(args)) if sample_args: sample = r.update_sample(sample, sample_args, args, api=api, path=path, session_file=session_file) samples[0] = sample # We get the fields of the sample if we haven't got # them yet and need them if sample and psa.needs_sample_fields(args): fields = psa.get_sample_fields(sample, csv_properties, args) if fields and args.export_fields: fields.summary_csv(os.path.join(path, args.export_fields)) sample_file(samples[0], fields, args, api, path=path, session_file=session_file) u.print_generated_files(path, log_file=session_file, verbosity=args.verbosity) if args.reports: clear_reports(path) if args.upload: upload_reports(args.reports, path)