if not all_netcdfs: print 'Extract failed for task: ',os.path.basename(u) continue # Process fields into single netcdf files for field in field_list: out_file=get_filename(u, field,output_dir,start_zip,end_zip,structure=args.structure,zip_freq=args.output_freq) print out_file netcdfs=all_netcdfs[field[0]] # List of netcdf files for stream in field (e.g. 'ga.pe') if not netcdfs: print 'Error, no files for requested file stream:',field[0] continue for i,nc_in_file in enumerate(netcdfs): if i==0: append=False else: append=True out_netcdf=process_netcdf(nc_in_file,temp_nc,field,append,zip_freq=args.output_freq) if not out_netcdf: break # Successfully created file: if out_netcdf: # First make the directory out_dir=os.path.dirname(out_file) if not os.path.exists(out_dir): os.makedirs(out_dir) # Rename temp file to out_netcdf shutil.move(temp_nc,out_file) print os.path.basename(out_file) # Remove netcdf files to stop temp directory getting too big for nc_list in all_netcdfs.itervalues(): for fname in nc_list:
# Extract zip files into temporary directory base_path, all_netcdfs = extract_url(u, field_list, output_dir, temp_dir, start_zip, end_zip) if not all_netcdfs: continue #something is wrong with this zip or files already exist, continue # Process fields into single netcdf files for field in field_list: netcdfs = all_netcdfs[field[ 0]] # List of netcdf files for stream in field (e.g. 'ga.pe') if not netcdfs: print 'Error, no files for requested file stream:', field[ 0] continue for nc_in_file in netcdfs: out_netcdf = process_netcdf(nc_in_file, base_path, field) if not out_netcdf: break print os.path.basename(out_netcdf) # Remove netcdf files to stop temp directory getting too big for nc_list in all_netcdfs.itervalues(): for fname in nc_list: os.remove(fname) except Exception, e: print 'Error extracting netcdf files', e # raise finally: # remove the temporary directory shutil.rmtree(temp_dir)