if not all_netcdfs: print 'Extract failed for task: ',os.path.basename(u) continue # Process fields into single netcdf files for field in field_list: out_file=get_filename(u, field,output_dir,start_zip,end_zip,structure=args.structure,zip_freq=args.output_freq) print out_file netcdfs=all_netcdfs[field[0]] # List of netcdf files for stream in field (e.g. 'ga.pe') if not netcdfs: print 'Error, no files for requested file stream:',field[0] continue for i,nc_in_file in enumerate(netcdfs): if i==0: append=False else: append=True out_netcdf=process_netcdf(nc_in_file,temp_nc,field,append,zip_freq=args.output_freq) if not out_netcdf: break # Successfully created file: if out_netcdf: # First make the directory out_dir=os.path.dirname(out_file) if not os.path.exists(out_dir): os.makedirs(out_dir) # Rename temp file to out_netcdf shutil.move(temp_nc,out_file) print os.path.basename(out_file) # Remove netcdf files to stop temp directory getting too big for nc_list in all_netcdfs.itervalues(): for fname in nc_list:
# Extract zip files into temporary directory
base_path, all_netcdfs = extract_url(u, field_list, output_dir,
temp_dir, start_zip, end_zip)
if not all_netcdfs:
continue #something is wrong with this zip or files already exist, continue
# Process fields into single netcdf files
for field in field_list:
netcdfs = all_netcdfs[field[
0]] # List of netcdf files for stream in field (e.g. 'ga.pe')
if not netcdfs:
print 'Error, no files for requested file stream:', field[
0]
continue
for nc_in_file in netcdfs:
out_netcdf = process_netcdf(nc_in_file, base_path, field)
if not out_netcdf:
break
print os.path.basename(out_netcdf)
# Remove netcdf files to stop temp directory getting too big
for nc_list in all_netcdfs.itervalues():
for fname in nc_list:
os.remove(fname)
except Exception, e:
print 'Error extracting netcdf files', e
# raise
finally:
# remove the temporary directory
shutil.rmtree(temp_dir)