def dump(self, outdir='.', outdat='photon.dat', fid=None, **kwargs): ''' remove photon from record, write TAU RESULT SCATTERED HISTORY 10.5f 10s 10i (10.5f,10.5f,10.5f) * nscat+1 index, bool, location of record to remove ''' from numpy import array, where, empty from lib import mkdir_p dbg((outdir, outdat), 4) mkdir_p(outdir) fopened = True if fid != None else False if not fopened: fname = '/'.join((outdir, outdat)) fid = open(fname, 'a') loc = self.history.T.flatten() val = [self.tau, self.result, self.scattered] val.extend(loc) #_make formatted print statement and write to file nk, nscat = self.history.shape fmt = '%10.5f%10s%10i ' + '%10.5f%10.5f%10.5f' * nscat + '\n' line = fmt % tuple(val) fid.write(line) if not fopened: fid.close()
def main(args): import optparse usage = """usage: %prog GOOGLE_BOOK_OR_ID Download a Google Book and create a PNG image for each page.""" parser = optparse.OptionParser(usage) parser.add_option('-s', '--page-start', dest='page_start', type="int", default=1, help='Start page') parser.add_option('-e', '--page-end', dest='page_end', type="int", default=None, help='End page') options, pargs = parser.parse_args(args) if not pargs: parser.print_usage() return 2 url = pargs[0] info = get_info_from_url(url) namespace = dict(title=info["title"], attribution=info["attribution"]) output_directory = "%(attribution)s - %(title)s" % namespace lib.mkdir_p(output_directory) for page_info, page, image_data in download_book(url, options.page_start - 1, options.page_end): filename = "%03d.png" % (page + 1) output_path = os.path.join(output_directory, filename) open(output_path, "wb").write(image_data) sys.stdout.write(output_path + "\n")
def wiki_create_page(request): project = request.matchdict['project'] page = request.matchdict['page'] wikiroot = request.registry.settings['wiki.root'] #determine the new page path: rootpath = os.path.join(wikiroot, project) filepath = os.path.join(rootpath,page) filepath += ".md" #TODO: markdown only for the moment #create the new page, and parent directories if needed if "/" in page: dirname = os.path.split(page)[0] dirpath = os.path.join(rootpath, dirname) log.debug("creating directory %s"%dirpath) mkdir_p(dirpath) log.debug("creating file %s"%filepath) f = open(filepath, "w") f.write("please set some content here") f.close() return HTTPFound(custom_route_path(request, "edit", project=project, page=page))
def setUp(self): self.config = testing.setUp() self.config.add_route('view_wiki', '/wiki/{project}/{page:.*}') self.config.add_route('edit', "/edit/{project}/{page:.*}") #create an empty repository self.tmpdir = tempfile.mkdtemp() self.repo = Repo.init(self.tmpdir) #populate with a new home page: with open("%s/%s"%(self.tmpdir, "Home.md"), "w") as f: f.write("hello wiki") #create a page in a subdirectory as well mkdir_p("%s/%s"%(self.tmpdir, "testsubdir")) home_md = open("%s/%s"%(self.tmpdir, "testsubdir/subdirfile.md"), "w") home_md.write("hello subdir file") #add this new page to the revision history: self.repo.stage(["Home.md"]) self.repo.do_commit("first revision", committer="john doe <*****@*****.**>") request = testing.DummyRequest() self.root = os.path.split(self.tmpdir)[0] self.projectname = os.path.split(self.tmpdir)[1] request.registry.settings['wiki.root'] = self.root request.matchdict['page'] = "Home" request.matchdict['project'] = self.projectname self.request = request
def plot(self, idx=None, ax=None, outdir='.', imgfmt='png', rho=False, limit_plot=True, clouds=None, **kwargs): ''' plot photon path idx, int, if supplied, only the listed photon paths will plot ax, plt.artist, matplotlib plot space outdir, str, output directory imgfmt, str, image file type rho, bool, instead of plotting path, plot density ''' from numpy import arange, array, tan, cos, pi from lib.tools import mkdir_p mkdir_p(outdir) x, y, z = self.history if self.line == None: #_new line if len(self.figure.axes) == 0: #_new plot completely self.axes = self.figure.add_subplot(111) self.axes.set_ylim(12, -2) self.axes.set_xlim(-20, 20) self.line, = self.axes.plot(x, z, linewidth=.3) else: #_update line self.line.set_xdata(x) self.line.set_ydata(z) #_expand plotting area if photons wander far if not limit_plot: cxmin, cxmax = self.axes.xaxis.get_view_interval() cymin, cymax = self.axes.yaxis.get_view_interval() xmax = max([cxmax, x.max()]) xmin = min([cxmin, x.min()]) self.axes.set_xlim(xmin, xmax) #_update cloud lines if clouds != None: x = [0, clouds.incident_x(cymax)] y = [0, cymax] clouds.ray.set_xdata(x) clouds.ray.set_ydata(y) clouds.base.set_xdata([cxmin, cxmax]) clouds.top.set_xdata([cxmin, cxmax]) #_only call draw if this is the only line, # otherwise do it in a wrapper loop if len(self.axes.lines) == 1: self.figure.canvas.draw() self.figure.show()
def main(args): import argparse parser = argparse.ArgumentParser() parser.add_argument('-s', '--page-start', dest='page_start', type=int, default=1, help='Start page') parser.add_argument('-e', '--page-end', dest='page_end', type=int, default=None, help='End page') parser.add_argument('-n', '--no-redownload', dest='noredownload', action="store_true", default=False, help='Do not re-download pages if they exist locally') parser.add_argument('-o', '--output-directory', dest='output_directory', default='', help='Output directory') parser.add_argument('-q', '--quiet', dest='quiet', action="store_true", default=False, help='Do not print messages to the terminal') parser.add_argument('url', help='GOOGLE_BOOK_OR_ID') args = parser.parse_args(args) url = args.url info = get_info_from_url(url) namespace = dict(title=info["title"], attribution=info["attribution"]) if args.output_directory: output_directory = args.output_directory else: output_directory = "%(attribution)s - %(title)s" % namespace lib.mkdir_p(output_directory) for page_info, page, image_data in\ download_book(url, args.page_start - 1, args.page_end): filename = "%03d.png" % (page + 1) output_path = os.path.join(output_directory, filename) if not ((os.path.isfile(output_path) and args.noredownload)): open(output_path, "wb").write(image_data) if not args.quiet: print 'Downloaded {}'.format(output_path.encode('utf-8')) elif not args.quiet: print 'Output file {} exists'.format(output_path.encode('utf-8'))
def main(args): import optparse usage = """usage: %prog GOOGLE_BOOK_OR_ID Download a Google Book and create a PNG image for each page.""" parser = optparse.OptionParser(usage) parser.add_option('-s', '--page-start', dest='page_start', type="int", default=1, help='Start page') parser.add_option('-e', '--page-end', dest='page_end', type="int", default=None, help='End page') parser.add_option('-n', '--no-redownload', dest='noredownload', action="store_true", default=False, help='Do not download pages if they exist') parser.add_option('-q', '--quiet', dest='quiet', action="store_true", default=False, help='Do not print messages to the terminal') options, pargs = parser.parse_args(args) if not pargs: parser.print_usage() return 2 url = pargs[0] info = get_info_from_url(url) namespace = dict(title=info["title"], attribution=info["attribution"]) output_directory = "%(attribution)s - %(title)s" % namespace lib.mkdir_p(output_directory) for page_info, page, image_data in\ download_book(url, options.page_start - 1, options.page_end): filename = "%03d.png" % (page + 1) output_path = os.path.join(output_directory, filename) if not ((os.path.isfile(output_path) and options.noredownload)): open(output_path, "wb").write(image_data) if not options.quiet: print 'Downloaded {}'.format(output_path) elif not options.quiet: print 'Output file {} exists'.format(output_path)
def run_main(drawlive=False, limit_plot=True, **namelist): import os import matplotlib.pyplot as plt from lib import photon_propagation from lib import mkdir_p from lib import Cloud as cloud from time import sleep #_loop over runs specified in the Namelist dictionary for run in namelist: #_turn on interactive window if drawlive: plt.ion() #_initialize output file outdir = 'output_{}'.format(run) mkdir_p(outdir) fname = os.path.join(outdir, 'photon.dat') print 'Writing photon data to {}'.format(fname) #_pull out keyword args and update kwargs = namelist[run] clouds = cloud(**kwargs) #_initialize cloud kwargs.update({ 'outdir': outdir, 'figure': clouds.figure, 'fid': open(fname, 'w'), 'axes': clouds.figure.axes[0], 'limit_plot': limit_plot, 'clouds': clouds, 'drawlive': drawlive }) #_calc how many loops need to be made photon_propagation(**kwargs) #_same image pname = os.path.join(outdir, 'photon.png') plt.savefig(pname) print 'Saving path image to {}'.format(pname) sleep(10)
def stageFiles(filterVolumes, filterTargetPath, filterImageIds, stageFolder): """ Make links from source files to clip stage folders. filterVolumes is a list of volumes as strings, e.g. ['5101','5102'], or None for all. filterTargetPath is system/craft/target/camera, e.g. 'Jupiter//Io', or None for all. filterImageIds is a string with start and stop imageIds, e.g. 'C1436454-C1528477' or None=all. stageFolder is config.clipsStageFolder or config.moviesStageFolder This fn is also used by vgMovies, so be careful editing it. """ # targetPathParts is [system, craft, target, camera], or None. # ntargetDirFiles is a dictionary that keeps track of how many files each folder contains. print 'Making links from source files' # print filterVolumes, targetPathParts # note: filterTargetPathParts = [pathSystem, pathCraft, pathTarget, pathCamera] targetPathParts = lib.parseTargetPath(filterTargetPath) if filterImageIds: imageIdStart, imageIdStop = filterImageIds.split('-') else: imageIdStart, imageIdStop = None, None # read some small dbs into memory retargetingInfo = lib.readCsv(config.dbRetargeting) # remapping listed targets framerateConstantInfo = lib.readCsv(config.dbFramerateConstants) # change framerate per target framerateInfo = lib.readCsv(config.dbFramerates) # change framerate per image centeringInfo = lib.readCsv(config.dbCentering) # turn centering on/off # keep track of number of files in each target subfolder, # so we can number files appropriately and know when to add titles ntargetDirFiles = {} # how many times should we duplicate the images? ncopies = 1 # default ncopiesMemory = {} # keyed on planet-spacecraft-target-camera # open positions.csv file for target angular size info (used to control effective framerate) csvPositions, fPositions = lib.openCsvReader(config.dbPositions) # open additions.csv for additional images to insert into sequence csvAdditions, fAdditions = lib.openCsvReader(config.dbAdditions) # iterate through all available images csvFiles, fFiles = lib.openCsvReader(config.dbFiles) for row in csvFiles: # get file info volume = row[config.colFilesVolume] fileId = row[config.colFilesFileId] filter = row[config.colFilesFilter] system = row[config.colFilesSystem] craft = row[config.colFilesCraft] target = row[config.colFilesTarget] camera = row[config.colFilesCamera] # relabel target field if necessary - see db/targets.csv for more info target = lib.retarget(retargetingInfo, fileId, target) # get expected angular size (as fraction of frame) imageFraction = lib.getImageFraction(csvPositions, fileId) # build a key targetKey = system + '-' + craft + '-' + target + '-' + camera # how many copies of this image do we want? # note: we need to do this even if we don't add this image, # because need to keep track of sticky overrides from framerates.csv. # ncopies = getNCopies(framerateConstantInfo, target, imageFraction, ncopiesMemory, ncopies = lib.getNCopies(framerateConstantInfo, target, imageFraction, ncopiesMemory, targetKey, framerateInfo, fileId) # check if this image matches the volume and target path the user specified on the cmdline volumeOk = (volume in filterVolumes if filterVolumes else True) targetOk = lib.targetMatches(targetPathParts, system, craft, target, camera) imageOk = (fileId >= imageIdStart and fileId <= imageIdStop) if filterImageIds else True ignoreTarget = (target in config.clipsIgnoreTargets) addImage = volumeOk and targetOk and imageOk and (not ignoreTarget) # <- note ANDs here # if targetOk: # if fileId=='C1474515': # print targetPathParts, system, craft, target, camera # print fileId, imageIdStart, imageIdStop # print volumeOk, targetOk, imageOk, ignoreTarget, addImage, ncopies if addImage: if ncopies > 0: # use annotated image if available, else mosaic or composite. # (don't use centered files or will get bw images mixed in with color). imageFilepath = lib.getFilepath('annotate', volume, fileId) if not os.path.isfile(imageFilepath): imageFilepath = lib.getFilepath('mosaic', volume, fileId) if not os.path.isfile(imageFilepath): imageFilepath = lib.getFilepath('composite', volume, fileId) # if image file exists, create subfolder and link image if os.path.isfile(imageFilepath): # get staging subfolder and make sure it exists # eg data/step09_clips/stage/Jupiter/Voyager1/Io/Narrow/ subfolder = system + '/' + craft + '/' + target + '/' + camera + '/' targetFolder = stageFolder + subfolder lib.mkdir_p(targetFolder) # get current file number in that folder, or start at 0 nfile = ntargetDirFiles.get(targetKey) or 0 # # if we haven't seen this subfolder before add titlepage a few times. # # titlepages are created in the previous step, vgTitle. # if config.includeTitles and nfile==0: # titleFilepath = lib.getFolder('titles') + subfolder + 'title' + \ # config.extension # ntitleCopies = config.videoFrameRate * config.titleSecondsToShow # lib.addImages(titleFilepath, targetFolder, ntitleCopies, # ntargetDirFiles, targetKey) print "Volume %s frame: %s x %d \r" % (volume, fileId, ncopies), # add links to file # note: mklink requires admin privileges, so must run in an admin console # eg imageFilepath=data/step04_centers/VGISS_5101/C1327321_centered.jpg lib.addImages(imageFilepath, targetFolder, ncopies, ntargetDirFiles, targetKey) # check for additional images in additions.csv rowAdditions = lib.getJoinRow(csvAdditions, config.colAdditionsFileId, fileId) while rowAdditions: # additions.csv has fileId,additionId,nframes,tx,ty,scale print fileId, rowAdditions additionId = rowAdditions[config.colAdditionsAdditionId] # eg C2684338_composite ncopies = int(rowAdditions[config.colAdditionsNFrames]) # tx = int(rowAdditions[config.colAdditionsTx]) # ty = int(rowAdditions[config.colAdditionsTy]) # scale = float(rowAdditions[config.colAdditionsScale]) # tx = int(lib.getCol(rowAdditions, config.colAdditionsTx, 0)) # ty = int(lib.getCol(rowAdditions, config.colAdditionsTy, 0)) # scale = int(lib.getCol(rowAdditions, config.colAdditionsScale, 1)) # get imagePath #. how get volume from fileId like C2684338? # eg data/step06_composites/VGISS_7206/C2684338_composite.jpg # get folder step06 from suffix, _composite. # could lookup the volume based on the imageId boundaries, and then grab # whatever image started with the given additionId so don't need filter. # other extraneous images would have a special prefix, eg 'images/'. # get staging subfolder and make sure it exists # eg data/step09_clips/stage/Jupiter/Voyager1/Io/Narrow/ subfolder = system + '/' + craft + '/' + target + '/' + camera + '/' # targetFolder = config.clipsStageFolder + subfolder targetFolder = stageFolder + subfolder lib.mkdir_p(targetFolder) # insert the additional image if additionId.startswith('images/'): print 'adding',additionId,targetKey filetitle = additionId[7:] # trim off images/ folder = lib.getFolder('additions') # data/images/ imageFilepath = folder + filetitle print imageFilepath # add nframes into stage lib.addImages(imageFilepath, targetFolder, ncopies, ntargetDirFiles, targetKey) else: # handle composite, mosaic, crop, annotate for stage in ['composite','mosaic','crop','annotate']: if additionId.endswith(stage): additionId = additionId.split('_')[0] # remove '_composite', etc #... need to look up correct volume here - might be in different one! imageFilepath = lib.getFilepath(stage, volume, additionId) break print imageFilepath if os.path.isfile(imageFilepath): # add nframes into stage lib.addImages(imageFilepath, targetFolder, ncopies, ntargetDirFiles, targetKey) else: print "warning: can't find image file",imageFilepath rowAdditions = lib.getJoinRow(csvAdditions, config.colAdditionsFileId, fileId) fAdditions.close() fPositions.close() fFiles.close() print
def vgTitle(targetPath=None): "Make titles for specified targetpaths" # what does the user want to focus on? targetPathParts = lib.parseTargetPath(targetPath) pathSystem, pathCraft, pathTarget, pathCamera = targetPathParts print 'Making titles for', targetPathParts targetPathSeen = {} systemPathSeen = {} # iterate through all available images csvFiles, fFiles = lib.openCsvReader(config.dbFiles) for row in csvFiles: volume = row[config.colFilesVolume] fileId = row[config.colFilesFileId] filter = row[config.colFilesFilter] system = row[config.colFilesSystem] craft = row[config.colFilesCraft] target = row[config.colFilesTarget] camera = row[config.colFilesCamera] # is this an image the user wants to see? doTarget = lib.targetMatches(targetPathParts, system, craft, target, camera) if target in config.targetsIgnore: doTarget = False if doTarget: # make sure we haven't seen this target before targetKey = system + craft + target + camera # eg JupiterVoyager1IoNarrow targetSeen = targetPathSeen.get(targetKey) if not targetSeen: # get subfolder and make sure it exists # eg data/step8_movies/Jupiter/Voyager1/Io/Narrow/ subfolder = system + '/' + craft + '/' + target + '/' + camera + '/' targetfolder = lib.getFolder('titles') + subfolder lib.mkdir_p(targetfolder) print subfolder + ' \r', # make title image title = target # eg Triton subtitle1 = camera + "-Angle Camera" # eg Narrow-Angle Camera subtitle2 = system + " System" # eg Neptune System subtitle3 = "Voyager " + craft[-1:] # eg Voyager 2 # img = libimg.makeTitlePage(title, subtitle1, subtitle2, subtitle3) img = makeTitlePage(title, subtitle1, subtitle2, subtitle3) # save it # note: ffmpeg requires file type to match that of other frames in movie, # so use config.extension here titlefilepath = targetfolder + 'title' + config.extension img.save(titlefilepath) # remember this targetpath targetPathSeen[targetKey] = True # make sure we haven't seen this system before systemKey = system + craft # eg JupiterVoyager1 systemSeen = systemPathSeen.get(systemKey) if not systemSeen: # get subfolder and make sure it exists # eg data/step8_movies/Jupiter/Voyager1/Io/Narrow/ subfolder = system + '/' + craft + '/' systemfolder = lib.getFolder('titles') + subfolder lib.mkdir_p(systemfolder) print subfolder + ' \r', # make title image title = system subtitle1 = "Voyager " + craft[-1:] # eg Voyager 2 subtitle2 = '' subtitle3 = '' # img = libimg.makeTitlePage(title, subtitle1, subtitle2, subtitle3, center=True) img = makeTitlePage(title, subtitle1, subtitle2, subtitle3, center=True) # save it # note: ffmpeg requires file type to match that of other frames in movie, # so use config.extension here titlefilepath = systemfolder + 'title' + config.extension img.save(titlefilepath) # remember this systempath systemPathSeen[systemKey] = True fFiles.close() print
def buildSegment(segmentId, subsegments): """ build an mp4 movie segment from its subsegments, building them if necessary. eg if segmentId='Jupiter-Voyager1-Ganymede', and subsegments= [['Jupiter-Voyager1-Ganymede', 'Jupiter-Voyager1-Ganymede-Narrow', 'C1460413-C1640725'], ['Jupiter-Voyager1-Ganymede', 'Jupiter-Voyager1-Ganymede-Wide', 'C1640141-C1640752']] then this would build Jupiter-Voyager1-Ganymede.mp4 from Jupiter-Voyager1-Ganymede-Narrow-C1460413-C1640725.mp4 Jupiter-Voyager1-Ganymede-Wide-C1640141-C1640752.mp4 building the latter if they don't already exist. """ print 'buildsegment',segmentId movieFolder = lib.getFolder('movies') pagesFolder = lib.getFolder('pages') stageFolder = config.moviesStageFolder filepaths = [] for subsegment in subsegments: print subsegment subsegmentId = subsegment[1].strip() # eg Neptune-Voyager2-Triton-Narrow contents = subsegment[2].strip() if len(subsegment)>2 else None # eg C1550829-C1618136 #.remove partname from segmentid # a = subsegmentId.split('@') # if len(a)>1: # partname = a[1] # print 'got partname',partname # get name of subsegment movie file subsegmentFiletitle = subsegmentId # eg 'Jupiter-Voyager1-Jupiter@Clouds' qualifier = '-' + contents if contents else '' # eg '-C1550829-C1618136' filepath = movieFolder + subsegmentFiletitle + qualifier + '.mp4' filepath = os.path.abspath(filepath) # build subsegment movie if doesn't already exist if not os.path.isfile(filepath): subsegmentPath = subsegmentId subsegmentPath = subsegmentPath.replace('-','/') subsegmentPath = subsegmentPath.replace('@','/') # eg jupiter/voyager1/jupiter/clouds subsegmentStageFolder = stageFolder + subsegmentPath + '/' # make stage folder print 'rmdir+mkdir',subsegmentStageFolder lib.rmdir(subsegmentStageFolder) lib.mkdir_p(subsegmentStageFolder) #. handle special subsegmentIds - Intro, Credits, Epilogue if subsegmentId in ['Intro', 'Prologue', 'Credits', 'Epilogue']: #. make a movie of credit jpg and add to filepaths pageFilepath = pagesFolder + subsegmentId + config.extension # pageFilepath = os.path.abspath(pageFilepath) # add links to file # targetFolder = stageFolder + subsegmentId + '/' # lib.rmdir(subsegmentStageFolder) # lib.mkdir_p(subsegmentStageFolder) # lib.addImages(pageFilepath, targetFolder, ncopies) # sourcePath = '../../../' + pageFilepath sourcePath = '../../../../' + pageFilepath ncopies = 50 #. param lib.makeSymbolicLinks(sourcePath, subsegmentStageFolder, ncopies) # build mp4 files from all staged images print 'makevideo with imagesToMp4 ->',filepath lib.imagesToMp4(subsegmentStageFolder, filepath) else: # stage images for ffmpeg print 'stagefiles', subsegmentPath, contents, stageFolder vgClips.stageFiles(None, subsegmentPath, contents, stageFolder) # build mp4 files from all staged images print 'makevideo with imagesToMp4 ->',filepath lib.imagesToMp4(subsegmentStageFolder, filepath) # add movie to filelist so can concatenate them later print 'add subsegment to filelist', filepath filepaths.append(filepath) # compile subsegment movies into single movie segmentFilepath = movieFolder + segmentId + '.mp4' print 'all subsegment movies created. now compile into single movie',segmentFilepath print filepaths lib.concatenateMovies(segmentFilepath, filepaths) # now remove intermediaries below a certain level, eg don't really want # Jupiter-Voyager1-Europa-Narrow hanging around # but if they're gone they'll have to be rebuilt each time... # print 'cleanup' # if len(segmentId.split('-'))>=4: # for filepath in filepaths: # lib.rm(filepath) print