def runsites(siteslist): global sites global radius global siteiter # Set the global variable sites=siteslist # Loop over each site for i in xrange(len(sites)): siteiter=i radius=sites[siteiter][3] print "Processing",sites[siteiter][0],"radius",radius inputdir= "data/geoebola" outputdir="data/geoebola-sites-"+sites[siteiter][0]+"-"+str(radius) tweetproc.process(runit,inputdir,outputdir,12)
# Apply keyword function to each file with a keyword of 'ebola' def runit(params): plist=list(params) plist.append("ebola") tweetproc.keyword(*plist) ''' # FIXME: This has been moved to util.py # Process all the files from input directory in parallel def process(funct,indir,outdir,numprocs): filelist=tweetproc.jsonindir(indir) infilelist=copy.copy(filelist) outfilelist=copy.copy(filelist) for i in xrange(len(filelist)): infilelist[i]=os.path.join(indir,filelist[i]) outfilelist[i]=os.path.join(outdir,filelist[i]) combinedfilelist=zip(infilelist,outfilelist) pool = Pool(processes=numprocs) pool.map(funct,combinedfilelist) pool.close() ''' inputdir="data/geo" outputdir="data/geoebola" tweetproc.process(runit,inputdir,outputdir,12)