Exemplo n.º 1
0
	def testlistfullpathdirectoryfalse(self):
		self.p = info('test')
		f1 = join(self.path,'File1.txt')
		f2 = join(self.path,'File2.txt')
		sub = join(self.path,'subdirectory')		
		d = [f1,f2,sub]
		self.assertFalse(self.p.listfiles() == d)
Exemplo n.º 2
0
def main():
    usage = "usage: %prog [options] arg"
    parser = OptionParser(usage)
    parser.add_option("-d", "--inputdirectory", action="store", dest="indir")
    parser.add_option("-f", "--mergedfilename", action="store", dest="file", help="Filename for merged file")
    parser.add_option("-o", "--outputdirectory", action="store", dest="outdir", default="~/", help="Default directory: "+abspath(expanduser('~/')))
    
    (options, args) = parser.parse_args()
    if not options.indir:
        parser.error('Input directory not provided')
    if not options.file:
        parser.error('Output merged filename not provided')    
    sourcedirectory = options.indir
    mergedfilename = options.file
    mergeddirectory = abspath(expanduser(options.outdir))

   
    filegroup = files( info(sourcedirectory).listfullpathfiles() )
    filegroup.mergeinto(join(mergeddirectory,mergedfilename))

    print "%s/%s"  % (mergeddirectory,mergedfilename)
Exemplo n.º 3
0
	def testlistfullpathfiles(self):
		self.p = info('test')
		f1 = join(self.path,'File1.txt')
		f2 = join(self.path,'File2.txt')
		d = [f1,f2] 
		self.assertTrue(self.p.listfullpathfiles() == d)
Exemplo n.º 4
0
	def testlisthiddendirectoryfalse(self):
		self.p = info('test')
	 	d = ['File1.txt','File2.txt','.hiddensubdirectory']
	 	self.assertFalse(self.p.listfiles() == d)
Exemplo n.º 5
0
	def testlisthiddenfilefalse(self):
		# tests for files starting with a .
		self.p = info('test')
	 	d = ['File1.txt','File2.txt','.hidden.file']
	 	self.assertFalse(self.p.listfiles() == d)
Exemplo n.º 6
0
	def testlistfiles(self):		
		self.p = info('test')
		d = ['File1.txt','File2.txt']
		self.assertTrue(self.p.listfiles() == d)
Exemplo n.º 7
0
	def testlistfullpathdirectoryfalse(self):
		self.p = info('~')
		sub = join(self.path,'subdirectory')		
		self.assertFalse(sub in self.p.listfullpathfiles())
Exemplo n.º 8
0
	def testlistfullpathfiles(self):
		self.p = info('~')
		f1 = join(self.path,'File1.txt')
		f2 = join(self.path,'File2.txt')
		self.assertTrue(f1 in self.p.listfullpathfiles() and f2 in self.p.listfullpathfiles())
Exemplo n.º 9
0
	def testlisthiddendirectoryfalse(self):
		# test for folders starting with a .
		self.p = info('~')
	 	d = '.hiddensubdirectory'
	 	self.assertFalse(d in self.p.listfiles())
Exemplo n.º 10
0
	def testlisthiddenfilefalse(self):
		# test for files starting with a .
		self.p = info('~')
	 	d = '.hidden.file'
	 	self.assertFalse(d in self.p.listfiles())
Exemplo n.º 11
0
	def testlistdirectoryfalse(self):
		self.p = info('~')
	 	d = 'subdirectory'
	 	self.assertFalse(d in self.p.listfiles())
Exemplo n.º 12
0
	def testlistfiles(self):		
		self.p = info('~')
		d = 'File1.txt'
		e = 'File2.txt'
		self.assertTrue(d in self.p.listfiles() and e in self.p.listfiles())
Exemplo n.º 13
0
	def testlisthiddendirectoryfalse(self):
		self.p = info('.')
	 	d = '.hiddensubdirectory'
	 	self.assertFalse(d in self.p.listfiles())
Exemplo n.º 14
0
def main():

    usage = "usage: %prog [options] arg"
    parser = OptionParser(usage)
    parser.add_option("-d", "--inputdirectory", action="store", dest="indir")
    parser.add_option("-c", "--connectionstring", action="store", dest="conn")
    parser.add_option("-k", "--freebaseapikey", action="store", dest="key")

    (options, args) = parser.parse_args()
    if not options.indir:
       	parser.error('Input directory not provided')
    argdir  = options.indir
    connStr = options.conn
    apiKey  = options.key
    ####################################################
    colsToKeep = ['airport','airline','flight','location','statusDateTime']
    airline_mapping = iatamapping()


    flist = info(argdir).listfullpathfiles()


    for f in flist:
        print (f)
        df = loadcsvtodf(f)
        
        if 'cork' in f:
            df = corktransf(df)
        elif 'shan' in f:
            df = shantransf(df)
        elif 'dubl' in f: 
            df = dubltransf(df)
        else:
            print ("Boink. Unrecognized file found")
            print f     
            df = DataFrame(None)

        if not df.empty:
            # Identify the actual departure date and time:
            # set to type datetime
            df['saveDateTime'] = pd.to_datetime(df['saveDateTime'])
            # status time : today's date + the time extracted from statusMessage 
            df['statusTime'] = df['statusMessage'].map(extracttime)
            # create a dateTime from statusTime(time) and saveDateTime(date)
                # and store it in statusDateTime
            df = buildDateTime(df,'statusTime','saveDateTime','statusDateTime')
            

            # Map flight numbers to airlines if no airline column provided
            cols = list(df.columns.values)
            if 'airline' not in cols:
                
                df['airline'] = df['flight']
                # strip the digits from the flight number
                df['airline'] = df['airline'].map(stripdigits)
                # map to an airline 
                df['airline'] = df['airline'].map(airline_mapping)
            
            # Drop unwanted columns
            updatedCols = list(df.columns.values)
            df = df.drop([l for l in updatedCols if l not in colsToKeep],axis=1)
            print (df.head()) 
            pdb.load_wrapper('timekeeping',df,connStr,if_exists='append') 
 
    return()