def test_log_filename(self):
# test whether things are properly logged in file
# process a random number of elements, and check log contents
args = [str(i) for i in range(random.randint(5, 100))]
log_filename = os.path.join(self.testdir, 'test.log')
processor = batchproc.BatchProcessor(args,
self.fun,
log_filename=log_filename)
processor.start()
# read log
with open(log_filename, 'r') as fd:
lines = fd.readlines()
os.unlink(log_filename)
# count number of files processed, marked with a . at the start
count = 0
print(lines)
for line in lines:
if line.startswith('.'):
count += 1
self.assertEqual(len(args), count)
def test_basic_functionality(self):
# test that the simplest use-case works
# test function that adds some characters to the end of a file
def append_zzz(filename):
with open(filename, 'a') as fd:
fd.write('zzz')
# create some files to test it on, containing only their own filename
filenames = [
os.path.join(self.testdir, 'a.txt'),
os.path.join(self.testdir, 'b.txt')
]
for file in filenames:
with open(file, 'w') as fd:
fd.write(file)
processor = batchproc.BatchProcessor(filenames, append_zzz)
processor.start()
# read contents of files into result
result = []
for file in filenames:
with open(file, 'r') as fd:
result.append(fd.read())
expected_result = [file + 'zzz' for file in filenames]
for file in filenames:
os.unlink(file)
self.assertEqual(result, expected_result)
def test_preprocessors(self):
# test that the preprocessor filters get applied in the correct order
# make some preprocessor functions where the order matters
# filter files with a short filename
def filter_short(files):
return [f for f in files if len(f) >= 4]
# make filenames longer
def add_length(files):
return [f + '1234' for f in files]
args = ['a', 'bcde', 'f']
log_filename = os.path.join(self.testdir, 'test.log')
pre1 = (filter_short, add_length)
pre2 = (add_length, filter_short)
# make two runs, one for each order of preprocessors
# use the same logfile so that results are both in there
processor = batchproc.BatchProcessor(args,
self.fun,
pre1,
log_filename=log_filename)
processor.start()
processor = batchproc.BatchProcessor(args,
self.fun,
pre2,
log_filename=log_filename)
processor.start()
# read log
with open(log_filename, 'r') as fd:
lines = fd.readlines()
os.unlink(log_filename)
# count number of files processed, marked with a . at the start
count = 0
print(lines)
for line in lines:
if line.startswith('.'):
count += 1
# log should contain 1 from the run with pre1, and 3 from the other
self.assertEqual(4, count)
def test_empty_list(self):
# see what happens if the list of files is empty
args = []
processor = batchproc.BatchProcessor(args, self.fun)
processor.start()
# passes test if no errors are raised
return
def test_lambda(self):
# test that it works when fun is a lambda
args = ['test']
lamfun = lambda filename: True
processor = batchproc.BatchProcessor(args, lamfun)
processor.start()
# passes test if no errors are raised
return
def test_continue_on_error(self):
# test both variations of the flag
def always_fail(filename):
return False
args = ['a', 'b', 'c', 'd']
log_filename = os.path.join(self.testdir, 'test.log')
# make two runs, one continuing after errors, and the other stopping after the first
# use the same logfile so that results are both in there
processor = batchproc.BatchProcessor(args,
always_fail,
log_filename=log_filename,
continue_on_error=True)
processor.start()
processor = batchproc.BatchProcessor(args,
always_fail,
log_filename=log_filename,
continue_on_error=False)
processor.start()
# read log
with open(log_filename, 'r') as fd:
lines = fd.readlines()
os.unlink(log_filename)
# count number of files processed, marked with a . at the start
count = 0
print(lines)
for line in lines:
if line.startswith('.'):
count += 1
# log should contain 1 from the run that stopped after errors, and all from the other run
self.assertEqual(len(args) + 1, count)
import batchproc as bp
# test function that adds some characters to the end of a file
def append_zzz(filename):
with open(filename, 'a') as fd:
fd.write('zzz')
if __name__ == "__main__":
# get list of files from user, either from command-line args or stdin
args = bp.get_file_list()
# write results to this file
log_filename = 'batch_process.log'
# optionally, a list of functions can be specified that are applied to the list of files before starting
# append_zzz only works on files, so expand_folder is used, which recursively replaces folders with their contents
file_filters = (bp.expand_folder, )
processor = bp.BatchProcessor(args,
append_zzz,
file_filters,
log_filename=log_filename)
# apply our data-processing function, append_zzz to all files specified by user
processor.start()
def extract_and_load_data(obj):
file = s3.Object('leafliink-data-interview-exercise', obj)
split_json = file.get()['Body'].read().decode('utf-8').splitlines()
data = [pd.json_normalize(json.loads(i)) for i in split_json]
all_data = pd.concat(data)
all_data.to_sql(
'table_name',
schema='table_schema',
con=con,
chunksize=10000,
method='multi',
if_exists='append',
index=False,
)
return obj + ' loaded at {}'.format(str(time.time()))
if __name__ == '__main__':
# Generate process to extract & load the data & write progress to a log file.
log_filename = 'batch_process.log'
processor = bp.BatchProcessor(objects,
extract_and_load_data,
log_filename=log_filename)
# apply our data-processing function, extract & load data from the s3 bucket.
processor.start()