def _file_to_split(self):
"""Function to return file handle for the file to split."""
file_to_split = None
if self._extract:
try:
file_to_split = extract_csv_from_zip(self._filename)
self._perform_filename_checks(file_to_split.name)
except zipfile.BadZipFile as e:
raise exceptions.ZipFileCheckException(
str(e),
statsd=self._statsd,
metrics_failures_root=self._metrics_failures_root)
else:
self._perform_filename_checks(self._filename)
file_to_split = open(self._filename, 'rb')
return file_to_split
def _populate_file_properties(config, file_list, run_id, perform_prevalidation,
logger):
"""Determine the attributes associated with the file."""
uncataloged_files = []
for file_name in file_list:
file_properties = file_name['file_properties']
file_path = file_name['file_path']
is_valid_zip = None
is_valid_format = None
files_to_delete = []
num_records = None
uncompressed_size_bytes = None
try:
# Validate zip file
extracted_file = extract_csv_from_zip(file_path)
is_valid_zip = True
if perform_prevalidation:
is_valid_format = _prevalidate_file(config, extracted_file,
file_path,
file_properties.file_type,
run_id,
file_name['schema'],
files_to_delete, logger)
num_records = sum(1 for _ in extracted_file)
with zipfile.ZipFile(file_path) as file_test:
uncompressed_size_bytes = file_test.getinfo(
extracted_file.name).file_size
except BadZipFile as err:
is_valid_zip = False
logger.warn('The zip file is invalid: {0}'.format(
file_properties.filename))
logger.warn('Zip check error: {0}'.format(str(err)))
except exceptions.PrevalidationCheckRawException as err:
is_valid_format = False
logger.warn(
'Pre-validation failed for file: {0} with error: {1}'.format(
file_path, str(err)))
finally:
logger.debug('Cleanup: deleting intermediate data files...')
for fn in files_to_delete:
logger.debug('Deleted intermediate file {0}'.format(fn))
remove(fn)
logger.debug('Cleanup: deleted intermediate data files')
# Compute MD5 hash
logger.info('Computing MD5 hash of the input file...')
with open(file_path, 'rb') as f:
md5 = compute_md5_hash(f)
logger.info('Computed MD5 hash')
# Fetch extra attributes (if any)
extra_attributes = _get_extra_attributes(file_path,
file_properties.file_type,
logger)
file_attributes = CatalogAttributes(
file_properties.filename, file_properties.file_type,
file_properties.modified_time,
file_properties.compressed_size_bytes, is_valid_zip,
is_valid_format, md5, extra_attributes, uncompressed_size_bytes,
num_records)
uncataloged_files.append(file_attributes)
return uncataloged_files
