def extract(data_store, record_selector, field_name, output_dir_path=None):
"""Extract specified field from selected records
Args:
data_store: a DataStore object
record_selector: a RecordSelector object. It defines which records
should be processed.
field_name: name of the field to be extracted. Can contain nested names,
e.g. 'field1.field2' .
output_dir_path: a FileSystemPath object. This is where the extracted
fields will be saved. If this is not given, the extracted fields
are printed to stdout.
"""
if output_dir_path is not None:
output_dir_path.make_dirs()
for record in record_selector.get_records(data_store):
datum = record.content
if datum: #check if there is any data to be written
parts = field_name.split('.')
for part in parts:
datum = datum[part]
if output_dir_path is not None:
try:
with output_dir_path.append(str(record.index)).open('w') \
as output:
output.write(to_byte_string(datum))
except Exception:
error("while processing record with index {}"\
.format(record.index))
raise
else:
print(to_byte_string(datum))
def __record_fulfills_condition(record, selection):
"""Checks whether a field in Avro records fulfills a condition
Args:
record: an Avro record as a Python dictionary
selection: an EqualitySelection object (optional)
Returns:
True if a record fulfill the condition
False otherwise
"""
if selection is None:
return True
key_parts = selection.get_key_parts()
value = selection.get_value()
content = record.content
try:
for key_part in key_parts:
content = content[key_part]
except KeyError:
error("Specified key not found in the schema: {}"\
.format(".".join(key_parts)))
raise
if content is None and value == "null":
return True
if unicode(content) == unicode(value):
return True
else:
return False
def get_schema(self):
"""Lazy accessor for data store schema
If schema is given as a run parameter, then returns this schema.
Otherwise extracts the schema from the Avro data store files.
"""
if self._schema:
return self._schema
else:
if not self._schema_path: #if there is no schema
paths = self.__get_paths_to_avro_files()
with DataFileReader(
paths[0].open("r"),
_FieldsOrderPreservingDatumReader()) as reader:
self._schema = avro.schema.parse(
reader.get_meta('avro.schema'))
return self._schema
else: #a schema is given
try:
self._schema = avro.schema.parse(
self._schema_path.open("r").read())
return self._schema
except TypeError:
error("supplied schema cannot be parsed!")
raise
def to_json(data_store, record_selector, printer, pretty=False):
"""Converts selected records to JSON.
Args:
data_store: a DataStore object
record_selector: a RecordSelector object that defines which records
should be processed.
printer: a Printer object that is used to print the JSON.
pretty: True if the output should be a valid, pretty-printed JSON.
"""
first_record = True
if pretty:
printer.print("[", end="")
for record in record_selector.get_records(data_store):
try:
if first_record:
first_record = False
else:
if pretty:
printer.print(",")
else:
printer.print("")
content = encapsulate_strings(record.content)
printer.print(dict_to_json(content, pretty), end="")
except Exception:
error("while processing record with index {}".format(record.index))
raise
if pretty:
printer.print("]")
else:
## The case when no records were present
if not first_record:
printer.print("")
def __record_fulfills_condition(record, selection):
"""Checks whether a field in Avro records fulfills a condition
Args:
record: an Avro record as a Python dictionary
selection: an EqualitySelection object (optional)
Returns:
True if a record fulfill the condition
False otherwise
"""
if selection is None:
return True
key_parts = selection.get_key_parts()
value = selection.get_value()
content = record.content
try:
for key_part in key_parts:
content = content[key_part]
except KeyError:
error("Specified key not found in the schema: {}"\
.format(".".join(key_parts)))
raise
if content is None and value=="null":
return True
if unicode(content) == unicode(value):
return True
else:
return False
def __get_value(datum, field_name):
parts = field_name.split('.')
try:
for part in parts:
datum = datum[part]
return datum
except KeyError:
error("Field '{}' is not defined in the data".format(field_name))
raise
def extract(data_store,
record_selector,
value_field,
name_field=None,
create_dirs=None,
output_dir_path=None):
"""Extract specified field from selected records
Args:
data_store: a DataStore object
record_selector: a RecordSelector object. It defines which records
should be processed.
value_field: name of the field to be extracted.
Can contain nested names, e.g. 'field1.field2' .
name_field: value of this field is used as a name of the file
containing extracted data for given row. If this is not given,
the name of the file is assumed to be the index of corresponding
record.
create_dirs: extracted fields corresponding to the same name
are to be placed in the same directory.
Name of directory is equal to the `name_field`;
names of the files inside are consecutive numbers.
output_dir_path: a FileSystemPath object. This is where the extracted
fields will be saved. If this is not given, the extracted fields
are printed to stdout.
"""
if output_dir_path is not None:
output_dir_path.make_dirs()
for record in record_selector.get_records(data_store):
datum = record.content
if datum: #check if there is any data to be written
field_value = __get_value(datum, value_field)
if output_dir_path is not None:
try:
output_name = __get_output_name(record.index, datum,
name_field)
output_path = output_dir_path.append(output_name)
file_path = None
if create_dirs:
file_path = __prepare_dir(output_path)
else:
file_path = __prepare_file(output_path, name_field)
with file_path.open("w") as output:
output.write(to_byte_string(field_value))
except Exception:
error("while processing record with index {}"\
.format(record.index))
raise
else:
print(to_byte_string(field_value))
def __prepare_dir(output_dir):
if output_dir.exists():
if not output_dir.is_dir():
error("File with name '{}' already exists. "\
"Unable to create a directory with the same name".\
format(output_dir))
raise Exception()
else:
output_dir.make_dirs()
file_names = output_dir.ls()
new_number = 0
if len(file_names) > 0:
file_numbers = [int(f) for f in file_names]
new_number = max(file_numbers) + 1
new_file_name = str(new_number)
return output_dir.append(new_file_name)
def __prepare_file(output_file, name_field):
if output_file.exists():
## There is a race condition between checking
## if the file exists and opening it for writing
## but I don't see an easy way to deal with it
## in a way that would work for HDFS as well as
## for local file system paths. Dealing with
## this problem doesn't seem to be important anyway.
error_string = "The file with name '{}' "\
"already exists.".format(output_file)
if name_field is not None:
error_string = error_string + " This is probably "\
"because the selected values of given field "\
"'{}' are not unique".format(name_field)
error(error_string)
raise Exception()
return output_file
def __iter__(self):
paths = self.__get_paths_to_avro_files()
prev_global_record_index = 0
for path in paths:
with DataFileReader(path.open("r"), _FieldsOrderPreservingDatumReader(
readers_schema=self.get_schema())) as reader:
prev_local_record_index = 0
try:
for record in reader:
yield record
prev_global_record_index = prev_global_record_index + 1
prev_local_record_index = prev_local_record_index + 1
except Exception:
error("processing record with index {} failed. "\
"This record comes from \"{}\" Avro file and in this "\
"file it has local index equal {}.".format(
prev_global_record_index+1, path,
prev_local_record_index+1))
raise
def get_schema(self):
"""Lazy accessor for data store schema
If schema is given as a run parameter, then returns this schema.
Otherwise extracts the schema from the Avro data store files.
"""
if self._schema:
return self._schema
else:
if not self._schema_path: #if there is no schema
paths = self.__get_paths_to_avro_files()
with DataFileReader(paths[0].open("r"), _FieldsOrderPreservingDatumReader()) as reader:
self._schema = avro.schema.parse(reader.get_meta('avro.schema'))
return self._schema
else: #a schema is given
try:
self._schema = avro.schema.parse(self._schema_path.open("r").read())
return self._schema
except TypeError:
error("supplied schema cannot be parsed!")
raise
def __iter__(self):
paths = self.__get_paths_to_avro_files()
prev_global_record_index = 0
for path in paths:
with DataFileReader(
path.open("r"),
_FieldsOrderPreservingDatumReader(
readers_schema=self.get_schema())) as reader:
prev_local_record_index = 0
try:
for record in reader:
yield record
prev_global_record_index = prev_global_record_index + 1
prev_local_record_index = prev_local_record_index + 1
except Exception:
error("processing record with index {} failed. "\
"This record comes from \"{}\" Avro file and in this "\
"file it has local index equal {}.".format(
prev_global_record_index+1, path,
prev_local_record_index+1))
raise
def __get_paths_to_avro_files(self):
paths = []
for file_name in self._datastore_path.ls():
## Ignore files starting with underscore.
## Such files are also ignored by default by map-reduce jobs.
## We're also ignoring files starting with dot to ignore
## ".svn" directories.
if not (fnmatch.fnmatch(file_name, "_*") or
fnmatch.fnmatch(file_name, ".*")):
path = self._datastore_path.append(file_name)
## We're not checking whether the path is a directory because
## such testing takes too much time
#if path.is_dir():
#continue
paths.append(path)
if len(paths) == 0:
raise error("Specified data store path is empty or is not valid")
paths.sort()
return paths
def __get_paths_to_avro_files(self):
paths = []
for file_name in self._datastore_path.ls():
## Ignore files starting with underscore.
## Such files are also ignored by default by map-reduce jobs.
## We're also ignoring files starting with dot to ignore
## ".svn" directories.
if not (fnmatch.fnmatch(file_name, "_*")
or fnmatch.fnmatch(file_name, ".*")):
path = self._datastore_path.append(file_name)
## We're not checking whether the path is a directory because
## such testing takes too much time
#if path.is_dir():
#continue
paths.append(path)
if len(paths) == 0:
raise error("Specified data store path is empty or is not valid")
paths.sort()
return paths