def create_index_file(index_file_name, database_list):
"""
Write the index (list of records) for a given system to a disk file.
:param index_file_name: index file name
:type index_file_name: str
:param database_list: list of databases for the given system
:type database_list: list
:return:
"""
record_name_list = []
for database_name in database_list:
print(database_name)
# Open database and macro substitution file (if any)
db = DatabaseFile(file_name=database_name)
m = read_subs_file(database_name)
# Create a list with all records in the databases
record_name_list.extend([r[0] for r in db.next_record_name()])
# print record_name_list
# Substitute macros if the macro substitution file was defined.
# There's no need to replace macros in the record fields since
# only record names are written to the index files.
if m is not None:
record_name_list = [m.replace_macros(r) for r in record_name_list]
# Write the index file
f = open(index_file_name, 'w')
f.write("\n".join(str(x.strip()) for x in record_name_list))
f.close()
def system_to_others(system, database_directory, channel_dict):
"""
Return references to records in other systems in the system's database file(s).
It looks for record references in the field values and when it finds one
it looks for it in the channel dictionary to determine what system it belongs to.
The output is a list of tuples with the following information: record name,
record field, referenced record name, referenced record field, list of systems
where the referenced record name exists.
:param system: system name
:type system: str
:param database_directory: database directory
:type database_directory: str
:param channel_dict: dictionary of known records
:type channel_dict: dict
:return: list tuples
:rtype: list
"""
output_list = []
# Loop over all databases in the data directory
for data_base_name in get_database_names(system, database_directory):
# Open database name by file name. Open macro substitution file (if any).
db = DatabaseFile(file_name=data_base_name)
m = read_subs_file(data_base_name)
# Loop over all records in the database
for record in db.next_record():
# Replace macros in the record name (if any)
assert isinstance(record, EpicsRecord)
record_name = record.get_name()
if m is not None:
record_name = m.replace_macros(record_name)
# Loop over all the fields in the record. If the field value contains anything
# matching a record name then append it to the output list, but only if it's
# in the dictionary of known records and it belongs to a system other than the
# one we are looking from.
for field_name, field_value in record.get_fields():
if m is not None:
field_value = m.replace_macros(field_value)
# print field_name, field_value
target_record_name, target_record_field = parse_field_value(field_value)
if target_record_name is not None:
# print '-', record_name, record_field
if target_record_name in channel_dict and system not in channel_dict[target_record_name]:
output_list.append((target_record_name, target_record_field,
# record.get_name(), field_name,
record_name, field_name,
channel_dict[target_record_name]))
# return remove_duplicates(output_list)
# print output_list
return output_list
def test_constructor(file_object):
try:
DatabaseFile()
except FileNotFoundError:
pass
try:
DatabaseFile(file_name='inexistent.db')
except FileNotFoundError:
pass
DatabaseFile(file_object)
DatabaseFile(f_in=file_object)
def epics_record():
"""
Fixture used to return a valid EpicsRecord object from single.db
:return: epics record
:rtype: EpicsRecord
"""
df = DatabaseFile(file_name=SINGLE_RECORD)
record = None
for record in df.next_record():
break
return record
def diff_files_internal(file_name1, file_name2, p_args):
"""
Open the two database files and call the the internal difference function
:param file_name1: file name 1
:type file_name1: str
:param file_name2: file name 2
:type file_name2: str
:param p_args: command line arguments
:type p_args: Namespace
:return:
"""
if debug_flag:
print('\n-- diff_databases', file_name1, file_name2)
try:
# f1 = open(file1, 'r')
# f2 = open(file2, 'r')
# diff_databases(f1, f2, file1, file2)
my_filter = diff_filter if p_args.clean else None
df1 = DatabaseFile(file_name=file_name1, filter_function=my_filter)
df2 = DatabaseFile(file_name=file_name2, filter_function=my_filter)
diff_databases(df1, df2, file_name1, file_name2)
df1.close()
df2.close()
except (OSError, IOError) as ex:
print(ex)
def others_to_system(system, sys_list, database_directory, channel_dict):
"""
:param system: system name we are looking for
:type system: str
:param sys_list: list of all systems
:type sys_list: list
:param database_directory: database directory
:type database_directory: str
:param channel_dict: dictionary of known records
:type channel_dict: dict
:return:
:rtype: dict
"""
print('others_to_system')
# Remove the system from the list of systems (we don't want references to itself)
other_systems = sys_list
other_systems.remove(system)
output_dict = {}
# Loop over all systems
for sys_name in other_systems:
# Loop over all databases for that system
for data_base_name in get_database_names(sys_name, database_directory):
# Open database by name
db = DatabaseFile(file_name=data_base_name)
m = read_subs_file(data_base_name)
# print '+', data_base_name, m
# Loop over all records in the database
for record in db.next_record():
assert isinstance(record, EpicsRecord)
# Loop over all fields in the database. If the field value contains anything
# matching a record name then add it to the output dictionary, but only if it's
# in the dictionary of known records and it belongs to a system we are looking for.
for field_name, field_value in record.get_fields():
if m is not None:
field_value = m.replace_macros(field_value)
target_record_name, target_record_field = parse_field_value(field_value)
if target_record_name is not None:
if target_record_name in channel_dict and system in channel_dict[target_record_name]:
# print sys_name, target_record_name, target_record_field
if sys_name in output_dict:
output_dict[sys_name].append((target_record_name, target_record_field,
record.get_name(), field_name))
else:
output_dict[sys_name] = [(target_record_name, target_record_field,
record.get_name(), field_name)]
# print output_dict
return output_dict
def database_file():
"""
Fixture used to return a valid DatabaseFile object
:return: database file
:rtype: DatabaseFile
"""
return DatabaseFile(file_name=SIMPLE_DATABASE)
def sort_database(f, file_name, p_args):
"""
This is the callback function for process_file_list.
Read the database file and print the database sorted by record and field names.
:param f: database file
:type f: TextIOBase
:param file_name: file name (needed, but not used)
:type file_name: str
:param p_args: command line arguments (not used)
:type p_args: Namespace
:return: None
"""
if debug_flag:
print('\n-- sort_database', f, file_name, p_args)
df = DatabaseFile(f)
db = df.read_database()
assert (isinstance(db, EpicsDatabase))
db.write_sorted_database(reverse=p_args.reverse)
df.close()
return
def list_records(f, file_name, p_args):
"""
This is the callback function for process_file_list.
It will get called once for each file in the input file list.
List records in a database file.
:param f: database file
:param file_name: file name (not used)
:param p_args: command line arguments
:return: None
"""
if debug_flag:
print('\n--list_records', f, file_name, p_args)
df = DatabaseFile(f)
for record_name, record_type in df.next_record_name():
if p_args.type:
print(record_name + ', ' + record_type)
else:
print(record_name)
df.close()
return
def larger_epics_database():
df = DatabaseFile(file_name=LARGER_DATABASE)
db = EpicsDatabase()
for record in df.next_record():
db.add_record(record)
return db
def epics_database():
df = DatabaseFile(file_name=SIMPLE_DATABASE)
db = EpicsDatabase()
for record in df.next_record():
db.add_record(record)
return db
def grep_file(f, file_name, p_args):
"""
This routine looks for matches in the record name, record type, field name and/or field value.
The command line options are used to select what type of matching is desired.
The output will have a valid database syntax.
:param f: database file
:param file_name: database file name
:param p_args: command line arguments
:type p_args: Namespace
"""
# Print debug information
if debug_flag:
print(file_name, p_args.pattern, file_name)
# This variable is used to control whether the file name should be printed embedded in the
# output database as a comment. When no matches are found no file name should be printed not
# be printed to avoid unwanted output.
more_than_one_file = True if len(p_args.files) > 1 else False
# The following variable controls whether only the file name should be printed when a match is found
# Flags to control the output
file_name_only = p_args.filename
# This variable is usd to control whether all the fields in a matching record
# should be printed when a match is found.
all_fields = p_args.all_fields
if debug_flag:
print('more_than_one_file, file_name_only =', more_than_one_file, file_name_only)
# Compile the pattern to make sure that there are no errors in it.
# This will speed up searches and will catch errors before processing files.
try:
p = re.compile(p_args.pattern, re.IGNORECASE if p_args.ignorecase else 0)
except Exception as ex:
print('Error while parsing regular expression', ex)
return
# Create the database file object
df = DatabaseFile(f)
if debug_flag:
print(df)
# Match field name or value?
match_fields = p_args.field_name or p_args.field_value
# Used to break out from two nested loops (Python doesn't have labeled breaks yet).
break_out = False
# Loop over all records in the file.
# The records will be processed in the same order as in the file.
for record in df.next_record():
assert (isinstance(record, EpicsRecord))
# This variable is used to determine whether the start of a record was printed or not.
# We only need the record name once per record. A match in a field name or value will
# print the record start if it was not printed already.
record_start_printed = False
# Get record and type
record_name, record_type = record.get_name(), record.get_type()
if debug_flag:
print(record_name, record_type)
# Check whether matching for record name and type is selected
# Print the record header if there's a record match
if (p_args.record_name and p.search(record_name)) or (p_args.record_type and p.search(record_type)):
# Print the file name and stop looking for more matches
if file_name_only:
print(file_name)
break
# Print a file header as comment if there are more than one input files
if more_than_one_file:
print(file_name_header(file_name))
more_than_one_file = False # file name header was printed
# Print the record start and mark it as printed
print(format_record_start(record_name, record_type))
record_start_printed = True # record start was printed
# If printing all fields then don't bother checking for matching fields
if all_fields:
print_all_fields(record)
print(format_record_end())
continue
# Match fields?
if match_fields:
for field_name, field_value in record.get_fields():
# Check matching for field name and/or value
if (p_args.field_name and p.search(field_name)) or (p_args.field_value and p.search(field_value)):
# Print the file name and stop looking for more matches
if file_name_only:
print(file_name)
break_out = True # stop record loop
break
# The record start and file header should be printed if it was done already.
if not record_start_printed:
if more_than_one_file:
print(file_name_header(file_name))
more_than_one_file = False # file name was printed
print(format_record_start(record_name, record_type))
record_start_printed = True # record start was printed
# If printing all fields then break the loop and continue with the next record
if all_fields:
print_all_fields(record)
break
else:
print(format_field(field_name, field_value))
# Break out from the record loop
if break_out:
break
# Print the record end if the start was printed
if record_start_printed:
print(format_record_end())
df.close()
return
def diff_files_external(file_name1, file_name2, p_args):
"""
Open the two database files, sort them and call the external difference program.
:param file_name1: file name 1
:param file_name2: file name 2
:param p_args: command line arguments
:return: None
"""
# Read databases into memory
try:
my_filter = diff_filter if p_args.clean else None
df1 = DatabaseFile(file_name=file_name1, filter_function=my_filter)
df2 = DatabaseFile(file_name=file_name2, filter_function=my_filter)
db1 = df1.read_database()
db2 = df2.read_database()
df1.close()
df2.close()
except (OSError, IOError) as ex:
print(ex)
return
# Create output file names by appending a time stamp
time_stamp = str(int(time.time()))
output_file_name1 = os.path.join(
TEMP_DIRECTORY,
os.path.basename(file_name1) + '_' + time_stamp + '_1')
output_file_name2 = os.path.join(
TEMP_DIRECTORY,
os.path.basename(file_name2) + '_' + time_stamp + '_2')
# Write the sorted databases to disk
for db, file_name in [(db1, output_file_name1), (db2, output_file_name2)]:
# print db, file_name
assert (isinstance(db, EpicsDatabase))
try:
f = open(file_name, 'w')
db.write_sorted_database(f_out=f)
f.close()
except (OSError, IOError) as ex:
print(ex)
return
# Call external program to diff the files
external_program = p_args.program[0]
print('calling', external_program, 'with', output_file_name1, 'and',
output_file_name2)
try:
subprocess.call(
[external_program, output_file_name1, output_file_name2])
except subprocess.CalledProcessError as ex:
print('error while calling' + external_program)
print(ex)
# Remove temporary files
try:
os.remove(output_file_name1)
os.remove(output_file_name2)
except OSError as ex:
print(ex)
return