def Main():
if len(sys.argv) != 2:
util.SendEmail(os.path.basename(sys.argv[0]),
"This script needs to be called with an email address as the only argument!\n", priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
try:
config = util.LoadConfigFile()
ftp_host = config.get("FTP", "host")
ftp_user = config.get("FTP", "username")
ftp_passwd = config.get("FTP", "password")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
# Download data from Crossref:
log_file_name = CreateLogFileName()
crossref_xml_file = "/tmp/crossref.xml"
os.unlink(crossref_xml_file)
util.ExecOrDie("/usr/local/bin/crossref_downloader", [ crossref_xml_file ], log_file_name)
# Upload the XML data to the BSZ FTP server:
ftp = util.FTPLogin(ftp_host, ftp_user, ftp_passwd)
try:
with open(crossref_xml_file, "rb") as xml_file:
ftp.storbinary("STOR crossref.xml", xml_file)
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
os.unlink(crossref_xml_file)
util.SendEmail("Crossref Data Import",
"Successfully imported Crossref data and uploaded it to the BSZ FTP server.", priority=5)
def Main():
if len(sys.argv) != 2:
util.SendEmail(
os.path.basename(sys.argv[0]),
"This script needs to be called with an email address as the only argument!\n",
priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
try:
config = util.LoadConfigFile()
ftp_host = config.get("FTP", "host")
ftp_user = config.get("FTP", "username")
ftp_passwd = config.get("FTP", "password")
directory_on_ftp_server = config.get("Upload",
"directory_on_ftp_server")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
marc_filename = "/tmp/crossref_marc.xml"
no_of_records = DownloadCrossrefData(marc_filename)
if no_of_records == 0:
email_msg_body = "No new records.\n\n"
else:
ftp = util.FTPLogin(ftp_host, ftp_user, ftp_passwd)
try:
ftp.cwd(directory_on_ftp_server)
except:
util.Error("failed to change directory on the FTP server to \"" +
directory_on_ftp_server + "\"!")
UploadFile(ftp, marc_filename, GenerateRemoteFilename())
email_msg_body = "Uploaded " + str(
no_of_records) + " MARC records to the BSZ FTP server.\n\n"
os.unlink(marc_filename)
util.SendEmail("BSZ Crossref File Upload", email_msg_body, priority=5)
def PurgeFiles(generations_to_keep, all_files):
regex = re.compile("\\d{6}")
map_to_lists = {}
for file_name in all_files:
normalised_name = regex.sub("YYMMDD", file_name)
if normalised_name in map_to_lists:
file_name_list = map_to_lists[normalised_name]
file_name_list.append(file_name)
map_to_lists[normalised_name] = file_name_list
else:
map_to_lists[normalised_name] = [file_name]
deleted_names = ""
for _, file_name_list in list(map_to_lists.items()):
if len(file_name_list) <= generations_to_keep:
continue
file_name_list = sorted(file_name_list, reverse=True)
while len(file_name_list) > generations_to_keep:
name_to_delete = file_name_list.pop()
os.unlink(name_to_delete)
deleted_names += name_to_delete + "\n"
if deleted_names:
util.SendEmail("File Purge", "Deleted:\n" + deleted_names, priority=5)
else:
util.SendEmail("File Purge", "Found no files to purge.\n", priority=5)
def Main():
util.default_email_sender = "*****@*****.**"
util.default_email_recipient = "*****@*****.**"
if len(sys.argv) != 2:
util.SendEmail("Create Refterm File (Kickoff Failure)",
"This script must be called with one argument,\n" +
"the default email recipient\n",
priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
# Download needed differential files
config = util.LoadConfigFile()
log_file_name = log_file_name = util.MakeLogFileName(
sys.argv[0], util.GetLogDirectory())
changelist_url = config.get("Unpaywall", "changelist_url")
api_key = config.get("Unpaywall", "api_key")
oadoi_download_directory = config.get("LocalConfig", "download_dir")
oadoi_imported_directory = oadoi_download_directory + "/imported/"
StartMongoDB()
json_update_objects = GetChangelists(changelist_url, api_key)
remote_update_files = GetRemoteUpdateFiles(json_update_objects)
local_update_files = GetLocalUpdateFiles(config, oadoi_download_directory)
download_lists = GetAllFilesStartingAtFirstMissingLocal(
remote_update_files, local_update_files)
DownloadUpdateFiles(download_lists['download'], json_update_objects,
api_key, oadoi_download_directory)
# Update the Database
ImportOADOIsToMongo(
GetImportFiles(config, oadoi_download_directory,
oadoi_imported_directory), oadoi_download_directory,
log_file_name)
# Generate the files to be used by the pipeline
share_directory = config.get("LocalConfig", "share_directory")
ixtheo_dois_file = config.get("LocalConfig", "ixtheo_dois_file")
ixtheo_urls_file = config.get("LocalConfig", "ixtheo_urls_file")
ExtractOADOIURLs(share_directory, ixtheo_dois_file, ixtheo_urls_file,
log_file_name)
ShareOADOIURLs(share_directory, ixtheo_urls_file)
krimdok_dois_file = config.get("LocalConfig", "krimdok_dois_file")
krimdok_urls_file = config.get("LocalConfig", "krimdok_urls_file")
ExtractOADOIURLs(share_directory, krimdok_dois_file, krimdok_urls_file,
log_file_name)
ShareOADOIURLs(share_directory, krimdok_urls_file)
DumpMongoDB(config, log_file_name)
StopMongoDB()
util.SendEmail("Update OADOI Data",
"Successfully created \"" + ixtheo_urls_file + "\" and \"" +
krimdok_urls_file + "\" in " + share_directory,
priority=5)
def Main():
if len(sys.argv) != 2:
util.SendEmail(
os.path.basename(sys.argv[0]),
"This script needs to be called with an email address as the only argument!\n",
priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
try:
config = util.LoadConfigFile()
ftp_host = config.get("FTP", "host")
ftp_user = config.get("FTP", "username")
ftp_passwd = config.get("FTP", "password")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
ftp = util.FTPLogin(ftp_host, ftp_user, ftp_passwd)
msg = []
download_cutoff_date = IncrementStringDate(
GetCutoffDateForDownloads(config))
complete_data_filenames = DownloadCompleteData(config, ftp,
download_cutoff_date, msg)
if complete_data_filenames is not None:
download_cutoff_date = ExtractDateFromFilename(
complete_data_filenames[0])
DownloadData(config, "Differenzabzug", ftp, download_cutoff_date, msg)
DownloadData(config, "Loeschlisten", ftp, download_cutoff_date, msg)
if config.has_section("Loeschlisten2"):
DownloadData(config, "Loeschlisten2", ftp, download_cutoff_date, msg)
if config.has_section("Hinweisabzug"):
DownloadData(config, "Hinweisabzug", ftp, "000000", msg)
if config.has_section("Errors"):
DownloadData(config, "Errors", ftp, download_cutoff_date, msg)
incremental_authority_cutoff_date = ShiftDateToTenDaysBefore(
download_cutoff_date)
if config.has_section("Normdatendifferenzabzug"):
if (not CurrentIncrementalAuthorityDumpPresent(
config, incremental_authority_cutoff_date)):
DownloadData(config, "Normdatendifferenzabzug", ftp,
incremental_authority_cutoff_date, msg)
else:
msg.append(
"Skipping Download of \"Normdatendifferenzabzug\" since already present\n"
)
CleanUpCumulativeCollection(config)
util.SendEmail("BSZ File Update", string.join(msg, ""), priority=5)
def Main():
util.default_email_sender = "*****@*****.**"
if len(sys.argv) != 3:
print(
"invalid arguments! usage: initiate_marc_pipeline.py <default email recipient> <MARC21 pipeline script name>"
)
util.SendEmail(
"MARC-21 Pipeline Kick-Off (Failure)",
"This script needs to be called with two arguments,\n" +
"the default email recipient and the name of the MARC-21\n" +
"pipeline script to be executed.\n",
priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
pipeline_script_name = sys.argv[2]
if not os.access(pipeline_script_name, os.X_OK):
print("Pipeline script not found or not executable: " +
pipeline_script_name)
util.SendEmail("MARC-21 Pipeline Kick-Off (Failure)",
"Pipeline script not found or not executable: \"" +
pipeline_script_name + "\"\n",
priority=1)
sys.exit(-1)
conf = util.LoadConfigFile()
link_name = conf.get("Misc", "link_name")
if FoundNewBSZDataFile(link_name):
if not FoundReftermMutex():
util.Error("No Refterm Mutex found")
bsz_data = util.ResolveSymlink(link_name)
if not bsz_data.endswith(".tar.gz"):
util.Error("BSZ data file must end in .tar.gz!")
file_name_list = util.ExtractAndRenameBSZFiles(bsz_data)
RunPipelineAndImportIntoSolr(pipeline_script_name, file_name_list[0],
conf)
util.SendEmail("MARC-21 Pipeline",
"Pipeline completed successfully.",
priority=5,
attachments=[solrmarc_log_summary, import_log_summary])
util.WriteTimestamp()
DeleteReftermMutex()
WriteImportFinishedFile()
else:
util.SendEmail("MARC-21 Pipeline Kick-Off",
"No new data was found.",
priority=5)
def Main():
if len(sys.argv) != 2:
util.Error("This script expects one argument: default_email_recipient")
util.default_email_recipient = sys.argv[1]
config = util.LoadConfigFile()
try:
deletion_list = config.get("Files", "loesch_liste")
complete_data = config.get("Files", "komplett_abzug")
differential_data = config.get("Files", "differenz_abzug")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
if not os.access(complete_data, os.R_OK):
util.Error("Fehlender oder nicht lesbarer Komplettabzug. (" +
complete_data + ")")
deletion_list_is_readable = os.access(deletion_list, os.R_OK)
if not deletion_list_is_readable:
deletion_list = None
differential_data_is_readable = os.access(differential_data, os.R_OK)
if not deletion_list_is_readable and not differential_data_is_readable:
util.Error(
"Fehlende oder nicht lesbare Löschliste und Differenzabzug..")
# Bail out if the most recent complete data set is at least as recent as the deletion list or the differential
# data:
complete_data_mtime = os.path.getmtime(complete_data)
deletion_list_mtime = None
if deletion_list_is_readable:
deletion_list_mtime = os.path.getmtime(deletion_list)
differential_data_mtime = None
if differential_data_is_readable:
differential_data_mtime = os.path.getmtime(differential_data)
if ((deletion_list_mtime is not None
and complete_data_mtime >= deletion_list_mtime)
or (differential_data_mtime is not None
and complete_data_mtime >= differential_data_mtime)):
util.SendEmail(
"Nichts zu tun!",
"Komplettabzug ist neuer als eventuell vorhandene Differenzabzüge.\n",
priority=5)
sys.exit(0)
data_dir = PrepareDataDirectory(
) # After this we're in the data directory...
util.ExtractAndRenameBSZFiles("../" + complete_data)
util.ExtractAndRenameBSZFiles("../" + differential_data, "Diff")
title_superior_norm_tuple = UpdateAllMarcFiles(
deletion_list) # ...and we're back in the original directory.
new_tarball_name = complete_data.replace(
"current",
datetime.date.today().strftime("%y%m%d"))
CreateNewTarballAndDeletePreviousTarball(new_tarball_name,
title_superior_norm_tuple,
complete_data)
util.RemoveLinkTargetAndLink(title_superior_norm_tuple[0])
util.RemoveLinkTargetAndLink(title_superior_norm_tuple[1])
util.RemoveLinkTargetAndLink(title_superior_norm_tuple[2])
util.Info("Successfully created updated MARC files.")
def Main():
if len(sys.argv) != 3:
util.SendEmail(os.path.basename(sys.argv[0]),
"This script needs to be called with an email address and the system type!\n", priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
system_type = sys.argv[2]
if system_type != "krimdok" and system_type != "relbib" and system_type != "ixtheo":
util.SendEmail(os.path.basename(sys.argv[0]),
"This system_type must be one of {krimdok,relbib,ixtheo}!\n", priority=1)
sys.exit(-1)
output_file = "/tmp/collect_solr_stats_data.csv"
util.Remove(output_file)
util.ExecOrDie("/usr/local/bin/collect_solr_stats_data", [ system_type, output_file ],
"/usr/local/var/log/tuefind/collect_solr_stats_data.log")
util.SendEmail("Solr Stats Collector", "Successfully generated Solr statistics and updated Ingo's MySQL database.", priority=5)
def Main():
if len(sys.argv) != 2:
util.SendEmail(
os.path.basename(sys.argv[0]),
"This script needs to be called with an email address as the only argument!\n",
priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
try:
config = util.LoadConfigFile()
sftp_host = config.get("SFTP", "host")
sftp_user = config.get("SFTP", "username")
sftp_keyfile = config.get("SFTP", "keyfile")
local_directory = config.get("Upload", "local_directory")
directory_on_sftp_server = config.get("Upload",
"directory_on_sftp_server")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
# Check directories with new Data
fulltext_files = GetExistingFiles(local_directory)
dirs_to_transfer = GetFulltextDirectoriesToTransfer(
local_directory, fulltext_files)
# If nothing to do
if not dirs_to_transfer:
util.SendEmail("Transfer Fulltexts",
"No directories to transfer",
priority=5)
return
# Transfer the data
util.ExecOrDie("/usr/local/bin/transfer_fulltext.sh", [
sftp_host, sftp_user, sftp_keyfile, local_directory,
directory_on_sftp_server
] + list(dirs_to_transfer))
# Clean up on the server
CleanUpFiles(fulltext_files)
email_msg_body = "Found Files:\n\n" + '\n'.join(
fulltext_files) + "\n\nTransferred directories:\n\n" + '\n'.join(
dirs_to_transfer)
util.SendEmail("Transfer Fulltexts", email_msg_body, priority=5)
def OptimizeSolrIndex(index):
try:
request = urllib.request.Request("http://localhost:8080/solr/" +
index + "/update?optimize=true")
response = urllib.request.urlopen(request, timeout=1800)
except:
util.SendEmail("MARC-21 Pipeline",
"Failed to optimize the SOLR index \"" + index + "\"!",
priority=1)
sys.exit(-1)
def GetMongoServiceDependingOnSystem():
distro = platform.linux_distribution()[0]
if re.search('CentOS', distro):
return 'mongod.service'
if re.search('Ubuntu', distro):
return 'mongodb.service'
util.SendEmail("Update OADOI Data",
"Cannot handle Distro \"" + distro + "\"",
priority=1)
sys.exit(-1)
def ClearSolrIndex(index):
try:
request = urllib2.Request(
"http://localhost:8080/solr/" + index +
"/update?wt=xml&stream.body=%3Cdelete%3E%3Cquery%3E*:*%3C/query%3E%3C/delete%3E"
)
response = urllib2.urlopen(request, timeout=60)
except:
util.SendEmail("MARC-21 Pipeline",
"Failed to clear the SOLR index \"" + index + "\"!",
priority=1)
sys.exit(-1)
def CleanUpFiles(fulltext_dirs):
today = datetime.datetime.today().strftime('%y%m%d')
backup_dir = "/usr/local/tmp/webdav_backup/" + today
if not os.path.exists(backup_dir):
os.makedirs(backup_dir)
for fulltext_dir in fulltext_dirs:
shutil.move(fulltext_dir, backup_dir)
else:
util.SendEmail(
"Transfer Fulltexts",
"Did not move directory to backup since directory for this day already present"
)
return
def Main():
if len(sys.argv) != 2:
util.SendEmail(os.path.basename(sys.argv[0]),
"This script needs to be called with an email address as the only argument!\n", priority=1)
ppns = []
currentCursorMark = '*'
while True:
jdata = GetDataFromGVI(currentCursorMark)
ppns = ExtractPPNs(jdata, ppns)
newCursorMark = ExtractNextCursorMark(jdata)
if currentCursorMark == newCursorMark:
break
currentCursorMark = newCursorMark
WriteListFile(ppns)
def ClearSolrIndex(index):
try:
url = "http://localhost:8080/solr/" + index + "/update"
values = "<delete><query>*:*</query></delete>"
data = values.encode('utf-8')
headers = {"Content-Type": "application/xml"}
request = urllib.request.Request(url, data, headers)
response = urllib.request.urlopen(request, timeout=300)
except Exception as e:
util.SendEmail("MARC-21 Pipeline",
"Failed to clear the SOLR index \"" + index + "\" [" +
str(e) + "]!",
priority=1)
sys.exit(-1)
def Main():
if len(sys.argv) != 2:
util.Error("This script expects one argument: default_email_recipient")
util.default_email_recipient = sys.argv[1]
config = util.LoadConfigFile()
try:
generations_to_keep = config.getint("PurgeFiles", "generations_to_keep")
except Exception as e:
util.Error("failed to read config file! ("+ str(e) + ")")
if generations_to_keep < 1:
util.Error("generations_to_keep must be at least 1!")
all_timestamped_files = glob.glob("*[0-9][0-9][0-9][0-9][0-9][0-9]*")
if not all_timestamped_files:
util.SendEmail("File Purge Failed", "No timestamped files found!", priority=1)
PurgeFiles(generations_to_keep, all_timestamped_files)
def Main():
if len(sys.argv) != 2:
util.SendEmail(
os.path.basename(sys.argv[0]),
"This script requires an email address as the only argument\n",
priority=1,
recipient=util.default_email_recipient)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
user = sys.argv[1]
try:
sql_config = util.LoadConfigFile(
"/usr/local/var/lib/tuelib/translations.conf")
sql_database = sql_config.get("Database", "sql_database")
sql_username = sql_config.get("Database", "sql_username")
sql_password = sql_config.get("Database", "sql_password")
except Exception as e:
util.Error("Failed to read sql_config file (" + str(e) + ")")
try:
config = util.LoadConfigFile()
archive_password = config.get("Passwords", "archive_password")
except Exception as e:
util.Error("Failed to read config file (" + str(e) + ")")
raw_dump_file = tmp_file_path + "/" + output_filename
DumpTranslationsDB(sql_database, sql_username, sql_password, raw_dump_file)
compressed_and_encrypted_dump_file = re.sub(
'\..*$', '',
raw_dump_file) + compressed_extension + encrypted_extension
CompressAndEncryptFile(raw_dump_file, compressed_and_encrypted_dump_file,
re.sub('^"|"$', '', archive_password))
MoveToDownloadPosition(compressed_and_encrypted_dump_file, web_server_path)
CleanUp(raw_dump_file)
servername = DetermineServerName()
NotifyUser(user, servername,
os.path.basename(compressed_and_encrypted_dump_file))
def Main():
util.default_email_recipient = sys.argv[1]
config = util.LoadConfigFile()
if config.has_option("Global", "validate_ssl_certificates"):
if not config.getboolean("Global", "validate_ssl_certificates"):
ssl._create_default_https_context = ssl._create_unverified_context
for section in config.sections():
if section == "Global":
continue
url = config.get(section, "url")
expected = None
if config.has_option(section, "expected"):
expected = config.get(section, "expected")
timeout = None
if config.has_option(section, "timeout"):
timeout = config.getfloat(section, "timeout")
if not RunTest(section, url, timeout, expected):
util.SendEmail("Black Box Test Failed!",
"Test " + section +
" failed!\n\n--Your friendly black box monitor",
"*****@*****.**",
priority=1)
ftp_host = config.get("FTP", "host")
ftp_user = config.get("FTP", "username")
ftp_passwd = config.get("FTP", "password")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
# Download data from Crossref:
log_file_name = CreateLogFileName()
crossref_xml_file = "/tmp/crossref.xml"
os.unlink(crossref_xml_file)
util.ExecOrDie("/usr/local/bin/crossref_downloader", [ crossref_xml_file ], log_file_name)
# Upload the XML data to the BSZ FTP server:
ftp = util.FTPLogin(ftp_host, ftp_user, ftp_passwd)
try:
with open(crossref_xml_file, "rb") as xml_file:
ftp.storbinary("STOR crossref.xml", xml_file)
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
os.unlink(crossref_xml_file)
util.SendEmail("Crossref Data Import",
"Successfully imported Crossref data and uploaded it to the BSZ FTP server.", priority=5)
try:
Main()
except Exception as e:
util.SendEmail("Crossref Data Import", "An unexpected error occurred: "
+ str(e) + "\n\n" + traceback.format_exc(20), priority=1)
def Main():
if len(sys.argv) != 4 and len(sys.argv) != 5:
util.SendEmail(os.path.basename(
sys.argv[0]
), "This script needs to be called with an email address, the beacon header file, an output "
"path and an optional ppn-filter file as arguments!\n",
priority=1)
sys.exit(-1)
util.default_email_recipient = sys.argv[1]
most_recent_authority_filename = GetMostRecentBSZFile(
"^Normdaten-(\d\d\d\d\d\d).mrc$")
if most_recent_authority_filename is None:
util.SendEmail("Beacon Generator",
"Found no matching authority files!",
priority=1)
most_recent_titles_filename = GetMostRecentBSZFile(
"^GesamtTiteldaten-(\d\d\d\d\d\d).mrc$")
if most_recent_titles_filename is None:
util.SendEmail("Beacon Generator",
"Found no matching title files!",
priority=1)
# Extract the GND numbers from the 035$a subfield of the MARC authority data for authors:
gnd_numbers_path = "/tmp/gnd_numbers"
util.ExecOrDie("/usr/local/bin/extract_person_gnd_numbers",
[most_recent_authority_filename], gnd_numbers_path)
# Count GND references in the title data:
gnd_counts_filename = "/tmp/gnd_counts"
if len(sys.argv) == 4:
util.ExecOrDie("/usr/local/bin/count_gnd_refs", [
gnd_numbers_path, most_recent_titles_filename, gnd_counts_filename
])
else:
util.ExecOrDie("/usr/local/bin/count_gnd_refs", [
"--control-number-list=" + sys.argv[4], gnd_numbers_path,
most_recent_titles_filename, gnd_counts_filename
])
# Generate a file with a timestamp in the Beacon format:
timestamp_filename = "/tmp/beacon_timestamp"
with open(timestamp_filename, "w") as timestamp_file:
timestamp_file.write("#TIMESTAMP: " + str(datetime.date.today()) +
"\n")
# Now generate the final output (header + counts):
if not util.ConcatenateFiles(
[sys.argv[2], timestamp_filename, gnd_counts_filename], sys.argv[3]):
util.SendEmail("Beacon Generator",
"An unexpected error occurred: could not write \"" +
sys.argv[3] + "\"!",
priority=1)
# Cleanup of temp files:
os.unlink(gnd_numbers_path)
os.unlink(timestamp_filename)
os.unlink(gnd_counts_filename)
util.SendEmail("Beacon File Generator",
"Successfully created a Beacon file.",
priority=5)
dirs_to_transfer = GetFulltextDirectoriesToTransfer(
local_directory, fulltext_files)
# If nothing to do
if not dirs_to_transfer:
util.SendEmail("Transfer Fulltexts",
"No directories to transfer",
priority=5)
return
# Transfer the data
util.ExecOrDie("/usr/local/bin/transfer_fulltext.sh", [
sftp_host, sftp_user, sftp_keyfile, local_directory,
directory_on_sftp_server
] + list(dirs_to_transfer))
# Clean up on the server
CleanUpFiles(fulltext_files)
email_msg_body = "Found Files:\n\n" + '\n'.join(
fulltext_files) + "\n\nTransferred directories:\n\n" + '\n'.join(
dirs_to_transfer)
util.SendEmail("Transfer Fulltexts", email_msg_body, priority=5)
try:
Main()
except Exception as e:
util.SendEmail("Transfer Fulltexts",
"An unexpected error occurred: " + str(e) + "\n\n" +
traceback.format_exc(20),
priority=1)
def Main():
if len(sys.argv) != 2:
util.Error("This script expects one argument: default_email_recipient")
util.default_email_recipient = sys.argv[1]
config = util.LoadConfigFile()
try:
generations_to_keep = config.getint("PurgeFiles",
"generations_to_keep")
except Exception as e:
util.Error("failed to read config file! (" + str(e) + ")")
if generations_to_keep < 1:
util.Error("generations_to_keep must be at least 1!")
all_timestamped_files = glob.glob("*[0-9][0-9][0-9][0-9][0-9][0-9]*")
if not all_timestamped_files:
util.SendEmail("File Purge Failed",
"No timestamped files found!",
priority=1)
PurgeFiles(generations_to_keep, all_timestamped_files)
try:
Main()
except Exception as e:
util.SendEmail("File Purge Failed",
"An unexpected error occurred: " + str(e) + "\n\n" +
traceback.format_exc(20),
priority=1)
def NotifyUser(user, server, filename):
util.SendEmail(
"New translations database dump provided",
"A new translation database dump has been provided for you at https://"
+ server + "/" + filename + "\n\n" + "Kind regards")
if FoundNewBSZDataFile(title_data_link_name):
start = datetime.datetime.now()
log_file_name = CreateLogFile()
title_data_file_orig = ExtractTitleDataMarcFile(title_data_link_name)
date_string = GetDateFromFilename(title_data_file_orig)
title_data_file = RenameTitleDataFile(title_data_file_orig, date_string)
atexit.register(CleanUp, title_data_file, log_file_name)
SetupTemporarySolrInstance(title_data_file, conf, log_file_name)
create_ref_term_process = multiprocessing.Process(target=CreateRefTermFile, name="Create Reference Terms File",
args=[ ref_data_archive, date_string, conf, log_file_name ])
create_serial_sort_term_process = multiprocessing.Process(target=CreateSerialSortDate, name="Serial Sort Date",
args=[ title_data_file, date_string, log_file_name ])
create_match_db_log_file_name = util.MakeLogFileName("create_match_db", util.GetLogDirectory())
create_match_db_process = multiprocessing.Process(target=CreateMatchDB, name="Create Match DB",
args=[ title_data_file, create_match_db_log_file_name ])
ExecuteInParallel(create_ref_term_process, create_serial_sort_term_process, create_match_db_process)
end = datetime.datetime.now()
duration_in_minutes = str((end - start).seconds / 60.0)
util.SendEmail("Create Refterm File", "Refterm file successfully created in " + duration_in_minutes + " minutes.", priority=5)
else:
util.SendEmail("Create Refterm File", "No new data was found.", priority=5)
try:
Main()
except Exception as e:
error_msg = "An unexpected error occurred: " + str(e) + "\n\n" + traceback.format_exc(20)
util.SendEmail("Create Refterm File", error_msg, priority=1)
sys.stderr.write(error_msg)
timestamp_filename = "/tmp/beacon_timestamp"
with open(timestamp_filename, "w") as timestamp_file:
timestamp_file.write("#TIMESTAMP: " + str(datetime.date.today()) +
"\n")
# Now generate the final output (header + counts):
if not util.ConcatenateFiles(
[sys.argv[2], timestamp_filename, gnd_counts_filename], sys.argv[3]):
util.SendEmail("Beacon Generator",
"An unexpected error occurred: could not write \"" +
sys.argv[3] + "\"!",
priority=1)
# Cleanup of temp files:
os.unlink(gnd_numbers_path)
os.unlink(timestamp_filename)
os.unlink(gnd_counts_filename)
util.SendEmail("Beacon File Generator",
"Successfully created a Beacon file.",
priority=5)
try:
Main()
except Exception as e:
util.SendEmail("Beacon Generator",
"An unexpected error occurred: " + str(e) + "\n\n" +
traceback.format_exc(20),
priority=1)
