def check():
comm.communicate(
level=2,
message="Checking URL fields of records in {0} ...".format(
options.tools_CSV_file_path),
output_streams=sys.stdout)
tools_CSV_data = read_tools_CSV()
test_results = []
progress_bar_obj = progressbar.ProgressBar()
for record in progress_bar_obj(tools_CSV_data):
# Retrieve the attribute values for this record for all attributes that are for URLs
URL = operator.itemgetter(*URL_record_columns)(record)
# Filter out empty URL strings
#URL = list(filter(None, URL))
if len(URL) > 0:
test_results += [
dict(zip(URL_record_columns, test_URLs(URL)))
] # list(zip(URL_record_columns, )) #, test_results)
data = test_results
return (data)
def signal_handler(signal, frame):
comm.communicate(level=1,
message="Manually interrupted.",
output_streams=sys.stderr)
sys.exit(0)
def test_mails(recorded_email_addresses, minimal_phase=1, maximal_phase=2):
comm.communicate(
level=2,
message="Checking mail fields of records in {0} ...".format(
tools_CSV_file_path),
output_streams=sys.stdout)
def send_out_verification_messages():
comm.communicate(level=1,
message="Sending out verification messages ...",
output_streams=sys.stdout)
for recorded_email_address in recorded_email_addresses:
email(from_addr=recorded_email_address,
to_addrs=options.test_mailbox)
def match_confirmations_with_recorded_email_addresses():
pass # X-
if minimal_phase <= 1 and maximal_phase >= 1:
send_out_verification_messages()
elif minimal_phase <= 2 and maximal_phase >= 2:
match_confirmations_with_recorded_email_addresses
else:
communicate(
level=1,
message=
"Unknown minimal and/or maximal phase(s) specified to test_mails(): {0} to {1}"
.format(minimal_phase, maximal_phase),
output_streams=sys.stderr)
def send_out_verification_messages():
comm.communicate(level=1,
message="Sending out verification messages ...",
output_streams=sys.stdout)
for recorded_email_address in recorded_email_addresses:
email(from_addr=recorded_email_address,
to_addrs=options.test_mailbox)
def main(*args, **kwargs):
comm.communicate(
level=1,
message="Tool to check integrity of CLARIN tools registry",
output_streams=sys.stdout)
prepare()
retrieve_tools_CSV()
# check_results is a list row objects for DictWriter
check_results = check()
write_check_results(check_results)
def write_check_results(rows):
comm.communicate(level=2,
message="Writing results of check to {0} ...".format(
options.output_CSV_file_path),
output_streams=sys.stdout)
with open(options.output_CSV_file_path, mode='wt') as CSV_file:
CSV_writer = csv.DictWriter(CSV_file,
fieldnames=URL_record_columns,
delimiter='\t',
quoting=csv.QUOTE_MINIMAL) # X-
CSV_writer.writeheader()
CSV_writer.writerows(rows)
def SMTP(from_addr, to_addrs, message):
comm.communicate(level=2,
message="Sending automatic probe e-mail [1] ... \n" +
message,
output_stream1=sys.stdout)
server = smtplib.SMTP(options.SMTP_server,
options.SMTP_port) #port 465 or 587
server.ehlo()
server.starttls()
server.ehlo()
server.login(from_addr, SMTP_password)
server.sendmail(from_addr=from_addr, to_addrs=to_addrs, msg=message)
server.close()
def download_file(destination_directory_path=None,
destination_file_name="",
file_mode=None,
base_URL="",
URL=""):
if base_URL == "" and URL == "":
# You need to specify either one of the URL arguments to this function.
raise URLError
elif URL == "":
URL = base_URL + destination_file_name
destination_file_path = os.path.join(destination_directory_path,
destination_file_name)
try:
comm.communicate(level=2,
message="Downloading {0} to {1} ...".format(
URL, destination_file_path),
output_streams=sys.stdout)
response = urllib.request.urlopen(URL)
except HTTPError as HTTP_error_obj:
comm.communicate(level=3,
message="HTTP Error: " + str(e.code) + " with: " +
URL,
output_streams=sys.stderr)
except URLError as URL_error_obj:
comm.communicate(level=3,
message="URL Error: " + str(e.reason) + " with: " +
URL,
output_streams=sys.stderr)
else:
with open(destination_file_path, "w" + file_mode) as destination_file:
destination_file.write(response.read())