def __init__(self,
driver,
user,
password,
dbname,
host,
url_list,
report_filename,
make_report,
be_quiet,
force,
web_user,
web_password,
compressed_dir=None):
# If no "--compressed-dir" parameter is set, use default
if compressed_dir is None:
compressed_dir = COMPRESSED_DIR
self.mail_parser = MailArchiveAnalyzer()
logging.basicConfig()
logging.getLogger('sqlalchemy.engine').setLevel(logging.WARN)
drv = url.URL(driver, user, password, host, database=dbname)
engine = create_engine(drv, encoding='utf8', convert_unicode=True)
Database.create_tables(engine, checkfirst=True)
Session = sessionmaker()
Session.configure(bind=engine)
session = Session()
self.db = Database()
self.db.set_session(session)
# User and password to make login in case the archives
# are set to private
self.web_user = web_user
self.web_password = web_password
# Don't show messages when retrieveing and analyzing files
self.be_quiet = be_quiet
# Force to download and parse any link found in the given URL
self.force = force
# URLs or local files to be analyzed
self.url_list = url_list
self.__check_mlstats_dirs(compressed_dir)
total_messages = 0
stored_messages = 0
non_parsed = 0
for mailing_list in url_list:
t, s, np = self.__analyze_mailing_list(mailing_list,
compressed_dir)
total_messages += t
stored_messages += s
non_parsed += np
self.__print_output("%d messages analyzed" % total_messages)
self.__print_output("%d messages stored in database %s" %
(stored_messages, dbname))
self.__print_output("%d messages ignored by the parser" % non_parsed)
difference = total_messages - stored_messages
if difference == 0 and non_parsed == 0:
self.__print_output("INFO: Everything seems to be ok.")
if difference > 0:
self.__print_output("WARNING: Some messages were parsed but "
"not stored")
if non_parsed > 0:
self.__print_output("WARNING: Some messages were ignored by "
"the parser (probably because they were "
"ill formed messages)")
if make_report:
report = Report()
report.set_session(session)
report.print_brief_report(report_filename=report_filename)
session.close()
def __init__(self, driver, user, password, dbname, host, admin_user,
admin_password, url_list, report_filename, make_report,
be_quiet, force, web_user, web_password):
self.mail_parser = MailArchiveAnalyzer()
self.db = create_database(driver=driver,
dbname=dbname,
username=user,
password=password,
hostname=host,
admin_user=admin_user,
admin_password=admin_password)
# Connect to database if exists, otherwise create it and connect
self.db.connect()
# User and password to make login in case the archives
# are set to private
self.web_user = web_user
self.web_password = web_password
# Don't show messages when retrieveing and analyzing files
self.be_quiet = be_quiet
# Force to download and parse any link found in the given URL
self.force = force
# URLs or local files to be analyzed
self.url_list = url_list
self.__check_mlstats_dirs()
total_messages = 0
stored_messages = 0
non_parsed = 0
for mailing_list in url_list:
t, s, np = self.__analyze_mailing_list(mailing_list)
total_messages += t
stored_messages += s
non_parsed += np
self.__print_output("%d messages analyzed" % total_messages)
self.__print_output("%d messages stored in database %s" %
(stored_messages, self.db.name))
self.__print_output("%d messages ignored by the parser" % non_parsed)
difference = total_messages - stored_messages
if difference == 0 and non_parsed == 0:
self.__print_output("INFO: Everything seems to be ok.")
if difference > 0:
self.__print_output("WARNING: Some messages were parsed but "
"not stored")
if non_parsed > 0:
self.__print_output("WARNING: Some messages were ignored by "
"the parser (probably because they were "
"ill formed messages)")
if make_report:
self.__print_brief_report(report_filename)
