def main():
"""
The scarping begins here!
Uses function from the Scraping_handler module (imported above)
Exceptions thrown in the craping_handler module module, bubbled and caught here
"""
logger.info("Scraping began")
args = parse_args()
configurations = parse_json()
try:
general_data, company_tab_data, ratings_tab_data = do_scraping(args, configurations)
except IOError as e:
print(e)
logger.error(f"===Something went wrong: {e}===")
sys.exit(1)
except ValueError as e:
logger.error(f"===Something went wrong: {e}===")
print(e)
sys.exit(1)
except KeyboardInterrupt:
logger.critical("Program stopped - User aborted")
print(f"Script stack - is that why you aborted?")
sys.exit(1)
# Save Final Result
try:
create_csv_res_file(company_tab_data, general_data, ratings_tab_data,
configurations['Scraping']['results_path'])
except Exception as e:
print(e)
logger.error(f"===Something went wrong: {e}===")
sys.exit(1)
# Create Database
create_database(configurations)
create_scarping_tables()
insert_values()
# Enrich with API
if args.api:
try:
create_api_table()
insert_values(where_from='api')
except Exception as e:
logger.error(f"===Something went wrong: {e}===")
print(e)
sys.exit(1)
parser = _get_parser_from_repository(repo)
parser.set_repository(repo, uri)
if parser is None:
printerr("Failed to create parser")
return 1
# TODO: check parser type == logfile type
db_exists = False
try:
printdbg("Creating database")
db = create_database(config.db_driver, config.db_database,
config.db_user, config.db_password,
config.db_hostname)
except AccessDenied, e:
printerr("Error creating database: %s", (e.message, ))
return 1
except DatabaseNotFound:
printerr("Database %s doesn't exist. It must be created before " + \
"running MininGit", (config.db_database,))
return 1
except DatabaseDriverNotSupported:
printerr("Database driver %s is not supported by MininGit",
(config.db_driver, ))
return 1
emg = _get_extensions_manager(config.extensions, config.hard_order)
profiler_start("Saving caches to disk")
self.__save_caches_to_disk()
profiler_stop("Saving caches to disk", delete=True)
self.cursor.close()
self.cnn.close()
self.cnn = None
if __name__ == '__main__':
import sys
from io import BytesIO
from Database import create_database, ICursor
uri = "http://svn.test-cvsanaly.org/svn/test"
db = create_database('mysql', 'dbcontenthandler', sys.argv[1], None,
'localhost')
cnn = db.connect()
tables = ['actions', 'branches', 'file_copies', 'file_links', 'files',
'people', 'repositories', 'scmlog', 'tag_revisions', 'tags']
cursor = cnn.cursor()
for table in tables:
query = "delete from %s" % (table)
cursor.execute(statement(query, db.place_holder))
cursor.close()
cnn.commit()
name = uri.rstrip("/").split("/")[-1].strip()
cursor = cnn.cursor()
rep = DBRepository(None, uri, name, 'svn')
parser.set_repository(repo, uri)
if parser is None:
printerr("Failed to create parser")
return 1
# TODO: check parser type == logfile type
db_exists = False
try:
printdbg("Creating database")
db = create_database(config.db_driver,
config.db_database,
config.db_user,
config.db_password,
config.db_hostname)
except AccessDenied, e:
printerr("Error creating database: %s", (e.message,))
return 1
except DatabaseNotFound:
printerr("Database %s doesn't exist. It must be created before " + \
"running MininGit", (config.db_database,))
return 1
except DatabaseDriverNotSupported:
printerr("Database driver %s is not supported by MininGit",
(config.db_driver,))
return 1
emg = _get_extensions_manager(config.extensions, config.hard_order)
profiler_stop("Saving caches to disk", delete=True)
self.cursor.close()
self.cnn.close()
self.cnn = None
if __name__ == "__main__":
import sys
from cStringIO import StringIO
from cPickle import dump, load
from Database import create_database, DBRepository, ICursor
uri = "http://svn.test-cvsanaly.org/svn/test"
db = create_database("mysql", "dbcontenthandler", sys.argv[1], None, "localhost")
cnn = db.connect()
tables = [
"actions",
"branches",
"file_copies",
"file_links",
"files",
"people",
"repositories",
"scmlog",
"tag_revisions",
"tags",
]
