def mine_repository(shortname, cvspath, username, password, database): """Mines a repository that you have direct access to. This should contain all of the RCS files for a given project. @param shortname - user friendly name for the archive @param cvspath - path to the CVS repository @param username - username to connect to the database as @param database - name of the database to connect to @param password - password to use for the database connection """ log.debug("mining repository [name=%s] [path=%s]", shortname, cvspath) db, st = dbutil.database_connect(username, password, database) # get the project id project_id = add_project(shortname, cvspath, st, db) cs = CommitSink(cvsfile=None) for path, dirs, files in os.walk(cvspath, topdown=True): # we only care about RCS files, not everything else for fn in [x for x in files if x[-2:] == ',v']: fname = path + os.sep + fn print fname fl = CVSFile(path=path, filename=fn) # get rid of the path elements and the ,v trailer fl.basename = fname[len(cvspath):-2].strip(os.sep) log.debug("fname: %s basename: %s", fname, fl.basename) cs.setCVSFile(fl) # apparently there are some files that are not valid, but in the set try: rcsparse.Parser().parse(open(fname), cs) except Exception, e: log.exception("Unable to parse file: %s", fname) if fl.headRevision != None: fl.removeAttics() storeCVSFile(fl, project_id, st, db)
def build_all_networks(username, password, database, dl=false): log.debug("Connecting to database") db, st = dbutil.database_connect(username, password, database) # db = PgSQL.connect(database="cvsminer") # st = db.cursor() log.debug("database connection succeeded") query = """SELECT project_id FROM project""" st.execute(query) res = st.fetchall() for x in res: if dl: build_network_dl(x[0], db, st) else: build_network_dynet(x[0], db, st)
action="store", default="cvsminer") parser.add_option("--host", dest="host", help="database host to connect to", action="store", default=None) parser.add_option("-p", "--password", dest="password", help="database password for user", action="store", default=None) (options, args) = parser.parse_args() if options.verbose: log.setLevel(logging.DEBUG) if (len(args) != 1): build_all_networks(options.username, options.password, options.database, dl=options.dl) else: db, st = dbutil.database_connect(username=options.username, password=options.password, databsae=options.database) for x in args: if options.dl: build_network_dl(int(x), db, st) else: build_network_dynet(int(x), db, st)