def mine_repository(shortname, cvspath, username, password, database):
"""Mines a repository that you have direct access to. This should contain all of the
RCS files for a given project.
@param shortname - user friendly name for the archive
@param cvspath - path to the CVS repository
@param username - username to connect to the database as
@param database - name of the database to connect to
@param password - password to use for the database connection
"""
log.debug("mining repository [name=%s] [path=%s]", shortname, cvspath)
db, st = dbutil.database_connect(username, password, database)
# get the project id
project_id = add_project(shortname, cvspath, st, db)
cs = CommitSink(cvsfile=None)
for path, dirs, files in os.walk(cvspath, topdown=True):
# we only care about RCS files, not everything else
for fn in [x for x in files if x[-2:] == ',v']:
fname = path + os.sep + fn
print fname
fl = CVSFile(path=path, filename=fn)
# get rid of the path elements and the ,v trailer
fl.basename = fname[len(cvspath):-2].strip(os.sep)
log.debug("fname: %s basename: %s", fname, fl.basename)
cs.setCVSFile(fl)
# apparently there are some files that are not valid, but in the set
try:
rcsparse.Parser().parse(open(fname), cs)
except Exception, e:
log.exception("Unable to parse file: %s", fname)
if fl.headRevision != None:
fl.removeAttics()
storeCVSFile(fl, project_id, st, db)
def build_all_networks(username, password, database, dl=false):
log.debug("Connecting to database")
db, st = dbutil.database_connect(username, password, database)
# db = PgSQL.connect(database="cvsminer")
# st = db.cursor()
log.debug("database connection succeeded")
query = """SELECT project_id FROM project"""
st.execute(query)
res = st.fetchall()
for x in res:
if dl:
build_network_dl(x[0], db, st)
else:
build_network_dynet(x[0], db, st)
action="store", default="cvsminer")
parser.add_option("--host", dest="host",
help="database host to connect to",
action="store", default=None)
parser.add_option("-p", "--password", dest="password",
help="database password for user",
action="store", default=None)
(options, args) = parser.parse_args()
if options.verbose:
log.setLevel(logging.DEBUG)
if (len(args) != 1):
build_all_networks(options.username,
options.password,
options.database,
dl=options.dl)
else:
db, st = dbutil.database_connect(username=options.username,
password=options.password,
databsae=options.database)
for x in args:
if options.dl:
build_network_dl(int(x), db, st)
else:
build_network_dynet(int(x), db, st)
