def execute_statement(statement,
parameters,
cursor,
db,
error_message,
exception=Exception):
"""Run a statement. Note that the cursor is *mutable*, and will contain
the results after running.
"""
if isinstance(db, SqliteDatabase):
import sqlite3.dbapi2
try:
return cursor.execute(statement, parameters)
except sqlite3.dbapi2.OperationalError as e:
printerr(error_message + ": %s", (e, ))
except Exception as e:
raise exception(e)
elif isinstance(db, MysqlDatabase):
import _mysql_exceptions
try:
return cursor.execute(statement, parameters)
except _mysql_exceptions.OperationalError as e:
printerr(error_message + ": %s", (e, ))
except Exception as e:
raise exception(e)
def send_email_quick(contents, to_addr):
try:
smtpObj = smtplib.SMTP('localhost')
smtpObj.sendmail("*****@*****.**", [to_addr], contents)
except smtplib.SMTPException as err:
printerr("Error: unable to send email, try configuring an SMTP server")
raise err
def run_extension(self, name, extension, repo, uri, db):
printout("Executing extension %s", (name,))
try:
extension.run(repo, uri, db)
except ExtensionRunError, e:
printerr("Error running extension %s: %s", (name, str(e)))
return False
def execute_statement(statement,
parameters,
write_cursor,
db,
error_message,
exception=Exception):
if isinstance(db, SqliteDatabase):
import sqlite3.dbapi2
try:
write_cursor.execute(statement, parameters)
except sqlite3.dbapi2.OperationalError as e:
printerr(error_message + ": %s", (e, ))
except Exception as e:
raise exception(e)
elif isinstance(db, MysqlDatabase):
import _mysql_exceptions
try:
write_cursor.execute(statement, parameters)
except _mysql_exceptions.OperationalError as e:
printerr(error_message + ": %s", (e, ))
except Exception as e:
raise exception(e)
def main(argv):
# Short (one letter) options. Those requiring argument followed by :
short_opts = "hVgqnf:l:s:u:p:d:H:w:e"
# Long options (all started by --). Those requiring argument followed by =
long_opts = ["help", "version", "debug", "quiet", "profile", "config-file=",
"repo-logfile=", "save-logfile=", "no-parse", "files=",
"db-user="******"db-password="******"db-hostname=", "db-database=", "db-driver=",
"extensions=", "metrics-all", "metrics-noerr", "list-extensions", "git-ref=", "writable-path="]
# Default options
debug = None
quiet = None
profile = None
configfile = None
no_parse = None
files = None
user = None
passwd = None
hostname = None
database = None
driver = None
logfile = None
save_logfile = None
writable_path = None
extensions = None
metrics_all = None
metrics_noerr = None
gitref = None
try:
opts, args = getopt.getopt(argv, short_opts, long_opts)
except getopt.GetoptError, e:
printerr(str(e))
return 1
def _get_uri_and_repo(path):
""" Get a URI and repositoryhandler object for a path.
This function returns a URI as a string, and the repositoryhandler
object that represents that URI. They are returned together as a tuple.
Args:
path: The path to the repository
"""
# Create repository
if path is not None:
try:
printdbg("Creating repositoryhandler instance")
repo = create_repository_from_path(path)
repo.timeout = 120
except RepositoryUnknownError:
printerr("Path %s doesn't seem to point to a repository " + \
"supported by cvsanaly", (path,))
sys.exit(1)
except Exception, e:
printerr("Unknown error creating repository for path %s (%s)",
(path, str(e)))
sys.exit(1)
uri = repo.get_uri_for_path(path)
return (uri, repo)
def main (argv):
# Short (one letter) options. Those requiring argument followed by :
short_opts = "hVgqnf:l:s:u:p:d:H:"
# Long options (all started by --). Those requiring argument followed by =
long_opts = ["help", "version", "debug", "quiet", "profile", "config-file=",
"repo-logfile=", "save-logfile=", "no-parse", "db-user="******"db-password="******"db-hostname=", "db-database=", "db-driver=", "extensions=",
"metrics-all", "metrics-noerr"]
# Default options
debug = None
quiet = None
profile = None
configfile = None
no_parse = None
user = None
passwd = None
hostname = None
database = None
driver = None
logfile = None
save_logfile = None
extensions = None
metrics_all = None
metrics_noerr = None
try:
opts, args = getopt.getopt (argv, short_opts, long_opts)
except getopt.GetoptError, e:
printerr (str (e))
return 1
def _get_uri_and_repo(path):
""" Get a URI and repositoryhandler object for a path.
This function returns a URI as a string, and the repositoryhandler
object that represents that URI. They are returned together as a tuple.
Args:
path: The path to the repository
"""
# Create repository
if path is not None:
try:
printdbg("Creating repositoryhandler instance")
repo = create_repository_from_path(path)
repo.timeout = 120
except RepositoryUnknownError:
printerr("Path %s doesn't seem to point to a repository " + \
"supported by cvsanaly", (path,))
sys.exit(1)
except Exception, e:
printerr("Unknown error creating repository for path %s (%s)",
(path, str(e)))
sys.exit(1)
uri = repo.get_uri_for_path(path)
return (uri, repo)
def run_extension (self, name, extension, repo, uri, db):
printout ("Executing extension %s", (name,))
try:
extension.run (repo, uri, db)
except ExtensionRunError, e:
printerr ("Error running extension %s: %s", (name, str (e)))
return False
def __init__(self, exts, hard_order=False):
self.exts = {}
self.hard_order = hard_order
order = 0
for ext in exts:
name = ext
if hard_order:
name = str(order) + name
order = order + 1
try:
self.exts[name] = get_extension(ext)
except ExtensionUnknownError,e:
printerr("Error importing extenstion %s: %s", (name, str(e)))
raise InvalidExtension(ext)
# Add dependencies
if not hard_order:
for dep in self.exts[ext].deps:
if dep not in self.exts.keys():
try:
self.exts[dep] = get_extension(dep)
except:
raise InvalidDependency(ext, dep)
def __init__(self, exts, hard_order=False):
self.exts = {}
self.hard_order = hard_order
order = 0
for ext in exts:
name = ext
if hard_order:
name = str(order) + name
order = order + 1
try:
self.exts[name] = get_extension(ext)
except ExtensionUnknownError, e:
printerr("Error importing extenstion %s: %s", (name, str(e)))
raise InvalidExtension(ext)
# Add dependencies
if not hard_order:
for dep in self.exts[ext].deps:
if dep not in self.exts.keys():
try:
self.exts[dep] = get_extension(dep)
except:
raise InvalidDependency(ext, dep)
def execute_statement(statement, parameters, cursor, db, error_message,
exception=Exception):
"""Run a statement. Note that the cursor is *mutable*, and will contain
the results after running.
"""
if isinstance(db, SqliteDatabase):
import sqlite3.dbapi2
try:
return cursor.execute(statement, parameters)
except sqlite3.dbapi2.OperationalError as e:
printerr(error_message + ": %s", (e,))
except Exception as e:
raise exception(e)
elif isinstance(db, MysqlDatabase):
import _mysql_exceptions
try:
return cursor.execute(statement, parameters)
except _mysql_exceptions.OperationalError as e:
printerr(error_message + ": %s", (e,))
except Exception as e:
raise exception(e)
def _get_extensions_manager(extensions, hard_order=False):
try:
printdbg("Starting ExtensionsManager")
emg = ExtensionsManager(extensions, hard_order=hard_order)
return emg
except InvalidExtension, e:
printerr("Invalid extension %s", (e.name, ))
sys.exit(1)
def run_extension(self, name, extension, change_file,understand_file,stat_file,rank_file,changemetric_file,predict_file,change_type):
# Trim off the ordering numeral before printing
try:
extension.run(change_file,understand_file,stat_file,rank_file,changemetric_file,predict_file,change_type)
except ExtensionRunError, e:
printerr("Error running extension %s: %s", (name, str(e)))
return False
def log_file_is_bzr(logfile):
retval = False
try:
f = open(logfile, 'r')
except IOError, e:
printerr(str(e))
return False
def log_file_is_bzr (logfile):
retval = False
try:
f = open (logfile, 'r')
except IOError, e:
printerr (str (e))
return False
def _get_extensions_manager(extensions, hard_order=False):
try:
printdbg("Starting ExtensionsManager")
emg = ExtensionsManager(extensions,
hard_order=hard_order)
return emg
except InvalidExtension, e:
printerr("Invalid extension %s", (e.name,))
sys.exit(1)
def scan_repo(repo_url, root):
repos_root = os.path.join(root, "repositories")
friendly_name = strip_suffix(os.path.basename(urlparse(repo_url).path), ".git")
results_file_path = os.path.join(root, "results", friendly_name)
repo = _checkout_if_not_present(repo_url, repos_root)
printerr("Scanning {name} for secrets".format(name=friendly_name))
_find_secrets(repo, results_file_path)
return
def run_extension(self, name, extension, repo, uri, db):
# Trim off the ordering numeral before printing
if self.hard_order:
name = name[1:]
printout("Executing extension %s", (name, ))
try:
extension.run(repo, uri, db)
except ExtensionRunError, e:
printerr("Error running extension %s: %s", (name, str(e)))
return False
def run_extension(self, name, extension, repo, uri, db):
# Trim off the ordering numeral before printing
if self.hard_order:
name = name[1:]
printout("Executing extension %s", (name,))
try:
extension.run(repo, uri, db)
except ExtensionRunError, e:
printerr("Error running extension %s: %s", (name, str(e)))
return False
def backout_extension(self, name, extension, repo, uri, db):
# Trim off the ordering numeral before printing
if self.hard_order:
name = name[1:]
printout("Backing out extension %s", (name,))
try:
extension.backout(repo, uri, db)
except (ExtensionRunError, ExtensionBackoutError), e:
printerr("Error backing out extension %s: %s", (name, str(e)))
return False
def main():
"""
"""
# Note: Default values for options are defined on
# configuration module
usage = 'Usage: %prog [options]'
try:
Config.set_config_options(usage)
except (ErrorLoadingConfig, InvalidConfig), e:
printerr(str(e))
sys.exit(2)
def main():
"""
"""
# Note: Default values for options are defined in
# configuration module
usage = 'Usage: %prog [options]'
try:
Config.set_config_options(usage)
except (ErrorLoadingConfig, InvalidConfig), e:
printerr(str(e))
sys.exit(2)
def backout_extension(self, name, extension, repo, uri, db):
# Trim off the ordering numeral before printing
if self.hard_order:
name = name[1:]
printout("Backing out extension %s", (name, ))
try:
extension.backout(repo, uri, db)
except (ExtensionRunError, ExtensionBackoutError), e:
printerr("Error backing out extension %s: %s", (name, str(e)))
return False
def _checkout_if_not_present(repo_url, repos_root):
"""Returns the existing repository if it exists,
otherwises puts a fresh clone in the right place
"""
repo_name = os.path.basename(urlparse(repo_url).path)
local_repo_path = os.path.join(repos_root, repo_name)
if os.path.isdir(local_repo_path):
return Repo(local_repo_path)
else:
printerr("Cloning new repo {repo} into {location}".format(repo=repo_name, location=local_repo_path))
return Repo.clone_from(repo_url, local_repo_path)
def update_organisation_repos(organisation, token, filename):
"""Searches a Github organisation for repositories that can be
cloned, writes discovered repo URLs to a local file."""
github_client = Github(token)
org = github_client.get_organization(organisation)
printerr("Looking up {org} repositories (may take some time)".format(org=organisation))
repos = org.get_repos("public")
clone_urls = _clone_urls(repos)
with open(filename, "w") as f:
_write_repo_list(clone_urls, f)
printerr("Wrote list of repositories to {location}".format(location=filename))
return
def create_parser_from_repository (repo):
if repo.get_type () == 'cvs':
p = CVSParser ()
elif repo.get_type () == 'svn':
p = SVNParser ()
elif repo.get_type () == 'git':
p = GitParser ()
elif repo.get_type () == 'bzr':
p = BzrParser ()
else:
printerr ("Error: Unsupported repository type: %s", (repo.get_type (),))
return None
return p
def create_parser_from_repository(repo):
if repo.get_type() == 'cvs':
p = CVSParser()
elif repo.get_type() == 'svn':
p = SVNParser()
elif repo.get_type() == 'git':
p = GitParser()
elif repo.get_type() == 'bzr':
p = BzrParser()
else:
printerr("Error: Unsupported repository type: %s", (repo.get_type(), ))
return None
return p
def main():
"""
"""
# Note: Default values for options are defined in
# configuration module
usage = 'Usage: %prog [options]'
# This s
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(message)s')
try:
Config.set_config_options(usage)
except (ErrorLoadingConfig, InvalidConfig), e:
printerr(str(e))
sys.exit(2)
def __init__(self, db, repo, uri, connection):
self.db = db
self.repo = repo
self.uri = uri
self.connection = connection
cursor = self.connection.cursor()
try:
self.repo_id = get_repo_id(self.uri, cursor, self.db)
except RepoNotFound:
# Repository not found, probably already backed out
printerr("Repository not found, is it in the database?")
self.repo_id = None
finally:
cursor.close()
def main(argv):
# Short (one letter) options. Those requiring argument followed by :
short_opts = "hVgqbnf:l:s:u:p:d:H:"
# Long options (all started by --). Those requiring argument followed by =
long_opts = [
"help", "version", "debug", "quiet", "profile", "config-file=",
"repo-logfile=", "save-logfile=", "no-parse", "db-user="******"db-password="******"db-hostname=", "db-database=", "db-driver=",
"extensions=", "hard-order", "metrics-all", "metrics-noerr",
"no-content", "branch=", "backout", "low-memory", "count-types=",
"analyze-merges", "hb-ignore-comments", "bugfixregexes=",
"bugfixregexes-case=", "dot-dir="
]
# Default options
debug = None
quiet = None
profile = None
configfile = None
no_parse = None
user = '******'
passwd = 'xybsnny'
hostname = None
database = 'eclipse'
driver = None
logfile = None
save_logfile = None
extensions = None
metrics_all = None
metrics_noerr = None
hard_order = None
low_memory = None
no_content = None
branch = None
backout = None
count_types = None
analyze_merges = None
hb_ignore_comments = None
bug_fix_regexes = None
bug_fix_regexes_case_sensitive = None
dot_dir = None
try:
opts, args = getopt.getopt(argv, short_opts, long_opts)
except getopt.GetoptError, e:
printerr(str(e))
return 1
def main(argv):
# Short (one letter) options. Those requiring argument followed by :
short_opts = "hVgqbnf:l:s:u:p:d:H:"
# Long options (all started by --). Those requiring argument followed by =
long_opts = ["help", "version", "debug", "quiet", "profile",
"config-file=", "repo-logfile=", "save-logfile=",
"no-parse", "db-user="******"db-password="******"db-hostname=",
"db-database=", "db-driver=", "extensions=", "hard-order",
"metrics-all", "metrics-noerr", "no-content", "branch=",
"backout", "low-memory", "count-types=", "analyze-merges",
"hb-ignore-comments", "bugfixregexes=", "bugfixregexes-case=",
"dot-dir="]
# Default options
debug = None
quiet = None
profile = None
configfile = None
no_parse = None
user = None
passwd = None
hostname = None
database = None
driver = None
logfile = None
save_logfile = None
extensions = None
metrics_all = None
metrics_noerr = None
hard_order = None
low_memory = None
no_content = None
branch = None
backout = None
count_types = None
analyze_merges = None
hb_ignore_comments = None
bug_fix_regexes = None
bug_fix_regexes_case_sensitive = None
dot_dir = None
try:
opts, args = getopt.getopt(argv, short_opts, long_opts)
except getopt.GetoptError, e:
printerr(str(e))
return 1
def blame(filename, args):
repo, uri, out, opts = args
filename = filename.strip(" \n")
if filename[-1] == "/":
return
p = create_parser(repo.get_type(), filename)
p.set_output_device(out)
def feed(line, p):
p.feed(line)
wid = repo.add_watch(BLAME, feed, p)
try:
repo.blame(os.path.join(uri, filename), rev=opts.revision, mc=not opts.fast)
except RepositoryCommandError, e:
printerr("Error getting blame information of path '%s': %s", (filename, e.error))
def execute_statement(statement, parameters, write_cursor, db, error_message, exception=Exception):
if isinstance(db, SqliteDatabase):
import sqlite3.dbapi2
try:
write_cursor.execute(statement, parameters)
except sqlite3.dbapi2.OperationalError as e:
printerr(error_message + ": %s", (e,))
except Exception as e:
raise exception(e)
elif isinstance(db, MysqlDatabase):
import _mysql_exceptions
try:
write_cursor.execute(statement, parameters)
except _mysql_exceptions.OperationalError as e:
printerr(error_message + ": %s", (e,))
except Exception as e:
raise exception(e)
def main(argv):
# Short (one letter) options. Those requiring argument followed by :
short_opts = "hVgqbnf:l:s:u:p:d:H:"
# Long options (all started by --). Those requiring argument followed by =
long_opts = [
"help", "version", "debug", "quiet", "profile", "config-file=",
"repo-logfile=", "save-logfile=", "no-parse", "db-user="******"db-password="******"db-hostname=", "db-database=", "db-driver=",
"extensions=", "hard-order", "metrics-all", "metrics-noerr",
"no-content", "branch=", "backout", "low-memory", "count-types="
]
# Default options
debug = None
quiet = None
profile = None
configfile = None
no_parse = None
user = None
passwd = None
hostname = None
database = None
driver = None
logfile = None
save_logfile = None
extensions = None
metrics_all = None
metrics_noerr = None
hard_order = None
low_memory = None
no_content = None
branch = None
backout = None
count_types = None
try:
opts, args = getopt.getopt(argv, short_opts, long_opts)
except getopt.GetoptError, e:
printerr(str(e))
return 1
def blame(filename, args):
repo, uri, out, opts = args
filename = filename.strip(' \n')
if filename[-1] == '/':
return
p = create_parser(repo.get_type(), filename)
p.set_output_device(out)
def feed(line, p):
p.feed(line)
wid = repo.add_watch(BLAME, feed, p)
try:
repo.blame(os.path.join(uri, filename),
rev=opts.revision,
mc=not opts.fast,
ignore_whitespaces=True)
except RepositoryCommandError, e:
printerr("Error getting blame information of path '%s': %s",
(filename, e.error))
def add_instance_to_group(inventory, group, server, outputs):
if len(SERVER_FILTER) > 0:
if server['_name'] in SERVER_FILTER:
if server['_name'] not in servers_already_filtered:
utils.printerr('Including server in inventory: %s' %
server['_name'])
servers_already_filtered[server['_name']] = True
else:
if server['_name'] not in servers_already_filtered:
utils.printerr('Omitting server from inventory: %s' %
server['_name'])
servers_already_filtered[server['_name']] = True
return inventory
if not group in inventory:
inventory[group] = {
'hosts': [],
'vars': {
'ansible_ssh_user': '******',
'projectname_ssh_keys': read_authorized_ssh_keys(),
}
}
projectname_hostnames = (
outputs.get('projectname_hostnames_proxied', []) +
outputs.get('projectname_hostnames_unproxied', []))
projectname_hostnames_full = map(
lambda h: (h + '.projectname.com').replace('@.', ''),
projectname_hostnames)
inventory[group]['hosts'].append(server['_name'])
inventory['_meta']['hostvars'][server['_name']] = {
'ansible_host': server['_public_ip'],
'projectname_users': outputs.get('projectname_users', []),
'projectname_hostnames': projectname_hostnames,
'projectname_hostnames_full': projectname_hostnames_full,
}
return inventory
return True
def backout_extension(self, name, extension, repo, uri, db):
# Trim off the ordering numeral before printing
if self.hard_order:
name = name[1:]
printout("Backing out extension %s", (name, ))
try:
extension.backout(repo, uri, db)
except (ExtensionRunError, ExtensionBackoutError), e:
printerr("Error backing out extension %s: %s", (name, str(e)))
return False
except NotImplementedError, e:
printerr("Extension %s doesn't have a backout function", (name, ))
return False
return True
def backout_extensions(self, repo, uri, db):
printdbg("Called backout extensions")
self.run_extensions(repo, uri, db, backout=True)
def run_extensions(self, repo, uri, db, backout=False):
done = []
list = sorted(self.exts)
for name, extension in [(ext, self.exts[ext]()) for ext in list]:
if name in done:
continue
"""
"""
# Note: Default values for options are defined in
# configuration module
usage = 'Usage: %prog [options]'
try:
Config.set_config_options(usage)
except (ErrorLoadingConfig, InvalidConfig), e:
printerr(str(e))
sys.exit(2)
try:
backend = Backend.create_backend(Config.backend)
except ImportError, e:
printerr("Backend ''" + Config.backend + "'' doesn't exist. " + str(e))
sys.exit(2)
printdbg("Bicho object created, options and backend initialized")
backend.run()
if Config.logtable:
try:
ilogger = IssueLogger.create_logger(Config.backend)
except ImportError, e:
printerr("Logger ''" + Config.backend + "'' doesn't exist. " + str(e))
sys.exit(2)
printdbg("Bicho logger object created")
ilogger.run()
if __name__ == "__main__":
main()
if (len(sys.argv) < 2):
exit(
"You must provide the minimum inhibition percentage at 1 micromol as argument."
)
min_inhibition_percentage = int(sys.argv[1])
assay = new_client.assay
# Filtramos los assay para quedarnos solo con los GSK
assay_results = assay.filter(description__contains='GSK_PKIS',
assay_type='B',
relationship_type='D').only(
['document_chembl_id'])
printerr('Number of Assay Search Results: ' + str(len(assay_results)))
# Vemos cuantos fármacos diferentes tenemos
doc_list = list()
for e in assay_results:
doc_list.append(e['document_chembl_id'])
doc_list = list(set(doc_list))
printerr('Documents found in search: ' + str(doc_list))
printerr('Filtering activities with doc id in previous query...')
activity = new_client.activity
# Hacemos otra búsqueda para obtener las interacciones
def start(self, new_line_cb, user_data=None):
if self.logfile is not None:
try:
self._read_from_logfile(new_line_cb, user_data)
except IOError, e:
printerr(str(e))
def _read_from_logfile(self, new_line_cb, user_data):
try:
f = open(self.logfile, 'r')
except IOError, e:
printerr(str(e))
return
from post_processing.logtable import IssuesLog
def main():
"""
"""
# Note: Default values for options are defined on
# configuration module
usage = 'Usage: %prog [options]'
try:
Config.set_config_options(usage)
except (ErrorLoadingConfig, InvalidConfig), e:
printerr(str(e))
sys.exit(2)
try:
backend = Backend.create_backend (Config.backend)
except ImportError, e:
printerr("Backend ''" + Config.backend + "'' not exists. " + str(e))
sys.exit(2)
printdbg ("Bicho object created, options and backend initialized")
backend.run()
il = IssuesLog(Config.backend)
il.run()
if __name__ == "__main__":
main()
def main (args):
parser = OptionParser (usage='%prog [ options ... ] URI [ FILES ]',
description='Analyze repository modifications',
version=VERSION)
parser.disable_interspersed_args()
parser.add_option ('-g', '--debug', dest='debug',
action="store_true", default=False,
help="Run in debug mode")
parser.add_option ('-c', '--config-file', dest='config_file',
metavar='FILE',
help="Use a custom configuration file")
parser.add_option ('-r', '--revision', dest='revision',
metavar='REV',
help='Revision to analyze (HEAD)')
parser.add_option ('-f', '--fast', dest='fast',
action="store_true", default=False,
help="Run faster but moves and copies are not detected")
parser.add_option ('-o', '--output', dest='output',
default = 'text',
help='Output type [text|db|xml|csv] (%default)')
add_outputs_options (parser)
# Save default values and pass an emtpy Values object to
# parser_args, so that default values are not set. We need it
# to know whether a value has been provided by the user or not
# After parsing the command line we complete the config options
# with the default values for the options that have not been set
# by the parser or by a config file
defaults = parser.get_default_values ()
options, args = parser.parse_args (args, values = Values())
try:
config = Config (options.config_file)
except AttributeError:
config = Config ()
config.update (options.__dict__)
config.add (defaults.__dict__)
if not args:
parser.error("missing required repository URI")
return 1
parser.destroy ()
if config.debug:
import repositoryhandler.backends
repositoryhandler.backends.DEBUG = True
uri = args[0]
files = args[1:]
files_from_stdin = (files and files[0] == '-')
# Create repository
path = uri_to_filename (uri)
if path is not None:
try:
repo = create_repository_from_path (path)
except RepositoryUnknownError:
printerr ("Path %s doesn't seem to point to a repository supported by guilty", (path,))
return 1
except Exception, e:
printerr ("Unknown error creating repository for path %s (%s)", (path, str (e)))
return 1
uri = repo.get_uri_for_path (path)
except RepositoryUnknownError:
printerr("Path %s doesn't seem to point to a repository " + \
"supported by cvsanaly", (path,))
sys.exit(1)
except Exception, e:
printerr("Unknown error creating repository for path %s (%s)",
(path, str(e)))
sys.exit(1)
uri = repo.get_uri_for_path(path)
return (uri, repo)
else:
uri = uri.strip('/')
repo = create_repository('svn', uri)
# Check uri actually points to a valid svn repo
if repo.get_last_revision(uri) is None:
printerr("URI %s doesn't seem to point to a valid svn repository",
(uri,))
sys.exit(1)
else:
return (uri, repo)
def _get_parser_from_repository(repo):
return create_parser_from_repository(repo)
def _get_extensions_manager(extensions, hard_order=False):
try:
printdbg("Starting ExtensionsManager")
emg = ExtensionsManager(extensions,
hard_order=hard_order)
return emg
except InvalidExtension, e:
printerr("Invalid extension %s", (e.name,))
if os.path.isfile(uri):
if logfile_is_svn(uri):
p = SVNParser()
elif logfile_is_cvs(uri):
p = CVSParser()
elif log_file_is_git(uri):
p = GitParser()
elif log_file_is_bzr(uri):
p = BzrParser()
assert p is not None
return p
printerr("Error: path %s doesn't look like a valid log file", (uri, ))
return None
def create_parser_from_repository(repo):
if repo.get_type() == 'cvs':
p = CVSParser()
elif repo.get_type() == 'svn':
p = SVNParser()
elif repo.get_type() == 'git':
p = GitParser()
elif repo.get_type() == 'bzr':
p = BzrParser()
else:
printerr("Error: Unsupported repository type: %s", (repo.get_type(), ))
return None
except RepositoryUnknownError:
printerr("Path %s doesn't seem to point to a repository " + \
"supported by cvsanaly", (path,))
sys.exit(1)
except Exception, e:
printerr("Unknown error creating repository for path %s (%s)",
(path, str(e)))
sys.exit(1)
uri = repo.get_uri_for_path(path)
return (uri, repo)
else:
uri = uri.strip('/')
repo = create_repository('svn', uri)
# Check uri actually points to a valid svn repo
if repo.get_last_revision(uri) is None:
printerr("URI %s doesn't seem to point to a valid svn repository",
(uri, ))
sys.exit(1)
else:
return (uri, repo)
def _get_parser_from_repository(repo):
return create_parser_from_repository(repo)
def _get_extensions_manager(extensions, hard_order=False):
try:
printdbg("Starting ExtensionsManager")
emg = ExtensionsManager(extensions, hard_order=hard_order)
return emg
except InvalidExtension, e:
def main(args):
parser = OptionParser(usage='%prog [ options ... ] URI [ FILES ]',
description='Analyze repository modifications',
version=VERSION)
parser.disable_interspersed_args()
parser.add_option('-g',
'--debug',
dest='debug',
action="store_true",
default=False,
help="Run in debug mode")
parser.add_option('-c',
'--config-file',
dest='config_file',
metavar='FILE',
help="Use a custom configuration file")
parser.add_option('-r',
'--revision',
dest='revision',
metavar='REV',
help='Revision to analyze (HEAD)')
parser.add_option('-f',
'--fast',
dest='fast',
action="store_true",
default=False,
help="Run faster but moves and copies are not detected")
parser.add_option('-o',
'--output',
dest='output',
default='text',
help='Output type [text|db|xml|csv] (%default)')
add_outputs_options(parser)
# Save default values and pass an emtpy Values object to
# parser_args, so that default values are not set. We need it
# to know whether a value has been provided by the user or not
# After parsing the command line we complete the config options
# with the default values for the options that have not been set
# by the parser or by a config file
defaults = parser.get_default_values()
options, args = parser.parse_args(args, values=Values())
try:
config = Config(options.config_file)
except AttributeError:
config = Config()
config.update(options.__dict__)
config.add(defaults.__dict__)
if not args:
parser.error("missing required repository URI")
return 1
parser.destroy()
if config.debug:
import repositoryhandler.backends
repositoryhandler.backends.DEBUG = True
uri = args[0]
files = args[1:]
files_from_stdin = (files and files[0] == '-')
# Create repository
path = uri_to_filename(uri)
if path is not None:
try:
repo = create_repository_from_path(path)
except RepositoryUnknownError:
printerr(
"Path %s doesn't seem to point to a repository supported by guilty",
(path, ))
return 1
except Exception, e:
printerr("Unknown error creating repository for path %s (%s)",
(path, str(e)))
return 1
uri = repo.get_uri_for_path(path)
except RepositoryUnknownError:
printerr(
"Path %s doesn't seem to point to a repository supported by guilty",
(path, ))
return 1
except Exception, e:
printerr("Unknown error creating repository for path %s (%s)",
(path, str(e)))
return 1
uri = repo.get_uri_for_path(path)
else:
uri = uri.strip('/')
repo = create_repository('svn', uri)
# Check uri actually points to a valid svn repo
if repo.get_last_revision(uri) is None:
printerr("URI %s doesn't seem to point to a valid svn repository",
(uri, ))
return 1
# Check we have a parser for the given repo
try:
p = create_parser(repo.get_type(), 'foo')
except ParserUnknownError:
printerr("%s repositories are not supported by guilty (yet)",
(repo.get_type(), ))
return 1
except Exception, e:
printerr("Unknown error creating parser for repository %s (%s)",
(repo.get_uri(), str(e)))
return 1
del p
if path is not None:
try:
repo = create_repository_from_path (path)
except RepositoryUnknownError:
printerr ("Path %s doesn't seem to point to a repository supported by guilty", (path,))
return 1
except Exception, e:
printerr ("Unknown error creating repository for path %s (%s)", (path, str (e)))
return 1
uri = repo.get_uri_for_path (path)
else:
uri = uri.strip ('/')
repo = create_repository ('svn', uri)
# Check uri actually points to a valid svn repo
if repo.get_last_revision (uri) is None:
printerr ("URI %s doesn't seem to point to a valid svn repository", (uri,))
return 1
# Check we have a parser for the given repo
try:
p = create_parser (repo.get_type (), 'foo')
except ParserUnknownError:
printerr ("%s repositories are not supported by guilty (yet)", (repo.get_type (),))
return 1
except Exception, e:
printerr ("Unknown error creating parser for repository %s (%s)", (repo.get_uri (), str (e)))
return 1
del p
try:
out = create_output_device (config.output)
def tree2set(t):
global _set
_set = set()
def func(node):
_set.add(node.get_str4conceptnet())
t.postorder_traverse(func)
return _set
word_embedding = WordEmbedding('./data/wordembedding')
snli = SNLI('./data/')
printerr("Before trim word embedding, " +
str(word_embedding.embeddings.size(0)) + " words")
word_embedding.trim_by_counts(snli.word_counts)
printerr("After trim word embedding, " +
str(word_embedding.embeddings.size(0)) + " words")
word_embedding.extend_by_counts(snli.train_word_counts)
printerr("After adding training words, " +
str(word_embedding.embeddings.size(0)) + " words")
phrases = set()
print 'Gathering phrases in train data...'
for data in snli.train:
phrases = phrases | tree2set(data['p_tree'])
phrases = phrases | tree2set(data['h_tree'])
print 'done'
printerr('Gathering phrases in dev data...')
for data in snli.dev:
elif opt in ("--git-ref", ):
gitref = value
if len(args) <= 0:
uri = os.getcwd()
else:
uri = args[0]
config = Config()
try:
if configfile is not None:
config.load_from_file(configfile)
else:
config.load()
except ErrorLoadingConfig, e:
printerr(e.message)
return 1
if debug is not None:
config.debug = debug
if quiet is not None:
config.quiet = quiet
if profile is not None:
config.profile = profile
if logfile is not None:
config.repo_logfile = logfile
if save_logfile is not None:
config.save_logfile = save_logfile
if writable_path is not None:
config.writable_path = writable_path
if no_parse is not None:
return True
def backout_extension(self, name, extension, repo, uri, db):
# Trim off the ordering numeral before printing
if self.hard_order:
name = name[1:]
printout("Backing out extension %s", (name,))
try:
extension.backout(repo, uri, db)
except (ExtensionRunError, ExtensionBackoutError), e:
printerr("Error backing out extension %s: %s", (name, str(e)))
return False
except NotImplementedError, e:
printerr("Extension %s doesn't have a backout function", (name,))
return False
return True
def backout_extensions(self, repo, uri, db):
printdbg("Called backout extensions")
self.run_extensions(repo, uri, db, backout=True)
def run_extensions(self, repo, uri, db, backout=False):
done = []
list = sorted(self.exts)
for name, extension in [(ext, self.exts[ext]()) for ext in list]:
if name in done:
continue
def main():
"""
"""
# Note: Default values for options are defined on
# configuration module
usage = 'Usage: %prog [options] <type> <URI>'
parser = OptionParser(usage=usage, description=info.DESCRIPTION,
version=info.VERSION)
# General options
parser.add_option('--cfg', dest='cfgfile', default=None,
help='Use a custom configuration file')
parser.add_option('-g', '--debug', action='store_true', dest='debug',
help='Enable debug mode')
parser.add_option('-d', '--delay', type='int', dest='delay',
help='Delay in seconds betweeen petitions to avoid been banned')
parser.add_option('-i', '--input', choices=['url', 'db'],
dest='input', help='Input format')
parser.add_option('-o', '--output', choices=['db'],
dest='output', help='Output format')
parser.add_option('-p', '--path', dest='path',
help='Path where downloaded URLs will be stored')
# Options for output database
group = OptionGroup(parser, 'Output database specific options')
group.add_option('--db-driver-out',
choices=['sqlite','mysql','postgresql'],
dest='db_driver_out', help='Output database driver')
group.add_option('--db-user-out', dest='db_user_out',
help='Database user name')
group.add_option('--db-password-out', dest='db_password_out',
help='Database user password')
group.add_option('--db-hostname-out', dest='db_hostname_out',
help='Name of the host where database server is running')
group.add_option('--db-port-out', dest='db_port_out',
help='Port of the host where database server is running')
group.add_option('--db-database-out', dest='db_database_out',
help='Output database name')
parser.add_option_group(group)
# Options for input database
group = OptionGroup(parser, 'Input database specific options')
group.add_option('--db-driver-in',
choices=['sqlite', 'mysql', 'postgresql'],
dest='db_driver_in', help='Input database driver')
group.add_option('--db-user-in', dest='db_user_in',
help='Database user name')
group.add_option('--db-password-in', dest='db_password_in',
help='Database user password')
group.add_option('--db-hostname-in', dest='db_hostname_in',
help='Name of the host where database server is running')
group.add_option('--db-port-in', dest='db_port_in',
help='Port of the host where database server is running')
group.add_option('--db-database-in', dest='db_database_in',
help='Input database name')
parser.add_option_group(group)
(options, args) = parser.parse_args()
if len(args) != 2:
parser.error('Incorrect number of arguments')
try:
config = set_config_options(args[0], args[1], options)
check_config(config)
except (ErrorLoadingConfig, InvalidConfig), e:
printerr(str(e))
sys.exit(2)
def history(root, message, notify, config):
results_dir = os.path.join(root, "results")
repo = Repo(results_dir)
repo.git.add(A=True)
contents = repo.git.diff(cached=True)
if contents:
printerr("Changes to secrets detected")
if notify:
if config.notification_email:
printerr("Notifying {address} of changes".format(
address=config.notification_email))
if config.pgp_key is None:
message = strip_ansi_sequences(contents)
else:
pgp_key = parse_key(config.pgp_key)
unencrypted = strip_ansi_sequences(contents)
message = "Dupin findings encrypted using key '{identity}'\nFingerprint: {fingerprint}\n\n{data}"\
.format(identity=pgp_key.userids[0].name,
fingerprint=pgp_key.fingerprint,
data=encrypt(unencrypted, pgp_key))
if config.smtp_configured():
send_email_properly(message, config.notification_email,
config.smtp_from, config.smtp_host,
config.smtp_username,
config.smtp_password)
else:
printerr(
"No SMTP configuration discovered, attempting to send email anyway"
)
send_email_quick(message, config.notification_email)
else:
printerr("Error: Missing notification email configuration")
sys.exit(1)
else:
print(contents)
printerr(
"Comitting log to repo at {location}".format(location=results_dir))
repo.index.commit(message)
else:
printerr("No new secrets found")
return
if os.path.isfile (uri):
if logfile_is_svn (uri):
p = SVNParser ()
elif logfile_is_cvs (uri):
p = CVSParser ()
elif log_file_is_git (uri):
p = GitParser ()
elif log_file_is_bzr (uri):
p = BzrParser ()
assert p is not None
return p
printerr ("Error: path %s doesn't look like a valid log file", (uri,))
return None
def create_parser_from_repository (repo):
if repo.get_type () == 'cvs':
p = CVSParser ()
elif repo.get_type () == 'svn':
p = SVNParser ()
elif repo.get_type () == 'git':
p = GitParser ()
elif repo.get_type () == 'bzr':
p = BzrParser ()
else:
printerr ("Error: Unsupported repository type: %s", (repo.get_type (),))
return None
elif opt in ("--metrics-noerr", ):
metrics_noerr = True
if len (args) <= 0:
uri = os.getcwd ()
else:
uri = args[0]
config = Config ()
try:
if configfile is not None:
config.load_from_file (configfile)
else:
config.load ()
except ErrorLoadingConfig, e:
printerr (e.message)
return 1
if debug is not None:
config.debug = debug
if quiet is not None:
config.quiet = quiet
if profile is not None:
config.profile = profile
if logfile is not None:
config.repo_logfile = logfile
if save_logfile is not None:
config.save_logfile = save_logfile
if no_parse is not None:
config.no_parse = no_parse
if driver is not None:
