def main(argv):
# Setup command line arguments.
parser = ModeArgsParser()
setupArgs(parser)
try:
flow = parser.parseArgs(argv[1], argv[2:])
except:
parser.printHelp(argv[0])
sys.exit()
# Make config point to the absolute path.
full_path = os.path.abspath(argv[0])
last_slash_index = full_path.rfind("/")
base_dir = full_path[0:last_slash_index]
global CONFIG_PATH
CONFIG_PATH = base_dir + "/" + CONFIG_PATH
# Set debugging.
try:
getArg(flow, "d", "debug")
Configurator.setDebugging(True)
except ArgException as err:
# # Debugging was not specified.
pass
if flow[parser.KEY_MODE] != ARGS_CONFIG:
# Load config.
Configurator.load(CONFIG_PATH)
if flow[parser.KEY_MODE] == ARGS_HELP:
parser.printHelp(argv[0])
sys.exit()
elif flow[parser.KEY_MODE] == ARGS_COMBINE_STATISTICS:
combineStatsFiles(flow)
elif flow[parser.KEY_MODE] == ARGS_CONSOLE:
id = 1
try:
id = getArg(flow, "id")
except:
pass
heap_size = Configurator.getHeapVal()
Neo4jHelper().setHeapsize(heap_size[0], heap_size[1])
Neo4jHelper().startConsole(os.path.abspath(flow["in"]), str(id))
elif flow[parser.KEY_MODE] == ARGS_SEARCH:
startSearchMode(flow)
elif flow[parser.KEY_MODE] == ARGS_CONTINUOUS_SEARCH:
startSearchMode(flow, continuous=True)
elif flow[parser.KEY_MODE] == ARGS_CONFIG:
Configurator().setupConfig(CONFIG_PATH, base_dir,
getArg(flow, "p", "path"))
elif flow[parser.KEY_MODE] == ARGS_PRINT_STATS:
printStats(flow)
elif flow[parser.KEY_MODE] == ARGS_PRINT_RESULTS:
printResults(flow)
else:
parser.printHelp(argv[0])
sys.exit()
def analyseDataHelper(args):
return Neo4jHelper.analyseData(args)
def analyseDataHelper(args):
return Neo4jHelper.analyseData(args)
def startSearchMode(flow, continuous=False):
flow["in"] = os.path.abspath(flow["in"])
if not os.path.exists(flow["in"]):
print "Given path (-in) does not exist."
sys.exit()
level = 0
multithreads = 0
neo4j_helper = Neo4jHelper()
heap_size = Configurator.getHeapVal()
neo4j_helper.setHeapsize(heap_size[0], heap_size[1])
if continuous:
# Continuous mode was specified, so read the config file
try:
stats_path = getArg(flow, "s", "statsfile")
except:
stats_path = DEFAULT_STATS_PATH
lock = multiprocessing.Lock()
shared_data = SharedData(stats_path, lock, in_path=flow["in"])
in_path = shared_data.getInPath()
if in_path != flow["in"]:
print(
"The given path with \"-in\" is not the path, "
"which was used before.")
_ = raw_input("Ctrl-C or Ctrl-D to abort. "
"Press any key to continue")
shared_data.setInPath(flow["in"])
Neo4jHelper.setStatisticsObj(shared_data)
try:
multithreads = int(getArg(flow, "m", "multithreading"))
except:
pass
code_path = getArg(flow, "q", "queries")
code = []
# Read given query.
if os.path.isfile(code_path):
with open(code_path, "r") as fh:
code.append(QueryFile(code_path, fh.read()))
elif os.path.isdir(code_path):
# Given path is a directory - get all files recursively inside the
# directory.
for path, _, files in os.walk(code_path):
for name in files:
file_path = os.path.join(path, name)
with open(file_path, "r") as fh:
code.append(QueryFile(file_path, fh.read()))
if not code:
# Did not find any file recursively inside 'code_path'.
print "Query-path (-q/--queries) does not contain any files."
sys.exit()
else:
# Path does not exist
print "Query-Path (-q/--queries) does not exist."
sys.exit()
try:
level = int(getArg(flow, "l", "level"))
except ArgException:
# Parameter "-l/--level" was not specified.
pass
if level == 0:
# Analyse specified file/files in specified directory with given
# gremlin query/queries.
Neo4jHelper.analyseData((code, flow["in"], 1))
else:
# Analyse folders 'level' levels under specified path.
try:
# Get the root directory of every project in a generator.
path_generator = getRootDirectories(flow["in"], level)
if continuous:
# Check if given in-path is the same as the one in the
# given stats file.
if in_path == flow["in"]:
# Skip generator elements if they were already
# analysed before.
projects_analysed_count = shared_data.getProjectsCount()
skipGeneratorElements(path_generator,
projects_analysed_count)
except Exception as err:
print "An exception occured: %s" % err
sys.exit()
projects_analysed = 0
if multithreads > 1:
# Multithreading was specified.
process_number_generator = ProcessIdGenerator()
# Start a lazy pool of processes.
pool = LazyMP().poolImapUnordered(
analyseDataHelper,
itertools.izip(
itertools.repeat(code),
path_generator,
process_number_generator.getGenerator([1, 2, 3, 4]),
), multithreads, process_number_generator)
# And let them work.
try:
while True:
# Let multiprocessing pool process all arguments.
pool.next()
projects_analysed += 1
except Exception as err:
# Done
print err
pass
else:
# No multithreading.
for path in path_generator:
neo4j_helper.analyseData((code, path, 1))
projects_analysed += 1
if projects_analysed == 0:
print "No project analysed for path: '%s'" % (flow["in"])
def startSearchMode(flow, continuous=False):
flow["in"] = os.path.abspath(flow["in"])
if not os.path.exists(flow["in"]):
print "Given path (-in) does not exist."
sys.exit()
level = 0
multithreads = 0
neo4j_helper = Neo4jHelper()
heap_size = Configurator.getHeapVal()
neo4j_helper.setHeapsize(heap_size[0], heap_size[1])
if continuous:
# Continuous mode was specified, so read the config file
try:
stats_path = getArg(flow, "s", "statsfile")
except:
stats_path = DEFAULT_STATS_PATH
lock = multiprocessing.Lock()
shared_data = SharedData(stats_path, lock, in_path=flow["in"])
in_path = shared_data.getInPath()
if in_path != flow["in"]:
print (
"The given path with \"-in\" is not the path, "
"which was used before."
)
_ = raw_input("Ctrl-C or Ctrl-D to abort. "
"Press any key to continue")
shared_data.setInPath(flow["in"])
Neo4jHelper.setStatisticsObj(shared_data)
try:
multithreads = int(getArg(flow, "m", "multithreading"))
except:
pass
code_path = getArg(flow, "q", "queries")
code = []
# Read given query.
if os.path.isfile(code_path):
with open(code_path, "r") as fh:
code.append(
QueryFile(code_path, fh.read())
)
elif os.path.isdir(code_path):
# Given path is a directory - get all files recursively inside the
# directory.
for path, _, files in os.walk(code_path):
for name in files:
file_path = os.path.join(path, name)
with open(file_path, "r") as fh:
code.append(
QueryFile(file_path, fh.read())
)
if not code:
# Did not find any file recursively inside 'code_path'.
print "Query-path (-q/--queries) does not contain any files."
sys.exit()
else:
# Path does not exist
print "Query-Path (-q/--queries) does not exist."
sys.exit()
try:
level = int(getArg(flow, "l", "level"))
except ArgException:
# Parameter "-l/--level" was not specified.
pass
if level == 0:
# Analyse specified file/files in specified directory with given
# gremlin query/queries.
Neo4jHelper.analyseData((
code, flow["in"], 1
))
else:
# Analyse folders 'level' levels under specified path.
try:
# Get the root directory of every project in a generator.
path_generator = getRootDirectories(flow["in"], level)
if continuous:
# Check if given in-path is the same as the one in the
# given stats file.
if in_path == flow["in"]:
# Skip generator elements if they were already
# analysed before.
projects_analysed_count = shared_data.getProjectsCount()
skipGeneratorElements(path_generator, projects_analysed_count)
except Exception as err:
print "An exception occured: %s" % err
sys.exit()
projects_analysed = 0
if multithreads > 1:
# Multithreading was specified.
process_number_generator = ProcessIdGenerator()
# Start a lazy pool of processes.
pool = LazyMP().poolImapUnordered(
analyseDataHelper, itertools.izip(
itertools.repeat(code), path_generator,
process_number_generator.getGenerator([1,2,3,4]),
),
multithreads,
process_number_generator
)
# And let them work.
try:
while True:
# Let multiprocessing pool process all arguments.
pool.next()
projects_analysed += 1
except Exception as err:
# Done
print err
pass
else:
# No multithreading.
for path in path_generator:
neo4j_helper.analyseData((
code, path, 1
))
projects_analysed += 1
if projects_analysed == 0:
print "No project analysed for path: '%s'" %(
flow["in"]
)
