def main(args):
fix_encoding()
supported_engines = initialize_engines()
if not args:
raise SystemExit(
"./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n"
"available engines: %s" % (','.join(supported_engines)))
elif args[0] == "--capabilities":
displayCapabilities(supported_engines)
return
elif len(args) < 3:
raise SystemExit(
"./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n"
"available engines: %s" % (','.join(supported_engines)))
# get only unique engines with set
engines_list = set(e.lower() for e in args[0].strip().split(','))
if 'all' in engines_list:
engines_list = supported_engines
else:
# discard un-supported engines
engines_list = [
engine for engine in engines_list if engine in supported_engines
]
if not engines_list:
# engine list is empty. Nothing to do here
return
cat = args[1].lower()
if cat not in CATEGORIES:
raise SystemExit(" - ".join(('Invalid category', cat)))
what = urllib.quote(' '.join(args[2:]))
if THREADED:
# child process spawning is controlled min(number of searches, number of cpu)
pool = Pool(min(len(engines_list), MAX_THREADS))
pool.map(run_search,
([globals()[engine], what, cat] for engine in engines_list))
else:
map(run_search,
([globals()[engine], what, cat] for engine in engines_list))
def parse_args(use_isolate_server, use_swarming):
"""Process arguments for the example scripts."""
os.chdir(ROOT_DIR)
colorama.init()
fix_encoding.fix_encoding()
parser = argparse.ArgumentParser(description=sys.modules['__main__'].__doc__)
if use_isolate_server:
parser.add_argument(
'-I', '--isolate-server', required=True,
metavar='URL', default=os.environ.get('ISOLATE_SERVER', ''),
help='Isolate server to use (default: ISOLATE_SERVER env var)')
if use_swarming:
task_name = (u'%s-%s-%s' % (
getpass.getuser(),
datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S'),
_hello_world)).encode('utf-8')
group = parser.add_argument_group('Swarming')
group.add_argument(
'-S', '--swarming',
metavar='URL', default=os.environ.get('SWARMING_SERVER', ''),
required=True,
help='Server to use (default: SWARMING_SERVER env var)')
group.add_argument(
'-t', '--task-name', default=task_name, metavar='NAME',
help='Task name, default is based on time; %(default)s')
group.add_argument(
'--idempotent', action='store_true',
help='Tells Swarming to reused previous task result if possible')
group.add_argument(
'-d', '--dimensions', nargs=2, action='append', default=[],
metavar='XX', help='Dimensions to request')
group.add_argument(
'--service-account',
help='Name of a service account to run the task as. Only literal "bot" '
'string can be specified currently (to run the task under bot\'s '
'account). Don\'t use task service accounts if not given '
'(default).')
group.add_argument(
'--priority', metavar='INT', type=int, help='Priority to use')
parser.add_argument('-v', '--verbose', action='count', default=0)
parser.add_argument(
'--normal', dest='which', action='store_const', const=u'hello_world')
parser.add_argument(
'--emoji', dest='which', action='store_const', const=_hello_world)
parser.set_defaults(which=_hello_world)
return parser.parse_args()
def main(args):
fix_encoding()
supported_engines = initialize_engines()
if not args:
raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n"
"available engines: %s" % (','.join(supported_engines)))
elif args[0] == "--capabilities":
displayCapabilities(supported_engines)
return
elif len(args) < 3:
raise SystemExit("./nova2.py [all|engine1[,engine2]*] <category> <keywords>\n"
"available engines: %s" % (','.join(supported_engines)))
#get only unique engines with set
engines_list = set(e.lower() for e in args[0].strip().split(','))
if 'all' in engines_list:
engines_list = supported_engines
else:
#discard un-supported engines
engines_list = [engine for engine in engines_list
if engine in supported_engines]
if not engines_list:
#engine list is empty. Nothing to do here
return
cat = args[1].lower()
if cat not in CATEGORIES:
raise SystemExit(" - ".join(('Invalid category', cat)))
what = urllib.quote(' '.join(args[2:]))
if THREADED:
#child process spawning is controlled min(number of searches, number of cpu)
pool = Pool(min(len(engines_list), MAX_THREADS))
pool.map(run_search, ([globals()[engine], what, cat] for engine in engines_list))
else:
map(run_search, ([globals()[engine], what, cat] for engine in engines_list))
def Main(argv):
"""Doesn't parse the arguments here, just find the right subcommand to
execute."""
if sys.hexversion < 0x02060000:
print >> sys.stderr, (
'\nYour python version %s is unsupported, please upgrade.\n' %
sys.version.split(' ', 1)[0])
return 2
if not sys.executable:
print >> sys.stderr, (
'\nPython cannot find the location of it\'s own executable.\n')
return 2
fix_encoding.fix_encoding()
disable_buffering()
colorama.init()
dispatcher = subcommand.CommandDispatcher(__name__)
try:
return dispatcher.execute(OptionParser(), argv)
except KeyboardInterrupt:
gclient_utils.GClientChildren.KillAllRemainingChildren()
raise
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
print >> sys.stderr, 'Error: %s' % str(e)
return 1
def Main(argv):
"""Doesn't parse the arguments here, just find the right subcommand to
execute."""
if sys.hexversion < 0x02060000:
print >> sys.stderr, (
'\nYour python version %s is unsupported, please upgrade.\n' %
sys.version.split(' ', 1)[0])
return 2
if not sys.executable:
print >> sys.stderr, (
'\nPython cannot find the location of it\'s own executable.\n')
return 2
fix_encoding.fix_encoding()
disable_buffering()
colorama.init()
dispatcher = subcommand.CommandDispatcher(__name__)
try:
return dispatcher.execute(OptionParser(), argv)
except KeyboardInterrupt:
gclient_utils.GClientChildren.KillAllRemainingChildren()
raise
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
print >> sys.stderr, 'Error: %s' % str(e)
return 1
# OptParser.description prefer nicely non-formatted strings.
parser.description = obj.__doc__ + '\n'
parser.set_usage('usage: %%prog %s %s' % (command, more))
def main(args=None):
# Do it late so all commands are listed.
# pylint: disable=E1101
parser = OptionParser(version=__version__)
if args is None:
args = sys.argv[1:]
if args:
command = Command(args[0])
if command:
# "fix" the usage and the description now that we know the subcommand.
gen_usage(parser, args[0])
return command(parser, args[1:])
# Not a known command. Default to help.
gen_usage(parser, 'help')
return CMDhelp(parser, args)
if __name__ == "__main__":
fix_encoding.fix_encoding()
try:
sys.exit(main())
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
sys.exit(1)
print ("Checkout path=%s" % scm_obj.project_path)
return 1
if "DEPS" in map(os.path.basename, patchset.filenames) and not options.ignore_deps:
gclient_root = gclient_utils.FindGclientRoot(full_dir)
if gclient_root and scm_type:
print ("A DEPS file was updated inside a gclient checkout, running gclient " "sync.")
gclient_path = os.path.join(BASE_DIR, "gclient")
if sys.platform == "win32":
gclient_path += ".bat"
with annotated_gclient.temp_filename(suffix="gclient") as f:
cmd = [gclient_path, "sync", "--nohooks", "--delete_unversioned_trees"]
if scm_type == "svn":
cmd.extend(["--revision", "BASE"])
if options.revision_mapping:
cmd.extend(["--output-json", f])
retcode = subprocess.call(cmd, cwd=gclient_root)
if retcode == 0 and options.revision_mapping:
revisions = annotated_gclient.parse_got_revision(f, options.revision_mapping)
annotated_gclient.emit_buildprops(revisions)
return retcode
return 0
if __name__ == "__main__":
fix_encoding.fix_encoding()
sys.exit(main())
def test_multiple_calls(self):
# Shouldn't do anything.
self.assertEquals(False, fix_encoding.fix_encoding())
print self.text.encode('utf-8')
print >> sys.stderr, self.text.encode('utf-8')
def test_unicode(self):
# Make sure printing unicode works.
print self.text
print >> sys.stderr, self.text
def test_default_encoding(self):
self.assertEquals('utf-8', sys.getdefaultencoding())
def test_win_console(self):
if sys.platform != 'win32':
return
# This should fail if not redirected, e.g. run directly instead of through
# the presubmit check. Can be checked with:
# python tests\fix_encoding_test.py
self.assertEquals(sys.stdout.__class__, fix_encoding.WinUnicodeOutput)
self.assertEquals(sys.stderr.__class__, fix_encoding.WinUnicodeOutput)
self.assertEquals(sys.stdout.encoding, sys.getdefaultencoding())
self.assertEquals(sys.stderr.encoding, sys.getdefaultencoding())
def test_multiple_calls(self):
# Shouldn't do anything.
self.assertEquals(False, fix_encoding.fix_encoding())
if __name__ == '__main__':
assert fix_encoding.fix_encoding()
unittest.main()
def test_multiple_calls(self): # Shouldn't do anything. self.assertEquals(False, fix_encoding.fix_encoding())
def test_unicode(self):
# Make sure printing unicode works.
print self.text
print >> sys.stderr, self.text
def test_default_encoding(self):
self.assertEquals('utf-8', sys.getdefaultencoding())
def test_win_console(self):
if sys.platform != 'win32':
return
# This should fail if not redirected, e.g. run directly instead of through
# the presubmit check. Can be checked with:
# python tests\fix_encoding_test.py
self.assertEquals(
sys.stdout.__class__, fix_encoding.WinUnicodeOutput)
self.assertEquals(
sys.stderr.__class__, fix_encoding.WinUnicodeOutput)
self.assertEquals(sys.stdout.encoding, sys.getdefaultencoding())
self.assertEquals(sys.stderr.encoding, sys.getdefaultencoding())
def test_multiple_calls(self):
# Shouldn't do anything.
self.assertEquals(False, fix_encoding.fix_encoding())
if __name__ == '__main__':
assert fix_encoding.fix_encoding()
unittest.main()
def search(searchengine, what, sort, maxresult=20):
fix_encoding()
engines = glob(path.join(path.dirname(__file__), 'engines', '*.py'))
enginclasslist = []
for engine in engines:
engi = path.basename(engine).split('.')[0].strip()
if len(engi) == 0 or engi.startswith('_'):
continue
if searchengine != engi and searchengine != 'all':
continue
#plugin.notify(searchengine)
try:
#import engines.[engine]
engine_module = __import__(".".join(("engines", engi)))
# #get low-level module
engine_module = getattr(engine_module, engi)
# #bind class name
engineclass = getattr(engine_module, engi[3:])
engineclass = engineclass()
enginclasslist.append(engineclass)
except:
pass
tasklist = Queue.Queue()
queueResult = Queue.Queue()
workers = []
for i in range(40):
worker = workerSearch(tasklist, queueResult)
workers.append(worker)
for engineclass in enginclasslist:
enginlevel = 0
if engineclass.page_result_count == 0:
engineclass.page_result_count = maxresult
pagecount = int(maxresult / engineclass.page_result_count)
if maxresult % engineclass.page_result_count > 0:
pagecount = pagecount + 1
for page in range(1, pagecount + 1):
tasklist.put({
'engine': engineclass,
'what': what,
'sort': sort,
'page': page,
'enginlevel': enginlevel
})
enginlevel = enginlevel + 1
for worker in workers:
worker.start()
for worker in workers:
worker.join()
resultlist = {}
while not queueResult.empty():
res_dict = queueResult.get_nowait()
rhash = getmagnethash(res_dict['link'])
if not resultlist.has_key(rhash):
resultlist[rhash] = res_dict
queueResult.task_done()
def getlevel(dict):
return dict['level']
return sorted(resultlist.values(), key=getlevel)
'''
