def __init__(self, index):
self.index = index
self.currentIp = ""
self.queue = MultiProcessingQueue()
self.p = threading.Thread(target=crawl, args=(self, ))
self.p.start()
from faraday.client.model.controller import ModelController
from faraday.client.plugins.controller import PluginController
from faraday.client.plugins.manager import PluginManager
from faraday.client.managers.mapper_manager import MapperManager
from faraday.client.managers.reports_managers import ReportProcessor
from faraday.server.models import Workspace
from faraday.client.persistence.server import server
from faraday.config.configuration import getInstanceConfiguration
CONF = getInstanceConfiguration()
logger = logging.getLogger(__name__)
UPLOAD_REPORTS_QUEUE = MultiProcessingQueue()
UPLOAD_REPORTS_CMD_QUEUE = MultiProcessingQueue()
upload_api = Blueprint('upload_reports', __name__)
class RawReportProcessor(Thread):
def __init__(self):
super(RawReportProcessor, self).__init__()
from faraday.client.start_client import setupPlugins
setupPlugins()
self.pending_actions = Queue()
try:
plugin_manager = PluginManager(
def main(argv=None): # IGNORE:C0111
"""Command line options."""
if argv is not None:
sys.argv.extend(argv)
program_name = os.path.basename(sys.argv[0])
program_version = "v%s" % __version__
program_build_date = str(__updated__)
program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
program_license = '''%s - %s
Created by user_name on %s.
Copyright 2013 Freedom. All rights reserved.
Licensed under the Apache License 2.0
http://www.apache.org/licenses/LICENSE-2.0
Distributed on an "AS IS" basis without warranties
or conditions of any kind, either express or implied.
USAGE
''' % (program_name, program_shortdesc, str(__date__))
# ----------------------------------------------------------------------------------------------------------------------
enable_workers = (cpu_count() > 1)
# ----------------------------------------------------------------------------------------------------------------------
# Setup argument parser
parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
parser.add_argument("-v", "--verbose", dest="verbose", action="count",
help="set verbosity level [default: %(default)s]")
parser.add_argument('-V', '--version', action='version', version=program_version_message)
parser.add_argument('-g', '--grades', default='A,B,C,D,E,F,G',
help="Comma seperated list of credit grades to test [default: %(default)s]")
parser.add_argument('-a', '--states', default='CA,AZ,FL,GA,IL,MD,NV,TX,NY',
help="Comma separated list of states to test [default: %(default)s]")
parser.add_argument('-t', '--terms', default='36,60',
help="Comma separated list of loan terms to test [default: %(default)s]")
parser.add_argument('-s', '--seed', default=100, help="Random Number Generator Seed [default: %(default)s]")
parser.add_argument('-d', '--data',
default="https://resources.lendingclub.com/LoanStats3a.csv.zip,https://resources.lendingclub.com/LoanStats3b.csv.zip,https://resources.lendingclub.com/LoanStats3c.csv.zip",
help="Comma separated download paths for the notes data files [default: %(default)s]")
parser.add_argument('-l', '--stats', default="LoanStats3a.csv.zip,LoanStats3b.csv.zip,LoanStats3c.csv.zip",
help="Comma separated list of input Loan Stats CSV files [default: %(default)s]")
parser.add_argument('-c', '--csvresults', default="lc_best.csv",
help="Output best results CSV file [default: %(default)s]")
parser.add_argument('-p', '--population_size', default=512, type=int, help="population size [default: %(default)s]")
parser.add_argument('-i', '--iterations', default=4096, type=int,
help="how many Genetic Algorithm iterations to perform [default: %(default)s]")
parser.add_argument('-e', '--elite_rate', default=0.05, type=float, help="elite rate [default: %(default)s]")
parser.add_argument('-m', '--mutation_rate', default=0.05, type=float, help="mutation rate [default: %(default)s]")
parser.add_argument('-k', '--check', default=False, action='store_true',
help="checking mode: open the CSV results file and filter the loans into a separate file [default: %(default)s]")
parser.add_argument('-r', '--checkresults', default="LoanStatsNewFiltered.csv",
help="file name for the filtered results used during checking mode [default: %(default)s]")
parser.add_argument('-z', '--zmq', default=True, action='store_true',
help="Use zmq libraries for multi-processing [default: %(default)s]")
parser.add_argument('-q', '--sqlite', default=1, type=int,
help="Use sqlite as the core processing engine for the backtesting [default: %(default)s]")
parser.add_argument('-f', '--fitness_sort_size', default=1000, type=int,
help="number of loans to limit the fitness sort size, the larger the longer and more optimal solution [default: %(default)s]")
parser.add_argument('-y', '--young_loans_in_days', default=3 * 30, type=int,
help="filter young loans if they are younger than specified number of days [default: %(default)s]")
parser.add_argument('-w', '--workers', default=enable_workers * cpu_count(), type=int,
help="number of workers defaults to the number of cpu cores [default: %(default)s]")
parser.add_argument('-b', '--work_batch', default=75, type=int,
help="size of work batch size to give to each worker [default: %(default)s]")
# Process arguments
args = parser.parse_args()
if args.population_size < args.workers:
args.workers = 1
enable_workers = 0
if enable_workers > 0 and args.zmq:
enable_zmq = utilities.check_for_pyzmq()
else:
enable_zmq = False
random.seed(args.seed)
csv_download_files = args.data.split(',')
csv_stats_files = args.stats.split(',')
for i in range(len(csv_stats_files)):
if os.path.exists(csv_stats_files[i]):
sys.stdout.write("Using %s as the stats file\n" % csv_stats_files[i])
else:
if i > len(csv_download_files):
sys.stderr.write("This file does not exist locally: ", csv_stats_files[i],
"but the corresponding download url does not exist either.",
"Make sure the the number of comma separated download urls",
"matches the comma seperate stats files", csv_stats_files)
sys.exit(-1)
sys.stdout.write("Downloading %s as data file\n" % csv_download_files[i])
utilities.download_data(csv_download_files[i], csv_stats_files[i])
if enable_workers:
if enable_zmq:
lcbt = ZmqLCBT(ConversionFilters, args, worker_idx=-1)
lcbt.initialize()
for worker_idx in range(args.workers):
Process(target=zmq_worker, args=(worker_idx, args)).start()
ga_test = ZmqGATest(BackTestFilters, lcbt, args)
ga_test.run()
else:
# only need this one to initialize the data
lcbt = LCBT(ConversionFilters, args, worker_idx=-1)
lcbt.initialize()
work_queue = MultiProcessingQueue()
response_queue = MultiProcessingQueue()
for worker_idx in range(args.workers):
mp_worker(worker_idx, args, work_queue, response_queue)
ga_test = ParallelGATest(BackTestFilters, lcbt, args, work_queue, response_queue)
ga_test.run()
else:
lcbt = LCBT(ConversionFilters, args, worker_idx=-1)
lcbt.initialize()
ga_test = GATest(BackTestFilters, lcbt, args)
ga_test.run()
return 0