def createWorker(self, clientSocket, address):
# workerId = hashlib.md5(str(address).encode('utf8')).hexdigest()
worker = Worker(clientSocket,
address,
self.db.Session,
sslContext=self.sslContext)
worker.start()
def run(self):
ae_res = []
stability_res = []
# Cross Validation loop
for i in range(len(self.folds)):
# get first fold #i info
fold = self.folds[i]
# set train, validation and test sets for this fold iteration
self.args.trainfiles = fold.train
self.args.valfile = fold.validation
self.args.testfile = fold.test
# all training logic is inside Worker (core/worker.py)
worker = Worker(i, self.conf, self.data_conf, self.cache, self.args,
verbose=self.verbose, inference=self.inference)
res, _ = worker.run()
# print results for each fold
if self.verbose:
partial_res = summary_angular_errors([r.error for r in res])
self._print_results(partial_res, 'fold '+str(i))
# accumulate results into lists
ae_res += res
# summary_angular_errors: computes mean, median, best 25%, etc...
results = summary_angular_errors([r.error for r in ae_res])
if self.verbose:
self._print_results(results, 'total')
return results
def main(args):
logging.info( args )
device = 'gpu' if args.gpu else 'cpu'
devices = device_lib.list_local_devices()
num_gpus = len([d for d in devices if '/gpu' in d.name])
env = gym.make(args.game)
env = Env(env, resized_width=84, resized_height=84, agent_history_length=4)
num_actions = len(env.gym_actions)
global_net = Network(num_actions, -1, 'cpu')
actor_networks = []
for t in range(args.threads):
device_index = 0 if device is 'cpu' else (t if args.threads <= num_gpus else 0)
n = Network(num_actions, t, device, device_index)
n.tie_global_net(global_net)
actor_networks.append(n)
sess = tf.Session(config=tf.ConfigProto(intra_op_parallelism_threads=args.threads, inter_op_parallelism_threads=args.threads))
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
if not os.path.exists(args.checkpoint_dir):
os.makedirs(args.checkpoint_dir)
threads = []
for t, net in enumerate(actor_networks):
e = Env(gym.make(args.game), net.width, net.height, net.depth)
w = Worker(t, e, net, sess, saver, args.checkpoint_dir)
w.start()
threads.append(w)
for t in threads:
t.join()
def main():
args = parser.parse_args()
if args.seed is not None:
random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.deterministic = True
warnings.warn('You have chosen to seed training. '
'This will turn on the CUDNN deterministic setting, '
'which can slow down your training considerably! '
'You may see unexpected behavior when restarting '
'from checkpoints.')
# load configuration file: epochs, loss function, etc... for this experiment
with open(args.configurationfile, 'r') as f:
conf = json.load(f)
# load datapath file: paths specific to the current machine
with open(args.datapath, 'r') as f:
data_conf = json.load(f)
# remove previous results
output_dir = os.path.join(args.outputfolder, args.dataset, args.subdataset,
conf['name'])
if not args.evaluate and not args.resume:
if os.path.exists(output_dir):
shutil.rmtree(output_dir)
# create output folder
os.makedirs(output_dir, exist_ok=True)
args.outputfolder = output_dir
# copy configuration file to output folder
shutil.copy(
args.configurationfile,
os.path.join(output_dir, os.path.basename(args.configurationfile)))
# we overwrite the stdout and stderr (standard output and error) to
# files in the output directory
sys.stdout = PrintLogger(os.path.join(output_dir, 'stdout.txt'),
sys.stdout)
sys.stderr = PrintLogger(os.path.join(output_dir, 'stderr.txt'),
sys.stderr)
fold = 0 # no folds, but we always use fold #0 for these experiments
cache = CacheManager(conf)
worker = Worker(fold, conf, data_conf, cache, args)
res, _ = worker.run()
# some datasets have no validation GT
if len(res) > 0:
# print angular errors statistics (mean, median, etc...)
generate_results(res, 'test')
def __init__(self):
super(Manager, self).__init__()
self.basePath = "db/db.json"
self.baseImagePath = "db/images"
self.hgigerbot = hgigerbot
self.imageManager = ImageManager(basePath=self.baseImagePath)
self.worker = Worker(self.baseImagePath, self.imageManager)
self.requestType = RequestType()
self.currentCell = {}
self.hints = []
self.hintSections = []
self.currentHint = {}
def test_24_hours_shifts(self):
# 1 hour to load, 1 hour to produce
# produce 2 every 2 hours
# ie 24 - 1 (1 to start the machine) / 2 = 10
machine, spec, stock = create_machine(stocking_zone_size=None)
factory = Factory()
factory.add_worker(Worker(working_hour = 8 * 60))
factory.add_worker(Worker(working_hour = 8 * 60))
factory.add_worker(Worker(working_hour = 8 * 60))
factory.add_production_unit(machine)
factory.run(24 * 60)
self.assertEquals(stock.count(), 720 - 1)
def runWorker(mapURL, threads): queue = Queue.Queue() # create a thread pool and give them a queue for i in range(threads): t = Worker(queue, str(i)) t.setDaemon(True) t.start() # give the queue some data for url in mapURL.values(): queue.put(url) # wait for the queue to finish queue.join()
def setUp(self):
self.unaffected_production_unit, spec, zone = create_machine(
material_type_input="yarn")
self.worker = Worker()
self.affected_production_unit = ProductionUnit(spec)
self.inputs = Material("yarn")
self.started_production_unit = ProductionUnit(spec)
self.started_production_unit.perform_next_operation(self.worker)
self.loaded_production_unit = ProductionUnit(spec)
self.loaded_production_unit.perform_next_operation(self.worker)
self.loaded_production_unit.perform_next_operation(self.worker)
config = {'rate_by_minute': 0.2}
spec_four = Specification()
spec_four.add(
MaterialInputConstraint(Material(type="flour", quantity=2)))
spec_four.add(
MaterialInputConstraint(Material(type="water", quantity=1)))
spec_four.add_output_material(Material("bread", 1))
self.four_a_pain = ProductionUnit(spec_four, config)
self.four_a_pain.perform_next_operation(self.worker)
self.four_a_pain.perform_next_operation(self.worker)
def setUp(self):
spec = Specification()
spec.add(MaterialInputConstraint(Material(type="wood", quantity=1)))
spec.add_output_material(Material("Furniture", 1))
self.machine = ProductionUnit(spec, config={"rate_by_minute":0.25})
self.worker = Worker()
self.input = Material("wood", quantity=3)
def main():
args = parser.parse_args()
# load configuration file for this experiment
with open(args.configurationfile, 'r') as f:
conf = json.load(f)
# load datapath file: paths specific to the current machine
with open(args.datapath, 'r') as f:
data_conf = json.load(f)
# create output folder
os.makedirs(args.outputfolder, exist_ok=True)
# copy configuration file to output folder
shutil.copy(
args.configurationfile,
os.path.join(args.outputfolder,
os.path.basename(args.configurationfile)))
# we overwrite the stdout and stderr (standard output and error) to
# files in the output directory
sys.stdout = PrintLogger(os.path.join(args.outputfolder, 'stdout.txt'),
sys.stdout)
sys.stderr = PrintLogger(os.path.join(args.outputfolder, 'stderr.txt'),
sys.stderr)
# used in core/worker.py to determine what to do
args.evaluate = True
args.resume = False
args.valfile = None
args.trainfiles = None
# init the cache manager: this caches images from the dataset
# to avoid reading them more than once
cache = CacheManager(conf, no_cache=True)
fold = 0 # no folds, but we always use fold #0 for these experiments
worker = Worker(fold, conf, data_conf, cache, args, inference=True)
res, _ = worker.run()
# print angular errors statistics (mean, median, etc...)
generate_results(res)
def test_add_skill_constraint_to_operation(self):
tech_production_unit, spec, zone = create_machine(
material_type_input="iron")
start_op = StartOperation(tech_production_unit, worker=self.worker)
start_op.add_constraint(SkillConstraint(skill_name="blacksmith"))
self.assertRaises(CannotPerformOperation, start_op.run)
blacksmith = Worker()
blacksmith.skills.append("blacksmith")
StartOperation(tech_production_unit, worker=blacksmith).run()
self.assertEquals(tech_production_unit.get_state(),
ProductionUnit.STARTED)
def run(args_obj):
# create worker thread
cmd_obj = Worker(args_obj=args_obj)
# create msg manager thread
msg_obj = MSGManager(cmd_obj)
msg_obj.start()
cmd_obj.start()
cmd_obj.join()
msg_obj.join()
def get_factory(yaml_conf):
yaml = load(yaml_conf)
factory = Factory(name=yaml["name"])
materials = create_materials(yaml)
for production_unit in yaml["production_units"]:
spec = create_spec(materials, production_unit)
config = {}
config["rate_by_minute"] = production_unit.get("rate", 1)
factory.add_production_unit(
ProductionUnit(spec=spec,
config=config,
name=production_unit["name"]))
for worker in yaml.get("workers", []):
working_hour = worker.get("working_hour", 8) * 60
factory.add_worker(Worker(working_hour=working_hour))
return factory
def main():
mapURL = {}
logo()
parser = argparse.ArgumentParser(description='Ragpicker Malware Crawler')
parser.add_argument("-a",
"--artwork",
help="Show artwork",
action="store_true",
required=False)
parser.add_argument("-p",
"--processes",
type=int,
default=3,
help="Number of processes (default=3, max=6)")
parser.add_argument("-u",
"--url",
help="Download and analysis from a single URL")
parser.add_argument("-d",
"--directory",
help="Load files from local directory")
parser.add_argument("-i",
"--info",
help="Print Ragpicker config infos",
action="store_true",
required=False)
parser.add_argument("-da",
"--delete",
help="Delete all stored data",
action="store_true")
parser.add_argument('--log-level',
default=logging.INFO,
help='logging level, default=logging.INFO')
parser.add_argument('--log-filename', help='logging filename')
parser.add_argument('--version',
action='version',
version='Ragpicker version ' + RAGPICKER_VERSION)
global args
args = parser.parse_args()
if args.artwork:
try:
while True:
time.sleep(1)
logo()
except KeyboardInterrupt:
return
if args.log_level:
log_conf = dict(
level=args.log_level,
format='%(levelname)s %(name)s %(module)s:%(lineno)d %(message)s')
if args.log_filename:
log_conf['filename'] = args.log_filename
log.info("log-filename: " + args.log_filename)
logging.basicConfig(**log_conf)
if args.delete:
worker = Worker()
worker.runDelete()
return
if args.info:
printRagpickerInfos(True)
return
if args.url:
log.info(color("Download and analysis from %s" % args.url, RED))
runWorker(args.url)
elif args.directory:
printRagpickerInfos()
log.info(
color("Load files from local directory %s" % args.directory, RED))
mapURL = getLocalFiles(args.directory)
else:
printRagpickerInfos()
# Malware URLs Crawlen
mapURL = runCrawler()
# Max Threads=6
if args.processes > 6:
args.processes = 6
log.info(color("Processes: " + str(args.processes), RED))
log.info(color("Process " + str(len(mapURL)) + " URLs", RED))
# Create Process Pool
pool = Pool(processes=args.processes)
# Malware Download, process and reporting
for url in mapURL.values():
pool.apply_async(runWorker, args=(url, ))
pool.close()
pool.join()
def runWorker(url):
log.info("Worker URL: " + url)
worker = Worker()
worker.run(url)
def test_factory_add_worker(self):
factory = Factory()
factory.add_worker(Worker())
self.assertEquals(len(factory.workers), 1)
class Manager(object):
def __init__(self):
super(Manager, self).__init__()
self.basePath = "db/db.json"
self.baseImagePath = "db/images"
self.hgigerbot = hgigerbot
self.imageManager = ImageManager(basePath=self.baseImagePath)
self.worker = Worker(self.baseImagePath, self.imageManager)
self.requestType = RequestType()
self.currentCell = {}
self.hints = []
self.hintSections = []
self.currentHint = {}
def inThread(self, function):
Thread(target=function).start()
def hintsCount(self):
return sum([len(cell["body"][1]) for cell in self.cells])
def setBuilder(self, builder):
self.builder = builder
def openCell(self, cellId):
self.currentCell = self.worker.findElementById(elements=self.cells,
id=cellId)
self.hintSections, self.hints = self.currentCell["body"][
0], self.currentCell["body"][1]
def openHint(self, hintId):
self.currentHint = [
hint for hint in self.hints if hint["id"] == hintId
][0]
def save(self):
with open(self.basePath, "w") as f:
json.dump([self.cellSections, self.cells],
f,
sort_keys=True,
indent=4)
def load(self):
with open(self.basePath) as f:
self.cellSections, self.cells = json.load(f)
def reception(self, type, **request):
self.worker.update(self)
# (↓) [-CELL SECTIONS-]
if type == self.requestType.ADD_CELL_SECTION:
self.cellSections = self.worker.addCellSection(**request)
elif type == self.requestType.DELETE_CELL_SECTION:
self.cellSections, self.cells = self.worker.deleteCellSection(
**request)
elif type == self.requestType.RENAME_CELL_SECTION:
self.cellSections, self.cells = self.worker.renameCellSection(
**request)
elif type == self.requestType.REICON_CELL_SECTION:
self.cellSections = self.worker.reiconCellSection(**request)
elif type == self.requestType.SWAP_CELL_SECTIONS:
self.cellSections = self.worker.swapCellSections(**request)
# (↓) [-CELL-]
elif type == self.requestType.ADD_CELL:
self.cells = self.worker.addCell(**request)
elif type == self.requestType.DELETE_CELL:
self.cells = self.worker.deleteCell(**request)
elif type == self.requestType.RENAME_CELL:
self.cells = self.worker.renameCell(**request)
elif type == self.requestType.MOVE_CELL:
self.cells = self.worker.moveCell(**request)
# (↓) [-HINT SECTIONS-]
elif type == self.requestType.ADD_HINT_SECTION:
self.cells = self.worker.addHintSection(**request)
elif type == self.requestType.DELETE_HINT_SECTION:
self.cells = self.worker.deleteHintSection(**request)
elif type == self.requestType.RENAME_HINT_SECTION:
self.cells = self.worker.renameHintSection(**request)
elif type == self.requestType.REICON_HINT_SECTION:
self.cells = self.worker.reiconHintSection(**request)
elif type == self.requestType.SWAP_HINT_SECTIONS:
self.cells = self.worker.swapHintSections(**request)
# (↓) [-HINTS-]
elif type == self.requestType.ADD_HINT:
self.cells = self.worker.addHint(**request)
elif type == self.requestType.DELETE_HINT:
self.cells = self.worker.deleteHint(**request)
elif type == self.requestType.RENAME_HINT:
self.cells = self.worker.renameHint(**request)
elif type == self.requestType.MOVE_HINT:
self.cells = self.worker.moveHint(**request)
# (↓) [-HINT-]
elif type == self.requestType.EDIT_TAGS:
self.cells = self.worker.editTags(**request)
elif type == self.requestType.ADD_PAGE:
self.cells = self.worker.addPage(**request)
elif type == self.requestType.DELETE_PAGE:
self.cells = self.worker.deletePage(**request)
elif type == self.requestType.RENAME_PAGE:
self.cells = self.worker.renamePage(**request)
elif type == self.requestType.SWAP_PAGES:
self.cells = self.worker.swapPages(**request)
elif type == self.requestType.EDIT_PAGE_TEXT:
self.cells = self.worker.editPageText(**request)
elif type == self.requestType.ADD_IMAGE:
self.cells = self.worker.addImage(**request)
elif type == self.requestType.DELETE_IMAGE:
self.cells = self.worker.deleteImage(**request)
self.inThread(function=self.save)
self.builder.rebuild()
def main():
mapURL = {}
logo()
parser = argparse.ArgumentParser(description='Ragpicker Malware Crawler')
parser.add_argument("-a", "--artwork", help="Show artwork", action="store_true", required=False)
parser.add_argument("-p", "--processes", type=int, default=3, help="Number of processes (default=3, max=6)")
parser.add_argument("-u", "--url", help="Download and analysis from a single URL")
parser.add_argument("-d", "--directory", help="Load files from local directory")
parser.add_argument("-i", "--info", help="Print Ragpicker config infos", action="store_true", required=False)
parser.add_argument("-da", "--delete", help="Delete all stored data", action="store_true")
parser.add_argument('--log-level', default=logging.INFO, help='logging level, default=logging.INFO')
parser.add_argument('--log-filename', help='logging filename')
parser.add_argument('--version', action='version', version='Ragpicker version ' + RAGPICKER_VERSION)
global args
args = parser.parse_args()
if args.artwork:
try:
while True:
time.sleep(1)
logo()
except KeyboardInterrupt:
return
if args.log_level:
log_conf = dict(level=args.log_level,
format='%(levelname)s %(name)s %(module)s:%(lineno)d %(message)s')
if args.log_filename:
log_conf['filename'] = args.log_filename
log.info("log-filename: " + args.log_filename)
logging.basicConfig(**log_conf)
if args.delete:
worker = Worker()
worker.runDelete()
return
if args.info:
printRagpickerInfos(True)
return
if args.url:
log.info(color("Download and analysis from %s" % args.url, RED))
runWorker(args.url)
elif args.directory:
printRagpickerInfos()
log.info(color("Load files from local directory %s" % args.directory, RED))
mapURL = getLocalFiles(args.directory)
else:
printRagpickerInfos()
# Malware URLs Crawlen
mapURL = runCrawler()
# Max Threads=6
if args.processes > 6:
args.processes = 6
log.info(color("Processes: " + str(args.processes), RED))
log.info(color("Process " + str(len(mapURL)) + " URLs", RED))
# Create Process Pool
pool = Pool(processes=args.processes)
# Malware Download, process and reporting
for url in mapURL.values():
pool.apply_async(runWorker, args=(url,))
pool.close()
pool.join()
def test_working_hour(self):
worker = Worker(working_hour=1)
worker.add_unit_of_work()
self.assertRaises(Event, worker.add_unit_of_work)
def test_working_hour(self):
eight_hour_worker = Worker(working_hour = 8 * 60)
self.assertRaises(Event, LoadOperation(Material(type="wood", quantity=1), production_unit=self.machine, worker=eight_hour_worker).run, during=8*60 + 1)
def main():
# Initialize worker
worker = Worker()
# Run worker
worker.run()
def setUp(self):
self.machine, spec, self.stock_zone = create_machine(material_type_input="wood", material_type_output="plank")
self.worker = Worker()
StartOperation(production_unit=self.machine, time_to_perform=1, worker=self.worker).run(during=1)
from multiprocessing import Process
from core.beat import Beat
from core.worker import Worker
if __name__ == '__main__':
beat = Beat()
beat_process = Process(target=beat.run)
beat_process.start()
worker = Worker()
worker_process = Process(target=worker.run)
worker_process.start()
beat_process.join()
worker_process.join()
def runWorker(url):
log.info("Worker URL: " + url)
worker = Worker()
worker.run(url)
