def main():
mode = Mode()
musicList = MusicList()
selectMusic = SelectMusic()
musicStart = MusicStart()
listUp = ListUp()
conf = Conf()
conf.existence()
listUp.listUp()
modeFlag = mode.selectMode()
while modeFlag == "EOF":
modeFlag = mode.selectMode()
if modeFlag == "loop":
print("\n")
musicList.musicList()
print("\n")
musicStart.loop()
if modeFlag == "single":
print("\n")
musicList.musicList()
number = selectMusic.selectMusic()
print("\n")
musicStart.single(number)
def init(f):
Conf.load(f)
conf = Conf.getConf()
monofont.init()
pygame.display.init()
# processing 'lie' section
globals.font_size = conf.getint("lie", "font_size")
globals.font = monofont.MonoFont(conf.get("lie", "font"), globals.font_size)
globals.scale_horizontally = conf.getint("lie", "scale_horizontally")
globals.darkest_gray = conf.getfloat("lie", "darkest_gray")
(globals.cell_width, globals.cell_height) = (globals.font.w, globals.font.h)
(globals.screen_width, globals.screen_height) = (conf.get("lie", "screen_width"), conf.get("lie", "screen_height"))
if globals.screen_width[-2:] == "px":
globals.screen_width = int(globals.screen_width[0:-2])
else:
globals.screen_width = globals.cell_width * int(globals.screen_width)
if globals.screen_height[-2:] == "px":
globals.screen_height = int(globals.screen_height[0:-2])
else:
globals.screen_height = globals.cell_height * int(globals.screen_height)
globals.message_buffer_height = conf.get("lie", "message_buffer_height")
if globals.message_buffer_height[-2:] == "px":
globals.message_buffer_height = int(globals.message_buffer_height[0:-2])
else:
globals.message_buffer_height = int(globals.message_buffer_height) * globals.cell_height
globals.grid_offset = globals.message_buffer_height
globals.screen = pygame.display.set_mode((globals.screen_width, globals.screen_height))
globals.screen.fill((0, 0, 0))
globals.background = globals.screen.copy()
globals.savefile_location = conf.get("lie", "savefile_location")
globals.wizard_mode = conf.getboolean("lie", "wizard_mode")
# processing 'logging' section
for name, level in conf.items("logging"):
logging.getLogger(name).setLevel(logging.getLevelName(level))
def run(conf_fname, sensor_queue, debug=False):
logging.basicConfig(
filename='climon.log',
format=
'%(asctime)s %(levelname)s MON[%(process)d/%(thread)d] %(message)s',
level=logging.DEBUG)
conf = Conf(conf_fname)
if 'monitor-interval' in conf.raw['common']:
db = database.WriteDB(conf.raw['common']['database'])
missing_stats = set()
queue_timestamp = monitor_timestamp = stats_timestamp = datetime.min
while True:
logging.debug('Queue size: %d', sensor_queue.qsize())
while not sensor_queue.empty():
try:
item = sensor_queue.get_nowait()
logging.debug('db.set(%r, %r, %r, %r)', item['sensor_id'],
item['timestamp'], item['metric'],
item['value'])
db.set(item['sensor_id'], item['timestamp'],
item['metric'], item['value'])
missing_stats.add((item['sensor_id'], item['timestamp']))
except queue.Empty:
logging.debug('empty sensor_queue')
break
if interval_over(monitor_timestamp,
int(conf.raw['common']['monitor-interval'])):
monitor_timestamp = datetime.utcnow()
for sensor_id, sensor in conf.iter_elements('sensor'):
log_sensor_data(db, sensor_id, sensor, monitor_timestamp)
missing_stats.add((sensor_id, monitor_timestamp))
for toggle_id, toggle in conf.iter_elements('toggle'):
log_toggle_state(db, toggle_id, toggle, monitor_timestamp)
missing_stats.add((toggle_id, monitor_timestamp))
if interval_over(stats_timestamp,
int(conf.raw['common']['stats-interval'])):
stats_timestamp = datetime.utcnow()
for id, timestamp in missing_stats:
db.update_stats(id, timestamp)
missing_stats = set()
logging.debug('Starting to sleep')
sleep_since(queue_timestamp,
int(conf.raw['common']['queue-interval']))
queue_timestamp = datetime.utcnow()
db.close()
else:
logging.debug('Monitor is idle.')
while True:
sleep(60)
def main():
botname = None
voiceword = None
ircserver = None
broadcastchan = None
try:
cfg = Conf.get('voicebot')
botname = cfg.get('name')
voiceword = cfg.get('voiceword')
ircserver = Conf.get('ircserver')
broadcastchan = Conf.get('broadcastchan')
except:
print "Error: Bad Configuration!"
print ""
print "You need a voicebot section with a name and voiceword configured"
print "Also, ircserver and broadcastchan are needed"
return 1
print "Voicebot is starting.."
irc = irclib.IRC()
irc.add_global_handler(
"pubmsg", lambda c, e: voiceThem(broadcastchan, voiceword, c, e), -20)
server = irc.server()
server.connect(ircserver[0], ircserver[1], botname)
server.join(broadcastchan)
print "Connected, joining eventloop."
irc.process_forever()
def __init__(self, app_name, abbrev=None):
self.app_name = app_name
self.abbrev = abbrev if abbrev else self.app_name[:4]
self.conf = Conf()
self.TABLES = {
ACTOR_PARTICIPANTS: self.get_actor_pariticipants,
ACTORS: self.get_actors,
ANSWERS: self.get_answers,
ARTIFACTS: self.get_artifacts,
CRITERIA: self.get_criteria,
EVAL_MODES: self.get_eval_modes,
ITEMS: self.get_items,
PARTICIPANTS: self.get_participants,
TASKS: self.get_tasks,
}
# TODO: It should be possible to grab this from the schema
self.UPDATE_ORDER = [
PARTICIPANTS,
ACTORS,
ACTOR_PARTICIPANTS,
CRITERIA,
EVAL_MODES,
TASKS,
ITEMS,
ARTIFACTS,
ANSWERS,
]
self._convert_id = lambda r: self.abbrev + '-' + '0' * (8 - len(
r)) + r if r != 'None' else None
def getLogger(cls, logfile="", conffile=""):
from conf import Conf
if (logfile == ""):
logfile = Conf.getconf("logdir", conffile=conffile) + Conf.getconf(
"logfile", conffile=conffile)
loglevel = Conf.getconf("loglevel", conffile=conffile)
rotate_log_size = Conf.getconf("rotate_log_size")
import logging, logging.handlers
logger = logging.getLogger()
if len(logger.handlers) < 1:
#fh = logging.FileHandler(filename="../../var/log/log2")
#logger.addHandler(fh)
rfh = logging.handlers.RotatingFileHandler(
filename=logfile,
maxBytes=rotate_log_size,
backupCount=Conf.getconf("backup_log_count"))
formatter = logging.Formatter(
'%(asctime)s - %(levelname)s - %(message)s')
rfh.setFormatter(formatter)
logger.addHandler(rfh)
#logging.basicConfig(filename="../../var/log/log2")
id_ = id(logger)
logger.setLevel(eval("logging." + loglevel))
logger.debug(
"return logger\n logfile[{logfile}]\n rotate_log_size[{rotate_log_size}]\n id[{id_}]"
.format(**locals()))
return logger
def main():
''' Start the main application interface '''
app=QtWidgets.QApplication(sys.argv)
# Initiate the global configuration
global conf
conf=Conf()
# Clean the log file
if(conf.get("clean_log")=="yes"):
open(LOG_FILE, "w").close()
# Show splash-screen
if(conf.get("splash_screen")=="yes"):
ss=True
else:
ss=False
if(ss):
splash=QtWidgets.QSplashScreen(QtWidgets.QPixmap("icons/splash.png"))
splash.show()
time.sleep(0.1)
app.processEvents()
splash.showMessage("Loading...")
# Create main window
w=Window()
if(ss):
splash.finish(w)
# Start application
sys.exit(app.exec_())
def visualizeSets_5(
cs=d.default_cs,
env=d.default_env,
agent=d.default_agent,
k=d.default_k_visualization,
ps=d.default_ps,
colors=d.default_colors_sets,
components=d.default_components,
loc=d.default_loc,
show=True,
conf=None,
test_chaos_theory=False,
):
if conf is None:
conf = Conf()
conf.test_chaos_theory = test_chaos_theory
trajs = []
for p in ps:
for c in cs:
trajs.append(getTraj(c, env, agent, p, conf=conf))
v = Visualisator()
v.show = show
name = tools.FileNaming.descrName(env, agent, c, conf)
filename = tools.FileNaming.imageTrajName(env.name, agent.name,
c, p, conf, k)
v.plotCompGP(trajs, colors=colors, name=name, components=components,
loc=loc, k=k, filename=filename)
class usage():
PROGRAM_USAGE = """DFF\nDigital Forensic Framework\n
Usage: """ + sys.argv[0] + """ [options]
Options:
-v --version display current version
-g --graphical launch graphical interface
-t --test=NAME start a specific test
-l --language=LANG use LANG as interface language
-h --help display this help message
-d --debug redirect IO to system console
--verbosity=LEVEL set verbosity level when debugging [0-3]
-c --config=FILEPATH use config file from FILEPATH
"""
VERSION = "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}"
def __init__(self, argv):
self.argv = argv
self.graphical = 0
self.test = ''
self.confPath = ''
self.debug = False
self.verbosity = 0
self.batch = None
# Configuration
self.main()
self.conf = Conf(self.confPath)
def main(self):
"""Check command line argument"""
try:
opts, args = getopt.getopt(self.argv, "vgdht:l:c:b:", [ "version", "graphical", "debug", "help", "test=", "language=", "verbosity=", "config=", "batch="])
except getopt.GetoptError:
self.usage()
for opt, arg in opts:
if opt in ("-h", "--help"):
self.usage()
elif opt in ("-g", "--graphical"):
self.graphical = 1
elif opt in ("-t", "--test"):
self.test = arg
elif opt in ("-l", "--language"):
self.conf.setLanguage(arg[:2])
elif opt in ("-v", "--version"):
print "dff version " + self.VERSION
sys.exit(1)
elif opt in ("-d", "--debug"):
self.debug = True
elif opt == "--verbosity":
self.verbosity = int(arg)
elif opt in ("-c", "--config"):
self.confPath = str(arg)
elif opt in ("-b", "--batch"):
self.batch = str(arg)
return
def usage(self):
"""Show usage"""
print self.PROGRAM_USAGE
sys.exit(2)
def main(argv):
parser = OptionParser(usage="usage: %prog [options]", version="%prog 0.1a")
parser.add_option("-a", "--app",
dest = "app",
help = "The application to build. Required.")
parser.add_option("-t", "--task",
dest = "task",
default = 'default',
help = "The task(s) that need(s) to be run. Optional. \
Comma separate to run multiple tasks. Default: `%default`")
parser.add_option("-p", "--path",
dest = "path", default = '.',
help = "Conche root path. Default: Current Working Directory. Optional. Default: `%default`")
(options, args) = parser.parse_args()
if len(args):
parser.error("Unexpected arguments encountered.")
if not options.app:
parser.error("You must specify an application.")
path = options.path
if path == ".":
path = os.getcwdu()
cnf = Conf(path)
cnf.run_task(options.app, options.task)
if argv == []:
print parser.format_option_help()
def getSparkLogger(cls, logfile="", conffile=""):
if (logfile == ""):
logfile = Conf.getconf("logdir", conffile=conffile) + Conf.getconf(
"logfile", conffile=conffile)
from pyspark import SparkContext
#sc = SparkContext.getOrCreate()
#log4j = sc._jvm.org.apache.log4j
#logger = log4j.LogManager.getLogger("myLogger")
#logger = log4j.LogManager.getLogger(__name__)
#logger = log4j.LogManager.getRootLogger()
#logger.appender.FILE.File="../../var/log/log"
#logger.setLevel(log4jLogger.Level.DEBUG)
#logger.info("aaaa")
#return logger
if not 'LOG_DIRS' in os.environ:
sys.stderr.write(
'Missing LOG_DIRS environment variable, pyspark logging disabled'
)
return
file = os.environ['LOG_DIRS'].split(',')[0] + '/log'
logging.basicConfig(
filename=file,
level=logging.INFO,
format=
'%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s'
)
logger = logging.getLogger()
return logger
def __init__(self, desc=False, name='MAINSERVER', parent=False, readLevel=-1, writeLevel=-1, kid_base='/'):
self._lock = Lock()
if not desc:
desc = collections.OrderedDict({'self': {}})
Scriptable.__init__(self)
self.log = logging
self.kid_base = kid_base
Conf.__init__(self, desc['self'])
if readLevel > 0:
self._readLevel = readLevel
if writeLevel > 0:
self._writeLevel = writeLevel
self.children = desc #.copy()
"""Child configuration dictionaries"""
self.children_obj = {}
"""Instantiated children configuration proxies"""
# Remove myself from children...
del self.children['self']
self.name = name
self._parent = parent
if not parent:
self._Method__name = name
else:
self._Method__name = parent._Method__name + self.separator + name
# if parent._parent:
# self._Method__name=parent._Method__name+self.separator+name
# else:
# self._Method__name=name
if self.has_key('devpath'):
self['devpath'] = name
self.autosort()
def main():
botname = None
voiceword = None
ircserver = None
broadcastchan = None
try:
cfg = Conf.get('voicebot')
botname = cfg.get('name')
voiceword = cfg.get('voiceword')
ircserver = Conf.get('ircserver')
broadcastchan = Conf.get('broadcastchan')
except:
print "Error: Bad Configuration!"
print ""
print "You need a voicebot section with a name and voiceword configured"
print "Also, ircserver and broadcastchan are needed"
return 1
print "Voicebot is starting.."
irc = irclib.IRC()
irc.add_global_handler("pubmsg", lambda c, e: voiceThem(broadcastchan, voiceword, c, e), -20)
server = irc.server()
server.connect(ircserver[0], ircserver[1], botname)
server.join(broadcastchan)
print "Connected, joining eventloop."
irc.process_forever()
def __init__(self):
self.SCHEDULER = BlockingScheduler()
conf = Conf()
confs = conf.get_confs()
for conf in confs:
self.add_corn_trigger(conf)
def readFilexClude(path, base=[]):
current=base.copy()
for line in open(path):
line=line.rstrip("\n")
if os.path.isdir(line) and ((line[-1]!="\\") if Conf.isWindows() else (line[-1]!="/")):
line+="\\" if Conf.isWindows() else "/"
current.append(line)
return current
def __init__(self, dev):
threading.Thread.__init__(self)
self.daemon = True
self.context = pyudev.Context()
self.dev = dev
self.mountpoint = Conf.get("mountpoint")
self.attrs = Conf.get("udevAttrs")
def __init__(self):
self.log = Log.getLogger()
self.driver = self.create_driver()
self.top_url = Conf.getconf("kakaku_top_page")
self.target_stores = Conf.getconf("target_stores")
self.extract_store_name = re.compile(r"\'")
self.warning_messages = False
self.log.debug(__class__.__name__ + "." +
sys._getframe().f_code.co_name + " start")
def __init__(self, dev):
threading.Thread.__init__(self)
self.daemon = True
self.context = pyudev.Context()
self.dev = dev
self.mountpoint = Conf.get("mountpoint")
self.attrs = Conf.get("udevAttrs")
def __init__(self):
sys.path.append(os.path.dirname(os.path.abspath(__file__)) + "/../lib/utils")
from conf import Conf
self.conf = Conf()
from log import Log as l
self.log = l.getLogger()
self.opt = Search_options()
self.log.debug("class " + __class__.__name__ + " created.")
class Alias:
''' Provides the class to handle symptom aliases '''
def __init__(self, conf=False):
'''
Initiates the alias object
Accepts a Conf object as parameter
'''
self.data={}
if(conf):
self.conf=conf
else:
self.conf=Conf()
self.compile()
def compile(self):
''' Compile the plaintext index files to a program usable format '''
# Loop over the alias files
for path, subdirs, files in os.walk(self.conf.get("alias_path")):
for name in files:
if(fnmatch(name, "*.txt")):
# Open the *.txt files
with open(self.conf.get("alias_path")+name, "r") as f:
for line in f:
# Ignore lines starting with #
line=line.rstrip().split("#")[0]
if(len(line)==0):
pass
else:
terms=[]
# Split words separated by ; and add to the terms
for i in line.split(";"):
if(i.strip()):
terms.append(i.strip())
# If alias present, add terms to the data
if(len(terms)==2):
self.data[terms[-1]]=terms[0]
elif(len(terms)>2):
t=terms.pop(0)
for i in terms:
self.data[i]=t
def get(self, term):
'''
Return the alias of the queried symptom
Parameter:
term - Queried string
Return value:
String containing the alias of the term
'''
if(term in self.data):
return self.data[term]
else:
return term
def __init__(self,
config=None,
verbose=False,
debug=False,
host=None,
db_name=None,
user=None,
password=None):
"""
Reads the config file and sets up a connection to the database
args
config -- path to the config file
verbose -- output useful messages
debug -- set to debug mode
host -- hostname or address (overrides the config file if given)
db -- name of database on server (overrides the config file if given)
user -- username to connect to database (overrides the config file if given)
password -- password to connect to database (overrides the config file if given)
"""
Conf.__init__(self)
self.verbose = verbose
self.debug = debug
self.module_dir = os.path.dirname(os.path.abspath(__file__))
if config is None:
config = os.path.join(self.module_dir, "config.yaml")
self.config = self.parse_config(yaml.safe_load(open(config)))
print("Connecting to database")
if host is None:
host = self.config.db.host
if db_name is None:
db_name = self.config.db.dbname
if user is None:
user = self.config.db.user
if password is None:
password = self.config.db.password
db_connection_string = " ".join([
"dbname=" + db_name, "user="******"host=" + host,
"password="******"DB connection: %s" % db_connection_string)
DBUtils.__init__(self, db_connection_string, self.verbose, self.debug)
self.sql_subs = self.make_bna_substitutions(self.config)
# mi/km
if "units" in self.config:
if self.config.units == "mi":
self.km = False
elif self.config.units == "km":
self.km = True
else:
raise ValueError("Invalid units \"{}\" in config".format(
self.config.units))
else:
self.km = False
class Alias:
''' Provides the class to handle symptom aliases '''
def __init__(self, conf=False):
'''
Initiates the alias object
Accepts a Conf object as parameter
'''
self.data = {}
if (conf):
self.conf = conf
else:
self.conf = Conf()
self.compile()
def compile(self):
''' Compile the plaintext index files to a program usable format '''
# Loop over the alias files
for path, subdirs, files in os.walk(self.conf.get("alias_path")):
for name in files:
if (fnmatch(name, "*.txt")):
# Open the *.txt files
with open(self.conf.get("alias_path") + name, "r") as f:
for line in f:
# Ignore lines starting with #
line = line.rstrip().split("#")[0]
if (len(line) == 0):
pass
else:
terms = []
# Split words separated by ; and add to the terms
for i in line.split(";"):
if (i.strip()):
terms.append(i.strip())
# If alias present, add terms to the data
if (len(terms) == 2):
self.data[terms[-1]] = terms[0]
elif (len(terms) > 2):
t = terms.pop(0)
for i in terms:
self.data[i] = t
def get(self, term):
'''
Return the alias of the queried symptom
Parameter:
term - Queried string
Return value:
String containing the alias of the term
'''
if (term in self.data):
return self.data[term]
else:
return term
def __init__(self, conf=False):
'''
Initiates the alias object
Accepts a Conf object as parameter
'''
self.data = {}
if (conf):
self.conf = conf
else:
self.conf = Conf()
self.compile()
def __init__(self, conf=False):
'''
Initiate the index object. Accepts a Conf object as an
optional parameter.
'''
if (conf):
self.conf = conf
else:
self.conf = Conf()
self.data = {}
self.compile()
def __init__(self, dbfile=""):
if (dbfile == ""):
App_Root = os.path.abspath(
os.path.join(
os.path.abspath(
os.path.join(
os.path.dirname(os.path.realpath(__file__)),
os.pardir)), os.pardir))
dbfile = os.path.join(App_Root, Conf("SqLite").read_sqlite())
else:
dbfile = os.path.join(dbfile, Conf("SqLite").read_sqlite())
self.dbfile = dbfile
def findInPath(self, d):
if not "PATH" in self._env: return ""
if not Conf.isWindows():
pp.replace(":", ";")
pp=self._env["PATH"].split(";")
for curr in pp:
x=os.path.abspath(os.path.join( curr, d))
if os.path.isfile(x):
return x
if Conf.isWindows() and os.path.isfile(x+".exe"):
return x+".exe"
return ""
def __init__(self, conf=False):
'''
The constructor optionally accepts a configuration.
If none is provided it creates a default configuration.
Parameters:
conf - A dictionary containing configuration options
'''
if (conf):
self._conf = conf
else:
self._conf = Conf()
self.clean = True
def _print(shell, title, message):
os.environ["ZENITY_DATADIR"] = os.path.abspath(Conf.ZENITY +
"\\..\\..\\share\\zenity\\")
if Conf.isWindows():
wd = os.getcwd()
os.chdir(os.path.abspath(Conf.ZENITY + "\\..\\.."))
try:
execSystem(
[Conf.ZENITY, '--error', '--text=' + message, '--title=' + title])
except FileNotFoundError as err:
if Conf.isWindows(): os.chdir(wd)
return CommandReturn(errors.MALFORMED_REQUEST, "Zenity Not found")
if Conf.isWindows(): os.chdir(wd)
def test_creation_file(self):
# delete directory if exist
self.delete_all()
# test if deleted correctly
self.assertEqual(self._default_directory.exists(), False)
self._cnf = Conf()
# test if created correctly
self.assertEqual(self._default_directory.exists(), True) # directory
self.assertEqual(
self._default_directory.joinpath(self._default_file_name).exists(),
True) #file
def __init__(self):
Ether2Any.__init__(self, tap=Conf.get('tunnelEthernet', True))
handlerConf = Conf.get('handler', {'allowFrom': None, 'allowTo': None})
self.mailHandler = NetMailHandler(self.dev, **handlerConf)
self.mailTo = Conf.get('mailTo', None)
self.mailFrom = Conf.get('mailFrom', None)
self.smtpConf = Conf.get('smtp')
smtpd = Conf.get("smtpd", {'enabled': False})
if smtpd['enabled']:
self.smtpd = SMTPServerThread(smtpd['listen'], self.mailHandler)
else:
self.smtpd = None
imapConf = Conf.get("imap", {'enabled': False})
if imapConf['enabled']:
self.imap = SimpleIMAPClient(imapConf, self.mailTo, self.mailHandler)
else:
self.imap = None
self.generator = SpamGenerator()
network = Conf.get('network', {'mtu': 1400})
self.dev.ifconfig(**network)
def move_to_vendor_page(self, vendor_button):
self.log.debug(__class__.__name__ + "." +
sys._getframe().f_code.co_name + " start.")
self.driver.get(vendor_button.get_attribute("href"),
warning_messages=self.warning_messages)
self.log.debug("wait start")
for sec in range(Conf.getconf("phantomJS_load_timeout")):
self.log.debug("wait redirect " + str(sec) + "[sec]")
if self.driver.title:
self.log.debug("move to shop page finished. page title: " +
self.driver.title)
break
time.sleep(Conf.getconf("vendor_page_wait_time"))
self.log.debug(__class__.__name__ + "." +
sys._getframe().f_code.co_name + " finished.")
def __init__(self, dev, networkQueue, writeLock):
threading.Thread.__init__(self)
self.daemon = True
self.quit = False
self.packetCounter = 0
self.dev = dev
self.writeLock = writeLock
self.networkQueue = networkQueue
self.mountpoint = Conf.get("mountpoint")
self.usbNetworkDir = Conf.get("usbNetworkDir")
self.networkFilePrefix = Conf.get("networkFilePrefix")
self.writePath = self.mountpoint + "/" + self.usbNetworkDir + "/"
def __init__(self, dev, networkQueue, writeLock):
threading.Thread.__init__(self)
self.daemon = True
self.quit = False
self.packetCounter = 0
self.dev = dev
self.writeLock = writeLock
self.networkQueue = networkQueue
self.mountpoint = Conf.get("mountpoint")
self.usbNetworkDir = Conf.get("usbNetworkDir")
self.networkFilePrefix = Conf.get("networkFilePrefix")
self.writePath = self.mountpoint + "/" + self.usbNetworkDir + "/"
def main():
import time
import numpy as np
from conf import Conf
cnf = Conf(exp_name='default')
model = Autoencoder(hmap_d=cnf.hmap_d).to(cnf.device)
model.load_w('/home/fabio/PycharmProjects/LoCO/models/weights/vha.pth')
print(model)
print(f'* number of parameters: {model.n_param}')
x = torch.rand((1, 14, cnf.hmap_d, cnf.hmap_h, cnf.hmap_w)).to(cnf.device)
print('\n--- ENCODING ---')
y = model.encode(x)
print(f'* input shape: {tuple(x.shape)}')
print(f'* output shape: {tuple(y.shape)}')
print(
f'* space savings: {100 - 100 * np.prod(tuple(y.shape)) / np.prod(tuple(x.shape)):.2f}%'
)
print(f'* factor: {np.prod(tuple(x.shape)) / np.prod(tuple(y.shape)):.2f}')
print('\n--- DECODING ---')
xd = model.decode(y)
print(f'* input shape: {tuple(y.shape)}')
print(f'* output shape: {tuple(xd.shape)}')
print('\n--- FORWARD ---')
y = model.forward(x)
print(f'* input shape: {tuple(x.shape)}')
print(f'* output shape: {tuple(y.shape)}')
def __init__(self):
self.connect = MySQLdb.connect(*Conf().read())
self.connect.set_character_set('utf8')
c = self.connect.cursor()
c.execute('SET NAMES utf8;')
c.execute('SET CHARACTER SET utf8;')
c.execute('SET character_set_connection=utf8;')
def main(exp_name):
# type: (str) -> None
cnf = Conf(exp_name=exp_name)
print(f'▶ Results of experiment \'{exp_name}\'')
results(cnf=cnf)
def main(exp_name, seed):
# type: (str, int) -> None
# if `exp_name` is None,
# ask the user to enter it
if exp_name is None:
exp_name = click.prompt('▶ experiment name', default='default')
# if `exp_name` contains a '@' character,
# the number following '@' is considered as
# the desired random seed for the experiment
split = exp_name.split('@')
if len(split) == 2:
seed = int(split[1])
exp_name = split[0]
cnf = Conf(seed=seed, exp_name=exp_name)
print(f'\n▶ Starting Experiment \'{exp_name}\' [seed: {cnf.seed}]')
if cnf.model_input == 'joint':
trainer = TrainerJoint(cnf=cnf)
elif cnf.model_input == 'detection':
trainer = TrainerDet(cnf=cnf)
else:
assert False, ''
trainer.run()
def main(exp_name, conf_file_path, seed):
# type: (str, str, int) -> None
# if `exp_name` is None,
# ask the user to enter it
if exp_name is None:
exp_name = click.prompt('▶ experiment name', default='default')
# if `exp_name` contains '!',
# `log_each_step` becomes `False`
log_each_step = True
if '!' in exp_name:
exp_name = exp_name.replace('!', '')
log_each_step = False
# if `exp_name` contains a '@' character,
# the number following '@' is considered as
# the desired random seed for the experiment
split = exp_name.split('@')
if len(split) == 2:
seed = int(split[1])
exp_name = split[0]
cnf = Conf(conf_file_path=conf_file_path,
seed=seed,
exp_name=exp_name,
log=log_each_step)
print(f'\n{cnf}')
print(f'\n▶ Starting Experiment \'{exp_name}\' [seed: {cnf.seed}]')
trainer = Trainer(cnf=cnf)
trainer.run()
def __init__(self,loghandler):
"""
"""
Conf.__init__(self)
self.__counterr = 0
self.__countwarn = 0
self.__notification_required = False
self.__logger = logging.getLogger(loghandler)
if not self.__logger.handlers:
self.__hdlr = logging.FileHandler(self.logfile)
self.__formatter = logging.Formatter('%(asctime)s [%(levelname)s] : %(message)s')
self.__hdlr.setFormatter(self.__formatter)
self.__logger.addHandler(self.__hdlr)
self.__logger.setLevel(logging.INFO)
def __init__(self, debug=False):
Ether2Any.__init__(self, tap=False)
self.debug = debug
network = Conf.get("network", {'mtu': 1400})
self.coder = Conf.get("coder", None)
self.twitterConf = Conf.get("twitter", None)
self.endpoint = self.twitterConf['endpoint']
if not self.endpoint:
print "No endpoint in configuration, please add one."
sys.exit(1)
self.dev.ifconfig(**network)
self.dev.up()
self._setupTwitter()
self.downstream = DownstreamThread(dev=self.dev, coder=self.coder, auth=self.auth, endpoint=self.endpoint, debug=self.debug)
self.downstream.start()
def __init__(self, dev, usbwriter, writeLock):
threading.Thread.__init__(self)
self.daemon = True
self.quit = False
self.packetCounter = 0
self.dev = dev
self.usbwriter = usbwriter
self.writeLock = writeLock
self.context = pyudev.Context()
self.mountpoint = Conf.get("mountpoint")
self.usbNetworkDir = Conf.get("usbNetworkDir")
self.networkFilePrefix = Conf.get("networkFilePrefix")
self.sync = Conf.get("sync")
self.writePath = self.mountpoint + "/" + self.usbNetworkDir + "/"
self.attrs = Conf.get("udevAttrs")
self.devname = None
def __init__(self):
Ether2Any.__init__(self, tap=False)
network = Conf.get("network", {'mtu': 1500})
self.dev.ifconfig(**network)
self.dev.up()
self.reader = UdevReader(self.dev)
self.reader.start()
def __init__(self,hostaddr=None,noop='noop'):
"""
"""
Conf.__init__(self)
logfile = self.logfile
c = Common()
c.banner()
c.client_hosts()
self.__host=hostaddr
self.__noop=noop
self.__counter = 0
self.__exclude = 0
self.__reader = open(logfile)
self.__startline = len(self.__reader.readlines())
self.__reader.close()
l = Logger('initiator')
l.logx('-'*70)
if self.__host=='all':
l.logx('Update command on all hosts executed')
self.update_all()
l.event_counter(self.__startline)
elif re.match(r'^\.\*$',self.__host):
print '\n[-]--> please use \'all\' instead of \'.*\'\n'
elif re.match(r'.*[\.\*\+\[\]]+.*',self.__host):
l.logx('Update regex command (%s) executed' % self.__host)
self.reget_host()
l.event_counter(self.__startline)
else:
l.logx('Update command on %s executed' % self.__host)
env.host_string=self.__host
self.update_host()
l.event_counter(self.__startline)
l.logx('End of execution')
def __init__(self, conf=False):
'''
Initiate the index object. Accepts a Conf object as an
optional parameter.
'''
if(conf):
self.conf=conf
else:
self.conf=Conf()
self.data={}
self.compile()
def __init__(self, conf=False):
'''
Initiates the alias object
Accepts a Conf object as parameter
'''
self.data={}
if(conf):
self.conf=conf
else:
self.conf=Conf()
self.compile()
def __init__(self, argv):
self.argv = argv
self.graphical = 0
self.test = ''
self.confPath = ''
self.debug = False
self.verbosity = 0
self.batch = None
# Configuration
self.main()
self.conf = Conf(self.confPath)
def __init__(self):
# device
Ether2Any.__init__(self, tap=True)
self.qrlog = self.setupLogging("QrNet")
self.mqueue = Queue.Queue()
self.squeue = Queue.Queue()
self.setTimeout(1)
network = Conf.get("network", {'mtu': 400})
self.packetDrop = Conf.get("packetDrop", 20)
self.dev.ifconfig(**network)
self.dev.up()
# thread starting...
gtk.gdk.threads_init()
self.cam = CamThread(self.dev, self.squeue, self.setupLogging("CamThread"))
self.cam.start()
self.display = DisplayThread(self.dev, self.mqueue, self.squeue, self.setupLogging("DisplayThread"))
self.display.start()
def __init__(self, conf=False):
'''
The constructor optionally accepts a configuration.
If none is provided it creates a default configuration.
Parameters:
conf - A dictionary containing configuration options
'''
if(conf):
self._conf=conf
else:
self._conf=Conf()
self.clean=True
def __init__(self,hostAddr=None,choice=None): """ """ Conf.__init__(self) logfile = self.logfile self.__choice = choice self.__host = hostAddr self.__counter = 0 env.host_string = self.__host if choice is None: print 'install or remove puppet ?' elif choice == 'install': self.install() elif choice == 'remove': self.remove() elif choice == 'restart': self.restart()
def __init__(self):
Ether2Any.__init__(self, tap=False)
network = Conf.get("network", {'mtu': 1400})
self.dev.ifconfig(**network)
self.dev.up()
self.networkQueue = Queue.Queue()
self.writeLock = threading.Lock()
self.writeLock.acquire()
self.usb = USBWriter(self.dev, self.networkQueue, self.writeLock)
self.usb.start()
self.udev = UdevHandler(self.dev, self.usb, self.writeLock)
self.udev.start()
def __init__(self, comp=False, conf=False):
"""
Initiate the diagnosis finder.
Parameters:
comp - Recompiles the data files if set to True
conf - Supply a Conf object
"""
if conf:
self.conf = conf
else:
self.conf = Conf()
self.compiler = Compile(conf)
if comp:
self.compiler.compile()
self.index = Index(conf)
def __init__(self, dev, squeue, camlog):
threading.Thread.__init__(self)
self.dev = dev
self.squeue = squeue
self.camlog = camlog
self.frame = 0
self.reportAfter = 20 # frames
self.quit = False
self.success = 0
self.lastPacket = ""
self.reader = highgui.cvCreateCameraCapture(Conf.get("camnum", 0))
self.scanner = zbar.ImageScanner()
self.scanner.parse_config('enable')
class Deploy:
def __init__(self):
self.config = Conf()
self.console = Console()
def path(self, option):
path = self.config.get(option)
return os.path.abspath(path);
def chown(self, path):
user = self.config.get('deploy_user')
self.console.run(['chown', '-R', user, path])
def linkdir(self, src, dst):
self.console.run(['ln', '-sfn', src, dst])
def version(self, deploy_path):
hash = self.console.run(['git', 'rev-parse', 'HEAD'], cwd=deploy_path);
return hash[0:8]
def hostname(self):
hostname = self.console.run(['hostname', '-f'], output=False);
return hostname
def sync(self, src, dst):
if os.path.exists(src) == False:
os.makedirs(src, 0755)
self.console.run([
'rsync',
'--links',
'--checksum',
'--whole-file',
'--recursive',
src.rstrip('/') + '/',
dst
])
def checkout(self):
deploy_path = self.path('release_path') + '/' + time.strftime('%Y%m%d%H%M%S')
if os.path.exists(deploy_path) == False:
os.makedirs(deploy_path, 0755)
gitclone = ' '.join([
'git',
'clone',
'--quiet',
'--recursive',
'--depth', '1',
'--branch', self.config.get('repo_branch'),
self.config.get('repo_url'),
deploy_path
])
sshadd = ' '.join([
'ssh-add',
self.config.get('deploy_key')
])
self.console.success('Fetching files')
self.console.execute('ssh-agent sh -c \'' + sshadd + '; ' + gitclone + '\'')
return deploy_path
def composer(self, deploy_path):
if os.path.exists(deploy_path + '/composer.json') == False:
return None
self.console.success('Installing composer dependencies')
self.console.run([
'composer',
'--quiet',
'--no-interaction',
'install',
'--prefer-dist',
'--no-dev',
'--optimize-autoloader'
], cwd=deploy_path);
def scripts(self, scripts_to_run, deploy_path):
if self.config.has(scripts_to_run) == False:
return
scripts = self.config.get(scripts_to_run)
for line in scripts:
command = line.replace('$deploy_path', deploy_path)
command = command.replace('$repo_branch', self.config.get('repo_branch'))
command = command.replace('$repo_url', self.config.get('repo_url'))
command = command.replace('$hostname', self.hostname())
self.console.run(shlex.split(command), cwd=deploy_path)
def clean(self):
release_path = self.path('release_path')
deployments = self.console.run(['ls', '-1tA', release_path])
deploys_to_keep = int(self.config.get('deploys_to_keep'))
for folder in deployments.splitlines()[deploys_to_keep:]:
self.console.run(['rm', '-rf', folder], cwd=release_path);
def rollback(self, version):
deploy_path = self.path('release_path') + '/' + version
if os.path.exists(deploy_path) == False:
self.console.error('Version does not exist')
return None
self.console.success('Rolling back')
self.linkdir(deploy_path, self.config.get('symlink'))
def deploy(self):
# checkout files
deploy_path = self.checkout()
# fetch resources
self.composer(deploy_path)
# static resources
if self.config.has('static_path'):
self.console.success('Copying static resources')
self.sync(self.path('static_path'), deploy_path)
if self.config.has('pre_scripts'):
self.console.success('Running pre-scripts')
self.scripts('pre_scripts', deploy_path)
self.console.success('Updating file owner')
self.chown(deploy_path)
self.console.success('Updating symlink')
self.linkdir(deploy_path, self.config.get('symlink'))
if self.config.has('post_scripts'):
self.console.success('Running post-scripts')
self.scripts('post_scripts', deploy_path)
self.console.success('Cleaning up old releases')
self.clean()
print >> stdout, "Usage: %s [ API-KEY ]" % sys.argv[0]
print >> stdout, __doc__.strip()
sys.exit(1)
def main():
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], "h", ["help", "solo", "force", "force-profile="])
except getopt.GetoptError, e:
usage(e)
apikey = None
force = False
force_profile = False
solo = False
conf = Conf()
for opt, val in opts:
if opt in ('-h', '--help'):
usage()
elif opt == '--force':
force = True
elif opt == '--force-profile':
force_profile = True
conf.force_profile = val
elif opt == "--solo":
solo = True
nbytes = sys.getsizeof(response)
print '%s:%s: %d bytes sent.' % (self.client_address[0], data, nbytes)
if __name__ == "__main__":
# Get the X-Plane path from the arguments
if len(sys.argv) > 1:
path = sys.argv[1]
else:
# Joanpc's personal debuggin options
if sys.platform == 'win32':
path = 'H:'
else:
path = '/Volumes/TO_GO/X-Plane 10'
conf = Conf(path)
logfile = open(os.sep.join([conf.respath, 'weatherServerLog.txt']), 'a')
sys.stderr = logfile
sys.stdout = logfile
print '---------------'
print 'Starting server'
print '---------------'
print sys.argv
try:
server = SocketServer.UDPServer(("localhost", conf.server_port), clientHandler)
except socket.error:
print "Can't bind address: %s, port: %d." % ("localhost", conf.server_port)
def __init__(self): self.config = Conf() self.console = Console()
class DDStorm:
""" Provides the class for finding differential diagnosis. """
conf = False
def __init__(self, comp=False, conf=False):
"""
Initiate the diagnosis finder.
Parameters:
comp - Recompiles the data files if set to True
conf - Supply a Conf object
"""
if conf:
self.conf = conf
else:
self.conf = Conf()
self.compiler = Compile(conf)
if comp:
self.compiler.compile()
self.index = Index(conf)
def dd(self, symptoms):
"""
Find the differential diagnosis list.
Parameter:
symptom - list of strings containing symptoms
Return value:
List of strings containing the differential diagnosis
"""
# Return empty list if symptom list is empty
if not symptoms:
return
# Find DD of first symptom and discard it
diff1 = self._getDiff(symptoms.pop(0))
# Loop through the rest of the list
for s in symptoms:
# Find DD of the current item in the list
diff2 = self._getDiff(s)
# List for temporary holding the DDs
temp = []
# Make both lists the same length by appending empty strings to the end
if len(diff1) > len(diff2):
diff2 += [""] * (len(diff1) - len(diff2))
elif len(diff2) > len(diff1):
diff1 += [""] * (len(diff2) - len(diff1))
# Loop over both lists
for (s1, s2) in zip(diff1, diff2):
# Add s1 to temp if s1 or any of its upstream ancestor is common to both list
if (s1 not in temp) and (len(s1) > 0):
if s1 in diff2:
temp.append(s1)
else:
us = self.index.upstream(s1)
for i in us:
if i in diff2:
temp.append(i)
# Add s2 to temp if s2 or any of its upstream ancestor is common to both list
if (s2 not in temp) and (len(s2) > 0):
if s2 in diff1:
temp.append(s2)
else:
us = self.index.upstream(s2)
for i in us:
if i in diff1:
temp.append(i)
# Copy temp to first list
diff1 = list(temp)
return diff1
def _getDiff(self, symptom):
""" Return differential diagnosis for a single symptom """
diff = []
symptom = symptom.lower().replace("_", " ").replace("-", " ")
if os.path.isfile(self.conf.get("module_path") + symptom + ".module"):
with open(self.conf.get("module_path") + symptom + ".module", "r") as mf:
for line in mf:
diff.append(line.strip())
return diff
def symptoms(self):
"""
Return a full list of available symptoms
Return value:
List of string containing symptoms
"""
symp = []
for n in os.listdir(self.conf.get("module_path")):
symp.append(os.path.splitext(os.path.basename(n))[0].capitalize())
return symp
class Compile:
'''
This class creates a compiler for the DDStorm
that compiles the text files containing list of
differential diagnosis to simplified modular
data files usable by the program.
'''
def __init__(self, conf=False):
'''
The constructor optionally accepts a configuration.
If none is provided it creates a default configuration.
Parameters:
conf - A dictionary containing configuration options
'''
if(conf):
self._conf=conf
else:
self._conf=Conf()
self.clean=True
def compile(self):
''' Compile the text files to DDStorm modules. '''
self.source=set()
self.custom=set()
self.alias=Alias(self._conf)
# Loop over library files and add *.txt files to source
for path, subdirs, files in os.walk(self._conf.get("library_path")):
for name in files:
if(fnmatch(name, "*.txt")):
self.source.add(os.path.join(path, name))
# Loop over custom files and add *.txt files to custom
for path, subdirs, files in os.walk(self._conf.get("custom_path")):
for name in files:
if(fnmatch(name, "*.txt")):
self.custom.add(os.path.join(path, name))
# Create module directory if not already present and delete all module files
if(not os.path.isdir(self._conf.get("module_path"))):
os.makedirs(self._conf.get("module_path"))
for f in os.listdir(self._conf.get("module_path")):
if(fnmatch(f, "*.module")):
os.unlink(self._conf.get("module_path")+f)
# Create a regex for calculating priority from filename
self.priorityRegex=re.compile("(?<=\.)\d+$")
# First sort files by priority then compile them to module
for src in self._sortPriority(self.source):
self._makeModule(src)
for src in self._sortPriority(self.custom):
self._makeModule(src)
def _sortPriority(self, files):
''' Sort data files based on their priority settings. '''
ls=[]
# Loop over the files
for addr in files:
# Format the file name
name=os.path.splitext(os.path.basename(addr))[0].lower().replace("_"," ").replace("-", " ")
# Search for priority tag on file name
m=re.search(self.priorityRegex, name)
# Add to ls as (symptom name, priority number, file name) with default priority of 100
if(m):
ls.append((name.replace("."+m.group(), ""), int(m.group()), addr))
else:
ls.append((name, 100, addr))
# Sort the file list, first by the symptom name, then by the priority number
ls.sort(reverse=True)
if(ls):
return(list(zip(*ls))[2])
else:
return ls
def _makeModule(self, src):
''' Create application usable modules from data files. '''
# Format the file name
module=os.path.splitext(os.path.basename(src))[0].lower().replace("_"," ").replace("-", " ")
# Remove the priority tag from file name
m=re.search(self.priorityRegex, module)
if(m):
module=module.replace("."+m.group(), "")
# Create the module file name
modFile=self._conf.get("module_path")+module+".module"
modFlag=False
# Loop over both files, the source data file and the target module file
with open(src, "r") as sf, open(modFile, "a") as tf:
# Ignore lines starting with ! or #, + and - has special meaning, write other lines to module. Log the errors.
for line in sf:
line=line.strip().split("#")[0]
if(len(line)==0):
pass
elif(line.startswith("!")):
pass
elif(line.startswith("#")):
pass
elif(line.startswith("+")):
modFlag=True
elif(line.startswith("-")):
modFlag=True
elif(line.replace(" ","").replace("-","").replace("_","").replace("'","").isalnum()):
print(self.alias.get(line).capitalize(), file=tf)
else:
self.clean=False
logging.warning("Syntax error in file '"+src+"': "+line)
# Deal with special lines
if(modFlag):
modFlag=False
with open(src, "r") as f:
for line in f:
line=line.strip().split("#")[0]
if(line[1:].replace(" ","").replace("-","").replace("_","").replace("'","").isalnum()):
# If line starts with + add it to the module file
if(line.startswith("+")):
with open(modFile, "r") as fn:
text=fn.read()
with open(modFile, "w") as fn:
print(self.alias.get(line[1:]).capitalize()+"\n"+text, file=fn)
# If line starts with - remove corresponding item from the module file
elif(line.startswith("-")):
with open(modFile, "r") as fn:
text=fn.read()
text=text.replace(self.alias.get(line[1:]).capitalize()+"\n", "")
with open(modFile, "w") as fn:
print(text, file=fn)
def is_clean(self):
'''Report if compilation ended successfully'''
return self.clean