def configure(self): if os.path.exists( os.path.join(self.build_dir, 'CMakeFiles/cmake.check_cache')): utils.output("vtktud build already configured.") return if not os.path.exists(self.build_dir): os.mkdir(self.build_dir) cmake_params = "-DBUILD_SHARED_LIBS=ON " \ "-DBUILD_CONTRIB=OFF " \ "-DBUILD_TESTING=OFF " \ "-DCMAKE_BACKWARDS_COMPATIBILITY=2.6 " \ "-DCMAKE_BUILD_TYPE=RelWithDebInfo " \ "-DCMAKE_INSTALL_PREFIX=%s " \ "-DVTK_DIR=%s" % (self.inst_dir, config.VTK_DIR) # we only add this under posix as a work-around to compile the # STLib code under g++ if os.name == 'posix': cmake_params = cmake_params + " -DCMAKE_CXX_FLAGS=-fpermissive " ret = utils.cmake_command(self.build_dir, self.source_dir, cmake_params) if ret != 0: utils.error("Could not configure vtktud. Fix and try again.")
def fires(jira, args): m = re.match(r'(\w+)?', args) if not m: return utils.not_valid_args(args) project_key = m.group(1) or config.get('jira_default_project') if not project_key: return utils.error('Project name is required') if not utils.check_project(jira, project_key): return utils.error('Project {} does not exist'.format(project_key)) try: query = 'project={0} and labels in (fire)'.format(project_key) issues = jira.search_issues(query) if not issues: return 'No issues found' return '\n'.join([utils.issue_info(issue) for issue in issues]) except JIRAError as e: response = utils.error('{} {}'.format(str(e.status_code), str(e.text))) return response
def close(jira, args): m = re.match(r'(\w+-\d+) ?(.*)', args) if not m: return utils.not_valid_args(args) issue_key = m.group(1) issue_status = 'Closed' comment = m.group(2) try: issue = jira.issue(issue_key) transitions = jira.transitions(issue) transition_id = utils.get_transition(transitions, issue_status) if issue_status == issue.fields.status.name: return utils.error('Issue already closed') if not transition_id: return utils.error('Operation not permitted') jira.transition_issue(issue, transition_id, comment=comment) issue = jira.issue(issue_key) return utils.issue_info(issue) except JIRAError as e: if e.status_code == 500: return utils.error('operation not permitted') response = utils.error('{} {}'.format(str(e.status_code), str(e.text))) return response
def checkGbcFilesEq(name, genfile, reffile): """check if non-zero bytes in binary files are equal""" with open(genfile, "rb") as fg, open(reffile, "rb") as fr: genimage = fg.read() refimage = fr.read()[6:] if genimage != refimage and genimage != refimage[:-1]: error("GPL image", "Image mismatch: " + name)
def status(jira, args): m = re.match(r'(\w+-\d+) (.*)', args) if not m: return utils.not_valid_args(args) issue_key = m.group(1) issue_status = m.group(2) try: issue = jira.issue(issue_key) statuses = jira.statuses() if issue_status not in [s.name for s in statuses]: return utils.error('Status {} does not exist'.format(issue_status)) if issue_status == issue.fields.status.name: return utils.error('Status {} already set'.format(issue_status)) transitions = jira.transitions(issue) transition_id = utils.get_transition(transitions, issue_status) if not transition_id: return utils.error('Operation not permitted') jira.transition_issue(issue, transition_id) issue = jira.issue(issue_key) return utils.issue_info(issue) except JIRAError as e: response = utils.error('{} {}'.format(str(e.status_code), str(e.text))) return response
def check_build_sanity(): if len(e('${BUILD_ROOT}')) > 38: error("Current path too long ({0} characters) for nullfs mounts during build", len(os.getcwd())) if e('${BE_ROOT}') in sh_str('mount'): error("You have dangling mounts inside {0}, did last build crash?", e('${BE_ROOT}'))
def check_port(name, port): debug('Checking for "{0}" command', name) for i in e('${PATH}').split(':'): if os.path.exists(e('${i}/${name}')): return error('Command {0} not found. Please run "pkg install {1}" or install from ports', name, port)
def add(u, v): a = [] if abs(len(u) - len(v)) < 1e-12: a = [sum(x) for x in zip(u, v)] else: utils.error('Vectors are of different length (utils_math: add)') return a
def create(self, opts): """Create a conf stanza.""" argv = opts.args count = len(argv) # unflagged arguments are conf, stanza, key. In this order # however, we must have a conf and stanza. cpres = True if count > 0 else False spres = True if count > 1 else False kpres = True if count > 2 else False if kpres: kvpair = argv[2].split("=") if len(kvpair) != 2: error("Creating a k/v pair requires key and value", 2) if not cpres and not spres: error("Conf name and stanza name is required for create", 2) name = argv[0] stan = argv[1] conf = self.service.confs[name] if not kpres: # create stanza conf.create(stan) return # create key/value pair under existing stanza stanza = conf[stan] stanza.submit(argv[2])
def configure_posix(self): os.chdir(self.build_dir) if os.path.exists("dcmdata/config.log"): utils.output("DCMTK already configured. Not redoing.") else: # we need to configure this without zlib, otherwise dcmtk # complains (at least on this system) about the symbol # inflateEnd not being available. ret = os.system("./configure --without-zlib " "--prefix=%s" % (self.inst_dir,)) if ret != 0: utils.error("Could not configure dcmtk. Fix and try again.") # now modify the generated config/Makefile.def to enable # building shared libraries as per # http://forum.dcmtk.org/viewtopic.php?t=19 repls = [ ("(^CFLAGS\s*=\s*)-O", "\\1-fPIC -O2"), ("(^CXXFLAGS\s*=\s*)-O", "\\1-fPIC -O2"), ("(^AR\s*=\s*)ar", "\\1gcc"), ("(^ARFLAGS\s*=\s*)cruv", "\\1-shared -o"), ("(^LIBEXT\s*=\s*)a", "\\1so"), ("(^RANLIB\s*=\s*)ranlib", "\\1:"), ] utils.re_sub_filter_file(repls, "config/Makefile.def")
def sub(u, v): a = [] if abs(len(u) - len(v)) < 1e-12: a = [ u[i]-v[i] for i in range(len(u)) ] else: utils.error('Vectors are of different length (utils_math: sub)') return a
def main(): usage = "usage: %prog <search>" opts = utils.parse(sys.argv[1:], {}, ".splunkrc", usage=usage) if len(opts.args) != 1: utils.error("Search expression required", 2) search = opts.args[0] service = connect(**opts.kwargs) try: result = service.get( "search/jobs/export", search=search, earliest_time="rt", latest_time="rt", search_mode="realtime") reader = results.ResultsReader(result.body) while True: kind = reader.read() if kind == None: break if kind == results.RESULT: event = reader.value pprint(event) except KeyboardInterrupt: print "\nInterrupted."
def build_nt(self): os.chdir(self.build_dir) # do check for some file if os.path.exists(os.path.join("dcmdata/libsrc", BUILD_TARGET, "dcmdata.lib")): utils.output("dcmtk::dcmdata already built. Skipping.") else: # Release buildtype (vs RelWithDebInfo) so we build with # /MD and not /MDd ret = utils.make_command("dcmtk.sln", install=False, project="dcmdata", win_buildtype=BUILD_TARGET) if ret != 0: utils.error("Could not build dcmtk::dcmdata.") if os.path.exists(os.path.join("ofstd/libsrc", BUILD_TARGET, "ofstd.lib")): utils.output("dcmtk::ofstd already built. Skipping.") else: # Release buildtype (vs RelWithDebInfo) so we build with # /MD and not /MDd ret = utils.make_command("dcmtk.sln", install=False, project="ofstd", win_buildtype=BUILD_TARGET) if ret != 0: utils.error("Could not build dcmtk::ofstd.")
def checkRecordsByLen(infile, fixed=None): """check records by encoded length""" refline = "*".join(["".join([chr(c) for c in xrange(64, 127)]) for _ in xrange(4)]) if fixed is None: with open(infile, "r") as f: records = f.readlines() else: with open(infile, "rb") as f: data = f.read() records = [data[i:i + fixed] for i in xrange(0, len(data), fixed)] for i, line in enumerate(records): if fixed: l = line if len(l) != fixed: error("VAR Records", "%s: Record %d length mismatch: %d != %d" % ( infile, i, len(l), fixed)) else: l = line[:-1] if line[-1] == "\n" else line l = l.rstrip() s = "!" + refline[:len(l) - 2] + chr(i + 49) if len(l) > 1 else "" if l != s: error("VAR Records", "%s: Record %i content mismatch" % (infile, i))
def parse_blog_parts(string): """Parse and convert blog parts. Argument: string: blog entry body string. """ ex_ref_char = re.compile('\&(?!amp;)') string = ex_ref_char.sub('&', string) string = string.replace('alt="no image"', '') try: xmltree = xml.etree.ElementTree.fromstring(string) except: utils.error(string) if xmltree.get('class') == 'amazlet-box': repl_amazon = parse_amazlet(xmltree) return repl_amazon if xmltree.get('class'): if xmltree.get('class').find('bbpBox') == 0: repl_twitter = parse_twitter(xmltree) return repl_twitter if str(xmltree.get('id')).find('__ss_') == 0: repl_slideshare = parse_slideshare(xmltree) return repl_slideshare if str(xmltree.get('href')).find('heyquiz.com') >= 0: repl_heyquiz = parse_heyquiz(xmltree) return repl_heyquiz
def SVDNorm(U, S, V, A): if np.linalg.norm(np.dot( np.dot(U, np.diag(S)), V) - A) >= 1e-10: nrm = str(np.linalg.norm(np.dot( np.dot(U, np.diag(S)), V) - A)) err = 'Something went wrong with the SVD, norm is ' + str(nrm) ut.error(err) return 0
def sendXML(self, xml): """ First, check wether the coq process is still running. Then it send the XML command, and finally it waits for the response """ if self.coqtop == None: utils.error('ERROR: The coqtop process is not running or died !') print('Trying to relaunch it ...') self.launchCoqtopProcess() try: self.coqtop.stdin.write(XMLFactory.tostring(xml, 'utf-8')) except IOError as e: utils.error('Cannot communicate with the coq process : ' + str(e)) self.coqtop = None return None response = '' file = self.coqtop.stdout.fileno() while True: try: response += os.read(file, 0x4000) try: t = XMLFactory.fromstring(response) return t except XMLFactory.ParseError: continue except OSError: return None
def enum_def(code): if len(code) == 0 or len(code[0]) <= 1: utils.error("?enum-def need an argument (enum name)") name = code[0][:-1] if not name in enums: utils.error("?enum-def unknown enum: %s" % name) _print_def_enum(name, enums[name])
def install(self): # first ez_install ############################################ os.chdir(config.working_dir) # we need to be elsewhere! ret = os.system('%s -c "import setuptools"' % (sys.executable,)) if ret == 0: utils.output('setuptools already installed. Skipping step.') else: utils.output('ImportError test shows that setuptools is not ' 'installed. Installing...') os.chdir(config.archive_dir) ret = os.system('%s %s' % (config.PYTHON_EXECUTABLE, EZ_BASENAME)) if ret != 0: utils.error('Error during setuptools install.') # then pip # ############################################## os.chdir(config.working_dir) # we need to be elsewhere! ret = os.system('%s -c "import pip"' % (sys.executable,)) if ret == 0: utils.output('pip already installed. Skipping step.') else: utils.output('ImportError test shows that pip is not ' 'installed. Installing...') os.chdir(config.archive_dir) ret = os.system('%s %s' % (config.PYTHON_EXECUTABLE, PIP_BASENAME)) if ret != 0: utils.error('Error during pip install.')
def pkgconfig_get_link_args(pkg, ucp='', system=True, static=True): havePcFile = pkg.endswith('.pc') pcArg = pkg if not havePcFile: if system: # check that pkg-config knows about the package in question run_command(['pkg-config', '--exists', pkg]) else: # look for a .pc file if ucp == '': ucp = default_uniq_cfg_path() pcfile = pkg + '.pc' # maybe needs to be an argument later? pcArg = os.path.join(get_cfg_install_path(pkg, ucp), 'lib', 'pkgconfig', pcfile) if not os.access(pcArg, os.R_OK): error("Could not find '{0}'".format(pcArg), ValueError) static_arg = [ ] if static: static_arg = ['--static'] libs_line = run_command(['pkg-config', '--libs'] + static_arg + [pcArg]); libs = libs_line.split() return libs
def run(self): if self.args.version: return subprocess.call(["javac"] + self.original_arguments) else: javac_cmd = ["javac", "-verbose", "-g"] if self.args.bootclasspath is not None: javac_cmd += ["-bootclasspath", self.args.bootclasspath] if self.args.classpath is None: classpath = utils.ANNOT_PROCESSOR_JAR else: classpath = os.pathsep.join([utils.ANNOT_PROCESSOR_JAR, self.args.classpath]) javac_cmd += ["-cp", classpath] if self.args.classes_out is not None: javac_cmd += ["-d", self.args.classes_out] javac_cmd += self.remaining_args javac_cmd.append("-J-Duser.language=en") with tempfile.NamedTemporaryFile(mode="w", suffix=".out", prefix="annotations_", delete=False) as annot_out: self.annotations_out = annot_out.name with tempfile.NamedTemporaryFile(mode="w", suffix=".out", prefix="javac_", delete=False) as file_out: self.verbose_out = file_out.name os.environ["INFER_ANNOTATIONS_OUT"] = self.annotations_out try: subprocess.check_call(javac_cmd, stderr=file_out) except subprocess.CalledProcessError: error_msg = "Javac compilation error with: \n\n{}\n" failing_cmd = [arg for arg in javac_cmd if arg != "-verbose"] utils.error(error_msg.format(failing_cmd)) subprocess.check_call(failing_cmd) return os.EX_OK
def post(self, username): username = urllib.unquote(username) if username != str(self.user) and not self.is_admin: error(self, 403); return if not self.is_admin and (self.request.get("is_member") or self.request.get("is_admin")): error(self, 403); return profile = db.Query(models.Profile).filter("user ="******"/user/create"); return screenname = self.request.get("screenname") if screenname: if screenname != profile.screenname: if db.Query(models.Profile).filter("screenname =", screenname).get(): self.misc_render("user_profile.html", message="This screen name is taken.", form_profile=profile, username=username); return profile.screenname = screenname if self.request.get("realname"): profile.realname = self.request.get("realname") if self.request.get("is_member"): profile.is_member = True else: profile.is_member = False if self.request.get("is_admin"): profile.is_admin = True else: profile.is_admin = False profile.put() self.redirect("/user/%s/profile" % username)
def output(record): print_record(record) for k in sorted(record.keys()): if k.endswith("_str"): continue # Ignore v = record[k] if v is None: continue # Ignore if isinstance(v, list): if len(v) == 0: continue v = ','.join([str(item) for item in v]) # Field renames k = { 'source': "status_source" }.get(k, k) if isinstance(v, str): format = '%s="%s" ' v = v.replace('"', "'") else: format = "%s=%r " result = format % (k, v) ingest.send(result) end = "\r\n---end-status---\r\n" try: ingest.send(end) except: error("There was an error with the TCP connection to Splunk.", 2)
def validate(featureclass, quiet=False): """ Checks if a feature class is suitable for uploading to Places. Requires arcpy and ArcGIS 10.x ArcView or better license Checks: geodatabase = good, shapefile = ok, others = fail geometry: polys, lines, point, no multipoints, patches, etc must have a spatial reference system geometryid (or alt) column = good othewise = ok if it has a placesid column it must be empty :rtype : basestring :param featureclass: The ArcGIS feature class to validate :param quiet: Turns off all messages :return: 'ok' if the feature class meets minimum requirements for upload 'good' if the feature class is suitable for syncing anything else then the feature class should not be used """ if not featureclass: if not quiet: utils.error("No feature class provided.") return 'no feature class' if not arcpy.Exists(featureclass): if not quiet: utils.error("Feature class not found.") return 'feature class not found' return 'ok'
def main(): try: args = parse_options() f = args.__dict__.get('infile') if f.find('~') == 0: infile = os.path.expanduser(f) else: infile = os.path.abspath(f) if args.__dict__.get('dstdir'): dstdir = args.__dict__.get('dstdir') else: # default: ~/tmp/hatena2rest/ dstdir = None if args.__dict__.get('retrieve'): retrieve_image_flag = True else: retrieve_image_flag = False processing.xml2rest(infile, dstdir, retrieve_image_flag) except RuntimeError as e: utils.error(e) return except UnboundLocalError as e: utils.error(e) return
def run(self, command, opts): """Dispatch the given command & args.""" handlers = {"create": self.create, "delete": self.delete, "list": self.list} handler = handlers.get(command, None) if handler is None: error("Unrecognized command: %s" % command, 2) handler(opts)
def init4places(featureclass, quiet=False): """ Sets up a Geodatabase feature class for syncing with Places. Adds a PlacesID column if it doesn't exist, turn on archiving if not already on. :rtype : bool :param featureclass: The ArcGIS feature class to validate :param quiet: Turns off all messages :return: True if successful, False otherwise """ if not featureclass: if not quiet: utils.error("No feature class provided.") return False if not arcpy.Exists(featureclass): if not quiet: utils.error("Feature class not found.") return False return True
def get(self, key): profile = db.get(key) if profile.avatar: self.response.headers['Content-Type'] = "image/png" self.response.out.write(profile.avatar) else: error(self, 404); return
def configure(self): if os.path.exists( os.path.join(self.build_dir, 'CMakeFiles/cmake.check_cache')): utils.output("vtkdevide build already configured.") return if not os.path.exists(self.build_dir): os.mkdir(self.build_dir) cmake_params = "-DBUILD_SHARED_LIBS=ON " \ "-DBUILD_TESTING=OFF " \ "-DCMAKE_BUILD_TYPE=RelWithDebInfo " \ "-DCMAKE_INSTALL_PREFIX=%s " \ "-DVTK_DIR=%s " \ "-DDCMTK_INCLUDE_PATH=%s " \ "-DDCMTK_LIB_PATH=%s " \ "-DPYTHON_EXECUTABLE=%s " \ "-DPYTHON_LIBRARY=%s " \ "-DPYTHON_INCLUDE_PATH=%s" % \ (self.inst_dir, config.VTK_DIR, config.DCMTK_INCLUDE, config.DCMTK_LIB, config.PYTHON_EXECUTABLE, config.PYTHON_LIBRARY, config.PYTHON_INCLUDE_PATH) ret = utils.cmake_command(self.build_dir, self.source_dir, cmake_params) if ret != 0: utils.error("Could not configure vtkdevide. Fix and try again.")
def checkFileSizes(files): for fn, fs in files: size = None with open(fn, "rb") as f: size = len(f.read()) if fs != size: error("Files", "Incorect file size " + fn + ": " + str(size))
def set_component_outputs(self): error("Attempted to set outputs via abstract function.")
pca = PCA(n_components=m, whiten=True) X_train_pca = pca.fit_transform(X_train_s) X_val_pca = pca.fit_transform(X_val_s) X_test_pca = pca.fit_transform(X_test_s) # Testing Linear_regression # With normalized data print("Linear regression with normalized data") LR = LinearRegression(lamb=0.1, delta=0.000015) LR.fit(X_train_n, Y_train, epochs=1000, Visual=True) Y_pred_1 = LR.predict(X_val_n) Y_pred_2 = LR.predict(X_test_n) error(Y_pred_1, Y_val, "val") error(Y_pred_2, Y_test, "test") #compare(Y_val, Y_pred) # With standardized data print("Linear regression with standardized data") LR = LinearRegression(lamb=0.1, delta=0.00001) LR.fit(X_train_s, Y_train, epochs=1000, Visual=True) Y_pred_1 = LR.predict(X_val_s) Y_pred_2 = LR.predict(X_test_s) error(Y_pred_1, Y_val, "val") error(Y_pred_2, Y_test, "test") compare(Y_test, Y_pred_2, 2018) # With standardized and orthogonalized data
def main(): if len(sys.argv) < 2: usage() else: start_time = time.time() rpad = 60 rpad_char = '+' # this is the default list of install packages # # you can override this by: # - specifying packages on the johannes command line # - specifying packages in the working dir johannes.py # (command line has preference over config file) # # capitalisation has to match the capitalisation of your # install package class, name of install package module is # exactly that, but all lower case, so e.g. MyModule will # become: install_packages.ip_mymodule.MyModule() # # johannes will: # - attempt to import the ip_name from install_packages # - instantiate ip_name.Name # ip_names = [ 'pip', 'NumPy', 'WXPython', 'matplotlib', 'CMake', 'DCMTK', 'VTK56', 'IPython', 'VTKTUDOSS', 'ITK', 'SWIG', 'CableSwig', 'WrapITK', 'ItkVtkGlue', 'itkPyBuffer', 'ITKTUDOSS', 'GDCM', 'DeVIDE', 'VTKDEVIDE', 'SetupEnvironment' ] try: optlist, args = getopt.getopt(sys.argv[1:], 'hm:p:dw:vt:', [ 'help', 'mode=', 'install-packages=', 'auto-deps', 'working-dir=', 'no-prereq-check', 'versions' 'target=' ]) except getopt.GetoptError, e: usage() return mode = 'everything' #ip_names = None working_dir = None profile = 'default' no_prereq_check = False ip_names_cli = False auto_deps = False target = None for o, a in optlist: if o in ('-h', '--help'): usage() return elif o in ('-m', '--mode'): if a in ('clean', 'clean_build'): mode = 'clean_build' else: mode = a elif o in ('--install-packages'): # list of package name to perform the action on ip_names = [i.strip() for i in a.split(',')] # remember that the user has specified ip_names on the command-line ip_names_cli = True elif o in ('-d', '--auto-deps'): auto_deps = True elif o in ('-w', '--working-dir'): working_dir = a elif o in ('--profile'): profile = a elif o in ('--no-prereq-check'): no_prereq_check = True elif o in ('-v', '--versions'): mode = 'show_versions' elif o in ('-t', '--target'): target = a # we need at LEAST a working directory if not working_dir: usage() return # init config (DURR) config.init(working_dir, profile) # set some variables we'll need to check later depending on # the configuration ip_dirs = [] # now try to read johannes config file from the working dir cp = ConfigParser.ConfigParser() # returns list of filenames successfully parsed cfgfns = cp.read(os.path.join(working_dir, 'johannes.cfg')) if cfgfns and cp.has_section('default'): if not ip_names_cli: # first packages that need to be installed # we only do this if the user has NOT specified install # packages on the command line. try: ip_names = [ i.strip() for i in cp.get('default', 'packages').split(',') ] except NoOptionError: pass # also try to read extra install package paths # this is also a comma separated list try: ip_dirs = [ i.strip() for i in cp.get('default', 'ip_dirs').split(',') ] except NoOptionError: pass # if user is asking for versions, we don't do the # prerequisites check as we're not going to build anything if mode == 'show_versions': no_prereq_check = True if os.name == 'nt' and not no_prereq_check: if not windows_prereq_check(working_dir): utils.output( 'Windows prerequisites do not check out. ' 'Fix and try again.', 70, '-') return else: utils.output('Windows prerequisites all good.', 70, '-') elif os.name == 'posix' and not no_prereq_check: if not posix_prereq_check(working_dir): utils.output( 'Posix prerequisites do not check out. ' 'Fix and try again.', 70, '-') return else: utils.output('Posix prerequisites all good.', 70, '-') # In case of a target, check whether the target is actually specified # in the ip list (does not check dependencies in case of auto-deps) if target != None: if not target in ip_names: utils.error( "Target '%s' was not found in the install package list." % target) # we're going to do some imports, so let's set the sys.path # correctly. # 1. first the default install packages dir config.ip_dir sys.path.insert(0, config.ip_dir) # 2. insert the extra specified paths BEFORE that, so they get # preference for uip_dir in ip_dirs: sys.path.insert(0, uip_dir) # now import only the specified packages ip_instance_list = [] imported_names = [] def import_ip(ip_name): # don't import more than once if ip_name in imported_names: return # turn Name into ip_name ip_name_l = 'ip_' + ip_name.lower() # import the module, but don't instantiate the ip class yet ip_m = __import__(ip_name_l) # import dependencies first if user has specified # auto-deps. if auto_deps: for dep in ip_m.dependencies: import_ip(dep) # instantiate ip_name.Name ip = getattr(ip_m, ip_name)() ip_instance_list.append(ip) imported_names.append(ip_name) print "%s imported from %s." % \ (ip_name, ip_m.__file__) # import all ip_names, including dependencies for ip_name in ip_names: import_ip(ip_name) # now check for dependencies and error if necessary # (in the case of auto_deps this will obviously be fine) deps_errors = [] for ip in ip_instance_list: n = ip.__class__.__name__ # there must be a more elegant way to get the module instance? deps = sys.modules[ip.__module__].dependencies for d in deps: # remember that if a package asks for "VTK", "VTK561" is also fine d_satisfied = False for ip_name in imported_names: if ip_name.startswith(d): d_satisfied = True # we don't have to finish more loops break elif ip_name == n: # this means we have reached the module whose deps # we're checking without satisfying dependency d, # which also means dependency problems, so we # can jut break out of the for loop break if not d_satisfied: deps_errors.append( '>>>>> Unsatisfied dependency: %s should be specified before %s' % (d, n)) if deps_errors: print "\n".join(deps_errors) utils.error("Unsatisfied dependencies. Fix and try again.") def get_stage(ip, n): utils.output("%s :: get()" % (n, ), rpad, rpad_char) ip.get() def unpack_stage(ip, n): utils.output("%s :: unpack()" % (n, ), rpad, rpad_char) ip.unpack() def configure_stage(ip, n): utils.output("%s :: configure()" % (n, ), rpad, rpad_char) ip.configure() def build_stage(ip, n): utils.output("%s :: build()" % (n, ), rpad, rpad_char) ip.build() def install_stage(ip, n): utils.output("%s :: install()" % (n, ), rpad, rpad_char) ip.install() def all_stages(ip, n): get_stage(ip, n) unpack_stage(ip, n) configure_stage(ip, n) build_stage(ip, n) install_stage(ip, n) if mode == 'show_versions': utils.output('Extracting all install_package versions.') print "python: %d.%d.%d (%s)" % \ (sys.version_info[0:3] + (config.PYTHON_EXECUTABLE,)) for ip in ip_instance_list: n = ip.__class__.__name__ if not n in ip_names: # n is a dependency, so do everything utils.output("%s (DEPENDENCY)" % (n, ), 70, '#') all_stages(ip, n) elif target != None and target != n: # A target has been specified (but this ip is not it), # so do everything for all other install packages we encounter. utils.output("%s (NON-TARGET)" % (n, ), 70, '#') all_stages(ip, n) elif mode == 'get_only': utils.output("%s GET_ONLY" % (n, ), 70, '#') utils.output("%s" % (n, ), 70, '#') get_stage(ip, n) elif mode == 'unpack_only': utils.output("%s UNPACK_ONLY" % (n, ), 70, '#') utils.output("%s" % (n, ), 70, '#') unpack_stage(ip, n) elif mode == 'configure_only': utils.output("%s CONFIGURE_ONLY" % (n, ), 70, '#') utils.output("%s" % (n, ), 70, '#') configure_stage(ip, n) elif mode == 'everything': utils.output("%s" % (n, ), 70, '#') all_stages(ip, n) elif mode == 'clean_build': utils.output("%s CLEAN_BUILD" % (n, ), 70, '#') ip.clean_build() elif mode == 'show_versions': print '%s: %s' % (n, ip.get_installed_version()) elif mode == 'rebuild': utils.output("%s REBUILD" % (n, ), 70, '#') # clean up ip.clean_build() # rebuild (executes all stages, as previous # stages are required and user will likely # need an install also) all_stages(ip, n) elif mode == 'reinstall': utils.output("%s REINSTALL" % (n, ), 70, '#') # clean up ip.clean_install() # reinstall all_stages(ip, n) else: utils.output("%s CUSTOM MODE" % (n, ), 70, '#') if hasattr(ip, mode): utils.output("%s :: %s()" % (n, mode), rpad, rpad_char) getattr(ip, mode)() else: utils.error("Mode not found: %s" % (mode, )) if mode != 'show_versions': # Print elapsed time and final message t = time.time() - start_time utils.output("Execution time (h:mm:ss): %d:%02d:%02d" % (int(t / 3600), int((t % 3600) / 60), t % 60)) utils.output("Now please read the RESULTS section of README.txt!")
def do_stat_sig_testing(methods, measures, label_aggregations, configs, results, limit_variables=None, run_mode="run"): testable_variables = list(configs[0].ddict.keys()) if limit_variables: testable_variables = [ x for x in testable_variables if x in limit_variables ] info("Running statistical tests on{} variables: {}".format( " all" if limit_variables is None else " specified", testable_variables)) for method, measure, label_aggregation in product(methods, measures, label_aggregations): info("Running statistical testing via {} on {} {}".format( method, label_aggregation, measure)) # get ids and variable values per configuration df_inputs = [] try: for run_id in results: # find corresp. configuration conf = [c for c in configs if c.id == run_id] error( "Num configurations found with id: {} is: {} during stat-testing!" .format(run_id, len(conf)), len(conf) != 1) conf = conf[0] # get variables df_row = {k: v for (k, v) in conf.ddict.items() if k != "id"} for score in results[run_id][run_mode][measure][ label_aggregation]['folds']: df_row["score"] = score df_inputs.append(deepcopy(df_row)) except: warning("Encountered invalid results accessors: {}".format( (run_mode, measure, label_aggregation))) continue data = pd.DataFrame(df_inputs) inst = instantiator.Instantiator() stat_test = inst.create(method) for v, variable in enumerate(testable_variables): info("Experiment variable {}/{}: {}".format( v + 1, len(testable_variables), variable)) if limit_variables is not None: if variable not in limit_variables: continue if len(data[variable]) == len(set(data[variable])): warning( "Skipping testing for parameter [{}] due to having only 1 observation per value" .format(variable)) continue if len(set(data[variable])) == 1: warning( "Skipping testing for parameter [{}] due to having only 1 unique parameter value: {}" .format(variable, data[variable].values[0])) continue stat_result = stat_test.run(data["score"], data[variable]) stat_test.report()
def main(input_path, only_report=False, force_dir=False, no_config_check=False, restart=False, is_testing_run=False, manual_config_tag=None): # settable parameters ############################################################ email = "*****@*****.**" passw = None ############################################################ # set the experiment parameters error("Non-existent input path: {} ".format(input_path), not exists(input_path)) if isdir(input_path): # assume a single .yml file in the directory ymls = [ x for x in listdir(input_path) if any( x.endswith(suff) for suff in [".yaml", ".yml"]) ] error( "Input path {} is a directory with no yaml configuration files.". format(input_path), not ymls) error( "Input path is a directory with more than one yaml configuration files." .format(input_path), len(ymls) > 1) config_file = join(input_path, ymls[0]) else: config_file = input_path # if input file is existing csv scores, just print them if config_file.endswith(".csv"): print_existing_csv_results(config_file) return conf = read_ordered_yaml(config_file) try: exps = conf[EXPERIMENTS_KEY_NAME] except KeyError: error( f"Need an [{EXPERIMENTS_KEY_NAME}] key for large-scale experiments." ) # folder to run experiments in run_dir = exps["run_folder"] if force_dir: warning( "Overriding experiment folder from yml value: {} to current dir: {}, due to force-dir" .format(run_dir, dirname(run_dir))) run_dir = dirname(input_path) if not isabs(run_dir): run_dir = join(os.getcwd(), run_dir) # dir checks # ---------- # virtualenv folder venv_dir = conf[EXPERIMENTS_KEY_NAME]["venv"] if "venv" in conf[ EXPERIMENTS_KEY_NAME] else None # results csv file # results_file = conf["experiments"]["results_file"] results_file = join(run_dir, "run_results.csv") if venv_dir and not exists(venv_dir): error("Virtualenv dir {} not found".format(venv_dir)) if not exists(run_dir): info("Run dir {} not found, creating.".format(run_dir)) makedirs(run_dir) else: error( "Specified a non-dir path as the running directory: {}".format( run_dir), not isdir(run_dir)) if restart: warning( "Specified restart, and experiment dir {} exists. Deleting!") rmtree(run_dir) makedirs(run_dir) # logging os.makedirs(run_dir, exist_ok=True) setup_simple_logging(conf["print"]["log_level"], logging_dir=run_dir) info("Generating configurations from source file {}".format(config_file)) # evaluation measures try: eval_measures = as_list(exps["measures"]) if "measures" in exps else [ "f1-score", "accuracy" ] print(eval_measures) aggr_measures = as_list(exps["label_aggregation"]) if "label_aggregation" in exps \ else ["macro", "micro"] stat_functions = as_list( exps["fold_aggregation"]) if "fold_aggregation" in exps else [ "mean" ] run_types = as_list( exps["run_types"]) if "run_types" in exps else ["run"] do_sstests = "sstests" in exps if not do_sstests: warning("No statistical tests specified.") else: sstests = ["tukeyhsd" ] if "names" not in exps["sstests"] else as_list( exps["sstests"]["names"]) sstests_measures = [ "f1-score" ] if "measures" not in exps["sstests"] else as_list( exps["sstests"]["measures"]) sstests_aggregations = [ "macro" ] if "aggregations" not in exps["sstests"] else as_list( exps["sstests"]["aggregations"]) sstests_limit_vars = None if "limit_variables" not in exps[ "sstests"] else as_list(exps["sstests"]["limit_variables"]) except Exception as ex: error( "Failed to read evaluation / testing options due to: [{}]".format( ex)) # folder where run scripts are sources_dir = exps["sources_dir"] if "sources_dir" in exps else os.getcwd() warning("Defaulting sources folder to the current directory: {}".format( sources_dir)) error( "Main module: {} not found. Is the sources dir ok?".format( join(sources_dir, "main.py")), not exists(join(sources_dir, "main.py"))) configs = make_configs(conf, run_dir, sources_dir) # check run id uniqueness if len(set([c.id for c in configs])) != len(configs): error("Duplicate run folders from the input: {}".format( [c.id for c in configs])) if len(set([c['folders']['run'] for c in configs])) != len(configs): error("Duplicate run folders from the input: {}".format( [c["folders"]["run"] for c in configs])) # if we're running a testing suite, filter out incompatible configs if is_testing_run: configs = filter_testing(configs, config_file) # mail do_send_mail = exps["send_mail"] if "send_mail" in exps else None if do_send_mail: passw = getpass.getpass() # copy the experiments configuration file in the target directory experiments_conf_path = join(run_dir, basename(config_file)) if exists(experiments_conf_path): # make sure it's the same effing config, unless check is overriden if not no_config_check: config_to_copy = OrderedDict( {k: v for (k, v) in conf.items() if k != EXPERIMENTS_KEY_NAME}) existing_exp_conf = read_ordered_yaml(experiments_conf_path) existing_exp_conf = OrderedDict({ k: v for (k, v) in existing_exp_conf.items() if k != EXPERIMENTS_KEY_NAME }) equal, diff = compare_dicts(config_to_copy, existing_exp_conf) if not equal: error( "The workflow contents derived from the original config [{}] differ from the ones in the experiment directory: [{}]!\nDifference is: {}" .format(config_file, experiments_conf_path, diff)) else: if not only_report: info("Copying experiments configuration at {}".format( experiments_conf_path)) with open(experiments_conf_path, "w") as f: write_ordered_dump(OrderedDict(conf), f) else: info( "Only-report run: will not copy experiment configuration at {}" .format(experiments_conf_path)) results, result_paths = {}, {} ################################################################################# skipped_configs = [] # prelim experiments for conf_index, conf in enumerate(configs): run_id = conf.id # prepend a configuration id tag, if supplied if manual_config_tag is not None: run_id += manual_config_tag experiment_dir = conf["folders"]["run"] + manual_config_tag else: experiment_dir = conf["folders"]["run"] info("Running experimens for configuration {}/{}: {}".format( conf_index + 1, len(configs), run_id)) completed_file = join(experiment_dir, "completed") error_file = join(experiment_dir, "error") # results to run folders, if not specified otherwise respath = join(experiment_dir, "results") if not isabs(respath): conf["folders"]["results"] = join(experiment_dir, respath) if exists(completed_file): info("Skipping completed experiment {}".format(run_id)) elif only_report: info("Only-report execution: skipping non-completed experiment {}". format(run_id)) skipped_configs.append(run_id) continue else: # run it if exists(error_file): os.remove(error_file) makedirs(experiment_dir, exist_ok=True) conf_path = join(experiment_dir, "config.yml") if exists(conf_path) and not no_config_check: warning("Configuration file at {} already exists!".format( conf_path)) existing = read_ordered_yaml(conf_path) equal, diff = compare_dicts(existing, conf) if not equal: error( "Different local config encountered: {} \nDifference: {}" .format(conf_path, diff)) #if not (OrderedDict(conf) == existing): # error("Different local config encountered at {}".format(conf_path)) else: with open(conf_path, "w") as f: write_ordered_dump(OrderedDict(conf), f) info("Configuration file: {}".format(conf_path)) # write the run script file script_path = join(experiment_dir, "run.sh") with open(script_path, "w") as f: if venv_dir: f.write("source \"{}/bin/activate\"".format(venv_dir)) f.write("cd \"{}\"\n".format(sources_dir)) f.write( "python3 \"{}\" \"{}\" && touch \"{}\" && exit 0\n".format( join(sources_dir, "main.py"), conf_path, completed_file)) f.write("touch '{}' && exit 1\n".format(error_file)) subprocess.run(["/usr/bin/env", "bash", script_path]) if exists(error_file): print("An error has occurred in the run, exiting.") info("An error has occurred in the run, exiting.") if do_send_mail: sendmail(email, passw, "an error occurred") exit(1) # read experiment results exp_res_file = join(experiment_dir, "results", "results.pkl") with open(exp_res_file, "rb") as f: res_data = pickle.load(f) results[run_id] = res_data result_paths[run_id] = exp_res_file # messages = [] total_results = {} # show results for stat in stat_functions: info("Results regarding {} statistic:".format(stat)) print_vals = {} for run_id in results: print_vals[run_id] = {} for m in eval_measures: for run in run_types: for ag in aggr_measures: try: results[run_id][run][m][ag] except KeyError: continue header = "{}.{}.{}.{}".format(run[:3], m[:3], ag[:3], stat) if stat in "var mean std".split(): val = results[run_id][run][m][ag][stat] if val is None: continue val = round(val, decimals=4) print_vals[run_id][header] = val # print'em info("SCORES:") print_dataframe_results(print_vals) total_results[stat] = print_vals info("Writing these results to file {}".format(results_file)) total_df = pd.DataFrame.from_dict(total_results, orient='index') if total_df.size == 0: info("No results parsed.") else: total_df.to_csv(results_file) if skipped_configs: for s, sk in enumerate(skipped_configs): info("Skipped incomplete config: {}/{} : {}".format( s + 1, len(skipped_configs), sk)) if do_sstests: do_stat_sig_testing(sstests, sstests_measures, sstests_aggregations, configs, results, sstests_limit_vars) # [info(msg) for msg in messages] if do_send_mail: sendmail(email, passw, "run complete.")
def validate(chplLocaleModel, chplComm): if chplLocaleModel == 'gpu' and chplComm != "none": error("The prototype GPU support does not work when CHPL_COMM is not set to\n\"none\".");
def produce_outputs(self): error("Attempted to produce outputs via abstract function.")
def main(): global options, device # Get the ENV context script_dir = os.path.dirname(__file__) env = os.environ.copy() # Set the input folder input_dir = os.path.expanduser(options.input_dir) if options.input_dir \ else os.path.join(script_dir, '..', 'data') vgg_path = os.path.join(input_dir, 'vgg', 'imagenet-vgg-verydeep-19.mat') coco_dir = os.path.join(input_dir, 'train') if not os.path.isdir(input_dir): fail('Failed to find the input folder at ' + input_dir) if not os.path.isfile(vgg_path): error('Failed to find the VGG model file at ' + vgg_path) fail( 'Please download it from http://www.vlfeat.org/matconvnet/models/beta16/imagenet-vgg-verydeep-19.mat' ) if not os.path.isdir(coco_dir): error('Failed to find the COCO 2014 training images in ' + coco_dir) fail( 'Plese download it from http://images.cocodataset.org/zips/train2014.zip' ) # Set the output folder output_dir = os.path.expanduser(options.output_dir) if options.output_dir \ else env.get('OUTPUT_DIR', os.path.join(script_dir, '..', 'output')) model_dir = os.path.join(output_dir, 'checkpoint') if os.path.isdir(output_dir): if not os.path.isdir(model_dir): info('Creating a folder to store checkpoint at ' + model_dir) os.makedirs(model_dir) else: info('Creating a folder to store checkpoint at ' + model_dir) os.makedirs(model_dir) # Set the TensorBoard folder log_dir = os.path.expanduser(options.log_dir) if options.log_dir \ else env.get('LOG_DIR', os.path.join(script_dir, '..', 'log')) if not os.path.isdir(log_dir): info('Creating a folder to store TensorBoard events at ' + log_dir) os.makedirs(log_dir) # Set the style image path style_path = os.path.expanduser(options.style_image) if os.path.isfile(options.style_image) \ else os.path.join(input_dir, 'style_images', options.style_image) style_name = os.path.basename(os.path.splitext(style_path)[0]) ckpt_path = os.path.join(model_dir, style_name + '.ckpt') if not os.path.isfile(style_path): fail('Failed to find the style image at ' + style_path) # Set hyper parameters batch_size = options.batch_size epochs = options.epoch lr = options.lr lambda_tv = options.lambda_tv lambda_feat = options.lambda_feat lambda_style = options.lambda_style # Print parsed arguments info('--------- Training parameters -------->') info('Style image path: ' + style_path) info('VGG model path: ' + vgg_path) info('Training image dir: ' + coco_dir) info('Checkpoint path: ' + ckpt_path) info('TensorBoard log dir: ' + log_dir) info('Training device: ' + device) info('Batch size: %d' % batch_size) info('Epoch count: %d' % epochs) info('Learning rate: ' + str(lr)) info('Lambda tv: ' + str(lambda_tv)) info('Lambda feat: ' + str(lambda_feat)) info('Lambda style: ' + str(lambda_style)) info('<-------- Training parameters ---------') # COCO images to train content_targets = list_jpgs(coco_dir) if len(content_targets) % batch_size != 0: content_targets = content_targets[:-(len(content_targets) % batch_size)] info('Total training data size: %d' % len(content_targets)) # Image shape image_shape = (224, 224, 3) batch_shape = (batch_size, ) + image_shape # Style target style_target = read_img(style_path) style_shape = (1, ) + style_target.shape with tf.device(device), tf.Session() as sess: # Compute gram maxtrix of style target style_image = tf.placeholder(tf.float32, shape=style_shape, name='style_image') vggstyletarget = vgg.net(vgg_path, vgg.preprocess(style_image)) style_vgg = vgg.get_style_vgg(vggstyletarget, style_image, np.array([style_target])) # Content target feature content_vgg = {} inputs = tf.placeholder(tf.float32, shape=batch_shape, name='inputs') content_net = vgg.net(vgg_path, vgg.preprocess(inputs)) content_vgg['relu4_2'] = content_net['relu4_2'] # Feature after transformation outputs = stylenet.net(inputs / 255.0) vggoutputs = vgg.net(vgg_path, vgg.preprocess(outputs)) # Compute feature loss loss_f = options.lambda_feat * vgg.total_content_loss( vggoutputs, content_vgg, batch_size) # Compute style loss loss_s = options.lambda_style * vgg.total_style_loss( vggoutputs, style_vgg, batch_size) # Total variation denoising loss_tv = options.lambda_tv * vgg.total_variation_regularization( outputs, batch_size, batch_shape) # Total loss total_loss = loss_f + loss_s + loss_tv train_step = tf.train.AdamOptimizer(options.lr).minimize(total_loss) # Create summary tf.summary.scalar('loss', total_loss) merged = tf.summary.merge_all() # Used to save model saver = tf.train.Saver() with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess: # Restore checkpoint if available sess.run(tf.global_variables_initializer()) ckpt = tf.train.get_checkpoint_state(model_dir) if ckpt and ckpt.model_checkpoint_path: info('Restoring from ' + ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) # Write the graph writer = tf.summary.FileWriter(log_dir, sess.graph) # Start to train total_step = 0 for epoch in range(epochs): info('epoch: %d' % epoch) step = 0 while step * batch_size < len(content_targets): time_start = time.time() # Load one batch batch = np.zeros(batch_shape, dtype=np.float32) for i, img in enumerate( content_targets[step * batch_size:(step + 1) * batch_size]): batch[i] = read_img(img, image_shape).astype( np.float32) # (224,224,3) # Proceed one step step += 1 total_step += 1 _, loss, summary = sess.run([train_step, total_loss, merged], feed_dict={inputs: batch}) time_elapse = time.time() - time_start if total_step % 5 == 0: info('[step {}] elapse time: {} loss: {}'.format( total_step, time_elapse, loss)) writer.add_summary(summary, total_step) # Write checkpoint if total_step % 2000 == 0: info('Saving checkpoint to ' + ckpt_path) saver.save(sess, ckpt_path, global_step=total_step) info('Saving final checkpoint to ' + ckpt_path) saver.save(sess, ckpt_path, global_step=total_step)
def check_eq(value1, value2, msg): if value1 != value2: error(msg, 'Count mismatch: expected %d, but got %d' % (value2, value1))
def get_component_inputs(self): error("Attempted to get inputs via abstract function.")
def main_denoising(wav_files, output_dir, verbose=False, **kwargs): """Perform speech enhancement for WAV files in ``wav_dir``. Parameters ---------- wav_files : list of str Paths to WAV files to enhance. output_dir : str Path to output directory for enhanced WAV files. verbose : bool, optional If True, print full stacktrace to STDERR for files with errors. kwargs Keyword arguments to pass to ``denoise_wav``. """ if not os.path.exists(output_dir): os.makedirs(output_dir) # Load global MVN statistics. global_mean_var = sio.loadmat(GLOBAL_MEAN_VAR_MATF) global_mean = global_mean_var['global_mean'] global_var = global_mean_var['global_var'] # Perform speech enhancement. for src_wav_file in wav_files: # Perform basic checks of input WAV. if not os.path.exists(src_wav_file): utils.error('File "%s" does not exist. Skipping.' % src_wav_file) continue if not utils.is_wav(src_wav_file): utils.error('File "%s" is not WAV. Skipping.' % src_wav_file) continue if utils.get_sr(src_wav_file) != SR: utils.error('Sample rate of file "%s" is not %d Hz. Skipping.' % (src_wav_file, SR)) continue if utils.get_num_channels(src_wav_file) != NUM_CHANNELS: utils.error('File "%s" is not monochannel. Skipping.' % src_wav_file) continue if utils.get_bitdepth(src_wav_file) != BITDEPTH: utils.error('Bitdepth of file "%s" is not %d. Skipping.' % (src_wav_file, BITDEPTH)) continue # Denoise. try: bn = os.path.basename(src_wav_file) dest_wav_file = os.path.join(output_dir, bn) denoise_wav(src_wav_file, dest_wav_file, global_mean, global_var, **kwargs) print('Finished processing file "%s".' % src_wav_file) except Exception as e: msg = 'Problem encountered while processing file "%s". Skipping.' % src_wav_file if verbose: msg = '%s Full error output:\n%s' % (msg, e) utils.error(msg) continue
def runtest(): """check command line interface""" # input and output files source = os.path.join(Dirs.sources, "ashello.asm") with open(Files.output, "wb") as f: xas(source, "-R", "-o", "-", stdout=f) xdm(Disks.asmsrcs, "-e", "ASHELLO-O", "-o", Files.reference) check_obj_code_eq(Files.output, Files.reference) with open(Files.output, "wb") as f: xas(source, "-R", "-i", "-o", "-", stdout=f) xdm(Disks.asmsrcs, "-e", "ASHELLO-I", "-o", Files.reference) check_image_files_eq(Files.output, Files.reference) with open(Files.output, "w") as f: xas(source, "-R", "-o", Files.output, "-L", "-", stdout=f) xdm(Disks.asmsrcs, "-e", "ASHELLO-L", "-o", Files.reference) check_list_files_eq(Files.output, Files.reference) source = os.path.join(Dirs.sources, "nonexisting") with open(Files.error, "w") as ferr: xas(source, "-i", "-R", "-o", Files.output, stderr=ferr, rc=1) with open(Files.error, "r") as ferr: errs = ferr.readlines() if len(errs) != 1 or errs[0][:10] != "File error": error("File error", "Incorrect file error message") # include path source = os.path.join(Dirs.sources, "ascopyi.asm") incls = os.path.join(Dirs.sources, "test") + "," + \ os.path.join(Dirs.sources, "test", "test") xas(source, "-i", "-I", incls, "-o", Files.output) with open(Files.output, "rb") as f: data = f.read() if len(data[6:]) != 20: error("Include paths", "Incorrect image length") # command-line definitions source = os.path.join(Dirs.sources, "asdef.asm") xas(source, "-b", "-D", "s1=1", "s3=3", "s2=4", "-o", Files.output) assert content(Files.output) == "\x01\x03" xas(source, "-b", "-D", "s1=2,s2=2,s3=3", "-o", Files.output) assert content(Files.output) == "\x02\x03" # various parameter combinations source = os.path.join(Dirs.sources, "asxbank1.asm") remove([Files.reference]) xas(source, "-b", "-o", Files.output, "-L", Files.reference) check_exists([Files.reference]) # text data output source = os.path.join(Dirs.sources, "ascart.asm") xas(source, "-b", "-R", "-o", Files.reference) xas(source, "-t", "a2", "-R", "-o", Files.output) check_bin_text_equal(Files.output, Files.reference) source = os.path.join(Dirs.sources, "asmtext.asm") xas(source, "-t", "a2", "-R", "-o", Files.output) check_instructions(Files.output, [";aorg>1000", "byte", ";aorg>2000", "byte"]) # symbols source = os.path.join(Dirs.sources, "assyms.asm") xas(source, "-b", "-R", "-o", Files.reference, "-E", Files.output) check_symbols(Files.output, (("START", ">0000"), ("S1", ">0001"), ("S2", ">0018"), ("VDPWA", ">8C02"))) # disable warnings source = os.path.join(Dirs.sources, "aswarn.asm") with open(Files.error, "w") as ferr: xas(source, "-b", "-R", "-w", "-o", Files.output, stderr=ferr, rc=0) if content_len(Files.error) > 0: error("warn", "warnings, even though disabled") # cleanup os.remove(Files.output) os.remove(Files.reference) os.remove(Files.error)
def get(flag='host'): if flag == 'host': platform_val = overrides.get('CHPL_HOST_PLATFORM') elif flag == 'target': platform_val = overrides.get('CHPL_TARGET_PLATFORM') if not platform_val: platform_val = get('host') else: raise error("Invalid flag: '{0}'".format(flag), ValueError) if not platform_val: # Check for cray platform. It is a cray platform if there is a # cle-release/CLEinfo config file with a known network value in it. cle_info_file = os.path.abspath( '/etc/opt/cray/release/cle-release') # CLE >= 6 if not os.path.exists(cle_info_file): cle_info_file = os.path.abspath( '/etc/opt/cray/release/CLEinfo') # CLE <= 5 if os.path.exists(cle_info_file): with open(cle_info_file, 'r') as fp: cle_info = fp.read() net_pattern = re.compile('^NETWORK=(?P<net>[a-zA-Z]+)$', re.MULTILINE) net_match = net_pattern.search(cle_info) if net_match is not None and len(net_match.groups()) == 1: net = net_match.group('net') if net.lower() == 'ari': platform_val = 'cray-xc' if not platform_val: network = os.environ.get('CRAYPE_NETWORK_TARGET', '') if network.startswith("slingshot") or network == "ofi": platform_val = 'hpe-cray-ex' if not platform_val: # uname() -> (system, node, release, version, machine, processor) uname = platform.uname() platform_val = uname[0].lower().replace('_', '') machine = uname[4] if platform_val == 'linux': if 'ppc' in machine: endianness = 'le' if 'le' in machine else '' bits = '64' if '64' in machine else '32' platform_val = 'linux_ppc_{0}{1}'.format(endianness, bits) elif machine == 'x86_64': build_64_as_32 = os.environ.get('CHPL_BUILD_X86_64_AS_32') if build_64_as_32 == "1": platform_val = "linux64_32" else: platform_val = "linux64" elif machine == 'aarch64': platform_val = "linux64" else: platform_val = "linux32" elif platform_val.startswith("cygwin"): if machine == 'x86_64': platform_val = "cygwin64" else: platform_val = "cygwin32" elif platform_val.startswith('netbsd'): if machine == 'amd64': platform_val = 'netbsd64' else: platform_val = 'netbsd32' return platform_val
def check_trunc(infile, lines, length): """check prefixes of lines""" with open(infile, 'r') as f: for i, inline in enumerate(f): if inline.rstrip() != lines[i].rstrip()[:length]: error('Truncated records', 'Record %d mismatch' % i)
def configure_embedding(self): # incorporate embeddings in the neural architecture error("Attempted to access abstract embedding configuration function.")
def new(message): # logging.info('(' + message.qq + '): ' + message.text) qq_group_id = int(message.group) _, tg_group_id, forward_index = get_forward_index(qq_group_id=qq_group_id) text = message.text # get message text # text, _ = cq_image_regex.subn('', text) # clear CQ:image in text # replace special characters text = decode_cq_escape(text) text = cq_emoji_regex.sub(lambda x: chr(int(x.group(1))), text) # replace [CQ:emoji,id=*] text = cq_face_regex.sub(lambda x: qq_emoji_list[int(x.group(1))] if int(x.group(1)) in qq_emoji_list else '\u2753', text) # replace [CQ:face,id=*] text = cq_bface_regex.sub('\u2753', text) # replace bface to '?' text = cq_sface_regex.sub( lambda x: qq_sface_list[int(x.group(1)) & 255] if int(x.group(1)) > 100000 and int(x.group(1)) & 255 in qq_sface_list else '\u2753', text) # replace [CQ:sface,id=*], https://cqp.cc/t/26206 def replace_name(qq_number): # replace each qq number with preset id qq_number = qq_number.group(1) if int(qq_number) == QQ_BOT_ID: return '@bot' result = '@' + get_qq_name(int(qq_number), forward_index) result = result.replace(':', ' ') return result text = CQAt.PATTERN.sub(replace_name, text) # replace CQAt to @username # send pictures to Telegram group pic_send_mode = 2 # mode = 0 -> direct mode: send cqlink to tg server # mode = 1 -> (deprecated) download mode: download to local,send local link to tg server # mode = 2 -> download mode: download to local, upload from disk to tg server message_parts = cq_image_simple_regex.split(text) message_parts_count = len(message_parts) image_num = message_parts_count - 1 if image_num == 0: # send plain text message with bold group member name full_msg_bold = '<b>' + get_qq_name(int( message.qq), forward_index) + '</b>: ' + text.strip().replace( '<', '<').replace('>', '>') global_vars.tg_bot.sendMessage(tg_group_id, full_msg_bold, parse_mode='HTML') else: if message_parts[0]: part_msg_bold = '<b>' + get_qq_name(int(message.qq), forward_index) + '</b>: ' +\ '(1/' + str(message_parts_count) + ')' + message_parts[0].strip().replace('<', '<').replace('>', '>') global_vars.tg_bot.sendMessage(tg_group_id, part_msg_bold, parse_mode='HTML') part_index = 1 else: message_parts.pop(0) message_parts_count -= 1 part_index = 0 for matches in CQImage.PATTERN.finditer(message.text): # replace QQ number to group member name, get full message text if message_parts_count == 1: part_msg = get_qq_name( int(message.qq), forward_index) + ': ' + message_parts[part_index].strip() else: part_msg = get_qq_name(int( message.qq), forward_index) + ': ' + '(' + str( part_index + 1) + '/' + str( message_parts_count ) + ')' + message_parts[part_index].strip() part_index += 1 decode_cq_escape(part_msg) filename = matches.group(1) url = cq_get_pic_url(filename) pic = url if pic_send_mode == 1: cq_download_pic(filename) pic = SERVER_PIC_URL + filename elif pic_send_mode == 2: cq_download_pic(filename) pic = open(os.path.join(CQ_IMAGE_ROOT, filename), 'rb') # gif pictures send as document if filename.lower().endswith('gif'): try: global_vars.tg_bot.sendDocument(tg_group_id, pic, caption=part_msg) except BadRequest: # when error occurs, download picture and send link instead error(message) traceback.print_exc() if pic_send_mode == 0: cq_download_pic(filename) pic = get_short_url(SERVER_PIC_URL + filename) global_vars.tg_bot.sendMessage(tg_group_id, pic + '\n' + part_msg) # jpg/png pictures send as photo else: try: # the first image in message attach full message text global_vars.tg_bot.sendPhoto(tg_group_id, pic, caption=part_msg) except BadRequest: # when error occurs, download picture and send link instead error(message) traceback.print_exc() if pic_send_mode == 0: cq_download_pic(filename) my_url = get_short_url(SERVER_PIC_URL + filename) pic = my_url global_vars.tg_bot.sendMessage(tg_group_id, pic + '\n' + part_msg) return True
import discord, re, utils, sqlite3, traceback, sys from discord.ext import commands from datetime import datetime try: from pyfiglet import print_figlet except ImportError: utils.error("PyFiglet is not installed, run pip3 install pyfiglet.", "Events", terminate=True) SLUR_REGEX = r"nigg(er|a)|f****t|tranny|crack(er|a)" INVITE_REGEX = r"(https?:\/\/)?(www\.)?((discord|invite)\.(gg|io|me|li)|discordapp\.com\/invite)\/.+[a-z]" NSFW_REGEX = r"(https?:\/\/)?(www\.)?(pornhub|redtube|youporn|tube8|pornmd|thumbzilla|modelhub)\.com" LINK_REGEX = r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\), ]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" INV_VALID_REGEX = r"(https?:\/\/)?(www\.)?(discord\.gg|discordapp\.com\/invite)\/(.[^\s]+)" class BasicEvents(commands.Cog): def __init__(self, bot): self.bot = bot def perm_cleanup(self, perm): return perm.replace("_", " ").title() @commands.Cog.listener() async def on_ready(self): print_figlet(self.bot.config.figlet)
def produce_outputs(self): # get input configuration data self.topk = None self.messages = [] self.input_parameters_dict = [dp for dp in self.data_pool.data if type(dp.data) == Dictionary][0].data.instances self.input_parameters = to_namedtuple(self.input_parameters_dict, "input_parameters") # get reference data by chain name output self.label_mapping = [] for mapping in self.params.label_mappings: # read json if type(mapping) is str: try: with open(mapping) as f: mapping = json.load(f) except: error("Requires json labelmapping or literal list") mapping_dict = {ix: val for (ix, val) in enumerate(mapping)} self.label_mapping.append(mapping_dict) # thresholding for th in self.params.thresholds: if th not in self.input_parameters_dict: self.result = {"results": [], "input_params": self.input_parameters_dict, "messages": [f"Threshold {th} missing from input parameters"]} return datapack = [x for x in self.data_pool.data if x.chain == self.params.data_chain][0] predictions, tagged_idx = [], [] for i, chain_name in enumerate(self.params.pred_chains): # predictions chain_preds = [x for x in self.data_pool.data if x.chain == chain_name][0] predictions.append(chain_preds) # for text data, keep just the words if type(datapack.data) == Text: data = [x["words"] for x in datapack.data.instances] res = [] predictions = [x.data.instances for x in predictions] num_all_ngrams = len(predictions[0]) num_steps = len(predictions) # compute thresholding values thresholding = np.zeros((num_all_ngrams, len(self.params.thresholds)), bool) # for i, th in enumerate(self.params.thresholds): # th_val = float(self.input_parameters_dict[th]) thresholding[:, 0] = predictions[0][:, 1] > float(self.input_parameters_dict[self.params.thresholds[0]]) thresholding[:, 1] = predictions[1][:, 1] > float(self.input_parameters_dict[self.params.thresholds[1]]) thresholding[:, 2] = np.any(predictions[2] > float(self.input_parameters_dict[self.params.thresholds[2]]), axis=1) ngram_tags = sorted([x for x in datapack.usages[0].tags if x.startswith("ngram_inst")]) with tictoc("Classification report building", announce=False): for n, ngram_tag in enumerate(ngram_tags): # indexes of the tokens for the current instance # to the entire data container original_instance_ix_data = datapack.usages[0].get_tag_instances(ngram_tag) inst_obj = {"instance": n, "data": [data[i] for i in original_instance_ix_data], "predictions": []} for local_word_idx, ix in enumerate(original_instance_ix_data): word_obj = {"word": data[ix], "word_idx": int(local_word_idx), "overall_predictions": {}} detailed = [] # for each step for step_idx in range(num_steps): preds = predictions[step_idx] step_name = self.params.pred_chains[step_idx] step_obj = {"name": step_name, "step_index": step_idx} survives = thresholding[ix, step_idx] step_preds = np.expand_dims(preds[ix, :], axis=0) scores, classes = self.get_topK_preds(step_preds, self.label_mapping[step_idx], self.params.only_report_labels[step_idx]) step_preds = {c: round(s, 4) for (c, s) in zip(classes[0], scores[0])} step_obj["step_preds"] = step_preds detailed.append(step_preds) modified, deleted, replaced = thresholding[ix, :] modify_obj = {"modified": int(modified), "prob": detailed[0]["modify"]} word_obj["overall_predictions"]["modify_prediction"] = modify_obj delete_obj = detailed[1] # replaced objs = [] for word, prob in detailed[2].items(): objs.append({"word": word, "prob": prob}) replace_obj = objs if modified: if deleted: # deleted word_obj["overall_predictions"]["delete_prediction"] = delete_obj elif replaced: word_obj["overall_predictions"]["replace_prediction"] = replace_obj if not self.omit_detailed_results(): word_obj["detailed_predictions"] = {"modify_prediction": modify_obj, "delete_prediction": delete_obj, "replace_prediction": replace_obj} inst_obj["predictions"].append(word_obj) res.append(inst_obj) self.result = {"results": res, "input_params": self.input_parameters_dict, "messages": self.messages}
def runInstruction(instr): ''' Run the given instruction. In case of block begin, the instructions will be collected and ran when the block end command is encountered. ''' global canvasWidth global canvasHeight global canvas global canvasImage global canvasBackground global inBlock global maxDuration global animationQueue if instr.data == 'canvas_size': if canvasWidth is not None and canvasHeight is not None: raise CanvasReadjustError( errorAtToken("Canvas size changed after drawing commands.", instr.children[0])) width = int(instr.children[0].children[0]) height = int(instr.children[1].children[0]) color = (255, 255, 255, 255) if len(instr.children) == 6: red = int(instr.children[2].children[0]) if red < 0 or red > 255: raise ColorOutOfRangeError( errorAtToken("0 <= color value <= 255", instr.children[2])) green = int(instr.children[3].children[0]) if green < 0 or green > 255: raise ColorOutOfRangeError( errorAtToken("0 <= color value <= 255", instr.children[3])) blue = int(instr.children[4].children[0]) if blue < 0 or blue > 255: raise ColorOutOfRangeError( errorAtToken("0 <= color value <= 255", instr.children[4])) alpha = int(instr.children[5].children[0]) if alpha < 0 or alpha > 255: alphaOutOfRangeError(instr.children[5]) color = (red, green, blue, alpha) canvasWidth = int(width) canvasHeight = int(height) canvasBackground = color elif instr.data == 'declare_image': checkCanvas(instr.children[0].children[0]) children = instr.children iPath = children[0].children[0][1:-1] id = children[1].children[0][1:-1] if id in imageData: raise DuplicateIdError( errorAtToken( "ID {} is already associated with an image.".format(id), children[1].children[0])) x = 0 if len(children) != 4 else int(children[2].children[0]) y = 0 if len(children) != 4 else int(children[3].children[0]) if not path.exists(iPath): raise ImagePathError( errorAtToken("Image at {} does not exist.".format(iPath), children[0].children[0])) if not path.isfile(iPath): raise ImagePathError( errorAtToken("{} is not a file".format(iPath), children[0].children[0])) try: im = Image.open(iPath).convert("RGBA") imageData[id] = im imageLocation[id] = (x, y) drawFrame() except PIL.UnidentifiedImageError: raise ImagePathError( errorAtToken( "Image at {} cannot be opened or identified.".format( iPath), children[0].children[0])) elif instr.data == 'move_object': checkCanvas(instr.children[0]) id = instr.children[0].children[0][1:-1] checkId(instr.children[0]) sx, sy = imageLocation[id] dx = int(instr.children[1].children[0]) dy = int(instr.children[2].children[0]) duration = instr.children[3].children[0] checkDuration(duration) duration = int(duration) animation = Move(sx, sy, dx, dy, duration) applyOrQueue(id, animation, duration) elif instr.data == 'rotate_object': checkCanvas(instr.children[0]) id = instr.children[0].children[0][1:-1] checkId(instr.children[0]) degrees = float(instr.children[1].children[0]) duration = instr.children[2].children[0] checkDuration(duration) duration = int(duration) animation = Rotate(degrees, duration, canvasBackground) applyOrQueue(id, animation, duration) elif instr.data == 'change_opacity': checkCanvas(instr.children[0]) id = instr.children[0].children[0][1:-1] checkId(instr.children[0]) destAlpha = int(instr.children[1].children[0]) if destAlpha < 0 or destAlpha > 255: alphaOutOfRangeError(instr.children[1]) duration = instr.children[2].children[0] checkDuration(duration) duration = int(duration) image = imageData[id] _, _, _, srcAlpha = image.getpixel((0, 0)) animation = Alpha(srcAlpha, destAlpha, duration) applyOrQueue(id, animation, duration) elif instr.data == 'scale_object': checkCanvas(instr.children[0]) id = instr.children[0].children[0][1:-1] checkId(instr.children[0]) sx = float(instr.children[1].children[0]) sy = float(instr.children[2].children[0]) duration = instr.children[3].children[0] checkDuration(duration) duration = int(duration) sWidth = imageData[id].width sHeight = imageData[id].height animation = Scale(sWidth, sHeight, duration, sx, sy) applyOrQueue(id, animation, duration) elif instr.data == 'wait': checkCanvas(instr.children[0]) duration = instr.children[0].children[0] checkDuration(duration) duration = int(duration) for i in range(duration): drawFrame() elif instr.data == 'delete_object': checkCanvas(instr.children[0]) id = instr.children[0].children[0][1:-1] checkId(instr.children[0]) del imageData[id] del imageLocation[id] drawFrame() elif instr.data == 'block_begin': if inBlock: raise NestedBlockError( error("Nested blocks are not supported", instr.line, 0)) inBlock = True elif instr.data == 'block_end': if not inBlock: raise BlockEndWithoutBeginError( error("No pairing begin block statement present", instr.line, 0)) applyAnimationsForDuration(animationQueue, maxDuration) inBlock = False maxDuration = 0 animationQueue.clear()
def forward(self, x): """Forward pass function""" # forward pass error("Attempted to access abstract forward function") return None
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING # IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. # ##################################################################### import os import sys from dsl import load_file from utils import sh, sh_str, info, error, env dsl = load_file('${BUILD_CONFIG}/repos.pyd', os.environ) def tag_repo(repo, tag): sh("git --git-dir=${repo}/.git tag ${tag}") if __name__ == '__main__': if len(sys.argv) < 2: error('Usage: apply-tag.py <tag>') tag = sys.argv[1] for i in dsl['repository'].values(): info('Tagging repository: {0}', i['name']) tag_repo(i)
def checkTrunc(infile, lines, length): """check prefixes of lines""" with open(infile, "r") as f: for i, inline in enumerate(f): if inline.rstrip() != lines[i].rstrip()[:length]: error("Truncated records", "Record %d mismatch" % i)
def usage(): error('%s [dataset]' % sys.argv[0])
def get(flag='target'): if flag == 'network': atomics_val = overrides.get('CHPL_NETWORK_ATOMICS') if not atomics_val: comm_val = chpl_comm.get() if comm_val in ['ofi', 'ugni'] and get('target') != 'locks': atomics_val = comm_val else: atomics_val = 'none' elif flag == 'target': atomics_val = overrides.get('CHPL_ATOMICS') if not atomics_val: compiler_val = chpl_compiler.get('target') platform_val = chpl_platform.get('target') # We default to C standard atomics (cstdlib) for gcc 5 and newer. # Some prior versions of gcc look like they support standard # atomics, but have buggy or missing parts of the implementation, # so we do not try to use cstdlib with gcc < 5. If support is # detected for clang (via preprocessor checks) we also default to # cstdlib atomics. For llvm-clang we always default to # cstdlib atomics. We know the llvm-clang will have compiler # support for atomics and llvm requires gcc 4.8 (or a compiler with # equivalent features) to be built so we know we'll have system # header support too. # # We support intrinsics for gcc, intel, cray and clang. gcc added # initial support in 4.1, and added support for 64-bit atomics on # 32-bit platforms with 4.8. clang and intel also support 64-bit # atomics on 32-bit platforms and the cray compiler will never run # on a 32-bit machine. # # For pgi or 32-bit platforms with an older gcc, we fall back to # locks if compiler_val in ['gnu', 'cray-prgenv-gnu', 'mpi-gnu']: version = get_compiler_version(flag) if version >= CompVersion('5.0'): atomics_val = 'cstdlib' elif version >= CompVersion('4.8'): atomics_val = 'intrinsics' elif version >= CompVersion( '4.1') and not platform_val.endswith('32'): atomics_val = 'intrinsics' elif compiler_val == 'intel' or compiler_val == 'cray-prgenv-intel': atomics_val = 'intrinsics' elif compiler_val == 'cray-prgenv-cray': atomics_val = 'cstdlib' elif compiler_val in ['allinea', 'cray-prgenv-allinea']: atomics_val = 'cstdlib' elif compiler_val == 'clang': if has_std_atomics(): atomics_val = 'cstdlib' else: atomics_val = 'intrinsics' elif compiler_val == 'llvm': atomics_val = 'cstdlib' # we can't use intrinsics, fall back to locks if not atomics_val: atomics_val = 'locks' else: error("Invalid flag: '{0}'".format(flag), ValueError) return atomics_val
def validate_llvm_config(): llvm_val = get() llvm_config = get_llvm_config() if llvm_val == 'system': if llvm_config == '' or llvm_config == 'none': error("CHPL_LLVM=system but could not find an installed LLVM" " with one of the supported versions: {0}".format( llvm_versions_string())) if (llvm_val == 'system' or (llvm_val == 'bundled' and os.path.exists(llvm_config))): version, config_error = check_llvm_config(llvm_config) if config_error: error("Problem with llvm-config at {0} -- {1}" .format(llvm_config, config_error)) if llvm_val == 'system': bindir = get_system_llvm_config_bindir() if not (bindir and os.path.isdir(bindir)): error("llvm-config command {0} provides missing bin dir {1}" .format(llvm_config, bindir)) clang_c = get_llvm_clang('c')[0] clang_cxx = get_llvm_clang('c++')[0] if not os.path.exists(clang_c): error("Missing clang command at {0}".format(clang_c)) if not os.path.exists(clang_cxx): error("Missing clang++ command at {0}".format(clang_cxx)) (noPackageErrors, package_err) = check_llvm_packages(llvm_config) if not noPackageErrors: error(package_err)
def api(): data = request.values.to_dict() try: command = re.split("\s+", data["text"]) slack_id = data["team_id"] team_domain = data["team_domain"] channel = data["channel_id"] except KeyError: abort(400) # ensuring that the request comes from slack if not valid_slack_request(request): return abort(404) team = db.session.query(Team).filter_by(slack_id=slack_id).first() if not team: return error( "You are not registered in our proxy server, try removig the app " "and adding it to slack again.") if command[0] == "help": fields = [ { "title": "`/pass` _or_ `/pass list`", "value": "To list the available passwords in this channel.", "short": True, }, { "title": "`/pass <secret>` or `/pass show <secret>`", "value": ("To retrieve a one time use link with the secret content, " "this link expires in 15 minutes."), "short": True, }, { "title": "`/pass insert <secret>`", "value": ("To retrieve the link with the editor to create the " "secret, this link expires in 15 minutes."), "short": True, }, { "title": "`/pass remove <secret>`", "value": ("To remove the secret from the group."), "short": True, }, { "title": "`/pass configure` or `/pass configure <server_url>`", "value": ("To setup the password storage, it is only necessary " "to execute it once."), "short": True, }, ] return jsonify({ "attachments": [{ "fallback": ("_Usage:_ https://github.com/talpor/password-scale"), "text": "*_Usage:_*", "fields": fields, }] }) if command[0] == "configure" and len(command) == 2: url = command[1] if not validators.url(url): return error("Invalid URL format, use: https://<domain>") if team.url: msg = ("This team is already configured, you want to replace " "the password server?") return jsonify({ "attachments": [{ "fallback": "This team already configured", "text": msg, "callback_id": "configure_password_server", "color": "warning", "actions": [ { "name": "reconfigure_server", "text": "Yes", "type": "button", "value": url, }, { "name": "no_reconfigure", "text": "No", "style": "danger", "type": "button", "value": "no", }, ], }] }) if not team.register_server(url): return error("Unable to retrieve the _public_key_ " "from the server".format(team_domain)) return success("{} team successfully configured!".format(team_domain)) if command[0] == "configure" and len(command) == 1 or not team.url: color = "warning" if team.url: msg = ( "*{}* team already have a server configured, if you want to " "swap select some of the options below".format(team.name)) elif command[0] == "configure": color = "good" msg = "What type of server do you want to use?" else: msg = ( "*{}* team does not have a password server configured, select " "one of the options below to start.".format(team_domain)) warning_msg = ( "This is a test server, any information stored on this server " "can be deleted at any moment without prior notice!") return jsonify({ "attachments": [{ "fallback": msg, "text": msg, "color": color, "callback_id": "configure_password_server", "actions": [ { "name": "use_demo_server", "text": "Use Test Server", "type": "button", "value": "no", "confirm": { "title": "Confirm", "text": warning_msg, "ok_text": "I understand", "dismiss_text": "No", }, }, { "text": "Request Private Server", "type": "button", "url": CONFIGURATION_GUIDE_URL, }, { "name": "no_configure", "text": "Later", "type": "button", "value": "no", }, ], }] }) if command[0] in ["", "list"]: try: dir_ls = cmd.list(team, channel) except SlashpassError as e: return error("_{}_".format(e.message)) if not dir_ls: return warning( "You have not passwords created for this channel, use " "`/pass insert <secret>` to create the first one!") return jsonify({ "attachments": [{ "fallback": dir_ls, "text": "Password Store\n{}".format(dir_ls), "footer": ("Use the command `/pass <key_name>` to retrieve " "some of the keys"), }] }) if command[0] == "insert" and len(command) == 2: app = command[1] token = cmd.generate_insert_token(team, channel, app) msg = "Adding password for *{}* in this channel".format(app) return jsonify({ "attachments": [{ "fallback": msg, "text": msg, "footer": "This editor will be valid for 15 minutes", "color": "good", "actions": [{ "text": "Open editor", "style": "primary", "type": "button", "url": "{}/insert/{}".format(SITE, token), }], }] }) if command[0] == "remove" and len(command) == 2: app = command[1] if cmd.remove(team, channel, app): return success( "The secret *{}* was removed successfully.".format(app)) return warning("Looks like the secret *{}* is not in your repository " ":thinking_face: use the command `/pass list` " "to verify your storage.".format(app)) if command[0] == "show" and len(command) == 2: app = command[1] else: app = command[0] onetime_link = cmd.show(team, channel, app) if onetime_link: return jsonify({ "attachments": [{ "fallback": "Password: {}".format(onetime_link), "text": "Password for *{}*".format(app), "footer": "This secret will be valid for 15 minutes", "color": "good", "actions": [{ "text": "Open secret", "style": "primary", "type": "button", "url": onetime_link, }], }] }) else: return warning("*{}* is not in the password store.".format(app))
def make(self): # make sure there exist enough labels LabelledLearner.make(self) error("Dataset supplied to classifier has only one label", ill_defined(self.num_labels, cannot_be=1))
return 1 / y.shape[1] * np.sum((y_mp - y)**2) def mean_absolute_error(y, y_mp): return 1 / y.shape[1] * np.sum(np.absolute(y_mp - y)) def negative_log_likelihood(y_mp): return -1 / y_mp.shape[1] * np.sum(np.log(y_mp + 1e-15)) if __name__ == '__main__': if len(sys.argv) != 2: usage() if not os.path.isfile(sys.argv[1]): error('no such file: %s' % sys.argv[1]) feature_number = 30 data = mp_train.read_data(sys.argv[1], feature_number) x, y = mp_train.split_xy(data) layers, neurons, wb, x_min, x_max = load_train_data() x = mp_train.scale(x, x_min, x_max) try: y_mp, za = mp_train.feedforward(wb, x, layers, neurons) l = mp_train.loss(y[0], y_mp[0]) except: error("invalid train data") print("binary cross entropy: %.4f" % l) mse = mean_squared_error(y, y_mp) print("mean squared error: %.4f" % mse) mae = mean_absolute_error(y, y_mp)
def make(self): error("Cannot apply {} to multilabel data.".format(self.name), self.do_multilabel) SKLClassifier.make(self)
header = header[6:] features = [{ "Ravenclaw": [], "Slytherin": [], "Gryffindor": [], "Hufflepuff": [] } for i in range(len(header))] is_numeric = [False for i in range(len(header))] for line in reader: for i, field in enumerate(line): if i == 1: house = field elif i >= 6 and field != "": features[i - 6][house] += [float(field)] except: error("invalid dataset") return (header, features) if __name__ == '__main__': # checks if len(sys.argv) != 2: usage() if not os.path.isfile(sys.argv[1]): error('no such file: %s' % sys.argv[1]) header, features = read_data(sys.argv[1]) index = 0 histogram(header[index], features[index])