def test2(): path = './data/list.txt' manifest_dic = fetcher.fetch_manifest_from_file(path) for manifest_url in manifest_dic.keys(): SSM = SmoothStreamingMedia() parser.parse(manifest_dic[manifest_url], manifest_url, SSM) fetcher.fetch_fragment(SSM)
def datestring_to_datetime(inp): """ Convert a date/time string do proper start (and optionally end) datetime """ if type(inp) in [str, unicode]: string = inp.strip() format = '{day:d}.{month:d}.{year:d} {hour:d}:{minute:d}-{hourend:d}:{minuteend:d}' p = parse.parse(format, string) if p is not None: out = datetime.datetime(p['year'], p['month'], p['day'], p['hour'], p['minute']) #sys.stdout.write("\nA p: %s, datetime: %s\n" % (p, out.isoformat())) return out else: format = '{day:d}.{month:d}.{year:d} {hour:d}:{minute:d}' p = parse.parse(format, string) if p is not None: out = datetime.datetime(p['year'], p['month'], p['day'], p['hour'], p['minute']) #sys.stdout.write("\nB p: %s, datetime: %s\n" % (p, out.isoformat())) return out else: format = '{day:d}.{month:d}.{year:d}' p = parse.parse(format, string) if p is not None: out = datetime.datetime(p['year'], p['month'], p['day'], 0, 0) #sys.stdout.write("\nB p: %s, datetime: %s\n" % (p, out.isoformat())) return out else: return None return inp
def __init__(self,item,parent): self.item = item self.parent = parent self.x = float(item.attrib['x']) self.y = float(item.attrib['y']) self.rot = 0.0 if 'rot' in item.attrib.keys(): rot = item.attrib['rot'] print rot rotParse = parse.parse('R{:d}',rot) if rotParse is None: rotParse = parse.parse('MR{:d}',rot) self.rot = float(rotParse[0])*(math.pi/180) rot = - self.parent.rot rot2 = -self.rot - self.parent.rot self.underMouse = False (self.x,self.y) = rotate(self.x,self.y,rot) self.dx = float(item.attrib['dx']) self.dy = float(item.attrib['dy']) (self.dx,self.dy) = rotate(self.dx,self.dy,rot2) (self.dx,self.dy) = (self.dx,self.dy) (self.x,self.y) = (self.x-self.dx/2,self.y-self.dy/2) self.name = item.attrib['name']
def main(): # parse command command_log = 'CIRCexplorer parameters: ' + ' '.join(sys.argv) if len(sys.argv) == 1: sys.exit(help_doc) elif sys.argv[1] == '--version' or sys.argv[1] == '-v': sys.exit(__version__) elif sys.argv[1] == 'align': import align align.align(docopt(align.__doc__, version=__version__), command=command_log, name='align') elif sys.argv[1] == 'parse': import parse parse.parse(docopt(parse.__doc__, version=__version__), command=command_log, name='parse') elif sys.argv[1] == 'annotate': import annotate annotate.annotate(docopt(annotate.__doc__, version=__version__), command=command_log, name='annotate') elif sys.argv[1] == 'assemble': import assemble assemble.assemble(docopt(assemble.__doc__, version=__version__), command=command_log, name='assemble') elif sys.argv[1] == 'denovo': import denovo denovo.denovo(docopt(denovo.__doc__, version=__version__), command=command_log, name='denovo') else: sys.exit(help_doc)
def parse_sig_command(msg, query): reply = "" if str(parse("sig til {unick} at {arg}", query)) != "None": p = parse("sig til {unick} at {arg}", query) reply = p['unick'] + ": " + p['arg'] else: reply = "Brug: sig til <person> at han/hun <besked>" replace_dict = ( ('jeg', msg['mucnick']), ('du', 'jeg'), ('han', 'du'), ('hun', 'du'), ('sig', 'dig'), ('min', msg['mucnik']), ('din', 'min'), ('sin', 'din'), ('dit', 'mit'), ('dine', 'mine'), ('sine', 'dine'), ) for old, new in replace_dict: reply = reply.replace(old, new) return reply
def test_parse(): """Assert the correct return values when parse is called to parse, group and retun desired information from a string. """ regex = "(?P<name>[a-zA-Z]+ ?[a-zA-Z]*)" assert parse.parse(regex, 'name', '07734 Hyewon Namkung') == 'Hyewon Namkung' assert not parse.parse(regex, 'name', '123450987*&!@#$')
def parse_install_output(output): """Parse output from pip download to get name and file mappings for all dependencies and their sub dependencies. This is required for proper file hashing with --require-hashes. """ output_sections = output.split('Collecting ') names = [] for section in output_sections: lines = section.split('\n') # Strip dependency parens from name line. e.g. package (from other_package) name = lines[0].split('(')[0] # Strip version specification. e.g. package; python-version=2.6 name = name.split(';')[0] for line in lines: r = parse.parse('Saved {file}', line.strip()) if r is None: r = parse.parse('Using cached {file}', line.strip()) if r is None: continue fname = r['file'].split(os.sep)[-1] # Unencode percent-encoded values like ``!`` in version number. fname = requests.compat.unquote(fname) names.append((fname, name.strip())) break return names
def test_biconditional(self): statement = parse("A<->B") self.assertIsNotNone(statement) self.assertEqual(statement.type, "<->") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "(A<->B)") statement = parse("A<->(B<->C)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "<->") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "<->") self.assertEqual(statement.__str__(), "(A<->(B<->C))") statement = parse("(A<->B)<->C") self.assertIsNotNone(statement) self.assertEqual(statement.type, "<->") self.assertEqual(statement.value1().type, "<->") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "((A<->B)<->C)") statement = parse("(A<->B)<->(C<->D)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "<->") self.assertEqual(statement.value1().type, "<->") self.assertEqual(statement.value2().type, "<->") self.assertEqual(statement.__str__(), "((A<->B)<->(C<->D))") statement = parse("A<->(B<->(C<->D))") self.assertIsNotNone(statement) self.assertEqual(statement.__str__(), "(A<->(B<->(C<->D)))")
def parse_for_req_prov_params(script_fpath): """ """ with open(script_fpath, "r") as f: all_lines = f.readlines() fnames_req_prov_dict = {} all_required_params = [] all_provided_params = [] for i in range(len(all_lines) - 1): if "@myFeature" in all_lines[i] and "def " in all_lines[i + 1]: reqs_provs_1 = parse( "@myFeature(requires={requires}, provides={provides})", all_lines[i].strip()) func_name = parse( "def {funcname}({args}):", all_lines[i + 1].strip()) fnames_req_prov_dict[func_name.named['funcname']] = { "requires": eval(reqs_provs_1.named["requires"]), "provides": eval(reqs_provs_1.named["provides"])} all_required_params = list(set( all_required_params + list(set(eval(reqs_provs_1.named["requires"]))))) all_provided_params = list(set( all_provided_params + list(set(eval(reqs_provs_1.named["provides"]))))) return (fnames_req_prov_dict, all_required_params, all_provided_params)
def parse_r_type(opcode, line): if opcode in ['add', 'sub', 'mul', 'and', 'or', 'xor', 'slt', 'seq']: return parse("{} ${}, ${}, ${}", line) if opcode in ['sll', 'srl', 'sra']: return parse("{} ${}, ${}, {}", line) if opcode == 'mv': return parse("{} ${}, ${}", line)
def __init__(self,element,pcb): self.element = element self.underMouse = False self.pcb = pcb self.libraryName = element.attrib['library'] self.partName = element.attrib['name'] self.packageName = element.attrib['package'] print element,element.tag,element.attrib self.x = float(element.attrib['x']) self.y = float(element.attrib['y']) self.rot = 0 self.pads = [] self.smds = [] if 'rot' in element.attrib: rot = element.attrib['rot'] print rot rotParse = parse.parse('R{:d}',rot) if rotParse is None: rotParse = parse.parse('MR{:d}',rot) self.rot = float(rotParse[0])*(math.pi/180) print self.rot self.drawingElements = [] self.findFromLibrary() self.loadFromLibrary() self.loadPads() self.loadSMDs()
def process_message(self, m): text = m.get('text') if text is None: return None sender = m['from']['first_name'] text = re.sub(' +', ' ', text).strip() # strip multiple spaces ###### two arguments ###### try: res = parse('/roll {:d} {:d}', text) except: res = None if res: faces = res.fixed[0] times = res.fixed[1] if faces >= 1 and times >= 1: out = [ random.randint(1, faces) for i in xrange(0, times)] return '%s rolled %s. The sum is %s.' % (sender, out, sum(out)) else: return 'Positive numbers please.' ###### one argument ###### try: res = parse('/roll {:d}', text) except: res = None if res: faces = res.fixed[0] if faces >= 1: return '%s rolled %s.' % (sender, random.randint(1, faces)) else: return 'A positive number please.' ###### zero argument ###### if text == '/roll': faces = 6 return '%s rolled %s.' % (sender, random.randint(1, faces)) return None
def test_implication(self): statement = parse("A->B") self.assertIsNotNone(statement) self.assertEqual(statement.type, "->") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "(A->B)") statement = parse("A->(B->C)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "->") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "->") self.assertEqual(statement.__str__(), "(A->(B->C))") statement = parse("(A->B)->C") self.assertIsNotNone(statement) self.assertEqual(statement.type, "->") self.assertEqual(statement.value1().type, "->") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "((A->B)->C)") statement = parse("(A->B)->(C->D)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "->") self.assertEqual(statement.value1().type, "->") self.assertEqual(statement.value2().type, "->") self.assertEqual(statement.__str__(), "((A->B)->(C->D))") statement = parse("A->(B->(C->D))") self.assertIsNotNone(statement) self.assertEqual(statement.__str__(), "(A->(B->(C->D)))")
def list_features_provided(script_fpath): """Parses script and returns a list of all features it provides. Parses decorator expression in custom feature definitions script, returning a list of all feature names generated by the various definitions in that script. Parameters ---------- script_fpath : str Path to custom features definition script. Returns ------- list of str List of feature names that the script will generate. """ with open(script_fpath, "r") as f: all_lines = f.readlines() fnames_req_prov_dict = {} all_required_params = [] all_provided_params = [] for i in range(len(all_lines) - 1): if "@myFeature" in all_lines[i] and "def " in all_lines[i + 1]: reqs_provs_1 = parse("@myFeature(requires={requires}, provides={provides})", all_lines[i].strip()) func_name = parse("def {funcname}({args}):", all_lines[i + 1].strip()) fnames_req_prov_dict[func_name.named["funcname"]] = { "requires": eval(reqs_provs_1.named["requires"]), "provides": eval(reqs_provs_1.named["provides"]), } all_required_params = list(set(all_required_params + list(set(eval(reqs_provs_1.named["requires"]))))) all_provided_params = list(set(all_provided_params + list(set(eval(reqs_provs_1.named["provides"]))))) return all_provided_params
def datestring_to_datetime(inp): """ Convert a date/time string do proper start (and optionally end) datetime """ berlin = timezone('Europe/Berlin') if type(inp) in [str, unicode]: string = inp.strip() format = '{day:d}.{month:d}.{year:d} {hour:d}:{minute:d}-{hourend:d}:{minuteend:d}' p = parse.parse(format, string) if p is not None: out = datetime.datetime(p['year'], p['month'], p['day'], p['hour'], p['minute'], tzinfo=berlin) return out else: format = '{day:d}.{month:d}.{year:d} {hour:d}:{minute:d}' p = parse.parse(format, string) if p is not None: out = datetime.datetime(p['year'], p['month'], p['day'], p['hour'], p['minute'], tzinfo=berlin) return out else: format = '{day:d}.{month:d}.{year:d}' p = parse.parse(format, string) if p is not None: out = datetime.datetime(p['year'], p['month'], p['day'], 0, 0, tzinfo=berlin) return out else: return None return inp
def test_width_str(self): res = parse.parse('{:.2}{:.2}', 'look') self.assertEqual(res.fixed, ('lo', 'ok')) res = parse.parse('{:2}{:2}', 'look') self.assertEqual(res.fixed, ('lo', 'ok')) res = parse.parse('{:4}{}', 'look at that') self.assertEqual(res.fixed, ('look', ' at that'))
def test_width_empty_input(self): res = parse.parse('{:.2}', '') self.assertIsNone(res) res = parse.parse('{:2}', 'l') self.assertIsNone(res) res = parse.parse('{:2d}', '') self.assertIsNone(res)
def parse_constraint(constraint): """ Parse a constraint of form into an info dictionary of form {'comparator': comparator, 'tag': tag, 'version': version, 'postfix': postfix} Args: constraint(string): The string representing the constratint Returns: dictionary: The information dictionary """ match = comparator_re.search(constraint) comparator = match.group(1) tag = match.group(2) version = None postfix = None parsed_results = parse('v{version}-{postfix}', tag) if parsed_results: version = parsed_results["version"] postfix = parsed_results["postfix"] else: parsed_results = parse('v{version}', tag) if parsed_results: version = parsed_results["version"] postfix = None return {'comparator': comparator, 'tag': tag, 'version': version, 'postfix': postfix}
def process(req, rep): (handle, tmp_in) = tempfile.mkstemp() (handle, tmp_out) = tempfile.mkstemp() # Update carry-over parameters rep.sequence = req.sequence rep.recipient = req.recipient rep.identifier = req.identifier # Write query sequence to file in FASTA format with open(tmp_in, 'w') as file: file.write('> x\n') file.write('%s\n' % req.sequence) params = { 'exe' : FLAGS.exe, 'db' : FLAGS.db, 'fasta' : tmp_in, 'out' : tmp_out } subprocess.check_call(hmmer_cmd.safe_substitute(params).split()) parse(tmp_out, rep) fix_alignments(rep.alignments) try: os.remove(tmp_in) os.remove(tmp_out) except: sys.stderr.write('Failed to remove one or more temporary files: %s, %s' % (tmp_in, tmp_out))
def test_conjunction(self): statement = parse("A^B") self.assertIsNotNone(statement) self.assertEqual(statement.type, "^") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "(A^B)") statement = parse("A^(B^C)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "^") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "^") self.assertEqual(statement.__str__(), "(A^(B^C))") statement = parse("(A^B)^C") self.assertIsNotNone(statement) self.assertEqual(statement.type, "^") self.assertEqual(statement.value1().type, "^") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "((A^B)^C)") statement = parse("(A^B)^(C^D)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "^") self.assertEqual(statement.value1().type, "^") self.assertEqual(statement.value2().type, "^") self.assertEqual(statement.__str__(), "((A^B)^(C^D))") statement = parse("A^(B^(C^D))") self.assertIsNotNone(statement) self.assertEqual(statement.__str__(), "(A^(B^(C^D)))")
def learn(*args): obs_ = args[0] gamma = 1 if (len(args) > 1): gamma = args[1] if (len(args) > 2): R = args[2] parsed = par.parse(obs_,R) else: parsed = par.parse(obs_) #parsed = [stateMap,actionMap,observations] stateMap = parsed[0] actMap = parsed[1] obs = parsed[2] model = mod.model(len(stateMap),len(actMap),obs) P = model[0] R = model[1] policy = sol.policy(P,gamma,R) #map integer policy and action back to strat = {} for i in range(0,len(policy)): strat[stateMap[i]] = actMap[policy[i]] #return strategy return strat
def test_disjunction(self): statement = parse("AvB") self.assertIsNotNone(statement) self.assertEqual(statement.type, "v") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "(AvB)") statement = parse("Av(BvC)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "v") self.assertEqual(statement.value1().type, "lit") self.assertEqual(statement.value2().type, "v") self.assertEqual(statement.__str__(), "(Av(BvC))") statement = parse("(AvB)vC") self.assertIsNotNone(statement) self.assertEqual(statement.type, "v") self.assertEqual(statement.value1().type, "v") self.assertEqual(statement.value2().type, "lit") self.assertEqual(statement.__str__(), "((AvB)vC)") statement = parse("(AvB)v(CvD)") self.assertIsNotNone(statement) self.assertEqual(statement.type, "v") self.assertEqual(statement.value1().type, "v") self.assertEqual(statement.value2().type, "v") self.assertEqual(statement.__str__(), "((AvB)v(CvD))") statement = parse("Av(Bv(CvD))") self.assertIsNotNone(statement) self.assertEqual(statement.__str__(), "(Av(Bv(CvD)))")
def parse_i_type(opcode, line): if opcode in ['sw', 'lw', 'thread_finished']: return [opcode] if opcode in ['ldc', 'ldi']: return parse("{} ${}, {}", line) if opcode == 'addi': return parse("{} ${}, ${}, {}", line)
def create_swig_xml(interface_filename, xml_filename): cmd = "swig -xml -xmllite -o %s %s" % (xml_filename, interface_filename) print(cmd) result = subprocess.call(cmd, shell=True) if result: sys.exit(result) usage = "usage: %prog [options] <header.h>" op = optparse.OptionParser(usage=usage) op.add_option("-o", "--output") op.add_option("-i", "--interface-file") op.add_option("-x", "--xml-file") op.add_option("-I", dest="includes", action="append", default=list()) op.add_option("-D", dest="defines", action="append", default=list()) (options, args) = op.parse_args(sys.argv[1:]) if len(args) < 1: print("No input file given", file=sys.stderr) sys.exit(1) header_filename = args[0] module_name, _ = os.path.splitext(os.path.basename(header_filename)) if options.interface_file is None: options.interface_file = module_name + ".i" if options.xml_file is None: options.xml_file = module_name + ".xml" if options.output is None: options.output = module_name + ".ffi" create_swig_interface(header_filename, options.interface_file, includes=options.includes, defines=options.defines) create_swig_xml(options.interface_file, options.xml_file) parse.parse(options.xml_file, options.output)
def test_dotted_type_conversion_pull_8(self): # test pull request 8 which fixes type conversion related to dotted # names being applied correctly r = parse.parse("{a.b:d}", "1") self.assertEqual(r["a.b"], 1) r = parse.parse("{a_b:w} {a.b:d}", "1 2") self.assertEqual(r["a_b"], "1") self.assertEqual(r["a.b"], 2)
def test_no_match(self): vals = [] vals.append(parse("a")) vals.append(parse("AB")) vals.append(parse("A B")) for val in vals: self.assertIsNone(val, msg="Invalid pattern Parsed!")
def test_reduce_literal(self): stmt = parse("A") red = parse("A") self.assertTrue(stmt.reduce(red)) stmt = parse("A") red = parse("~A") self.assertFalse(stmt.reduce(red))
def enterREPL(): while(True): inp = raw_input(str(utils.lines) + "> ").strip() if(inp == "quit"): return parse.parse(inp)
def test_named_aligned_typed(self): # pull a named, typed values out of string r = parse.parse("hello {number:<d} {things}", "hello 12 people") self.assertEqual(r.named, dict(number=12, things="people")) r = parse.parse("hello {number:>d} {things}", "hello 12 people") self.assertEqual(r.named, dict(number=12, things="people")) r = parse.parse("hello {number:^d} {things}", "hello 12 people") self.assertEqual(r.named, dict(number=12, things="people"))
def test_dotted_type_conversion_pull_8(self): # test pull request 8 which fixes type conversion related to dotted # names being applied correctly r = parse.parse('{a.b:d}', '1') self.assertEqual(r['a.b'], 1) r = parse.parse('{a_b:w} {a.b:d}', '1 2') self.assertEqual(r['a_b'], '1') self.assertEqual(r['a.b'], 2)
def identify(self, filepath, url=None): """ Identifies a file of the layer :param filepath: filepath from AMQP :param url: fully qualified URL of file :returns: `list` of file properties """ super().identify(filepath, url) self.model = 'cansips' LOGGER.debug('Loading model information from store') self.file_dict = json.loads(self.store.get_key(self.model)) filename_pattern = self.file_dict[self.model]['filename_pattern'] tmp = parse(filename_pattern, os.path.basename(filepath)) file_pattern_info = { 'resolution': tmp.named['resolution'], 'wx_variable': '{}_{}_{}'.format(tmp.named['wx_variable'], tmp.named['pressure'], tmp.named['pres_value']), 'year_': tmp.named['YYYY'], 'month_': tmp.named['MM'] } LOGGER.debug('Defining the different file properties') self.wx_variable = file_pattern_info['wx_variable'] if self.wx_variable not in self.file_dict[self.model]['variable']: msg = 'Variable "{}" not in ' \ 'configuration file'.format(self.wx_variable) LOGGER.warning(msg) return False weather_var = self.file_dict[self.model]['variable'][self.wx_variable] self.geomet_layers = weather_var['geomet_layers'] date_format = '%Y%m' self.date_ = '{}{}'.format(file_pattern_info['year_'], file_pattern_info['month_']) reference_datetime = datetime.strptime(self.date_, date_format) self.date_ = reference_datetime self.model_run = '{}Z'.format(reference_datetime.strftime('%H')) for band in self.file_dict[self.model]['bands']: dict_bands = self.file_dict[self.model]['bands'] fhi = dict_bands[band]['forecast_interval'] fhi = re.sub('[^0-9]', '', fhi) forecast_hour_datetime = reference_datetime + \ relativedelta(months=int(fhi)) elevation = weather_var['elevation'] member = dict_bands[band]['member'] mem_str = str(member).zfill(2) for layer, layer_config in self.geomet_layers.items(): layer_name = layer.format(mem_str) str_mr = re.sub('[^0-9]', '', reference_datetime.strftime(DATE_FORMAT)) str_fh = re.sub('[^0-9]', '', forecast_hour_datetime.strftime(DATE_FORMAT)) identifier = '{}-{}-{}'.format(layer_name, str_mr, str_fh) vrt = 'vrt://{}?bands={}'.format(self.filepath, band) begin, end, interval = layer_config['forecast_hours'].split( '/') interval = int(re.sub("[^0-9]", "", interval)) feature_dict = { 'layer_name': layer_name, 'filepath': vrt, 'identifier': identifier, 'reference_datetime': reference_datetime.strftime(DATE_FORMAT), 'forecast_hour_datetime': forecast_hour_datetime.strftime(DATE_FORMAT), # noqa 'member': member, 'model': self.model, 'elevation': elevation, 'expected_count': None, 'forecast_hours': { 'begin': begin, 'end': end, 'interval': layer_config['forecast_hours'].split('/')[2] }, 'static_model_run': { 'begin': layer_config['begin'] }, 'layer_config': layer_config, 'register_status': True } self.items.append(feature_dict) return True
def parse(self): # Read sheet file sheet = open(self.sheet_loc, 'r') read_lines = sheet.readlines() sheet.close() # Find sheet START and END try: start_index = read_lines.index('START\n') except: raise SheetParserError( 'No START token.', 'Sheet should be placed between START and END') else: read_lines = read_lines[start_index + 1:] try: end_index = read_lines.index('END\n') except: raise SheetParserError( 'No END token.', 'Sheet should be placed between START and END') else: read_lines = read_lines[:end_index] # Check bpm bpm_parse = parse('bpm: {:d}', read_lines[0]) if bpm_parse is None: raise SheetParserError( 'No bpm.', 'bpm should be specified after START. ex) bpm: 120') bpm = bpm_parse[0] if bpm > 180: raise SheetParserError('Too big bpm', 'bpm should be less or equal than 180.') if bpm < 40: raise SheetParserError('Too small bpm', 'bpm should be greater or equal than 40') # Set freq self.freq = bpm / 60.0 # Check bar length bar_parse = parse('bar: {:d}/{:d}', read_lines[1]) if bar_parse is None: raise SheetParserError( 'No bar.', 'bar should be specified after bpm. ex) bar: 4/4') # Calculate bar length base_beat = bar_parse[1] num_of_beat = bar_parse[0] self.length_of_bar = self.beat_checker(base_beat, bpm, 0) * num_of_beat # Exclude bpm and bar read_lines = read_lines[2:] # Find lines START and END lines = [] # Found lines line = [] # List to store notes line_started = False for i, token in enumerate(read_lines): # If comment, pass if token.startswith('#'): continue if (not line_started) and (token == 'LINESTART\n'): line_started = True line = [] elif line_started and token == 'LINEEND\n': if not len(line) > 0: raise SheetParserError( 'Line {}'.format(i), 'Line should consist of at least one note.') else: line_started = False lines.append(line) elif token == 'BARSTART\n' or token == 'BAREND\n': line.append(token) else: p = parse('({}, {:d})', token) if p is None: raise SheetParserError('Line {} : {}'.format(i, token), 'Note format error.') line.append(token) if line_started: raise SheetParserError('Line error', 'Line not finished after LINESTART') if not len(lines) > 0: raise SheetParserError('Line error', 'At least one line should exist.') # Convert lines converted_lines = [] for i, line in enumerate(lines): converted_lines.append(self.parse_line(line, i + 1, bpm)) # Return return Sheet(bpm, converted_lines)
def find_latest_remote(self): """ Used to update the published dict CommandLine: python -m ibeis.algo.verif.vsone find_latest_remote Example: >>> # DISABLE_DOCTEST >>> from ibeis.algo.verif.vsone import * # NOQA >>> self = Deployer() >>> task_clf_names = self.find_latest_remote() """ base_url = 'https://{remote}/public/models/pairclf'.format( **self.publish_info) import requests import bs4 resp = requests.get(base_url) soup = bs4.BeautifulSoup(resp.text, 'html.parser') table = soup.findAll('table')[0] def parse_bs_table(table): n_columns = 0 n_rows = 0 column_names = [] # Find number of rows and columns # we also find the column titles if we can for row in table.find_all('tr'): td_tags = row.find_all('td') if len(td_tags) > 0: n_rows += 1 if n_columns == 0: n_columns = len(td_tags) # Handle column names if we find them th_tags = row.find_all('th') if len(th_tags) > 0 and len(column_names) == 0: for th in th_tags: column_names.append(th.get_text()) # Safeguard on Column Titles if len(column_names) > 0 and len(column_names) != n_columns: raise Exception( "Column titles do not match the number of columns") columns = column_names if len(column_names) > 0 else range( 0, n_columns) import pandas as pd df = pd.DataFrame(columns=columns, index=list(range(0, n_rows))) row_marker = 0 for row in table.find_all('tr'): column_marker = 0 columns = row.find_all('td') for column in columns: df.iat[row_marker, column_marker] = column.get_text().strip() column_marker += 1 if len(columns) > 0: row_marker += 1 return df df = parse_bs_table(table) # Find all available models df = df[df['Name'].map(lambda x: x.endswith('.cPkl'))] # df = df[df['Last modified'].map(len) > 0] fname_fmt = self.fname_fmtstr + '.cPkl' task_clf_candidates = ut.ddict(list) import parse for idx, row in df.iterrows(): fname = basename(row['Name']) result = parse.parse(fname_fmt, fname) if result: task_key = result.named['task_key'] species = result.named['species'] task_clf_candidates[(species, task_key)].append(idx) task_clf_fnames = ut.ddict(dict) for key, idxs in task_clf_candidates.items(): species, task_key = key # Find the classifier most recently created max_idx = ut.argmax(df.loc[idxs]['Last modified'].tolist()) fname = df.loc[idxs[max_idx]]['Name'] task_clf_fnames[species][task_key] = fname print('published = ' + ut.repr2(task_clf_fnames, nl=2)) return task_clf_fnames
def run_RGBDFusion(output_root, root_path, cfg): o3d.utility.set_verbosity_level(o3d.utility.VerbosityLevel.Error) def FusionFromRGBD(frame_paths, intrinsics, cfg): ''' frame_paths: [(color_path, depth_path, cam_path)] ''' read = o3d.io.read_image volume = o3d.integration.ScalableTSDFVolume( voxel_length=cfg.tsdf_cubic_size / 512.0, sdf_trunc=0.04, color_type=o3d.integration.TSDFVolumeColorType.RGB8) # first pose is the canonical pose pose_base2world = np.loadtxt(frame_paths[0][2], dtype=np.float32) pose_world2base = np.linalg.inv(pose_base2world) for tup in frame_paths: cp, dp, camp = tup # current pose to canonical pose pose_cam2world = np.loadtxt(camp, dtype=np.float32) pose_cam2base = pose_world2base @ pose_cam2world # read rgbd image rgbd_image = o3d.geometry.RGBDImage.create_from_color_and_depth(\ read(cp), read(dp), cfg.depth_scale, cfg.depth_trunc, False) # fusion to canonical volume volume.integrate(rgbd_image, intrinsic, np.linalg.inv(pose_cam2base)) pc = volume.extract_point_cloud() pc.estimate_normals() return pc, pose_base2world # scene under root folders get_folders = lambda s: list( filter(lambda f: os.path.isdir(f), glob.glob(s + '/*'))) color_id = lambda x: parse('frame-{:d}.color.png', os.path.basename(x))[0] depth_id = lambda x: parse('frame-{:d}.depth.png', os.path.basename(x))[0] pose_id = lambda x: parse('frame-{:d}.pose.txt', os.path.basename(x))[0] scenes = get_folders(root_path) for s in scenes: frag_counter = 0 scene_name = os.path.basename(s) output_dir = os.path.join(output_root, scene_name) os.makedirs(output_dir, exist_ok=True) # scene camera intrinsics intrinsic = read_intrinsic(os.path.join(s, 'camera-intrinsics.txt'), cfg.width, cfg.height) # seq under scene folders for seq in get_folders(s): if os.path.basename(seq).startswith('seq'): # here contain the color, depth, pose images cpaths = glob.glob(os.path.join(seq, "*.color.png")) dpaths = glob.glob(os.path.join(seq, "*.depth.png")) ppaths = glob.glob(os.path.join(seq, "*.pose.txt")) cpaths.sort(key=color_id) dpaths.sort(key=depth_id) ppaths.sort(key=pose_id) # sanity check assert len(cpaths) == len(dpaths) and len(cpaths) == len( ppaths) frame_paths = list(zip(cpaths, dpaths, ppaths)) # loop over n frames for fusion nframes = cfg.frames_per_frag head = 0 tail = min(nframes, len(cpaths)) while tail <= len(cpaths): print("Processing %d:%d/%d at scene %s..." % (head, tail, len(cpaths), scene_name)) pc, pose = FusionFromRGBD(frame_paths[head:tail], intrinsic, cfg) # ------ debug only ---------------------- # points = np.asarray(pc.points)[::10] # points = to_hom_np(points.T).T @ pose.T # points = points[:,:3] # save_ply(os.path.join(output_dir, 'aligned_cloud_bin_%d.ply'%frag_counter), points) # print("Successfully written fused and aligned point cloud #%d for scene %s"%(frag_counter, scene_name)) # ---------------------------------------- np.savetxt( os.path.join(output_dir, 'cloud_bin_%d_pose.txt' % frag_counter), pose) if o3d.io.write_point_cloud( os.path.join(output_dir, 'cloud_bin_%d.ply' % frag_counter), pc): print( "Successfully written fused point cloud #%d for scene %s" % (frag_counter, scene_name)) # update counter frag_counter += 1 head = tail tail += nframes return None
def generate(filename, api="", kernel_path="", script_name=None, line_length=False, distributed_memory=DISTRIBUTED_MEMORY): '''Takes a GungHo algorithm specification as input and outputs the associated generated algorithm and psy codes suitable for compiling with the specified kernel(s) and GungHo infrastructure. Uses the :func:`parse.parse` function to parse the algorithm specification, the :class:`psyGen.PSy` class to generate the PSy code and the :class:`algGen.Alg` class to generate the modified algorithm code. :param str filename: The file containing the algorithm specification. :param str kernel_path: The directory from which to recursively search for the files containing the kernel source (if different from the location of the algorithm specification) :param str script_name: A script file that can apply optimisations to the PSy layer (can be a path to a file or a filename that relies on the PYTHONPATH to find the module). :param bool line_length: A logical flag specifying whether we care about line lengths being longer than 132 characters. If so, the input (algorithm and kernel) code is checked to make sure that it conforms. The default is False. :param bool distributed_memory: A logical flag specifying whether to generate distributed memory code. The default is set in the config.py file. :return: The algorithm code and the psy code. :rtype: ast :raises IOError: if the filename or search path do not exist For example: >>> from generator import generate >>> alg, psy = generate("algspec.f90") >>> alg, psy = generate("algspec.f90", kernel_path="src/kernels") >>> alg, psy = generate("algspec.f90", script_name="optimise.py") >>> alg, psy = generate("algspec.f90", line_length=True) >>> alg, psy = generate("algspec.f90", distributed_memory=False) ''' if api == "": api = DEFAULTAPI else: if api not in SUPPORTEDAPIS: raise GenerationError( "generate: Unsupported API '{0}' specified. Supported " "types are {1}.".format(api, SUPPORTEDAPIS)) if not os.path.isfile(filename): raise IOError("file '{0}' not found".format(filename)) if (len(kernel_path) > 0) and (not os.access(kernel_path, os.R_OK)): raise IOError("kernel search path '{0}' not found".format(kernel_path)) try: from algGen import Alg ast, invoke_info = parse(filename, api=api, invoke_name="invoke", kernel_path=kernel_path, line_length=line_length) psy = PSyFactory(api, distributed_memory=distributed_memory).\ create(invoke_info) if script_name is not None: sys_path_appended = False try: # a script has been provided filepath, filename = os.path.split(script_name) if filepath: # a path to a file has been provided # we need to check the file exists if not os.path.isfile(script_name): raise IOError( "script file '{0}' not found".format(script_name)) # it exists so we need to add the path to the python # search path sys_path_appended = True sys.path.append(filepath) filename, fileext = os.path.splitext(filename) if fileext != '.py': raise GenerationError( "generator: expected the script file '{0}' to have " "the '.py' extension".format(filename)) try: transmod = __import__(filename) except ImportError: raise GenerationError( "generator: attempted to import '{0}' but script file " "'{1}' has not been found".format( filename, script_name)) except SyntaxError: raise GenerationError( "generator: attempted to import '{0}' but script file " "'{1}' is not valid python".format( filename, script_name)) if callable(getattr(transmod, 'trans', None)): try: psy = transmod.trans(psy) except Exception: exc_type, exc_value, exc_traceback = sys.exc_info() lines = traceback.format_exception( exc_type, exc_value, exc_traceback) e_str = '{\n' +\ ''.join(' ' + line for line in lines[2:]) + '}' raise GenerationError( "Generator: script file '{0}'\nraised the " "following exception during execution " "...\n{1}\nPlease check your script".format( script_name, e_str)) else: raise GenerationError( "generator: attempted to import '{0}' but script file " "'{1}' does not contain a 'trans()' function".format( filename, script_name)) except Exception as msg: if sys_path_appended: os.sys.path.pop() raise msg if sys_path_appended: os.sys.path.pop() alg = Alg(ast, psy) except Exception: raise return alg.gen, psy.gen
#!/usr/bin/python """ Purpose: pip install parse """ import parse result = parse.parse("{greeting}, the time is {now:tt}", "Hello, the time is 6:30 PM") print(result) print(result.named["greeting"])
def _demap_project(item_id): result = parse.parse('P{configid}-{pid}', item_id) return int(result['pid'])
def load(path): @with_pattern(r"[a-z-]+") # match letters and dashes def _name_pattern(text): return text[:-1] # remove trailing dash return [parse("{name:name}{id:d}[{hash:l}]", line, dict(name=_name_pattern)) for line in open(path).readlines()]
from sys import argv, exit import re from parse import parse if len(argv) != 3: print('usage: {} <reference.xml> <out-txt-file>'.format(argv[0])) exit(1) plugin = parse(argv[1]) functions = [] variables = [] with open(argv[2], 'w') as ftxt: for cmd in plugin.commands: if plugin.short_name: func = 'sim{}.{}'.format(plugin.short_name, cmd.name) else: func = '{}{}'.format(plugin.command_prefix, cmd.name) functions.append(func) for enum in plugin.enums: for item in enum.items: if plugin.short_name: prefix = 'sim{}.{}.'.format(plugin.short_name, enum.name) else: prefix = enum.item_prefix variables.append(prefix + item) ftxt.write('{}\n\n{}\n'.format(' '.join(functions), ' '.join(variables)))
def train(self, data): """Training function. Parameters ---------- data_tr : tuple Training data. data_va : tuple Validation data. x_va : ndarray Validation data. y_va : ndarray Validation labels. """ print("Initializing...") self.sess.run(tf.global_variables_initializer()) # ---------------------------------------- # Resume data if it already exists latest_checkpoint = tf.train.latest_checkpoint( self.res_dir_tr) b_resume = latest_checkpoint is not None if b_resume: # Restore network print("Restoring from {}...".format( self.res_dir_tr)) self.saver_cur.restore( self.sess, latest_checkpoint ) # restore number of steps so far step = self.sess.run(self.global_step) # restore best validation result if os.path.exists(self.va_res_file): with open(self.va_res_file, "r") as ifp: dump_res = ifp.read() dump_res = parse( "{best_va_res:e}\n", dump_res) best_va_res = dump_res["best_va_res"] if os.path.exists(self.va_res_file_ours_ransac): with open(self.va_res_file_ours_ransac, "r") as ifp: dump_res = ifp.read() dump_res = parse( "{best_va_res:e}\n", dump_res) best_va_res_ours_ransac = dump_res["best_va_res"] else: print("Starting from scratch...") step = 0 best_va_res = -1 best_va_res_ours_ransac = -1 # ---------------------------------------- if self.config.data_name.startswith("oan"): data_loader = iter(data["train"]) else: # Unpack some data for simple coding xs_tr = data["train"]["xs"] ys_tr = data["train"]["ys"] Rs_tr = data["train"]["Rs"] ts_tr = data["train"]["ts"] T1s_tr = data["train"]["T1s"] T2s_tr = data["train"]["T2s"] K1s_tr = data["train"]["K1s"] K2s_tr = data["train"]["K2s"] # ---------------------------------------- # The training loop batch_size = self.config.train_batch_size max_iter = self.config.train_iter for step in trange(step, max_iter, ncols=self.config.tqdm_width): # ---------------------------------------- # Batch construction # Get a random training batch if self.config.data_name.startswith("oan"): try: data_dict = next(data_loader) except StopIteration: data_loader = iter(data["train"]) data_dict = next(data_loader) xs_b = data_dict["xs"] ys_b = data_dict["ys"] Rs_b = data_dict["Rs"].reshape(-1, 9) ts_b = data_dict["ts"].reshape(-1, 3) T1s_b = data_dict["T1s"] T2s_b = data_dict["T2s"] K1s_b = data_dict["K1s"] K2s_b = data_dict["K2s"] else: ind_cur = np.random.choice( len(xs_tr), batch_size, replace=False) # Use minimum kp in batch to construct the batch numkps = np.array([xs_tr[_i].shape[1] for _i in ind_cur]) cur_num_kp = numkps.min() # Actual construction of the batch xs_b = np.array( [xs_tr[_i][:, :cur_num_kp, :] for _i in ind_cur] ).reshape(batch_size, 1, cur_num_kp, 4) ys_b = np.array( [ys_tr[_i][:cur_num_kp, :] for _i in ind_cur] ).reshape(batch_size, cur_num_kp, 2) Rs_b = np.array( [Rs_tr[_i] for _i in ind_cur] ).reshape(batch_size, 9) ts_b = np.array( [ts_tr[_i] for _i in ind_cur] ).reshape(batch_size, 3) if self.config.use_fundamental > 0: T1s_b = np.array( [T1s_tr[_i] for _i in ind_cur]) T2s_b = np.array( [T2s_tr[_i] for _i in ind_cur]) K1s_b = np.array( [K1s_tr[_i] for _i in ind_cur]) K2s_b = np.array( [K2s_tr[_i] for _i in ind_cur]) # ---------------------------------------- # Train # Feed Dict feed_dict = { self.x_in: xs_b, self.y_in: ys_b, self.R_in: Rs_b, self.t_in: ts_b, self.is_training: True, } # add use_fundamental if self.config.use_fundamental > 0: feed_dict[self.T1_in] = T1s_b feed_dict[self.T2_in] = T2s_b feed_dict[self.K1_in] = K1s_b feed_dict[self.K2_in] = K2s_b # Fetch fetch = { "optim": self.optim, "loss": self.loss, "precision": self.precision, "recall": self.recall, } # Check if we want to write summary and check validation b_write_summary = ((step + 1) % self.config.report_intv) == 0 b_validate = ((step + 1) % self.config.val_intv) == 0 if b_write_summary or b_validate: fetch["summary"] = self.summary_op fetch["global_step"] = self.global_step # Run optimization # res = self.sess.run(fetch, feed_dict=feed_dict) try: res = self.sess.run(fetch, feed_dict=feed_dict) except (ValueError, tf.errors.InvalidArgumentError): print("Backward pass had numerical errors. " "This training batch is skipped!") continue # Write summary and save current model if b_write_summary: self.summary_tr.add_summary( res["summary"], global_step=res["global_step"]) self.saver_cur.save( self.sess, self.save_file_cur, global_step=self.global_step, write_meta_graph=False) # ---------------------------------------- # Validation if b_validate: va_res = 0 cur_global_step = res["global_step"] score = self.last_logit # defaul score: local attention if self.config.weight_opt == "sigmoid_softmax": score = [self.last_logit, self.logit_softmax, self.last_weights] test_process_ins = [self.x_in, self.y_in, self.R_in, self.t_in, self.is_training] if self.config.use_fundamental > 0: test_process_ins += [self.T1_in, self.T2_in, self.K1_in, self.K2_in] va_res, va_res_ours_ransac = test_process( "valid", self.sess, cur_global_step, self.summary_op, self.summary_va, test_process_ins, None, None, None, self.logits, self.e_hat, self.loss, self.precision, self.recall, self.last_e_hat, score, self.last_x_in, data["valid"], self.res_dir_va, self.config, True) # Higher the better if va_res > best_va_res: print( "Saving best model with va_res = {}".format( va_res)) best_va_res = va_res # Save best validation result with open(self.va_res_file, "w") as ofp: ofp.write("{:e}\n".format(best_va_res)) # Save best model self.saver_best.save( self.sess, self.save_file_best, write_meta_graph=False, ) if va_res_ours_ransac > best_va_res_ours_ransac: print( "Saving best model with va_res_ours_ransac = {}".format( va_res_ours_ransac)) best_va_res_ours_ransac = va_res_ours_ransac # Save best validation result with open(self.va_res_file_ours_ransac, "w") as ofp: ofp.write("{:e}\n".format(best_va_res_ours_ransac)) # Save best model self.saver_best.save( self.sess, self.save_file_best_ours_ransac, write_meta_graph=False, )
# start 8:23, 1. 8:45, 2. 8:55 from collections import defaultdict from pathlib import Path from parse import parse all_ingredients = set() all_allergens = set() ingredients_list = [] with Path('input.txt').open() as file: for line in file: results = parse('{ingredients} (contains {allergens})', line.strip()) allergens = set(results['allergens'].split(', ')) ingredients = set(results['ingredients'].split()) ingredients_list.append((ingredients, allergens)) all_allergens |= allergens all_ingredients |= ingredients possible_mapping = {ing: set(all_allergens) for ing in all_ingredients} for ings, als in ingredients_list: for ing in all_ingredients - ings: possible_mapping[ing] -= als good_ingredients = {ing for ing, als in possible_mapping.items() if len(als) == 0} print(sum(1 for ings, _ in ingredients_list for ing in ings if ing in good_ingredients)) bad_ingredients = {ing: als for ing, als in possible_mapping.items() if als} resolved = {}
def find_handler(self, request_path): for path, handler in self.routes.items(): parse_result = parse.parse(path, request_path) if parse_result is not None: return handler, parse_result.named return None, None
from parse import parse LOG = '[2018-05-06T12:58:00.714611] - SALE - PRODUCT: 1345 - PRICE: $09.99' FORMAT = '[{date}] - SALE - PRODUCT: {product} - PRICE: ${price}' # Run parse and check the results: result = parse(FORMAT, LOG) # OUTPUT # <Result () {'date': '2018-05-06T12:58:00.714611', 'product': '1345', 'price': '09.99'}> # The results are all strings. Define the types to be parsed: FORMAT = '[{date:ti}] - SALE - PRODUCT: {product:d} - PRICE: ${price:05.2f}' result = parse(FORMAT, LOG) # OUTPUT # <Result () {'date': datetime.datetime(2018, 5, 6, 12, 58, 0, 714611), 'product': 1345, 'price': 9.99}> # Define a custom type for the price to avoid issues with the float type: from decimal import Decimal def price(string): return Decimal(string) FORMAT = '[{date:ti}] - SALE - PRODUCT: {product:d} - PRICE: ${price:price}' result = parse(FORMAT, LOG, {'price': price}) print(result) # <Result () {'date': datetime.datetime(2018, 5, 6, 12, 58, 0, 714611), 'product': 1345, 'price': Decimal('9.99')}> print(result['date']) # output : 2018-05-06 12:58:00.714611
def parse_line(self, line, i, bpm): converted_notes = [] merge = False current_bar_length = 0 bar_start = False for i, note in enumerate(line): if note == 'BARSTART\n': current_bar_length = 0 if bar_start: raise SheetParserError('LINE {} : {}'.format(i, note), 'Bar error.') bar_start = True continue if note == 'BAREND\n': if not bar_start: raise SheetParserError('LINE {} : {}'.format(i, note), 'Bar error.') bar_start = False left_length = self.length_of_bar - current_bar_length if left_length > 0: converted_notes.append(('rest', left_length, 5)) continue elif left_length == 0: continue else: raise SheetParserError( 'LINE {} : {}'.format(i, note), 'Bar length exceeds. {:d} < {:d}'.format( self.length_of_bar, current_bar_length)) # Parse note p = parse('({}, {:d}, {:d})', note) if p is None: p = parse('({}, {:d})', note) if p is None: raise SheetParserError('LINE {} : {}'.format(i, note), 'Note format error.') else: tone, beat = p volume = self.DEFAULT_VOLUME else: tone, beat, volume = p # If note need to be merged with previous note, if merge: tone = self.tone_checker(tone, i) prev_tone = converted_notes[-1][0] if tone != prev_tone: raise SheetParserError( 'LINE {} : {}'.format(i, note), 'Different tones can\'t be merged.') beat_ms = self.beat_checker(beat, bpm, i) prev_note = converted_notes[-1] converted_notes[-1] = (prev_note[0], prev_note[1] + beat_ms, prev_note[2]) merge = False current_bar_length += beat_ms # If note is 'rest' note, elif tone == 'rest': beat_ms = self.beat_checker(beat, bpm, i) converted_notes.append((tone, beat_ms, volume)) current_bar_length += beat_ms # If note is merge, set merge flag & continue elif tone == 'MERGE': merge = True continue elif tone == 'ITER': notesnum, iternum = beat, volume iter_notes = converted_notes[-notesnum:] # Calculate iterlen iterlen = 0 for note in iter_notes: iterlen += note[1] for i in range(iternum - 1): iter_notes.extend(iter_notes[0:notesnum]) converted_notes = converted_notes[:-notesnum] + iter_notes current_bar_length += iterlen * (iternum - 1) # O.w. else: beat_ms = self.beat_checker(beat, bpm, i) tone = self.tone_checker(tone, i) converted_notes.append((tone, beat_ms, volume)) current_bar_length += beat_ms return converted_notes
def main(): data = p.parse(p.MY_FILE, ",") return create_map(data)
def convert_from_v1(hub_dict, resolution=128): weightname_dict = { 'weight_u': 'u0', 'weight_bar': 'weight', 'bias': 'bias' } convnum_dict = {'conv0': 'conv1', 'conv1': 'conv2', 'conv_sc': 'conv_sc'} attention_blocknum = {128: 3, 256: 4, 512: 3}[resolution] hub2me = { 'linear.weight': 'shared.weight', # This is actually the shared weight # Linear stuff 'G_linear.module.weight_bar': 'linear.weight', 'G_linear.module.bias': 'linear.bias', 'G_linear.module.weight_u': 'linear.u0', # output layer stuff 'ScaledCrossReplicaBN.weight': 'output_layer.0.gain', 'ScaledCrossReplicaBN.bias': 'output_layer.0.bias', 'ScaledCrossReplicaBN.running_mean': 'output_layer.0.stored_mean', 'ScaledCrossReplicaBN.running_var': 'output_layer.0.stored_var', 'colorize.module.weight_bar': 'output_layer.2.weight', 'colorize.module.bias': 'output_layer.2.bias', 'colorize.module.weight_u': 'output_layer.2.u0', # Attention stuff 'attention.gamma': 'blocks.%d.1.gamma' % attention_blocknum, 'attention.theta.module.weight_u': 'blocks.%d.1.theta.u0' % attention_blocknum, 'attention.theta.module.weight_bar': 'blocks.%d.1.theta.weight' % attention_blocknum, 'attention.phi.module.weight_u': 'blocks.%d.1.phi.u0' % attention_blocknum, 'attention.phi.module.weight_bar': 'blocks.%d.1.phi.weight' % attention_blocknum, 'attention.g.module.weight_u': 'blocks.%d.1.g.u0' % attention_blocknum, 'attention.g.module.weight_bar': 'blocks.%d.1.g.weight' % attention_blocknum, 'attention.o_conv.module.weight_u': 'blocks.%d.1.o.u0' % attention_blocknum, 'attention.o_conv.module.weight_bar': 'blocks.%d.1.o.weight' % attention_blocknum, } # Loop over the hub dict and build the hub2me map for name in hub_dict.keys(): if 'GBlock' in name: if 'HyperBN' not in name: # it's a conv out = parse.parse('GBlock.{:d}.{}.module.{}', name) blocknum, convnum, weightname = out if weightname not in weightname_dict: continue # else hyperBN in out_name = 'blocks.%d.0.%s.%s' % ( blocknum, convnum_dict[convnum], weightname_dict[weightname]) # Increment conv number by 1 else: # hyperbn not conv BNnum = 2 if 'HyperBN_1' in name else 1 if 'embed' in name: out = parse.parse('GBlock.{:d}.{}.module.{}', name) blocknum, gamma_or_beta, weightname = out if weightname not in weightname_dict: # Ignore weight_v continue out_name = 'blocks.%d.0.bn%d.%s.%s' % ( blocknum, BNnum, 'gain' if 'gamma' in gamma_or_beta else 'bias', weightname_dict[weightname]) else: out = parse.parse('GBlock.{:d}.{}.bn.{}', name) blocknum, dummy, mean_or_var = out if 'num_batches_tracked' in mean_or_var: continue out_name = 'blocks.%d.0.bn%d.%s' % ( blocknum, BNnum, 'stored_mean' if 'mean' in mean_or_var else 'stored_var') hub2me[name] = out_name # Invert the hub2me map me2hub = {hub2me[item]: item for item in hub2me} new_dict = {} dimz_dict = {128: 20, 256: 20, 512: 16} for item in me2hub: # Swap input dim ordering on batchnorm bois to account for my arbitrary change of ordering when concatenating Ys and Zs if ('bn' in item and 'weight' in item) and ( 'gain' in item or 'bias' in item) and ('output_layer' not in item): new_dict[item] = torch.cat([ hub_dict[me2hub[item]][:, -128:], hub_dict[me2hub[item]][:, :dimz_dict[resolution]] ], 1) # Reshape the first linear weight, bias, and u0 elif item == 'linear.weight': new_dict[item] = hub_dict[me2hub[item]].contiguous().view( 4, 4, 96 * 16, -1).permute(2, 0, 1, 3).contiguous().view(-1, dimz_dict[resolution]) elif item == 'linear.bias': new_dict[item] = hub_dict[me2hub[item]].view( 4, 4, 96 * 16).permute(2, 0, 1).contiguous().view(-1) elif item == 'linear.u0': new_dict[item] = hub_dict[me2hub[item]].view( 4, 4, 96 * 16).permute(2, 0, 1).contiguous().view(1, -1) elif me2hub[ item] == 'linear.weight': # THIS IS THE SHARED WEIGHT NOT THE FIRST LINEAR LAYER # Transpose shared weight so that it's an embedding new_dict[item] = hub_dict[me2hub[item]].t() elif 'weight_u' in me2hub[item]: # Unsqueeze u0s new_dict[item] = hub_dict[me2hub[item]].unsqueeze(0) else: new_dict[item] = hub_dict[me2hub[item]] return new_dict
root = None for token in doc: if token.dep_ == 'ROOT': root = token if root == None: break summary_eres.append(dep_parse(root)) return summary_eres if __name__ == '__main__': import sys import parse data = parse.parse() if len(sys.argv) != 2: print( "Please run command with desired conversation/summary ID, EX: 'py summ_depparse.py 0'" ) elif not int(sys.argv[1]) in range(0, 45): print("Error with given argument") else: id = int(sys.argv[1]) for summ in data[id].summaries: print("Summary:") print(summ) eres = parse_summaries(summ) print(eres) print("\n")
else: try: A=int(columns[3].text) Z=int(columns[0].text) except ValueError: continue abundance=columns[4].text.strip() # print("This is not a child") abundance=abundance.replace(" ", ""); abundance=abundance.replace(" ", ""); # if abundance[0]=='[': # p=parse("[{:f},{:f}]", abundance) # if not p: # continue # abundance_low=p[0] # abundance_high=p[1] # print(f"got low {abundance_low:f} and high {abundance_high:f}") p=parse("{:g}", abundance) if p: abundance_single=p[0] p=parse("{:f}({d})", abundance) if p: abundance_single=p[0] p=parse("[{:f},{:f}]", abundance) if p: abundance_low=p[0] abundance_high=p[1] abundance_single=(p[0]+p[1])/2 #TODO: not the greatest choice? print(f"{Z:3d} {A:3d} {abundance_single:5f} {abundance_low:5f} {abundance_high:5f}")
return blocks # read a file, replace stuff, write it out # only overwrite files if the contents will change def make_file(f_in, f_out, **args): f_tmp = '%s.swp' % f_out t = pyratemp.Template(filename=f_in) with open(f_tmp, 'w') as out: out.write(t(**args).encode()) if os.path.exists(f_out) and filecmp.cmp(f_tmp, f_out): # Nothing changed, don't touch the file. os.remove(f_tmp) return shutil.move(f_tmp, f_out) devs = [] for config_file in CONFIGS: with open(config_file, 'r') as f: dev = parse.parse(f) cpp_name = 'devices/%s.cpp' % dev.name h_name = 'devices/%s.h' % dev.name make_file('templates/dev.cpp', cpp_name, dev=dev, blocks=make_blocks(dev)) make_file('templates/dev.h', h_name, dev=dev, blocks=make_blocks(dev)) devs.append(dev) make_file('templates/devices.cpp', 'devices/devices.cpp', devs=devs)
def identify(self, filepath, url=None): """ Identifies a file of the layer :param filepath: filepath from AMQP :param url: fully qualified URL of file :returns: `list` of file properties """ super().identify(filepath, url) self.model = 'radar' LOGGER.debug('Loading model information from store') self.file_dict = json.loads(self.store.get_key(self.model)) filename_pattern = self.file_dict[self.model]['filename_pattern'] tmp = parse(filename_pattern, os.path.basename(filepath)) file_pattern_info = { 'wx_variable': tmp.named['precipitation_type'], 'time_': tmp.named['YYYYMMDDhhmm'], } LOGGER.debug('Defining the different file properties') self.wx_variable = file_pattern_info['wx_variable'] if self.wx_variable not in self.file_dict[self.model]['variable']: msg = 'Variable "{}" not in ' 'configuration file'.format( self.wx_variable ) LOGGER.warning(msg) return False time_format = '%Y%m%d%H%M' self.date_ = datetime.strptime(file_pattern_info['time_'], time_format) layer_config = self.file_dict[self.model]['variable'][self.wx_variable] layer_name = layer_config['geomet_layers'] member = self.file_dict[self.model]['variable'][self.wx_variable][ 'members' ] elevation = self.file_dict[self.model]['variable'][self.wx_variable][ 'elevation' ] str_fh = re.sub('[^0-9]', '', self.date_.strftime(DATE_FORMAT)) identifier = '{}-{}'.format(layer_name, str_fh) date_format = DATE_FORMAT feature_dict = { 'layer_name': layer_name, 'layer_config': layer_config, 'filepath': self.filepath, 'identifier': identifier, 'reference_datetime': None, 'forecast_hour_datetime': self.date_.strftime(date_format), 'member': member, 'model': self.model, 'elevation': elevation, 'expected_count': None, 'layer_config': layer_config, 'register_status': True, 'refresh_config': True, } self.items.append(feature_dict) return True
attributes = [{'handicapped-infants': 'n', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': 'y', 'education-spending': 'y', 'duty-free-exports': 'n', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'y', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': '?', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'n', 'Class': 'republican'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': '?', 'superfund-right-to-sue': 'y', 'education-spending': 'y', 'duty-free-exports': 'n', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'n', 'Class': 'republican'}, {'handicapped-infants': '?', 'export-administration-act-south-africa': 'n', 'superfund-right-to-sue': 'y', 'education-spending': 'n', 'duty-free-exports': 'n', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': '?', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'y', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': 'y', 'education-spending': 'n', 'duty-free-exports': 'n', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'n', 'el-salvador-aid': '?', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'y', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'n', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'y', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': 'y', 'education-spending': '?', 'duty-free-exports': 'y', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'n', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'y', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': 'y', 'education-spending': 'n', 'duty-free-exports': 'y', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'n', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': '?', 'education-spending': 'n', 'duty-free-exports': 'y', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': 'y', 'education-spending': 'n', 'duty-free-exports': '?', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'n', 'Class': 'republican'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': 'y', 'superfund-right-to-sue': 'y', 'education-spending': 'y', 'duty-free-exports': 'n', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'n', 'Class': 'republican'}, {'handicapped-infants': 'y', 'export-administration-act-south-africa': '?', 'superfund-right-to-sue': 'n', 'education-spending': 'n', 'duty-free-exports': '?', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'n', 'el-salvador-aid': 'n', 'religious-groups-in-schools': 'n', 'mx-missile': 'y', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'y', 'water-project-cost-sharing': 'y', 'crime': 'n', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': 'n', 'superfund-right-to-sue': 'y', 'education-spending': '?', 'duty-free-exports': 'n', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'n', 'mx-missile': 'n', 'synfuels-corporation-cutback': '?', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'n', 'Class': 'republican'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': '?', 'superfund-right-to-sue': 'y', 'education-spending': '?', 'duty-free-exports': '?', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'y', 'el-salvador-aid': 'y', 'religious-groups-in-schools': 'y', 'mx-missile': 'n', 'synfuels-corporation-cutback': 'y', 'anti-satellite-test-ban': 'n', 'water-project-cost-sharing': 'y', 'crime': 'y', 'adoption-of-the-budget-resolution': 'n', 'Class': 'republican'}, {'handicapped-infants': 'n', 'export-administration-act-south-africa': '?', 'superfund-right-to-sue': 'y', 'education-spending': 'n', 'duty-free-exports': '?', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'n', 'physician-fee-freeze': 'n', 'el-salvador-aid': 'n', 'religious-groups-in-schools': 'n', 'mx-missile': 'y', 'synfuels-corporation-cutback': 'n', 'anti-satellite-test-ban': 'y', 'water-project-cost-sharing': 'y', 'crime': 'n', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}, {'handicapped-infants': 'y', 'export-administration-act-south-africa': '?', 'superfund-right-to-sue': 'n', 'education-spending': '?', 'duty-free-exports': 'y', 'aid-to-nicaraguan-contras': 'n', 'immigration': 'y', 'physician-fee-freeze': 'n', 'el-salvador-aid': 'n', 'religious-groups-in-schools': 'y', 'mx-missile': '?', 'synfuels-corporation-cutback': 'y', 'anti-satellite-test-ban': 'y', 'water-project-cost-sharing': 'y', 'crime': 'n', 'adoption-of-the-budget-resolution': 'y', 'Class': 'democrat'}] assert(ID3.findBestAttribute(attributes) == 'adoption-of-the-budget-resolution') print "Passed test: find_best_attribute_test()\n" data = parse('house_votes_84.data') find_best_attribute_test() a = [1,2,3,4,5,6,7] random.shuffle(a) print a print a[1] random.shuffle(a) print a print a[1] a[1]
def _devcheck_manage_snapshots(workdir, recent=5, factor=10, dry=True): """ Sometimes netharn produces too many snapshots. The Monitor class attempts to prevent this, but its not perfect. So, sometimes you need to manually clean up. This code snippet serves as a template for doing so. I recommend using IPython to do this following this code as a guide. Unfortunately, I don't have a safe automated way of doing this yet. The basic code simply lists all snapshots that you have. Its then your job to find a huerstic to remove the ones you don't need. Note: # Idea for more automatic method In the future, we should use monitor to inspect the critical points of all metric curves and include any epoch that is at those cricial points. A cricial point is defined as one where there is a significant change in trajectory. Basically, we try to fit a low-degree polynomial or piecewise linear function to the metric curves, and we take the places where there is a significant change from a global perspective. # Specify your workdir workdir = ub.truepath('~/work/voc_yolo2') """ USE_RANGE_HUERISTIC = True run_dpath = join(workdir, 'fit', 'runs') snapshot_dpaths = list(glob.glob(join(run_dpath, '**/torch_snapshots'))) all_keep = [] all_remove = [] for snapshot_dpath in snapshot_dpaths: snapshots = sorted(glob.glob(join(snapshot_dpath, '_epoch_*.pt'))) epoch_to_snap = { int(parse.parse('{}_epoch_{num:d}.pt', path).named['num']): path for path in snapshots } existing_epochs = sorted(epoch_to_snap.keys()) # print('existing_epochs = {}'.format(ub.repr2(existing_epochs))) toremove = [] tokeep = [] if USE_RANGE_HUERISTIC: # My Critieron is that I'm only going to keep the two latest and # I'll also keep an epoch in the range [0,50], [50,100], and # [100,150], and so on. existing_epochs = sorted(existing_epochs) dups = ub.find_duplicates(np.array(sorted(existing_epochs)) // factor, k=0) keep_idxs = [max(idxs) for _, idxs in dups.items()] keep = set(ub.take(existing_epochs, keep_idxs)) keep.update(existing_epochs[-recent:]) if existing_epochs and existing_epochs[0] != 0: keep.update(existing_epochs[0:1]) print('keep = {!r}'.format(sorted(keep))) for epoch, path in epoch_to_snap.items(): if epoch in keep: tokeep.append(path) else: toremove.append(path) print('Keep {}/{} from {}'.format(len(keep), len(existing_epochs), snapshot_dpath)) all_keep += [tokeep] all_remove += [toremove] # print('all_keep = {}'.format(ub.repr2(all_keep, nl=2))) # print('all_remove = {}'.format(ub.repr2(all_remove, nl=2))) """ pip install send2trash import send2trash send2trash.send2trash(path) """ total = 0 for path in ub.flatten(all_remove): total += os.path.getsize(path) total_mb = total / 2 ** 20 if dry: print('Cleanup would free {!r} MB'.format(total_mb)) print('Use -f to confirm and force cleanup') else: print('About to free {!r} MB'.format(total_mb)) for path in ub.flatten(all_remove): ub.delete(path, verbose=True)
input = open("input.txt").readlines() from parse import parse fmt = "position=<{x:6d}, {y:6d}> velocity=<{vx:3d}, {vy:2d}>" pts = [parse(fmt, l) for l in input] x = [pt["x"] for pt in pts] y = [pt["y"] for pt in pts] vx = [pt["vx"] for pt in pts] vy = [pt["vy"] for pt in pts] print(x[0], y[0], vx[0], vy[0]) num = 0 den = 0 for i in range(len(pts)): for j in range(len(pts)): if i == j: continue num += x[i] * (vx[i] - vx[j]) + x[j] * (vx[j] - vx[i]) + y[i] * ( vy[i] - vy[j]) + y[j] * (vy[j] - vy[i]) den += vx[i]**2 + vx[j]**2 - 2 * vx[i] * vx[j] + vy[i]**2 + vy[ j]**2 - 2 * vy[i] * vy[j] t_min = round(-num / den) print(t_min) result = [[x[i] + vx[i] * t_min, y[i] + vy[i] * t_min] for i in range(len(x))] print(result[0])
tn.write(passwd.encode('ascii') + b"\n") tn.write(b"knvram -a |grep system/admin\n") tn.write(b"uptime\n") tn.write(b"killall telnetd\n") tn.write(b"exit\n") response = tn.read_all() #print(response) token = str(response).split('\\r\\n') #print (token) for i in range(len(token)): res = parse.parse("system/adminid={}", token[i]) if (res): print("adminid:" + res[0]) res = parse.parse("system/adminpw={}", token[i]) if (res): print("adminpw:" + res[0]) res = parse.parse("{} up {} load average: {}", token[i]) if (res): print("up:" + res[1]) #tn.write(b"exit\n") #print (response) #print(tn.read_all().decode("ascii"))
from aocd import data from parse import parse a = 16807 b = 48271 d = 2147483647 parsed = parse('Generator A starts with {a0:d}\nGenerator B starts with {b0:d}', data) a0 = parsed.named['a0'] b0 = parsed.named['b0'] def gen(m, x0): x = x0 while True: x = (x * m) % d yield x def gen2(m, x0, d): for x in gen(m, x0): if x % d == 0: yield x def score(gena, genb, n=40000000): x = 0 for i in range(n): a, b = next(gena), next(genb) x += a & 0xffff == b & 0xffff return x assert score(gen(a, 65), gen(b, 8921)) == 588 assert score(gen2(a, 65, 4), gen2(b, 8921, 8), n=5000000) == 309
import os import json import parse text = requests.get( "http://mountainview.gov/depts/comdev/planning/activeprojects/list.asp" ).text links = re.findall( "http://www.mountainview.gov/civicax/filebank/blobdload.aspx\?BlobID=\d+", text) os.system("mkdir -p data") all_projects = {} for l in [links[-1]]: print("Downloading: " + l) pdf_file = "data/" + re.search("BlobID=([\d]+)", l).group(1) + ".pdf" os.system("curl -L " + l + " -o " + pdf_file) os.system("pdftotext -layout " + pdf_file) txt_file = pdf_file.replace(".pdf", ".txt") # Clean bad UTF-8 characters os.system("iconv -f utf-8 -t utf-8 -c < " + txt_file + " > tmp.txt") os.system("mv tmp.txt " + txt_file) parsed = parse.parse(txt_file, l) for p in parsed: all_projects[p["title"]] = p with open("data/projects.json", "w") as f: json.dump(all_projects.values(), f, sort_keys=True, indent=4)
def __init__(self, id=None, session=None, _adaptor=None, _adaptor_state={}, _ttype=None): """ __init__(id=None, session=None) Create / reconnect to a resource. :param id: id of the resource :type id: :class:`saga.Url` :param session: :class:`saga.Session` Resource class instances are usually created by calling :func:`acquire` on the :class:`saga.resource.Manager` class. Already acquired resources are identified by a string typed identifier. This constructor accepts such an identifier to create another representation of the same resource. As the resource itself is new newly acquired, it can be in any state. In particular, it can be in a final state, and thus be unusable. Further, the resource may already have expired or failed, and the information about it may have been purged -- in that case the id will not be valid any longer, and a :class:`saga.BadParameter` exception will be raised. The session parameter is interpreted exactly as the session parameter on the :class:`saga.resource.Manager` constructor. """ # set attribute interface properties import radical.saga.attributes as sa self._attributes_extensible(False) self._attributes_camelcasing(True) # register properties with the attribute interface self._attributes_register(c.ID, None, sa.ENUM, sa.SCALAR, sa.READONLY) self._attributes_register(c.RTYPE, None, sa.ENUM, sa.SCALAR, sa.READONLY) self._attributes_register(c.STATE, None, sa.ENUM, sa.SCALAR, sa.READONLY) self._attributes_register(c.STATE_DETAIL, None, sa.STRING, sa.SCALAR, sa.READONLY) self._attributes_register(c.ACCESS, None, sa.URL, sa.SCALAR, sa.READONLY) self._attributes_register(c.MANAGER, None, sa.URL, sa.SCALAR, sa.READONLY) self._attributes_register(c.DESCRIPTION, None, sa.ANY, sa.SCALAR, sa.READONLY) self._attributes_set_enums(c.STATE, [ c.UNKNOWN, c.PENDING, c.ACTIVE, c.CANCELED, c.EXPIRED, c.FAILED, c.FINAL ]) self._attributes_set_enums(c.RTYPE, [c.COMPUTE, c.STORAGE, c.NETWORK]) self._attributes_set_getter(c.ID, self.get_id) self._attributes_set_getter(c.RTYPE, self.get_rtype) self._attributes_set_getter(c.STATE, self.get_state) self._attributes_set_getter(c.STATE_DETAIL, self.get_state_detail) self._attributes_set_getter(c.ACCESS, self.get_access) self._attributes_set_getter(c.MANAGER, self.get_manager) self._attributes_set_getter(c.DESCRIPTION, self.get_description) # FIXME: we need the ID to be or to include an URL, as we don't have # a scheme otherwise, which means we can't select an adaptor. Duh! :-/ # FIXME: documentation for attributes is missing. # param checks scheme = None if not id: if 'resource_schema' not in _adaptor_state: raise se.BadParameter("Cannot initialize resource without id" % self.rtype) else: scheme = _adaptor_state['resource_schema'] else: # ID is formatted as '[manager-url]-[resource-id]' import parse # FIXME: use regex to reduce number of dependencies res = parse.parse('[{}]-[{}]', id) url = ru.Url(res[0]) scheme = url.scheme.lower() if not session: session = ss.Session(default=True) self._base = super(Resource, self) self._base.__init__(scheme, _adaptor, _adaptor_state, id, session, ttype=_ttype)
def processGraph(filepath, bestPart, logFile): statsByTol = {} statsByTol["timeMean"] = [] statsByTol["timeMin"] = [] statsByTol["coarsenLevel"] = [] for tolerance in tolerances: metricsPath = "metrics/group{}.txt".format(secrets.token_urlsafe(10)) print("running sgpar on {} with tol {}, comparing to {}, data logged in {}".format(filepath, tolerance, bestPart, metricsPath)) err = os.system(sysCall.format(filepath, metricsPath, bestPart, tolerance)) if(err == 256): print("error code: {}".format(err)) print("error produced by:") print(sysCall.format(filepath, metricsPath, bestPart, tolerance)) else: totalTimes = [] coarsenTimes = [] refineTimes = [] multilevel = [] cnt = 0 with open(metricsPath) as fp: for line in fp: parsed = parse(form, line) refineData = parsed[0].split() niters = refineData[0::2] niters.reverse() maxIterReached = refineData[1::2] maxIterReached.reverse() totalTimes.append(float(parsed[1])) coarsenTimes.append(float(parsed[2])) refineTimes.append(float(parsed[3])) multilevelResults = parsed[4].split() multilevelEdgeCuts = multilevelResults[0::2] multilevelEdgeCuts.reverse() multilevelSwaps = multilevelResults[1::2] multilevelSwaps.reverse() data = zip(niters, maxIterReached, multilevelEdgeCuts, multilevelSwaps) dataCount = 0 for datum in data: dataCount += 1 if len(multilevel) < dataCount: multilevel.append(MultilevelData()) multilevel[dataCount-1].niters.append(int(datum[0])) multilevel[dataCount-1].maxIters.append(int(datum[1])) multilevel[dataCount-1].edgeCuts.append(int(datum[2])) multilevel[dataCount-1].swaps.append(int(datum[3])) cnt += 1 levelCount = 0 for level in multilevel: levelCount += 1 if len(statsByTol["coarsenLevel"]) < levelCount: levelStats = MultilevelStats() statsByTol["coarsenLevel"].append(levelStats) levelStats = statsByTol["coarsenLevel"][levelCount - 1] levelStats.refineMean.append(mean(level.niters)) levelStats.refineMin.append(min(level.niters)) if len(level.niters) > 1: levelStats.refineStdDev.append(stdev(level.niters)) else: levelStats.refineStdDev.append(0) levelStats.edgeCutMean.append(mean(level.edgeCuts)) levelStats.edgeCutMin.append(min(level.edgeCuts)) if len(level.edgeCuts) > 1: levelStats.edgeCutStdDev.append(stdev(level.edgeCuts)) else: levelStats.edgeCutStdDev.append(0) levelStats.swapsMean.append(mean(level.swaps)) levelStats.swapsMin.append(min(level.swaps)) if len(level.swaps) > 1: levelStats.swapsStdDev.append(stdev(level.swaps)) else: levelStats.swapsStdDev.append(0) levelStats.maxItersReached.append(sum(level.maxIters)) statsByTol["timeMean"].append(mean(totalTimes)) statsByTol["timeMin"].append(min(totalTimes)) output = open(logFile, "w") print("tolerances: {}".format(' '.join(tolerances)), file=output) print("mean total time: {}".format(concatStatWithSpace(statsByTol["timeMean"])), file=output) print("min total time: {}".format(concatStatWithSpace(statsByTol["timeMin"])), file=output) printStat("coarsen level {} mean refine iterations: {}", [level.refineMean for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} min refine iterations: {}", [level.refineMin for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} refine iterations std deviation: {}", [level.refineStdDev for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} times max iter reached: {}", [level.maxItersReached for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} mean edge cut: {}", [level.edgeCutMean for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} min edge cut: {}", [level.edgeCutMin for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} edge cut std deviation: {}", [level.edgeCutStdDev for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} mean swaps to best partition: {}", [level.swapsMean for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} min swaps to best partition: {}", [level.swapsMin for level in statsByTol["coarsenLevel"]], output) printStat("coarsen level {} swaps to best partition std deviation: {}", [level.swapsStdDev for level in statsByTol["coarsenLevel"]], output) print("end {} processing".format(filepath))
def run(self): while True: request = self.socket.recv_string() # download filename if request.startswith("download"): filename = str(parse.parse("download {}", request)[0]) keeper_data = self.videos.get(filename) if keeper_data is None: self.socket.send_json(None) continue keeper_ips = keeper_data[0] file_size = self.videos[filename][3] ips = [] ports = [] for ip in keeper_ips: if not self.keepers[ip][-1]: continue for port, busy in self.keepers[ip][0].items(): if not busy: ips.append(ip) ports.append(port) set_busy(self.keepers, self.lk, ip, port, True) break self.socket.send_json((ips, ports, file_size)) elif request.startswith("upload"): keepers_used_storage = {} for ip in self.keepers.keys(): if not self.keepers[ip][-1]: continue keepers_used_storage[ip] = 0 for file, data in self.videos.items(): for ip in data[0]: if not self.keepers[ip][-1]: continue keepers_used_storage[ip] += data[3] keepers_ips = [ ip for ip, used_storage in sorted( keepers_used_storage.items(), key=lambda item: item[1]) ] chosen_ip = None chosen_port = None for ip in keepers_ips: if not self.keepers[ip][-1]: continue for port, busy in self.keepers[ip][0].items(): if not busy: chosen_ip = ip chosen_port = port break if chosen_ip is not None: break if chosen_ip is not None: self.socket.send_json((chosen_ip, chosen_port)) set_busy(self.keepers, self.lk, chosen_ip, chosen_port, True) else: self.socket.send_json(None) elif request.startswith("successfully_uploaded"): parsed = parse.parse( "successfully_uploaded: {} {} {} {} {} {}", request) filename = str(parsed[0]) data_keeper_ip = parsed[1] data_keeper_port = int(parsed[2]) user_id = int(parsed[3]) file_path = str(parsed[4]) size = int(parsed[5]) add_video(self.videos, self.lv, filename, {data_keeper_ip: True}, user_id, file_path, size) set_busy(self.keepers, self.lk, data_keeper_ip, data_keeper_port, False) self.socket.send_string("OK") print('User: {} uploaded "{}" successfully to {}:{}'.format( user_id, filename, data_keeper_ip, data_keeper_port)) elif request.startswith("successfully_downloaded"): parsed = parse.parse("successfully_downloaded: {} {}", request) data_keeper_ip = parsed[0] data_keeper_port = int(parsed[1]) set_busy(self.keepers, self.lk, data_keeper_ip, data_keeper_port, False) self.socket.send_string("OK")
from parse import parse from gm_hmm.src.genHMM import GenHMMclassifier, save_model from gm_hmm.src.ref_hmm import GaussianHMMclassifier if __name__ == "__main__": usage = "Aggregate models from several classes.\n" \ "Usage: python bin/aggregate_models.py models/epoch1.mdl" if len(sys.argv) != 2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": print(usage) sys.exit(1) out_mdl_file = sys.argv[1] # Find the class digit get_sort_key = lambda x: parse("{}class{:d}.mdlc", x)[1] # find the model used, 'gen' or 'gaus' model_type = parse("{}/exp/{}/{:d}feats/{}", os.getcwd())[1] # Find all trained classes submodels in_mdlc_files = sorted(glob.glob( out_mdl_file.replace(".mdl", "_class*.mdlc")), key=get_sort_key) if model_type == 'gaus': mdl = GaussianHMMclassifier(mdlc_files=in_mdlc_files) assert (all( [int(h.iclass) == int(i) + 1 for i, h in enumerate(mdl.hmms)])) with open(out_mdl_file, "wb") as handle: pkl.dump(mdl, handle) elif model_type == 'gen':